Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RDFInterfaceUtils.cxx
Go to the documentation of this file.
1// Author: Enrico Guiraud, Danilo Piparo CERN 02/2018
2
3/*************************************************************************
4 * Copyright (C) 1995-2018, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#include <ROOT/RDataSource.hxx>
14#include <ROOT/RDF/RDisplay.hxx>
21#include <ROOT/RDF/Utils.hxx>
22#include <string_view>
23#include <TBranch.h>
24#include <TClass.h>
25#include <TClassEdit.h>
26#include <TDataType.h>
27#include <TError.h>
28#include <TLeaf.h>
29#include <TObjArray.h>
30#include <TPRegexp.h>
31#include <TROOT.h>
32#include <TString.h>
33#include <TTree.h>
34#include <TVirtualMutex.h>
35
36// pragma to disable warnings on Rcpp which have
37// so many noise compiling
38#if defined(__GNUC__)
39#pragma GCC diagnostic push
40#pragma GCC diagnostic ignored "-Woverloaded-virtual"
41#pragma GCC diagnostic ignored "-Wshadow"
42#endif
43#include "lexertk.hpp"
44#if defined(__GNUC__)
45#pragma GCC diagnostic pop
46#endif
47
48#include <algorithm>
49#include <cassert>
50#include <cstdlib> // for size_t
51#include <iterator> // for back_insert_iterator
52#include <map>
53#include <memory>
54#include <set>
55#include <sstream>
56#include <stdexcept>
57#include <string>
58#include <type_traits> // for remove_reference<>::type
59#include <typeinfo>
60#include <unordered_map>
61#include <unordered_set>
62#include <utility> // for pair
63#include <vector>
64
65namespace ROOT {
66namespace Detail {
67namespace RDF {
68class RDefineBase;
69} // namespace RDF
70namespace Internal {
71namespace RDF {
72class RJittedAction;
73}
74} // namespace Internal
75} // namespace Detail
76
77} // namespace ROOT
78
79namespace {
82
83/// A string expression such as those passed to Filter and Define, digested to a standardized form
84struct ParsedExpression {
85 /// The string expression with the dummy variable names in fVarNames in place of the original column names
86 std::string fExpr;
87 /// The list of valid column names that were used in the original string expression.
88 /// Duplicates are removed and column aliases (created with Alias calls) are resolved.
89 ColumnNames_t fUsedCols;
90 /// The list of variable names used in fExpr, with same ordering and size as fUsedCols
91 ColumnNames_t fVarNames;
92};
93
94/// Look at expression `expr` and return a pair of (column names used, aliases used)
95std::pair<ColumnNames_t, ColumnNames_t>
96FindUsedColsAndAliases(const std::string &expr, const ColumnNames_t &treeBranchNames,
97 const ROOT::Internal::RDF::RColumnRegister &colRegister, const ColumnNames_t &dataSourceColNames)
98{
99 lexertk::generator tokens;
100 const auto tokensOk = tokens.process(expr);
101 if (!tokensOk) {
102 const auto msg = "Failed to tokenize expression:\n" + expr + "\n\nMake sure it is valid C++.";
103 throw std::runtime_error(msg);
104 }
105
106 std::unordered_set<std::string> usedCols;
107 std::unordered_set<std::string> usedAliases;
108
109 // iterate over tokens in expression and fill usedCols and usedAliases
110 const auto nTokens = tokens.size();
111 const auto kSymbol = lexertk::token::e_symbol;
112 for (auto i = 0u; i < nTokens; ++i) {
113 const auto &tok = tokens[i];
114 // lexertk classifies '&' as e_symbol for some reason
115 if (tok.type != kSymbol || tok.value == "&" || tok.value == "|") {
116 // token is not a potential variable name, skip it
117 continue;
118 }
119
120 ColumnNames_t potentialColNames({tok.value});
121
122 // if token is the start of a dot chain (a.b.c...), a.b, a.b.c etc. are also potential column names
123 auto dotChainKeepsGoing = [&](unsigned int _i) {
124 return _i + 2 <= nTokens && tokens[_i + 1].value == "." && tokens[_i + 2].type == kSymbol;
125 };
126 while (dotChainKeepsGoing(i)) {
127 potentialColNames.emplace_back(potentialColNames.back() + "." + tokens[i + 2].value);
128 i += 2; // consume the tokens we looked at
129 }
130
131 // in an expression such as `a.b`, if `a` is a column alias add it to `usedAliases` and
132 // replace the alias with the real column name in `potentialColNames`.
133 const auto maybeAnAlias = potentialColNames[0]; // intentionally a copy as we'll modify potentialColNames later
134 const auto &resolvedAlias = colRegister.ResolveAlias(maybeAnAlias);
135 if (resolvedAlias != maybeAnAlias) { // this is an alias
136 usedAliases.insert(maybeAnAlias);
137 for (auto &s : potentialColNames)
138 s.replace(0, maybeAnAlias.size(), resolvedAlias);
139 }
140
141 // find the longest potential column name that is an actual column name
142 // (potential columns are sorted by length, so we search from the end to find the longest)
143 auto isRDFColumn = [&](const std::string &col) {
144 if (colRegister.IsDefineOrAlias(col) || IsStrInVec(col, treeBranchNames) ||
145 IsStrInVec(col, dataSourceColNames))
146 return true;
147 return false;
148 };
149 const auto longestRDFColMatch = std::find_if(potentialColNames.crbegin(), potentialColNames.crend(), isRDFColumn);
150 if (longestRDFColMatch != potentialColNames.crend())
151 usedCols.insert(*longestRDFColMatch);
152 }
153
154 return {{usedCols.begin(), usedCols.end()}, {usedAliases.begin(), usedAliases.end()}};
155}
156
157/// Substitute each '.' in a string with '\.'
158std::string EscapeDots(const std::string &s)
159{
160 TString out(s);
161 TPRegexp dot("\\.");
162 dot.Substitute(out, "\\.", "g");
163 return std::string(std::move(out));
164}
165
166TString ResolveAliases(const TString &expr, const ColumnNames_t &usedAliases,
167 const ROOT::Internal::RDF::RColumnRegister &colRegister)
168{
169 TString out(expr);
170
171 for (const auto &alias : usedAliases) {
172 const auto &col = colRegister.ResolveAlias(alias);
173 TPRegexp replacer("\\b" + EscapeDots(alias) + "\\b");
174 replacer.Substitute(out, col, "g");
175 }
176
177 return out;
178}
179
180ParsedExpression ParseRDFExpression(std::string_view expr, const ColumnNames_t &treeBranchNames,
181 const ROOT::Internal::RDF::RColumnRegister &colRegister,
182 const ColumnNames_t &dataSourceColNames)
183{
184 // transform `#var` into `R_rdf_sizeof_var`
185 TString preProcessedExpr(expr);
186 // match #varname at beginning of the sentence or after not-a-word, but exclude preprocessor directives like #ifdef
187 TPRegexp colSizeReplacer(
188 "(^|\\W)#(?!(ifdef|ifndef|if|else|elif|endif|pragma|define|undef|include|line))([a-zA-Z_][a-zA-Z0-9_]*)");
189 colSizeReplacer.Substitute(preProcessedExpr, "$1R_rdf_sizeof_$3", "g");
190
191 ColumnNames_t usedCols;
192 ColumnNames_t usedAliases;
193 std::tie(usedCols, usedAliases) =
194 FindUsedColsAndAliases(std::string(preProcessedExpr), treeBranchNames, colRegister, dataSourceColNames);
195
196 const auto exprNoAliases = ResolveAliases(preProcessedExpr, usedAliases, colRegister);
197
198 // when we are done, exprWithVars willl be the same as preProcessedExpr but column names will be substituted with
199 // the dummy variable names in varNames
200 TString exprWithVars(exprNoAliases);
201
202 ColumnNames_t varNames(usedCols.size());
203 for (auto i = 0u; i < varNames.size(); ++i)
204 varNames[i] = "var" + std::to_string(i);
205
206 // sort the vector usedColsAndAliases by decreasing length of its elements,
207 // so in case of friends we guarantee we never substitute a column name with another column containing it
208 // ex. without sorting when passing "x" and "fr.x", the replacer would output "var0" and "fr.var0",
209 // because it has already substituted "x", hence the "x" in "fr.x" would be recognized as "var0",
210 // whereas the desired behaviour is handling them as "var0" and "var1"
211 std::sort(usedCols.begin(), usedCols.end(),
212 [](const std::string &a, const std::string &b) { return a.size() > b.size(); });
213 for (const auto &col : usedCols) {
214 const auto varIdx = std::distance(usedCols.begin(), std::find(usedCols.begin(), usedCols.end(), col));
215 TPRegexp replacer("\\b" + EscapeDots(col) + "\\b");
216 replacer.Substitute(exprWithVars, varNames[varIdx], "g");
217 }
218
219 return ParsedExpression{std::string(std::move(exprWithVars)), std::move(usedCols), std::move(varNames)};
220}
221
222/// Return the static global map of Filter/Define functions that have been jitted.
223/// It's used to check whether a given expression has already been jitted, and
224/// to look up its associated variable name if it is.
225/// Keys in the map are the body of the expression, values are the name of the
226/// jitted variable that corresponds to that expression. For example, for:
227/// auto f1(){ return 42; }
228/// key would be "(){ return 42; }" and value would be "f1".
229std::unordered_map<std::string, std::string> &GetJittedExprs() {
230 static std::unordered_map<std::string, std::string> jittedExpressions;
231 return jittedExpressions;
232}
233
234std::string
235BuildFunctionString(const std::string &expr, const ColumnNames_t &vars, const ColumnNames_t &varTypes)
236{
237 assert(vars.size() == varTypes.size());
238
239 TPRegexp re(R"(\breturn\b)");
240 const bool hasReturnStmt = re.MatchB(expr);
241
242 static const std::vector<std::string> fundamentalTypes = {
243 "int",
244 "signed",
245 "signed int",
246 "Int_t",
247 "unsigned",
248 "unsigned int",
249 "UInt_t",
250 "double",
251 "Double_t",
252 "float",
253 "Float_t",
254 "char",
255 "Char_t",
256 "unsigned char",
257 "UChar_t",
258 "bool",
259 "Bool_t",
260 "short",
261 "short int",
262 "Short_t",
263 "long",
264 "long int",
265 "long long int",
266 "Long64_t",
267 "unsigned long",
268 "unsigned long int",
269 "ULong64_t",
270 "std::size_t",
271 "size_t",
272 "Ssiz_t"
273 };
274
275 std::stringstream ss;
276 ss << "(";
277 for (auto i = 0u; i < vars.size(); ++i) {
278 std::string fullType;
279 const auto &type = varTypes[i];
280 if (std::find(fundamentalTypes.begin(), fundamentalTypes.end(), type) != fundamentalTypes.end()) {
281 // pass it by const value to help detect common mistakes such as if(x = 3)
282 fullType = "const " + type + " ";
283 } else {
284 // We pass by reference to avoid expensive copies
285 // It can't be const reference in general, as users might want/need to call non-const methods on the values
286 fullType = type + "& ";
287 }
288 ss << fullType << vars[i] << ", ";
289 }
290 if (!vars.empty())
291 ss.seekp(-2, ss.cur);
292
293 if (hasReturnStmt)
294 ss << "){";
295 else
296 ss << "){return ";
297 ss << expr << "\n;}";
298
299 return ss.str();
300}
301
302/// Declare a function to the interpreter in namespace R_rdf, return the name of the jitted function.
303/// If the function is already in GetJittedExprs, return the name for the function that has already been jitted.
304std::string DeclareFunction(const std::string &expr, const ColumnNames_t &vars, const ColumnNames_t &varTypes)
305{
307
308 const auto funcCode = BuildFunctionString(expr, vars, varTypes);
309 auto &exprMap = GetJittedExprs();
310 const auto exprIt = exprMap.find(funcCode);
311 if (exprIt != exprMap.end()) {
312 // expression already there
313 const auto funcName = exprIt->second;
314 return funcName;
315 }
316
317 // new expression
318 const auto funcBaseName = "func" + std::to_string(exprMap.size());
319 const auto funcFullName = "R_rdf::" + funcBaseName;
320
321 const auto toDeclare = "namespace R_rdf {\nauto " + funcBaseName + funcCode + "\nusing " + funcBaseName +
322 "_ret_t = typename ROOT::TypeTraits::CallableTraits<decltype(" + funcBaseName +
323 ")>::ret_type;\n}";
325
326 // InterpreterDeclare could throw. If it doesn't, mark the function as already jitted
327 exprMap.insert({funcCode, funcFullName});
328
329 return funcFullName;
330}
331
332/// Each jitted function comes with a func_ret_t type alias for its return type.
333/// Resolve that alias and return the true type as string.
334std::string RetTypeOfFunc(const std::string &funcName)
335{
336 const auto dt = gROOT->GetType((funcName + "_ret_t").c_str());
337 R__ASSERT(dt != nullptr);
338 const auto type = dt->GetFullTypeName();
339 return type;
340}
341
342[[noreturn]] void
343ThrowJitBuildActionHelperTypeError(const std::string &actionTypeNameBase, const std::type_info &helperArgType)
344{
345 int err = 0;
346 const char *cname = TClassEdit::DemangleTypeIdName(helperArgType, err);
347 std::string actionHelperTypeName = cname;
348 delete[] cname;
349 if (err != 0)
350 actionHelperTypeName = helperArgType.name();
351
352 std::string exceptionText =
353 "RDataFrame::Jit: cannot just-in-time compile a \"" + actionTypeNameBase + "\" action using helper type \"" +
354 actionHelperTypeName +
355 "\". This typically happens in a custom `Fill` or `Book` invocation where the types of the input columns have "
356 "not been specified as template parameters and the ROOT interpreter has no knowledge of this type of action "
357 "helper. Please add template parameters for the types of the input columns to avoid jitting this action (i.e. "
358 "`df.Fill<float>(..., {\"x\"})`, where `float` is the type of `x`) or declare the action helper type to the "
359 "interpreter, e.g. via gInterpreter->Declare.";
360
361 throw std::runtime_error(exceptionText);
362}
363
364} // anonymous namespace
365
366namespace ROOT {
367namespace Internal {
368namespace RDF {
369
370/// Take a list of column names, return that list with entries starting by '#' filtered out.
371/// The function throws when filtering out a column this way.
372ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action)
373{
374 ColumnNames_t columnListWithoutSizeColumns;
375 ColumnNames_t filteredColumns;
376 std::copy_if(columnNames.begin(), columnNames.end(), std::back_inserter(columnListWithoutSizeColumns),
377 [&](const std::string &name) {
378 if (name[0] == '#') {
379 filteredColumns.emplace_back(name);
380 return false;
381 } else {
382 return true;
383 }
384 });
385
386 if (!filteredColumns.empty()) {
387 std::string msg = "Column name(s) {";
388 for (auto &c : filteredColumns)
389 msg += c + ", ";
390 msg[msg.size() - 2] = '}';
391 msg += "will be ignored. Please go through a valid Alias to " + action + " an array size column";
392 throw std::runtime_error(msg);
393 }
394
395 return columnListWithoutSizeColumns;
396}
397
398std::string ResolveAlias(const std::string &col, const std::map<std::string, std::string> &aliasMap)
399{
400 const auto it = aliasMap.find(col);
401 if (it != aliasMap.end())
402 return it->second;
403
404 // #var is an alias for R_rdf_sizeof_var
405 if (col.size() > 1 && col[0] == '#')
406 return "R_rdf_sizeof_" + col.substr(1);
407
408 return col;
409}
410
411void CheckValidCppVarName(std::string_view var, const std::string &where)
412{
413 bool isValid = true;
414
415 if (var.empty())
416 isValid = false;
417 const char firstChar = var[0];
418
419 // first character must be either a letter or an underscore
420 auto isALetter = [](char c) { return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); };
421 const bool isValidFirstChar = firstChar == '_' || isALetter(firstChar);
422 if (!isValidFirstChar)
423 isValid = false;
424
425 // all characters must be either a letter, an underscore or a number
426 auto isANumber = [](char c) { return c >= '0' && c <= '9'; };
427 auto isValidTok = [&isALetter, &isANumber](char c) { return c == '_' || isALetter(c) || isANumber(c); };
428 for (const char c : var)
429 if (!isValidTok(c))
430 isValid = false;
431
432 if (!isValid) {
433 const auto objName = where == "Define" ? "column" : "variation";
434 const auto error = "RDataFrame::" + where + ": cannot define " + objName + " \"" + std::string(var) +
435 "\". Not a valid C++ variable name.";
436 throw std::runtime_error(error);
437 }
438}
439
440std::string DemangleTypeIdName(const std::type_info &typeInfo)
441{
442 int dummy(0);
443 char *tn = TClassEdit::DemangleTypeIdName(typeInfo, dummy);
444 std::string tname(tn);
445 free(tn);
446 return tname;
447}
448
449ColumnNames_t
450ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName)
451{
452 const auto theRegexSize = columnNameRegexp.size();
453 std::string theRegex(columnNameRegexp);
454
455 const auto isEmptyRegex = 0 == theRegexSize;
456 // This is to avoid cases where branches called b1, b2, b3 are all matched by expression "b"
457 if (theRegexSize > 0 && theRegex[0] != '^')
458 theRegex = "^" + theRegex;
459 if (theRegexSize > 0 && theRegex[theRegexSize - 1] != '$')
460 theRegex = theRegex + "$";
461
462 ColumnNames_t selectedColumns;
463
464 // Since we support gcc48 and it does not provide in its stl std::regex,
465 // we need to use TPRegexp
466 TPRegexp regexp(theRegex);
467 for (auto &&colName : colNames) {
468 if ((isEmptyRegex || regexp.MatchB(colName.c_str())) && !IsInternalColumn(colName)) {
469 selectedColumns.emplace_back(colName);
470 }
471 }
472
473 if (selectedColumns.empty()) {
474 std::string text(callerName);
475 if (columnNameRegexp.empty()) {
476 text = ": there is no column available to match.";
477 } else {
478 text = ": regex \"" + std::string(columnNameRegexp) + "\" did not match any column.";
479 }
480 throw std::runtime_error(text);
481 }
482 return selectedColumns;
483}
484
485/// Throw if column `definedColView` is already there.
486void CheckForRedefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister,
487 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
488{
489 const std::string definedCol(definedColView); // convert to std::string
490
491 std::string error;
492 if (colRegister.IsAlias(definedCol))
493 error = "An alias with that name, pointing to column \"" + colRegister.ResolveAlias(definedCol) +
494 "\", already exists in this branch of the computation graph.";
495 else if (colRegister.IsDefineOrAlias(definedCol))
496 error = "A column with that name has already been Define'd. Use Redefine to force redefinition.";
497 // else, check if definedCol is in the list of tree branches. This is a bit better than interrogating the TTree
498 // directly because correct usage of GetBranch, FindBranch, GetLeaf and FindLeaf can be tricky; so let's assume we
499 // got it right when we collected the list of available branches.
500 else if (std::find(treeColumns.begin(), treeColumns.end(), definedCol) != treeColumns.end())
501 error =
502 "A branch with that name is already present in the input TTree/TChain. Use Redefine to force redefinition.";
503 else if (std::find(dataSourceColumns.begin(), dataSourceColumns.end(), definedCol) != dataSourceColumns.end())
504 error =
505 "A column with that name is already present in the input data source. Use Redefine to force redefinition.";
506
507 if (!error.empty()) {
508 error = "RDataFrame::" + where + ": cannot define column \"" + definedCol + "\". " + error;
509 throw std::runtime_error(error);
510 }
511}
512
513/// Throw if column `definedColView` is _not_ already there.
514void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister,
515 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
516{
517 const std::string definedCol(definedColView); // convert to std::string
518 std::string error;
519
520 if (colRegister.IsAlias(definedCol)) {
521 error = "An alias with that name, pointing to column \"" + colRegister.ResolveAlias(definedCol) +
522 "\", already exists. Aliases cannot be Redefined or Varied.";
523 }
524
525 if (error.empty()) {
526 const bool isAlreadyDefined = colRegister.IsDefineOrAlias(definedCol);
527 // check if definedCol is in the list of tree branches. This is a bit better than interrogating the TTree
528 // directly because correct usage of GetBranch, FindBranch, GetLeaf and FindLeaf can be tricky; so let's assume we
529 // got it right when we collected the list of available branches.
530 const bool isABranch = std::find(treeColumns.begin(), treeColumns.end(), definedCol) != treeColumns.end();
531 const bool isADSColumn =
532 std::find(dataSourceColumns.begin(), dataSourceColumns.end(), definedCol) != dataSourceColumns.end();
533
534 if (!isAlreadyDefined && !isABranch && !isADSColumn)
535 error = "No column with that name was found in the dataset. Use Define to create a new column.";
536 }
537
538 if (!error.empty()) {
539 error = "RDataFrame::" + where + ": cannot redefine or vary column \"" + definedCol + "\". " + error;
540 throw std::runtime_error(error);
541 }
542}
543
544/// Throw if the column has systematic variations attached.
545void CheckForNoVariations(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister)
546{
547 const std::string definedCol(definedColView);
548 const auto &variationDeps = colRegister.GetVariationDeps(definedCol);
549 if (!variationDeps.empty()) {
550 const std::string error =
551 "RDataFrame::" + where + ": cannot redefine column \"" + definedCol +
552 "\". The column depends on one or more systematic variations and re-defining varied columns is not supported.";
553 throw std::runtime_error(error);
554 }
555}
556
557void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames)
558{
559 if (nTemplateParams != nColumnNames) {
560 std::string err_msg = "The number of template parameters specified is ";
561 err_msg += std::to_string(nTemplateParams);
562 err_msg += " while ";
563 err_msg += std::to_string(nColumnNames);
564 err_msg += " columns have been specified.";
565 throw std::runtime_error(err_msg);
566 }
567}
568
569/// Choose between local column names or default column names, throw in case of errors.
570const ColumnNames_t
571SelectColumns(unsigned int nRequiredNames, const ColumnNames_t &names, const ColumnNames_t &defaultNames)
572{
573 if (names.empty()) {
574 // use default column names
575 if (defaultNames.size() < nRequiredNames)
576 throw std::runtime_error(
577 std::to_string(nRequiredNames) + " column name" + (nRequiredNames == 1 ? " is" : "s are") +
578 " required but none were provided and the default list has size " + std::to_string(defaultNames.size()));
579 // return first nRequiredNames default column names
580 return ColumnNames_t(defaultNames.begin(), defaultNames.begin() + nRequiredNames);
581 } else {
582 // use column names provided by the user to this particular transformation/action
583 if (names.size() != nRequiredNames) {
584 auto msg = std::to_string(nRequiredNames) + " column name" + (nRequiredNames == 1 ? " is" : "s are") +
585 " required but " + std::to_string(names.size()) + (names.size() == 1 ? " was" : " were") +
586 " provided:";
587 for (const auto &name : names)
588 msg += " \"" + name + "\",";
589 msg.back() = '.';
590 throw std::runtime_error(msg);
591 }
592 return names;
593 }
594}
595
596ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns,
597 const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns)
598{
599 ColumnNames_t unknownColumns;
600 for (auto &column : requiredCols) {
601 const auto isBranch = std::find(datasetColumns.begin(), datasetColumns.end(), column) != datasetColumns.end();
602 if (isBranch)
603 continue;
604 if (definedCols.IsDefineOrAlias(column))
605 continue;
606 const auto isDataSourceColumn =
607 std::find(dataSourceColumns.begin(), dataSourceColumns.end(), column) != dataSourceColumns.end();
608 if (isDataSourceColumn)
609 continue;
610 unknownColumns.emplace_back(column);
611 }
612 return unknownColumns;
613}
614
615std::vector<std::string> GetFilterNames(const std::shared_ptr<RLoopManager> &loopManager)
616{
617 return loopManager->GetFiltersNames();
618}
619
620ParsedTreePath ParseTreePath(std::string_view fullTreeName)
621{
622 // split name into directory and treename if needed
623 std::string_view dirName = "";
624 std::string_view treeName = fullTreeName;
625 const auto lastSlash = fullTreeName.rfind('/');
626 if (std::string_view::npos != lastSlash) {
627 dirName = treeName.substr(0, lastSlash);
628 treeName = treeName.substr(lastSlash + 1, treeName.size());
629 }
630 return {std::string(treeName), std::string(dirName)};
631}
632
633std::string PrettyPrintAddr(const void *const addr)
634{
635 std::stringstream s;
636 // Windows-friendly
637 s << std::hex << std::showbase << reinterpret_cast<size_t>(addr);
638 return s.str();
639}
640
641/// Book the jitting of a Filter call
642std::shared_ptr<RDFDetail::RJittedFilter>
643BookFilterJit(std::shared_ptr<RDFDetail::RNodeBase> *prevNodeOnHeap, std::string_view name, std::string_view expression,
644 const ColumnNames_t &branches, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds)
645{
646 const auto &dsColumns = ds ? ds->GetColumnNames() : ColumnNames_t{};
647
648 const auto parsedExpr = ParseRDFExpression(expression, branches, colRegister, dsColumns);
649 const auto exprVarTypes =
650 GetValidatedArgTypes(parsedExpr.fUsedCols, colRegister, tree, ds, "Filter", /*vector2rvec=*/true);
651 const auto funcName = DeclareFunction(parsedExpr.fExpr, parsedExpr.fVarNames, exprVarTypes);
652 const auto type = RetTypeOfFunc(funcName);
653 if (type != "bool")
654 std::runtime_error("Filter: the following expression does not evaluate to bool:\n" + std::string(expression));
655
656 // definesOnHeap is deleted by the jitted call to JitFilterHelper
658 const auto definesOnHeapAddr = PrettyPrintAddr(definesOnHeap);
659 const auto prevNodeAddr = PrettyPrintAddr(prevNodeOnHeap);
660
661 const auto jittedFilter = std::make_shared<RDFDetail::RJittedFilter>(
662 (*prevNodeOnHeap)->GetLoopManagerUnchecked(), name,
663 Union(colRegister.GetVariationDeps(parsedExpr.fUsedCols), (*prevNodeOnHeap)->GetVariations()));
664
665 // Produce code snippet that creates the filter and registers it with the corresponding RJittedFilter
666 // Windows requires std::hex << std::showbase << (size_t)pointer to produce notation "0x1234"
667 std::stringstream filterInvocation;
668 filterInvocation << "ROOT::Internal::RDF::JitFilterHelper(" << funcName << ", new const char*["
669 << parsedExpr.fUsedCols.size() << "]{";
670 for (const auto &col : parsedExpr.fUsedCols)
671 filterInvocation << "\"" << col << "\", ";
672 if (!parsedExpr.fUsedCols.empty())
673 filterInvocation.seekp(-2, filterInvocation.cur); // remove the last ",
674 // lifetime of pointees:
675 // - jittedFilter: heap-allocated weak_ptr to the actual jittedFilter that will be deleted by JitFilterHelper
676 // - prevNodeOnHeap: heap-allocated shared_ptr to the actual previous node that will be deleted by JitFilterHelper
677 // - definesOnHeap: heap-allocated, will be deleted by JitFilterHelper
678 filterInvocation << "}, " << parsedExpr.fUsedCols.size() << ", \"" << name << "\", "
679 << "reinterpret_cast<std::weak_ptr<ROOT::Detail::RDF::RJittedFilter>*>("
680 << PrettyPrintAddr(MakeWeakOnHeap(jittedFilter)) << "), "
681 << "reinterpret_cast<std::shared_ptr<ROOT::Detail::RDF::RNodeBase>*>(" << prevNodeAddr << "),"
682 << "reinterpret_cast<ROOT::Internal::RDF::RColumnRegister*>(" << definesOnHeapAddr << ")"
683 << ");\n";
684
685 auto lm = jittedFilter->GetLoopManagerUnchecked();
686 lm->ToJitExec(filterInvocation.str());
687
688 return jittedFilter;
689}
690
691/// Book the jitting of a Define call
692std::shared_ptr<RJittedDefine> BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm,
693 RDataSource *ds, const RColumnRegister &colRegister,
694 const ColumnNames_t &branches,
695 std::shared_ptr<RNodeBase> *upcastNodeOnHeap)
696{
697 auto *const tree = lm.GetTree();
698 const auto &dsColumns = ds ? ds->GetColumnNames() : ColumnNames_t{};
699
700 const auto parsedExpr = ParseRDFExpression(expression, branches, colRegister, dsColumns);
701 const auto exprVarTypes =
702 GetValidatedArgTypes(parsedExpr.fUsedCols, colRegister, tree, ds, "Define", /*vector2rvec=*/true);
703 const auto funcName = DeclareFunction(parsedExpr.fExpr, parsedExpr.fVarNames, exprVarTypes);
704 const auto type = RetTypeOfFunc(funcName);
705
706 auto definesCopy = new RColumnRegister(colRegister);
707 auto definesAddr = PrettyPrintAddr(definesCopy);
708 auto jittedDefine = std::make_shared<RDFDetail::RJittedDefine>(name, type, lm, colRegister, parsedExpr.fUsedCols);
709
710 std::stringstream defineInvocation;
711 defineInvocation << "ROOT::Internal::RDF::JitDefineHelper<ROOT::Internal::RDF::DefineTypes::RDefineTag>(" << funcName
712 << ", new const char*[" << parsedExpr.fUsedCols.size() << "]{";
713 for (const auto &col : parsedExpr.fUsedCols) {
714 defineInvocation << "\"" << col << "\", ";
715 }
716 if (!parsedExpr.fUsedCols.empty())
717 defineInvocation.seekp(-2, defineInvocation.cur); // remove the last ",
718 // lifetime of pointees:
719 // - lm is the loop manager, and if that goes out of scope jitting does not happen at all (i.e. will always be valid)
720 // - jittedDefine: heap-allocated weak_ptr that will be deleted by JitDefineHelper after usage
721 // - definesAddr: heap-allocated, will be deleted by JitDefineHelper after usage
722 defineInvocation << "}, " << parsedExpr.fUsedCols.size() << ", \"" << name
723 << "\", reinterpret_cast<ROOT::Detail::RDF::RLoopManager*>(" << PrettyPrintAddr(&lm)
724 << "), reinterpret_cast<std::weak_ptr<ROOT::Detail::RDF::RJittedDefine>*>("
725 << PrettyPrintAddr(MakeWeakOnHeap(jittedDefine))
726 << "), reinterpret_cast<ROOT::Internal::RDF::RColumnRegister*>(" << definesAddr
727 << "), reinterpret_cast<std::shared_ptr<ROOT::Detail::RDF::RNodeBase>*>("
728 << PrettyPrintAddr(upcastNodeOnHeap) << "));\n";
729
730 lm.ToJitExec(defineInvocation.str());
731 return jittedDefine;
732}
733
734/// Book the jitting of a DefinePerSample call
735std::shared_ptr<RJittedDefine> BookDefinePerSampleJit(std::string_view name, std::string_view expression,
736 RLoopManager &lm, const RColumnRegister &colRegister,
737 std::shared_ptr<RNodeBase> *upcastNodeOnHeap)
738{
739 const auto funcName = DeclareFunction(std::string(expression), {"rdfslot_", "rdfsampleinfo_"},
740 {"unsigned int", "const ROOT::RDF::RSampleInfo"});
741 const auto retType = RetTypeOfFunc(funcName);
742
743 auto definesCopy = new RColumnRegister(colRegister);
744 auto definesAddr = PrettyPrintAddr(definesCopy);
745 auto jittedDefine = std::make_shared<RDFDetail::RJittedDefine>(name, retType, lm, colRegister, ColumnNames_t{});
746
747 std::stringstream defineInvocation;
748 defineInvocation << "ROOT::Internal::RDF::JitDefineHelper<ROOT::Internal::RDF::DefineTypes::RDefinePerSampleTag>("
749 << funcName << ", nullptr, 0, ";
750 // lifetime of pointees:
751 // - lm is the loop manager, and if that goes out of scope jitting does not happen at all (i.e. will always be valid)
752 // - jittedDefine: heap-allocated weak_ptr that will be deleted by JitDefineHelper after usage
753 // - definesAddr: heap-allocated, will be deleted by JitDefineHelper after usage
754 defineInvocation << "\"" << name << "\", reinterpret_cast<ROOT::Detail::RDF::RLoopManager*>(" << PrettyPrintAddr(&lm)
755 << "), reinterpret_cast<std::weak_ptr<ROOT::Detail::RDF::RJittedDefine>*>("
756 << PrettyPrintAddr(MakeWeakOnHeap(jittedDefine))
757 << "), reinterpret_cast<ROOT::Internal::RDF::RColumnRegister*>(" << definesAddr
758 << "), reinterpret_cast<std::shared_ptr<ROOT::Detail::RDF::RNodeBase>*>("
759 << PrettyPrintAddr(upcastNodeOnHeap) << "));\n";
760
761 lm.ToJitExec(defineInvocation.str());
762 return jittedDefine;
763}
764
765/// Book the jitting of a Vary call
766std::shared_ptr<RJittedVariation>
767BookVariationJit(const std::vector<std::string> &colNames, std::string_view variationName,
768 const std::vector<std::string> &variationTags, std::string_view expression, RLoopManager &lm,
769 RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches,
770 std::shared_ptr<RNodeBase> *upcastNodeOnHeap, bool isSingleColumn)
771{
772 auto *const tree = lm.GetTree();
773 const auto &dsColumns = ds ? ds->GetColumnNames() : ColumnNames_t{};
774
775 const auto parsedExpr = ParseRDFExpression(expression, branches, colRegister, dsColumns);
776 const auto exprVarTypes =
777 GetValidatedArgTypes(parsedExpr.fUsedCols, colRegister, tree, ds, "Vary", /*vector2rvec=*/true);
778 const auto funcName = DeclareFunction(parsedExpr.fExpr, parsedExpr.fVarNames, exprVarTypes);
779 const auto type = RetTypeOfFunc(funcName);
780
781 if (type.rfind("ROOT::VecOps::RVec", 0) != 0)
782 throw std::runtime_error(
783 "Jitted Vary expressions must return an RVec object. The following expression returns a " + type +
784 " instead:\n" + parsedExpr.fExpr);
785
786 auto colRegisterCopy = new RColumnRegister(colRegister);
787 const auto colRegisterAddr = PrettyPrintAddr(colRegisterCopy);
788 auto jittedVariation = std::make_shared<RJittedVariation>(colNames, variationName, variationTags, type, colRegister,
789 lm, parsedExpr.fUsedCols);
790
791 // build invocation to JitVariationHelper
792 // arrays of strings are passed as const char** plus size.
793 // lifetime of pointees:
794 // - lm is the loop manager, and if that goes out of scope jitting does not happen at all (i.e. will always be valid)
795 // - jittedVariation: heap-allocated weak_ptr that will be deleted by JitDefineHelper after usage
796 // - definesAddr: heap-allocated, will be deleted by JitDefineHelper after usage
797 std::stringstream varyInvocation;
798 varyInvocation << "ROOT::Internal::RDF::JitVariationHelper<" << (isSingleColumn ? "true" : "false") << ">("
799 << funcName << ", new const char*[" << parsedExpr.fUsedCols.size() << "]{";
800 for (const auto &col : parsedExpr.fUsedCols) {
801 varyInvocation << "\"" << col << "\", ";
802 }
803 if (!parsedExpr.fUsedCols.empty())
804 varyInvocation.seekp(-2, varyInvocation.cur); // remove the last ", "
805 varyInvocation << "}, " << parsedExpr.fUsedCols.size();
806 varyInvocation << ", new const char*[" << colNames.size() << "]{";
807 for (const auto &col : colNames) {
808 varyInvocation << "\"" << col << "\", ";
809 }
810 varyInvocation.seekp(-2, varyInvocation.cur); // remove the last ", "
811 varyInvocation << "}, " << colNames.size() << ", new const char*[" << variationTags.size() << "]{";
812 for (const auto &tag : variationTags) {
813 varyInvocation << "\"" << tag << "\", ";
814 }
815 varyInvocation.seekp(-2, varyInvocation.cur); // remove the last ", "
816 varyInvocation << "}, " << variationTags.size() << ", \"" << variationName
817 << "\", reinterpret_cast<ROOT::Detail::RDF::RLoopManager*>(" << PrettyPrintAddr(&lm)
818 << "), reinterpret_cast<std::weak_ptr<ROOT::Internal::RDF::RJittedVariation>*>("
819 << PrettyPrintAddr(MakeWeakOnHeap(jittedVariation))
820 << "), reinterpret_cast<ROOT::Internal::RDF::RColumnRegister*>(" << colRegisterAddr
821 << "), reinterpret_cast<std::shared_ptr<ROOT::Detail::RDF::RNodeBase>*>("
822 << PrettyPrintAddr(upcastNodeOnHeap) << "));\n";
823
824 lm.ToJitExec(varyInvocation.str());
825 return jittedVariation;
826}
827
828// Jit and call something equivalent to "this->BuildAndBook<ColTypes...>(params...)"
829// (see comments in the body for actual jitted code)
830std::string JitBuildAction(const ColumnNames_t &cols, std::shared_ptr<RDFDetail::RNodeBase> *prevNode,
831 const std::type_info &helperArgType, const std::type_info &at, void *helperArgOnHeap,
832 TTree *tree, const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds,
833 std::weak_ptr<RJittedAction> *jittedActionOnHeap)
834{
835 // retrieve type of action as a string
836 auto actionTypeClass = TClass::GetClass(at);
837 if (!actionTypeClass) {
838 std::string exceptionText = "An error occurred while inferring the action type of the operation.";
839 throw std::runtime_error(exceptionText);
840 }
841 const std::string actionTypeName = actionTypeClass->GetName();
842 const std::string actionTypeNameBase = actionTypeName.substr(actionTypeName.rfind(':') + 1);
843
844 // retrieve type of result of the action as a string
845 const auto helperArgTypeName = TypeID2TypeName(helperArgType);
846 if (helperArgTypeName.empty()) {
847 ThrowJitBuildActionHelperTypeError(actionTypeNameBase, helperArgType);
848 }
849
850 auto definesCopy = new RColumnRegister(colRegister); // deleted in jitted CallBuildAction
851 auto definesAddr = PrettyPrintAddr(definesCopy);
852
853 // Build a call to CallBuildAction with the appropriate argument. When run through the interpreter, this code will
854 // just-in-time create an RAction object and it will assign it to its corresponding RJittedAction.
855 std::stringstream createAction_str;
856 createAction_str << "ROOT::Internal::RDF::CallBuildAction<" << actionTypeName;
857 const auto columnTypeNames =
858 GetValidatedArgTypes(cols, colRegister, tree, ds, actionTypeNameBase, /*vector2rvec=*/true);
859 for (auto &colType : columnTypeNames)
860 createAction_str << ", " << colType;
861 // on Windows, to prefix the hexadecimal value of a pointer with '0x',
862 // one need to write: std::hex << std::showbase << (size_t)pointer
863 createAction_str << ">(reinterpret_cast<std::shared_ptr<ROOT::Detail::RDF::RNodeBase>*>("
864 << PrettyPrintAddr(prevNode) << "), new const char*[" << cols.size() << "]{";
865 for (auto i = 0u; i < cols.size(); ++i) {
866 if (i != 0u)
867 createAction_str << ", ";
868 createAction_str << '"' << cols[i] << '"';
869 }
870 createAction_str << "}, " << cols.size() << ", " << nSlots << ", reinterpret_cast<shared_ptr<" << helperArgTypeName
871 << ">*>(" << PrettyPrintAddr(helperArgOnHeap)
872 << "), reinterpret_cast<std::weak_ptr<ROOT::Internal::RDF::RJittedAction>*>("
873 << PrettyPrintAddr(jittedActionOnHeap)
874 << "), reinterpret_cast<ROOT::Internal::RDF::RColumnRegister*>(" << definesAddr << "));";
875 return createAction_str.str();
876}
877
878bool AtLeastOneEmptyString(const std::vector<std::string_view> strings)
879{
880 for (const auto &s : strings) {
881 if (s.empty())
882 return true;
883 }
884 return false;
885}
886
887std::shared_ptr<RNodeBase> UpcastNode(std::shared_ptr<RNodeBase> ptr)
888{
889 return ptr;
890}
891
892/// Given the desired number of columns and the user-provided list of columns:
893/// * fallback to using the first nColumns default columns if needed (or throw if nColumns > nDefaultColumns)
894/// * check that selected column names refer to valid branches, custom columns or datasource columns (throw if not)
895/// * replace column names from aliases by the actual column name
896/// Return the list of selected column names.
897ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns,
898 const RColumnRegister &colRegister, RDataSource *ds)
899{
900 auto selectedColumns = SelectColumns(nColumns, columns, lm.GetDefaultColumnNames());
901
902 for (auto &col : selectedColumns) {
903 col = colRegister.ResolveAlias(col);
904 }
905
906 // Complain if there are still unknown columns at this point
907 const auto unknownColumns = FindUnknownColumns(selectedColumns, lm.GetBranchNames(), colRegister,
908 ds ? ds->GetColumnNames() : ColumnNames_t{});
909
910 if (!unknownColumns.empty()) {
911 using namespace std::string_literals;
912 std::string errMsg = "Unknown column"s + (unknownColumns.size() > 1 ? "s: " : ": ");
913 for (auto &unknownColumn : unknownColumns)
914 errMsg += '"' + unknownColumn + "\", ";
915 errMsg.resize(errMsg.size() - 2); // remove last ", "
916 throw std::runtime_error(errMsg);
917 }
918
919 return selectedColumns;
920}
921
922std::vector<std::string> GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister,
923 TTree *tree, RDataSource *ds, const std::string &context,
924 bool vector2rvec)
925{
926 auto toCheckedArgType = [&](const std::string &c) {
927 RDFDetail::RDefineBase *define = colRegister.GetDefine(c);
928 const auto colType = ColumnName2ColumnTypeName(c, tree, ds, define, vector2rvec);
929 if (colType.rfind("CLING_UNKNOWN_TYPE", 0) == 0) { // the interpreter does not know this type
930 const auto msg =
931 "The type of custom column \"" + c + "\" (" + colType.substr(19) +
932 ") is not known to the interpreter, but a just-in-time-compiled " + context +
933 " call requires this column. Make sure to create and load ROOT dictionaries for this column's class.";
934 throw std::runtime_error(msg);
935 }
936 return colType;
937 };
938 std::vector<std::string> colTypes;
939 colTypes.reserve(colNames.size());
940 std::transform(colNames.begin(), colNames.end(), std::back_inserter(colTypes), toCheckedArgType);
941 return colTypes;
942}
943
944/// Return a bitset each element of which indicates whether the corresponding element in `selectedColumns` is the
945/// name of a column that must be defined via datasource. All elements of the returned vector are false if no
946/// data-source is present.
947std::vector<bool> FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedCols)
948{
949 const auto nColumns = requestedCols.size();
950 std::vector<bool> mustBeDefined(nColumns, false);
951 for (auto i = 0u; i < nColumns; ++i)
952 mustBeDefined[i] = std::find(definedCols.begin(), definedCols.end(), requestedCols[i]) == definedCols.end();
953 return mustBeDefined;
954}
955
957{
958 std::unordered_set<std::string> uniqueCols;
959 for (auto &col : cols) {
960 if (!uniqueCols.insert(col).second) {
961 const auto msg = "Error: column \"" + col +
962 "\" was passed to Snapshot twice. This is not supported: only one of the columns would be "
963 "readable with RDataFrame.";
964 throw std::logic_error(msg);
965 }
966 }
967}
968
969/// Return copies of colsWithoutAliases and colsWithAliases with size branches for variable-sized array branches added
970/// in the right positions (i.e. before the array branches that need them).
971std::pair<std::vector<std::string>, std::vector<std::string>>
972AddSizeBranches(const std::vector<std::string> &branches, TTree *tree, std::vector<std::string> &&colsWithoutAliases,
973 std::vector<std::string> &&colsWithAliases)
974{
975 if (!tree) // nothing to do
976 return {std::move(colsWithoutAliases), std::move(colsWithAliases)};
977
978 assert(colsWithoutAliases.size() == colsWithAliases.size());
979
980 auto nCols = colsWithoutAliases.size();
981 // Use index-iteration as we modify the vector during the iteration.
982 for (std::size_t i = 0u; i < nCols; ++i) {
983 const auto &colName = colsWithoutAliases[i];
984 if (!IsStrInVec(colName, branches))
985 continue; // this column is not a TTree branch, nothing to do
986
987 auto *b = tree->GetBranch(colName.c_str());
988 if (!b) // try harder
989 b = tree->FindBranch(colName.c_str());
990 assert(b != nullptr);
991 auto *leaves = b->GetListOfLeaves();
992 if (b->IsA() != TBranch::Class() || leaves->GetEntries() != 1)
993 continue; // this branch is not a variable-sized array, nothing to do
994
995 TLeaf *countLeaf = static_cast<TLeaf *>(leaves->At(0))->GetLeafCount();
996 if (!countLeaf || IsStrInVec(countLeaf->GetName(), colsWithoutAliases))
997 continue; // not a variable-sized array or the size branch is already there, nothing to do
998
999 // otherwise we must insert the size in colsWithoutAliases _and_ colsWithAliases
1000 colsWithoutAliases.insert(colsWithoutAliases.begin() + i, countLeaf->GetName());
1001 colsWithAliases.insert(colsWithAliases.begin() + i, countLeaf->GetName());
1002 ++nCols;
1003 ++i; // as we inserted an element in the vector we iterate over, we need to move the index forward one extra time
1004 }
1005
1006 return {std::move(colsWithoutAliases), std::move(colsWithAliases)};
1007}
1008
1010{
1011 std::set<std::string> uniqueCols;
1012 columnNames.erase(
1013 std::remove_if(columnNames.begin(), columnNames.end(),
1014 [&uniqueCols](const std::string &colName) { return !uniqueCols.insert(colName).second; }),
1015 columnNames.end());
1016}
1017
1018} // namespace RDF
1019} // namespace Internal
1020} // namespace ROOT
#define b(i)
Definition RSha256.hxx:100
#define c(i)
Definition RSha256.hxx:101
#define a(i)
Definition RSha256.hxx:99
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
#define R__ASSERT(e)
Definition TError.h:118
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char cname
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
Option_t Option_t TPoint TPoint const char text
char name[80]
Definition TGX11.cxx:110
R__EXTERN TVirtualMutex * gROOTMutex
Definition TROOT.h:63
#define gROOT
Definition TROOT.h:406
#define R__LOCKGUARD(mutex)
#define free
Definition civetweb.c:1539
The head node of a RDF computation graph.
const ColumnNames_t & GetBranchNames()
Return all valid TTree::Branch names (caching results for subsequent calls).
void ToJitExec(const std::string &) const
const ColumnNames_t & GetDefaultColumnNames() const
Return the list of default columns – empty if none was provided when constructing the RDataFrame.
A binder for user-defined columns, variations and aliases.
bool IsDefineOrAlias(std::string_view name) const
Check if the provided name is tracked in the names list.
RDFDetail::RDefineBase * GetDefine(const std::string &colName) const
Return the RDefine for the requested column name, or nullptr.
bool IsAlias(const std::string &name) const
Return true if the given column name is an existing alias.
std::string ResolveAlias(std::string_view alias) const
Return the actual column name that the alias resolves to.
std::vector< std::string > GetVariationDeps(const std::string &column) const
Get the names of all variations that directly or indirectly affect a given column.
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual const std::vector< std::string > & GetColumnNames() const =0
Returns a reference to the collection of the dataset's column names.
static TClass * Class()
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Definition TClass.cxx:2968
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
Definition TLeaf.h:57
const char * GetName() const override
Returns name of object.
Definition TNamed.h:47
Bool_t MatchB(const TString &s, const TString &mods="", Int_t start=0, Int_t nMaxMatch=10)
Definition TPRegexp.h:78
Basic string class.
Definition TString.h:139
A TTree represents a columnar dataset.
Definition TTree.h:79
const ColumnNames_t SelectColumns(unsigned int nRequiredNames, const ColumnNames_t &names, const ColumnNames_t &defaultNames)
Choose between local column names or default column names, throw in case of errors.
void CheckForNoVariations(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister)
Throw if the column has systematic variations attached.
ParsedTreePath ParseTreePath(std::string_view fullTreeName)
void CheckForRedefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is already there.
void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is not already there.
std::shared_ptr< RJittedDefine > BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a Define call.
void CheckValidCppVarName(std::string_view var, const std::string &where)
std::string ColumnName2ColumnTypeName(const std::string &colName, TTree *, RDataSource *, RDefineBase *, bool vector2rvec=true)
Return a string containing the type of the given branch.
Definition RDFUtils.cxx:222
void RemoveDuplicates(ColumnNames_t &columnNames)
ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns, const RColumnRegister &colRegister, RDataSource *ds)
Given the desired number of columns and the user-provided list of columns:
std::shared_ptr< RNodeBase > UpcastNode(std::shared_ptr< RNodeBase > ptr)
std::string TypeID2TypeName(const std::type_info &id)
Returns the name of a type starting from its type_info An empty string is returned in case of failure...
Definition RDFUtils.cxx:99
bool IsStrInVec(const std::string &str, const std::vector< std::string > &vec)
Definition RDFUtils.cxx:417
std::string ResolveAlias(const std::string &col, const std::map< std::string, std::string > &aliasMap)
std::vector< std::string > GetFilterNames(const std::shared_ptr< RLoopManager > &loopManager)
std::string PrettyPrintAddr(const void *const addr)
void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames)
bool AtLeastOneEmptyString(const std::vector< std::string_view > strings)
std::shared_ptr< RDFDetail::RJittedFilter > BookFilterJit(std::shared_ptr< RDFDetail::RNodeBase > *prevNodeOnHeap, std::string_view name, std::string_view expression, const ColumnNames_t &branches, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds)
Book the jitting of a Filter call.
std::vector< T > Union(const std::vector< T > &v1, const std::vector< T > &v2)
Return a vector with all elements of v1 and v2 and duplicates removed.
Definition Utils.hxx:266
std::string JitBuildAction(const ColumnNames_t &cols, std::shared_ptr< RDFDetail::RNodeBase > *prevNode, const std::type_info &helperArgType, const std::type_info &at, void *helperArgOnHeap, TTree *tree, const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds, std::weak_ptr< RJittedAction > *jittedActionOnHeap)
bool IsInternalColumn(std::string_view colName)
Whether custom column with name colName is an "internal" column such as rdfentry_ or rdfslot_.
Definition RDFUtils.cxx:363
ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action)
Take a list of column names, return that list with entries starting by '#' filtered out.
void InterpreterDeclare(const std::string &code)
Declare code in the interpreter via the TInterpreter::Declare method, throw in case of errors.
Definition RDFUtils.cxx:315
std::shared_ptr< RJittedVariation > BookVariationJit(const std::vector< std::string > &colNames, std::string_view variationName, const std::vector< std::string > &variationTags, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap, bool isSingleColumn)
Book the jitting of a Vary call.
void CheckForDuplicateSnapshotColumns(const ColumnNames_t &cols)
ColumnNames_t ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName)
std::pair< std::vector< std::string >, std::vector< std::string > > AddSizeBranches(const std::vector< std::string > &branches, TTree *tree, std::vector< std::string > &&colsWithoutAliases, std::vector< std::string > &&colsWithAliases)
Return copies of colsWithoutAliases and colsWithAliases with size branches for variable-sized array b...
std::vector< bool > FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedCols)
Return a bitset each element of which indicates whether the corresponding element in selectedColumns ...
std::shared_ptr< RJittedDefine > BookDefinePerSampleJit(std::string_view name, std::string_view expression, RLoopManager &lm, const RColumnRegister &colRegister, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a DefinePerSample call.
std::vector< std::string > GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds, const std::string &context, bool vector2rvec)
ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns, const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns)
std::vector< std::string > ColumnNames_t
This file contains a specialised ROOT message handler to test for diagnostic in unit tests.
char * DemangleTypeIdName(const std::type_info &ti, int &errorCode)
Demangle in a portable way the type id name.