Logo ROOT  
Reference Guide
InterfaceUtils.hxx
Go to the documentation of this file.
1// Author: Enrico Guiraud, Danilo Piparo CERN 02/2018
2
3/*************************************************************************
4 * Copyright (C) 1995-2018, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#ifndef ROOT_RDF_TINTERFACE_UTILS
12#define ROOT_RDF_TINTERFACE_UTILS
13
14#include "RColumnRegister.hxx"
15#include <ROOT/RDF/RAction.hxx>
16#include <ROOT/RDF/ActionHelpers.hxx> // for BuildAction
18#include <ROOT/RDF/RDefine.hxx>
20#include <ROOT/RDF/RFilter.hxx>
21#include <ROOT/RDF/Utils.hxx>
27#include <ROOT/RStringView.hxx>
29#include <ROOT/TypeTraits.hxx>
30#include <TError.h> // gErrorIgnoreLevel
31#include <TH1.h>
32#include <TROOT.h> // IsImplicitMTEnabled
33
34#include <deque>
35#include <functional>
36#include <map>
37#include <memory>
38#include <string>
39#include <type_traits>
40#include <typeinfo>
41#include <vector>
42#include <unordered_map>
43
44class TObjArray;
45class TTree;
46namespace ROOT {
47namespace Detail {
48namespace RDF {
49class RNodeBase;
50}
51}
52namespace RDF {
53template <typename T>
54class RResultPtr;
55template<typename T, typename V>
56class RInterface;
58class RDataSource;
59} // namespace RDF
60
61} // namespace ROOT
62
63/// \cond HIDDEN_SYMBOLS
64
65namespace ROOT {
66namespace Internal {
67namespace RDF {
68using namespace ROOT::Detail::RDF;
69using namespace ROOT::RDF;
70namespace TTraits = ROOT::TypeTraits;
71
72std::string DemangleTypeIdName(const std::type_info &typeInfo);
73
75ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName);
76
77/// An helper object that sets and resets gErrorIgnoreLevel via RAII.
78class RIgnoreErrorLevelRAII {
79private:
80 int fCurIgnoreErrorLevel = gErrorIgnoreLevel;
81
82public:
83 RIgnoreErrorLevelRAII(int errorIgnoreLevel) { gErrorIgnoreLevel = errorIgnoreLevel; }
84 ~RIgnoreErrorLevelRAII() { gErrorIgnoreLevel = fCurIgnoreErrorLevel; }
85};
86
87/****** BuildAction overloads *******/
88
89// clang-format off
90/// This namespace defines types to be used for tag dispatching in RInterface.
91namespace ActionTags {
92struct Histo1D{};
93struct Histo2D{};
94struct Histo3D{};
95struct HistoND{};
96struct Graph{};
97struct GraphAsymmErrors{};
98struct Profile1D{};
99struct Profile2D{};
100struct Min{};
101struct Max{};
102struct Sum{};
103struct Mean{};
104struct Fill{};
105struct StdDev{};
106struct Display{};
107struct Snapshot{};
108struct Book{};
109}
110// clang-format on
111
112template <typename T, bool ISV6HISTO = std::is_base_of<TH1, std::decay_t<T>>::value>
113struct HistoUtils {
114 static void SetCanExtendAllAxes(T &h) { h.SetCanExtend(::TH1::kAllAxes); }
115 static bool HasAxisLimits(T &h)
116 {
117 auto xaxis = h.GetXaxis();
118 return !(xaxis->GetXmin() == 0. && xaxis->GetXmax() == 0.);
119 }
120};
121
122template <typename T>
123struct HistoUtils<T, false> {
124 static void SetCanExtendAllAxes(T &) {}
125 static bool HasAxisLimits(T &) { return true; }
126};
127
128// Generic filling (covers Histo2D, Histo3D, HistoND, Profile1D and Profile2D actions, with and without weights)
129template <typename... ColTypes, typename ActionTag, typename ActionResultType, typename PrevNodeType>
130std::unique_ptr<RActionBase>
131BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &h, const unsigned int nSlots,
132 std::shared_ptr<PrevNodeType> prevNode, ActionTag, const RColumnRegister &colRegister)
133{
134 using Helper_t = FillHelper<ActionResultType>;
135 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
136 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
137}
138
139// Histo1D filling (must handle the special case of distinguishing FillHelper and BufferedFillHelper
140template <typename... ColTypes, typename PrevNodeType>
141std::unique_ptr<RActionBase>
142BuildAction(const ColumnNames_t &bl, const std::shared_ptr<::TH1D> &h, const unsigned int nSlots,
143 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Histo1D, const RColumnRegister &colRegister)
144{
145 auto hasAxisLimits = HistoUtils<::TH1D>::HasAxisLimits(*h);
146
147 if (hasAxisLimits) {
148 using Helper_t = FillHelper<::TH1D>;
149 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
150 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
151 } else {
152 using Helper_t = BufferedFillHelper;
153 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
154 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
155 }
156}
157
158template <typename... ColTypes, typename PrevNodeType>
159std::unique_ptr<RActionBase>
160BuildAction(const ColumnNames_t &bl, const std::shared_ptr<TGraph> &g, const unsigned int nSlots,
161 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Graph, const RColumnRegister &colRegister)
162{
163 using Helper_t = FillTGraphHelper;
164 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
165 return std::make_unique<Action_t>(Helper_t(g, nSlots), bl, std::move(prevNode), colRegister);
166}
167
168template <typename... ColTypes, typename PrevNodeType>
169std::unique_ptr<RActionBase>
170BuildAction(const ColumnNames_t &bl, const std::shared_ptr<TGraphAsymmErrors> &g, const unsigned int nSlots,
171 std::shared_ptr<PrevNodeType> prevNode, ActionTags::GraphAsymmErrors, const RColumnRegister &colRegister)
172{
173 using Helper_t = FillTGraphAsymmErrorsHelper;
174 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
175 return std::make_unique<Action_t>(Helper_t(g, nSlots), bl, std::move(prevNode), colRegister);
176}
177
178// Min action
179template <typename ColType, typename PrevNodeType, typename ActionResultType>
180std::unique_ptr<RActionBase>
181BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &minV, const unsigned int nSlots,
182 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Min, const RColumnRegister &colRegister)
183{
184 using Helper_t = MinHelper<ActionResultType>;
185 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
186 return std::make_unique<Action_t>(Helper_t(minV, nSlots), bl, std::move(prevNode), colRegister);
187}
188
189// Max action
190template <typename ColType, typename PrevNodeType, typename ActionResultType>
191std::unique_ptr<RActionBase>
192BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &maxV, const unsigned int nSlots,
193 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Max, const RColumnRegister &colRegister)
194{
195 using Helper_t = MaxHelper<ActionResultType>;
196 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
197 return std::make_unique<Action_t>(Helper_t(maxV, nSlots), bl, std::move(prevNode), colRegister);
198}
199
200// Sum action
201template <typename ColType, typename PrevNodeType, typename ActionResultType>
202std::unique_ptr<RActionBase>
203BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &sumV, const unsigned int nSlots,
204 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Sum, const RColumnRegister &colRegister)
205{
206 using Helper_t = SumHelper<ActionResultType>;
207 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
208 return std::make_unique<Action_t>(Helper_t(sumV, nSlots), bl, std::move(prevNode), colRegister);
209}
210
211// Mean action
212template <typename ColType, typename PrevNodeType>
213std::unique_ptr<RActionBase>
214BuildAction(const ColumnNames_t &bl, const std::shared_ptr<double> &meanV, const unsigned int nSlots,
215 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Mean, const RColumnRegister &colRegister)
216{
217 using Helper_t = MeanHelper;
218 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
219 return std::make_unique<Action_t>(Helper_t(meanV, nSlots), bl, std::move(prevNode), colRegister);
220}
221
222// Standard Deviation action
223template <typename ColType, typename PrevNodeType>
224std::unique_ptr<RActionBase>
225BuildAction(const ColumnNames_t &bl, const std::shared_ptr<double> &stdDeviationV, const unsigned int nSlots,
226 std::shared_ptr<PrevNodeType> prevNode, ActionTags::StdDev, const RColumnRegister &colRegister)
227{
228 using Helper_t = StdDevHelper;
229 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
230 return std::make_unique<Action_t>(Helper_t(stdDeviationV, nSlots), bl, prevNode, colRegister);
231}
232
233// Display action
234template <typename... ColTypes, typename PrevNodeType>
235std::unique_ptr<RActionBase> BuildAction(const ColumnNames_t &bl, const std::shared_ptr<RDisplay> &d,
236 const unsigned int, std::shared_ptr<PrevNodeType> prevNode,
237 ActionTags::Display, const RDFInternal::RColumnRegister &colRegister)
238{
239 using Helper_t = DisplayHelper<PrevNodeType>;
240 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
241 return std::make_unique<Action_t>(Helper_t(d, prevNode), bl, prevNode, colRegister);
242}
243
244struct SnapshotHelperArgs {
245 std::string fFileName;
246 std::string fDirName;
247 std::string fTreeName;
248 std::vector<std::string> fOutputColNames;
250};
251
252// Snapshot action
253template <typename... ColTypes, typename PrevNodeType>
254std::unique_ptr<RActionBase>
255BuildAction(const ColumnNames_t &colNames, const std::shared_ptr<SnapshotHelperArgs> &snapHelperArgs,
256 const unsigned int nSlots, std::shared_ptr<PrevNodeType> prevNode, ActionTags::Snapshot,
257 const RColumnRegister &colRegister)
258{
259 const auto &filename = snapHelperArgs->fFileName;
260 const auto &dirname = snapHelperArgs->fDirName;
261 const auto &treename = snapHelperArgs->fTreeName;
262 const auto &outputColNames = snapHelperArgs->fOutputColNames;
263 const auto &options = snapHelperArgs->fOptions;
264
265 auto makeIsDefine = [&] {
266 std::vector<bool> isDef;
267 isDef.reserve(sizeof...(ColTypes));
268 for (auto i = 0u; i < sizeof...(ColTypes); ++i)
269 isDef[i] = colRegister.IsDefineOrAlias(colNames[i]);
270 return isDef;
271 };
272 std::vector<bool> isDefine = makeIsDefine();
273
274 std::unique_ptr<RActionBase> actionPtr;
276 // single-thread snapshot
277 using Helper_t = SnapshotHelper<ColTypes...>;
278 using Action_t = RAction<Helper_t, PrevNodeType>;
279 actionPtr.reset(
280 new Action_t(Helper_t(filename, dirname, treename, colNames, outputColNames, options, std::move(isDefine)),
281 colNames, prevNode, colRegister));
282 } else {
283 // multi-thread snapshot
284 using Helper_t = SnapshotHelperMT<ColTypes...>;
285 using Action_t = RAction<Helper_t, PrevNodeType>;
286 actionPtr.reset(new Action_t(
287 Helper_t(nSlots, filename, dirname, treename, colNames, outputColNames, options, std::move(isDefine)),
288 colNames, prevNode, colRegister));
289 }
290 return actionPtr;
291}
292
293// Book with custom helper type
294template <typename... ColTypes, typename PrevNodeType, typename Helper_t>
295std::unique_ptr<RActionBase>
296BuildAction(const ColumnNames_t &bl, const std::shared_ptr<Helper_t> &h, const unsigned int /*nSlots*/,
297 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Book, const RColumnRegister &colRegister)
298{
299 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
300 return std::make_unique<Action_t>(Helper_t(std::move(*h)), bl, std::move(prevNode), colRegister);
301}
302
303/****** end BuildAndBook ******/
304
305template <typename Filter>
306void CheckFilter(Filter &)
307{
308 using FilterRet_t = typename RDF::CallableTraits<Filter>::ret_type;
310 "filter expression returns a type that is not convertible to bool");
311}
312
313ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action);
314
315void CheckValidCppVarName(std::string_view var, const std::string &where);
316
317void CheckForRedefinition(const std::string &where, std::string_view definedCol, const RColumnRegister &colRegister,
318 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns);
319
320void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister,
321 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns);
322
323void CheckForNoVariations(const std::string &where, std::string_view definedColView,
324 const RColumnRegister &colRegister);
325
326std::string PrettyPrintAddr(const void *const addr);
327
328std::shared_ptr<RJittedFilter> BookFilterJit(std::shared_ptr<RNodeBase> *prevNodeOnHeap, std::string_view name,
329 std::string_view expression, const ColumnNames_t &branches,
330 const RColumnRegister &colRegister, TTree *tree, RDataSource *ds);
331
332std::shared_ptr<RJittedDefine> BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm,
333 RDataSource *ds, const RColumnRegister &colRegister,
334 const ColumnNames_t &branches, std::shared_ptr<RNodeBase> *prevNodeOnHeap);
335
336std::shared_ptr<RJittedDefine> BookDefinePerSampleJit(std::string_view name, std::string_view expression,
337 RLoopManager &lm, const RColumnRegister &colRegister,
338 std::shared_ptr<RNodeBase> *upcastNodeOnHeap);
339
340std::shared_ptr<RJittedVariation>
341BookVariationJit(const std::vector<std::string> &colNames, std::string_view variationName,
342 const std::vector<std::string> &variationTags, std::string_view expression, RLoopManager &lm,
343 RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches,
344 std::shared_ptr<RNodeBase> *upcastNodeOnHeap, bool isSingleColumn);
345
346std::string JitBuildAction(const ColumnNames_t &bl, std::shared_ptr<RDFDetail::RNodeBase> *prevNode,
347 const std::type_info &art, const std::type_info &at, void *rOnHeap, TTree *tree,
348 const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds,
349 std::weak_ptr<RJittedAction> *jittedActionOnHeap);
350
351// Allocate a weak_ptr on the heap, return a pointer to it. The user is responsible for deleting this weak_ptr.
352// This function is meant to be used by RInterface's methods that book code for jitting.
353// The problem it solves is that we generate code to be lazily jitted with the addresses of certain objects in them,
354// and we need to check those objects are still alive when the generated code is finally jitted and executed.
355// So we pass addresses to weak_ptrs allocated on the heap to the jitted code, which is then responsible for
356// the deletion of the weak_ptr object.
357template <typename T>
358std::weak_ptr<T> *MakeWeakOnHeap(const std::shared_ptr<T> &shPtr)
359{
360 return new std::weak_ptr<T>(shPtr);
361}
362
363// Same as MakeWeakOnHeap, but create a shared_ptr that makes sure the object is definitely kept alive.
364template <typename T>
365std::shared_ptr<T> *MakeSharedOnHeap(const std::shared_ptr<T> &shPtr)
366{
367 return new std::shared_ptr<T>(shPtr);
368}
369
370bool AtLeastOneEmptyString(const std::vector<std::string_view> strings);
371
372/// Take a shared_ptr<AnyNodeType> and return a shared_ptr<RNodeBase>.
373/// This works for RLoopManager nodes as well as filters and ranges.
374std::shared_ptr<RNodeBase> UpcastNode(std::shared_ptr<RNodeBase> ptr);
375
376ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns,
377 const RColumnRegister &validDefines, RDataSource *ds);
378
379std::vector<std::string> GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister,
380 TTree *tree, RDataSource *ds, const std::string &context,
381 bool vector2rvec);
382
383std::vector<bool> FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedDSCols);
384
385template <typename T>
386void AddDSColumnsHelper(const std::string &colName, RLoopManager &lm, RDataSource &ds, RColumnRegister &colRegister)
387{
388 if (colRegister.IsDefineOrAlias(colName) || !ds.HasColumn(colName) ||
389 lm.HasDataSourceColumnReaders(colName, typeid(T)))
390 return;
391
392 const auto nSlots = lm.GetNSlots();
393 std::vector<std::unique_ptr<RColumnReaderBase>> colReaders;
394 colReaders.reserve(nSlots);
395
396 const auto valuePtrs = ds.GetColumnReaders<T>(colName);
397 if (!valuePtrs.empty()) { // we are using the old GetColumnReaders mechanism in this RDataSource
398 for (auto *ptr : valuePtrs)
399 colReaders.emplace_back(new RDSColumnReader<T>(ptr));
400
401 } else { // using the new GetColumnReaders mechanism
402 // TODO consider changing the interface so we return all of these for all slots in one go
403 for (auto slot = 0u; slot < lm.GetNSlots(); ++slot)
404 colReaders.emplace_back(ds.GetColumnReaders(slot, colName, typeid(T)));
405 }
406
407 lm.AddDataSourceColumnReaders(colName, std::move(colReaders), typeid(T));
408}
409
410/// Take list of column names that must be defined, current map of custom columns, current list of defined column names,
411/// and return a new map of custom columns (with the new datasource columns added to it)
412template <typename... ColumnTypes>
413void AddDSColumns(const std::vector<std::string> &requiredCols, RLoopManager &lm, RDataSource &ds,
414 TTraits::TypeList<ColumnTypes...>, RColumnRegister &colRegister)
415{
416 // hack to expand a template parameter pack without c++17 fold expressions.
417 using expander = int[];
418 int i = 0;
419 (void)expander{(AddDSColumnsHelper<ColumnTypes>(requiredCols[i], lm, ds, colRegister), ++i)..., 0};
420}
421
422// this function is meant to be called by the jitted code generated by BookFilterJit
423template <typename F, typename PrevNode>
424void JitFilterHelper(F &&f, const char **colsPtr, std::size_t colsSize, std::string_view name,
425 std::weak_ptr<RJittedFilter> *wkJittedFilter, std::shared_ptr<PrevNode> *prevNodeOnHeap,
426 RColumnRegister *colRegister) noexcept
427{
428 if (wkJittedFilter->expired()) {
429 // The branch of the computation graph that needed this jitted code went out of scope between the type
430 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
431 delete wkJittedFilter;
432 delete colRegister;
433 delete prevNodeOnHeap;
434 return;
435 }
436
437 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
438 delete[] colsPtr;
439
440 const auto jittedFilter = wkJittedFilter->lock();
441
442 // mock Filter logic -- validity checks and Define-ition of RDataSource columns
443 using Callable_t = std::decay_t<F>;
445 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
446 constexpr auto nColumns = ColTypes_t::list_size;
447 CheckFilter(f);
448
449 auto &lm = *jittedFilter->GetLoopManagerUnchecked(); // RLoopManager must exist at this time
450 auto ds = lm.GetDataSource();
451
452 if (ds != nullptr)
453 AddDSColumns(cols, lm, *ds, ColTypes_t(), *colRegister);
454
455 jittedFilter->SetFilter(
456 std::unique_ptr<RFilterBase>(new F_t(std::forward<F>(f), cols, *prevNodeOnHeap, *colRegister, name)));
457 // colRegister points to the columns structure in the heap, created before the jitted call so that the jitter can
458 // share data after it has lazily compiled the code. Here the data has been used and the memory can be freed.
459 delete colRegister;
460 delete prevNodeOnHeap;
461 delete wkJittedFilter;
462}
463
464namespace DefineTypes {
465struct RDefineTag {};
466struct RDefinePerSampleTag {};
467}
468
469template <typename F>
470auto MakeDefineNode(DefineTypes::RDefineTag, std::string_view name, std::string_view dummyType, F &&f,
471 const ColumnNames_t &cols, RColumnRegister &colRegister, RLoopManager &lm)
472{
473 return std::unique_ptr<RDefineBase>(new RDefine<std::decay_t<F>, ExtraArgsForDefine::None>(
474 name, dummyType, std::forward<F>(f), cols, colRegister, lm));
475}
476
477template <typename F>
478auto MakeDefineNode(DefineTypes::RDefinePerSampleTag, std::string_view name, std::string_view dummyType, F &&f,
479 const ColumnNames_t &, RColumnRegister &, RLoopManager &lm)
480{
481 return std::unique_ptr<RDefineBase>(
482 new RDefinePerSample<std::decay_t<F>>(name, dummyType, std::forward<F>(f), lm));
483}
484
485// Build a RDefine or a RDefinePerSample object and attach it to an existing RJittedDefine
486// This function is meant to be called by jitted code right before starting the event loop.
487// If colsPtr is null, build a RDefinePerSample (it has no input columns), otherwise a RDefine.
488template <typename RDefineTypeTag, typename F>
489void JitDefineHelper(F &&f, const char **colsPtr, std::size_t colsSize, std::string_view name, RLoopManager *lm,
490 std::weak_ptr<RJittedDefine> *wkJittedDefine, RColumnRegister *colRegister,
491 std::shared_ptr<RNodeBase> *prevNodeOnHeap) noexcept
492{
493 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
494 auto doDeletes = [&] {
495 delete wkJittedDefine;
496 delete colRegister;
497 delete prevNodeOnHeap;
498 delete[] colsPtr;
499 };
500
501 if (wkJittedDefine->expired()) {
502 // The branch of the computation graph that needed this jitted code went out of scope between the type
503 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
504 doDeletes();
505 return;
506 }
507
508 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
509
510 auto jittedDefine = wkJittedDefine->lock();
511
512 using Callable_t = std::decay_t<F>;
513 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
514
515 auto ds = lm->GetDataSource();
516 if (ds != nullptr)
517 AddDSColumns(cols, *lm, *ds, ColTypes_t(), *colRegister);
518
519 // will never actually be used (trumped by jittedDefine->GetTypeName()), but we set it to something meaningful
520 // to help devs debugging
521 const auto dummyType = "jittedCol_t";
522 // use unique_ptr<RDefineBase> instead of make_unique<NewCol_t> to reduce jit/compile-times
523 std::unique_ptr<RDefineBase> newCol{
524 MakeDefineNode(RDefineTypeTag{}, name, dummyType, std::forward<F>(f), cols, *colRegister, *lm)};
525 jittedDefine->SetDefine(std::move(newCol));
526
527 doDeletes();
528}
529
530template <bool IsSingleColumn, typename F>
531void JitVariationHelper(F &&f, const char **colsPtr, std::size_t colsSize, const char **variedCols,
532 std::size_t variedColsSize, const char **variationTags, std::size_t variationTagsSize,
533 std::string_view variationName, RLoopManager *lm,
534 std::weak_ptr<RJittedVariation> *wkJittedVariation, RColumnRegister *colRegister,
535 std::shared_ptr<RNodeBase> *prevNodeOnHeap) noexcept
536{
537 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
538 auto doDeletes = [&] {
539 delete[] colsPtr;
540 delete[] variedCols;
541 delete[] variationTags;
542
543 delete wkJittedVariation;
544 delete colRegister;
545 delete prevNodeOnHeap;
546 };
547
548 if (wkJittedVariation->expired()) {
549 // The branch of the computation graph that needed this jitted variation went out of scope between the type
550 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
551 doDeletes();
552 return;
553 }
554
555 const ColumnNames_t inputColNames(colsPtr, colsPtr + colsSize);
556 std::vector<std::string> variedColNames(variedCols, variedCols + variedColsSize);
557 std::vector<std::string> tags(variationTags, variationTags + variationTagsSize);
558
559 auto jittedVariation = wkJittedVariation->lock();
560
561 using Callable_t = std::decay_t<F>;
562 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
563
564 auto ds = lm->GetDataSource();
565 if (ds != nullptr)
566 AddDSColumns(inputColNames, *lm, *ds, ColTypes_t(), *colRegister);
567
568 // use unique_ptr<RDefineBase> instead of make_unique<NewCol_t> to reduce jit/compile-times
569 std::unique_ptr<RVariationBase> newVariation{new RVariation<std::decay_t<F>, IsSingleColumn>(
570 std::move(variedColNames), variationName, std::forward<F>(f), std::move(tags), jittedVariation->GetTypeName(),
571 *colRegister, *lm, std::move(inputColNames))};
572 jittedVariation->SetVariation(std::move(newVariation));
573
574 doDeletes();
575}
576
577/// Convenience function invoked by jitted code to build action nodes at runtime
578template <typename ActionTag, typename... ColTypes, typename PrevNodeType, typename HelperArgType>
579void CallBuildAction(std::shared_ptr<PrevNodeType> *prevNodeOnHeap, const char **colsPtr, std::size_t colsSize,
580 const unsigned int nSlots, std::shared_ptr<HelperArgType> *helperArgOnHeap,
581 std::weak_ptr<RJittedAction> *wkJittedActionOnHeap, RColumnRegister *colRegister) noexcept
582{
583 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
584 auto doDeletes = [&] {
585 delete[] colsPtr;
586 delete helperArgOnHeap;
587 delete wkJittedActionOnHeap;
588 // colRegister must be deleted before prevNodeOnHeap because their dtor needs the RLoopManager to be alive
589 // and prevNodeOnHeap is what keeps it alive if the rest of the computation graph is already out of scope
590 delete colRegister;
591 delete prevNodeOnHeap;
592 };
593
594 if (wkJittedActionOnHeap->expired()) {
595 // The branch of the computation graph that needed this jitted variation went out of scope between the type
596 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
597 doDeletes();
598 return;
599 }
600
601 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
602
603 auto jittedActionOnHeap = wkJittedActionOnHeap->lock();
604
605 // if we are here it means we are jitting, if we are jitting the loop manager must be alive
606 auto &prevNodePtr = *prevNodeOnHeap;
607 auto &loopManager = *prevNodePtr->GetLoopManagerUnchecked();
608 using ColTypes_t = TypeList<ColTypes...>;
609 constexpr auto nColumns = ColTypes_t::list_size;
610 auto ds = loopManager.GetDataSource();
611 if (ds != nullptr)
612 AddDSColumns(cols, loopManager, *ds, ColTypes_t(), *colRegister);
613
614 auto actionPtr = BuildAction<ColTypes...>(cols, std::move(*helperArgOnHeap), nSlots, std::move(prevNodePtr),
615 ActionTag{}, *colRegister);
616 jittedActionOnHeap->SetAction(std::move(actionPtr));
617
618 doDeletes();
619}
620
621/// The contained `type` alias is `double` if `T == RInferredType`, `U` if `T == std::container<U>`, `T` otherwise.
623struct RMinReturnType {
624 using type = T;
625};
626
627template <>
628struct RMinReturnType<RInferredType, false> {
629 using type = double;
630};
631
632template <typename T>
633struct RMinReturnType<T, true> {
634 using type = TTraits::TakeFirstParameter_t<T>;
635};
636
637// return wrapper around f that prepends an `unsigned int slot` parameter
638template <typename R, typename F, typename... Args>
639std::function<R(unsigned int, Args...)> AddSlotParameter(F &f, TypeList<Args...>)
640{
641 return [f](unsigned int, Args... a) mutable -> R { return f(a...); };
642}
643
644template <typename ColType, typename... Rest>
645struct RNeedJittingHelper {
646 static constexpr bool value = RNeedJittingHelper<Rest...>::value;
647};
648
649template <typename... Rest>
650struct RNeedJittingHelper<RInferredType, Rest...> {
651 static constexpr bool value = true;
652};
653
654template <typename T>
655struct RNeedJittingHelper<T> {
656 static constexpr bool value = false;
657};
658
659template <>
660struct RNeedJittingHelper<RInferredType> {
661 static constexpr bool value = true;
662};
663
664template <typename ...ColTypes>
665struct RNeedJitting {
666 static constexpr bool value = RNeedJittingHelper<ColTypes...>::value;
667};
668
669template <>
670struct RNeedJitting<> {
671 static constexpr bool value = false;
672};
673
674///////////////////////////////////////////////////////////////////////////////
675/// Check preconditions for RInterface::Aggregate:
676/// - the aggregator callable must have signature `U(U,T)` or `void(U&,T)`.
677/// - the merge callable must have signature `U(U,U)` or `void(std::vector<U>&)`
678template <typename R, typename Merge, typename U, typename T, typename decayedU = std::decay_t<U>,
679 typename mergeArgsNoDecay_t = typename CallableTraits<Merge>::arg_types_nodecay,
680 typename mergeArgs_t = typename CallableTraits<Merge>::arg_types,
681 typename mergeRet_t = typename CallableTraits<Merge>::ret_type>
682void CheckAggregate(TypeList<U, T>)
683{
684 constexpr bool isAggregatorOk =
686 static_assert(isAggregatorOk, "aggregator function must have signature `U(U,T)` or `void(U&,T)`");
687 constexpr bool isMergeOk =
688 (std::is_same<TypeList<decayedU, decayedU>, mergeArgs_t>::value && std::is_same<decayedU, mergeRet_t>::value) ||
689 (std::is_same<TypeList<std::vector<decayedU> &>, mergeArgsNoDecay_t>::value &&
691 static_assert(isMergeOk, "merge function must have signature `U(U,U)` or `void(std::vector<U>&)`");
692}
693
694///////////////////////////////////////////////////////////////////////////////
695/// This overload of CheckAggregate is called when the aggregator takes more than two arguments
696template <typename R, typename T>
697void CheckAggregate(T)
698{
699 static_assert(sizeof(T) == 0, "aggregator function must take exactly two arguments");
700}
701
702///////////////////////////////////////////////////////////////////////////////
703/// Check as many template parameters were passed as the number of column names, throw if this is not the case.
704void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames);
705
706/// Return local BranchNames or default BranchNames according to which one should be used
707const ColumnNames_t SelectColumns(unsigned int nArgs, const ColumnNames_t &bl, const ColumnNames_t &defBl);
708
709/// Check whether column names refer to a valid branch of a TTree or have been `Define`d. Return invalid column names.
710ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns,
711 const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns);
712
713/// Returns the list of Filters defined in the whole graph
714std::vector<std::string> GetFilterNames(const std::shared_ptr<RLoopManager> &loopManager);
715
716/// Returns the list of Filters defined in the branch
717template <typename NodeType>
718std::vector<std::string> GetFilterNames(const std::shared_ptr<NodeType> &node)
719{
720 std::vector<std::string> filterNames;
721 node->AddFilterName(filterNames);
722 return filterNames;
723}
724
725struct ParsedTreePath {
726 std::string fTreeName;
727 std::string fDirName;
728};
729
730ParsedTreePath ParseTreePath(std::string_view fullTreeName);
731
732// Check if a condition is true for all types
733template <bool...>
734struct TBoolPack;
735
736template <bool... bs>
737using IsTrueForAllImpl_t = typename std::is_same<TBoolPack<bs..., true>, TBoolPack<true, bs...>>;
738
739template <bool... Conditions>
740struct TEvalAnd {
741 static constexpr bool value = IsTrueForAllImpl_t<Conditions...>::value;
742};
743
744// Check if a class is a specialisation of stl containers templates
745// clang-format off
746
747template <typename>
748struct IsList_t : std::false_type {};
749
750template <typename T>
751struct IsList_t<std::list<T>> : std::true_type {};
752
753template <typename>
754struct IsDeque_t : std::false_type {};
755
756template <typename T>
757struct IsDeque_t<std::deque<T>> : std::true_type {};
758// clang-format on
759
761
762void TriggerRun(ROOT::RDF::RNode &node);
763
764template <typename T>
765struct InnerValueType {
766 using type = T; // fallback for when T is not a nested RVec
767};
768
769template <typename Elem>
770struct InnerValueType<ROOT::VecOps::RVec<ROOT::VecOps::RVec<Elem>>> {
771 using type = Elem;
772};
773
774template <typename T>
775using InnerValueType_t = typename InnerValueType<T>::type;
776
777std::pair<std::vector<std::string>, std::vector<std::string>>
778AddSizeBranches(const std::vector<std::string> &branches, TTree *tree, std::vector<std::string> &&colsWithoutAliases,
779 std::vector<std::string> &&colsWithAliases);
780
781void RemoveDuplicates(ColumnNames_t &columnNames);
782
783} // namespace RDF
784} // namespace Internal
785
786namespace Detail {
787namespace RDF {
788
789/// The aliased type is `double` if `T == RInferredType`, `U` if `T == container<U>`, `T` otherwise.
790template <typename T>
791using MinReturnType_t = typename RDFInternal::RMinReturnType<T>::type;
792
793template <typename T>
794using MaxReturnType_t = MinReturnType_t<T>;
795
796template <typename T>
797using SumReturnType_t = MinReturnType_t<T>;
798
799} // namespace RDF
800} // namespace Detail
801} // namespace ROOT
802
803/// \endcond
804
805#endif
#define d(i)
Definition: RSha256.hxx:102
#define f(i)
Definition: RSha256.hxx:104
#define h(i)
Definition: RSha256.hxx:106
#define R(a, b, c, d, e, f, g, h, i)
Definition: RSha256.hxx:110
Int_t gErrorIgnoreLevel
Definition: TError.cxx:34
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t g
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition: TGX11.cxx:110
The head node of a RDF computation graph.
void AddDataSourceColumnReaders(const std::string &col, std::vector< std::unique_ptr< RColumnReaderBase > > &&readers, const std::type_info &ti)
RDataSource * GetDataSource() const
unsigned int GetNSlots() const
RLoopManager * GetLoopManagerUnchecked() final
bool HasDataSourceColumnReaders(const std::string &col, const std::type_info &ti) const
Return true if AddDataSourceColumnReaders was called for column name col.
A binder for user-defined columns, variations and aliases.
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual bool HasColumn(std::string_view colName) const =0
Checks if the dataset has a certain column.
std::vector< T ** > GetColumnReaders(std::string_view columnName)
Called at most once per column by RDF.
The public interface to the RDataFrame federation of classes.
Definition: RInterface.hxx:103
A "std::vector"-like collection of values implementing handy operation to analyse them.
Definition: RVec.hxx:1456
@ kAllAxes
Definition: TH1.h:75
An array of TObjects.
Definition: TObjArray.h:31
A TTree represents a columnar dataset.
Definition: TTree.h:79
T Sum(const RVec< T > &v, const T zero=T(0))
Sum elements of an RVec.
Definition: RVec.hxx:1878
RVec< T > Filter(const RVec< T > &v, F &&f)
Create a new collection with the elements passing the filter expressed by the predicate.
Definition: RVec.hxx:2102
basic_string_view< char > string_view
#define F(x, y, z)
const ColumnNames_t SelectColumns(unsigned int nRequiredNames, const ColumnNames_t &names, const ColumnNames_t &defaultNames)
Choose between local column names or default column names, throw in case of errors.
void CheckForNoVariations(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister)
Throw if the column has systematic variations attached.
ParsedTreePath ParseTreePath(std::string_view fullTreeName)
void CheckForRedefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is already there.
void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is not already there.
std::shared_ptr< RJittedDefine > BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a Define call.
void CheckValidCppVarName(std::string_view var, const std::string &where)
void RemoveDuplicates(ColumnNames_t &columnNames)
ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns, const RColumnRegister &colRegister, RDataSource *ds)
Given the desired number of columns and the user-provided list of columns:
std::shared_ptr< RNodeBase > UpcastNode(std::shared_ptr< RNodeBase > ptr)
std::vector< std::string > GetFilterNames(const std::shared_ptr< RLoopManager > &loopManager)
std::string PrettyPrintAddr(const void *const addr)
void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames)
std::string DemangleTypeIdName(const std::type_info &typeInfo)
bool AtLeastOneEmptyString(const std::vector< std::string_view > strings)
std::shared_ptr< RDFDetail::RJittedFilter > BookFilterJit(std::shared_ptr< RDFDetail::RNodeBase > *prevNodeOnHeap, std::string_view name, std::string_view expression, const ColumnNames_t &branches, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds)
Book the jitting of a Filter call.
std::string JitBuildAction(const ColumnNames_t &cols, std::shared_ptr< RDFDetail::RNodeBase > *prevNode, const std::type_info &helperArgType, const std::type_info &at, void *helperArgOnHeap, TTree *tree, const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds, std::weak_ptr< RJittedAction > *jittedActionOnHeap)
ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action)
Take a list of column names, return that list with entries starting by '#' filtered out.
std::shared_ptr< RJittedVariation > BookVariationJit(const std::vector< std::string > &colNames, std::string_view variationName, const std::vector< std::string > &variationTags, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap, bool isSingleColumn)
Book the jitting of a Vary call.
void CheckForDuplicateSnapshotColumns(const ColumnNames_t &cols)
ColumnNames_t ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName)
std::pair< std::vector< std::string >, std::vector< std::string > > AddSizeBranches(const std::vector< std::string > &branches, TTree *tree, std::vector< std::string > &&colsWithoutAliases, std::vector< std::string > &&colsWithAliases)
Return copies of colsWithoutAliases and colsWithAliases with size branches for variable-sized array b...
std::vector< bool > FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedCols)
Return a bitset each element of which indicates whether the corresponding element in selectedColumns ...
std::shared_ptr< RJittedDefine > BookDefinePerSampleJit(std::string_view name, std::string_view expression, RLoopManager &lm, const RColumnRegister &colRegister, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a DefinePerSample call.
std::vector< std::string > GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds, const std::string &context, bool vector2rvec)
ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns, const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns)
void TriggerRun(ROOT::RDF::RNode &node)
Trigger the execution of an RDataFrame computation graph.
void(off) SmallVectorTemplateBase< T
double T(double x)
Definition: ChebyshevPol.h:34
std::vector< std::string > ColumnNames_t
void function(const Char_t *name_, T fun, const Char_t *docstring=0)
Definition: RExports.h:167
ROOT type_traits extensions.
Definition: TypeTraits.hxx:21
This file contains a specialised ROOT message handler to test for diagnostic in unit tests.
Bool_t IsImplicitMTEnabled()
Returns true if the implicit multi-threading in ROOT is enabled.
Definition: TROOT.cxx:558
Short_t Max(Short_t a, Short_t b)
Returns the largest of a and b.
Definition: TMathBase.h:250
Double_t Mean(Long64_t n, const T *a, const Double_t *w=0)
Returns the weighted mean of an array a with length n.
Definition: TMath.h:1087
Short_t Min(Short_t a, Short_t b)
Returns the smallest of a and b.
Definition: TMathBase.h:198
Double_t StdDev(Long64_t n, const T *a, const Double_t *w=0)
Definition: TMath.h:525
Definition: tree.py:1
A collection of options to steer the creation of the dataset on file.
Lightweight storage for a collection of types.
Definition: TypeTraits.hxx:25
TArc a
Definition: textangle.C:12