Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RLoopManager.cxx
Go to the documentation of this file.
1/*************************************************************************
2 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
3 * All rights reserved. *
4 * *
5 * For the licensing terms see $ROOTSYS/LICENSE. *
6 * For the list of contributors see $ROOTSYS/README/CREDITS. *
7 *************************************************************************/
8
9#include "RConfigure.h" // R__USE_IMT
10#include "ROOT/RDataSource.hxx"
12#include "ROOT/InternalTreeUtils.hxx" // GetTreeFullPaths
15#include "ROOT/RDF/RDefineReader.hxx" // RDefinesWithReaders
21#include "ROOT/RDF/RVariationReader.hxx" // RVariationsWithReaders
22#include "ROOT/RLogger.hxx"
23#include "ROOT/RNTuple.hxx"
24#include "ROOT/RNTupleDS.hxx"
25#include "RtypesCore.h" // Long64_t
26#include "TStopwatch.h"
27#include "TBranchElement.h"
28#include "TBranchObject.h"
29#include "TChain.h"
30#include "TEntryList.h"
31#include "TFile.h"
32#include "TFriendElement.h"
33#include "TInterpreter.h"
34#include "TROOT.h" // IsImplicitMTEnabled, gCoreMutex, R__*_LOCKGUARD
35#include "TTreeReader.h"
36#include "TTree.h" // For MaxTreeSizeRAII. Revert when #6640 will be solved.
37
38#include "ROOT/RTTreeDS.hxx"
39
40#ifdef R__USE_IMT
43#include "ROOT/RSlotStack.hxx"
44#endif
45
46#include "ROOT/InternalIOUtils.hxx"
47#include "TSystem.h"
48
49#include <algorithm>
50#include <atomic>
51#include <cassert>
52#include <functional>
53#include <iostream>
54#include <memory>
55#include <stdexcept>
56#include <string>
57#include <sstream>
58#include <thread>
59#include <unordered_map>
60#include <vector>
61#include <set>
62#include <limits> // For MaxTreeSizeRAII. Revert when #6640 will be solved.
63
64using namespace ROOT::Detail::RDF;
65using namespace ROOT::Internal::RDF;
66
67namespace {
68/// A helper function that returns all RDF code that is currently scheduled for just-in-time compilation.
69/// This allows different RLoopManager instances to share these data.
70/// We want RLoopManagers to be able to add their code to a global "code to execute via cling",
71/// so that, lazily, we can jit everything that's needed by all RDFs in one go, which is potentially
72/// much faster than jitting each RLoopManager's code separately.
73std::string &GetCodeToJit()
74{
75 static std::string code;
76 return code;
77}
78
79std::string &GetCodeToDeclare()
80{
81 static std::string code;
82 return code;
83}
84
85// Signature of all helper functions that are created by JIT helpers, see
86// Book*Jit and JitBuildAction in RDFInterfaceUtils.cxx
87using JitHelperFunc_t = void (*)(const std::vector<std::string> &, ROOT::Internal::RDF::RColumnRegister &,
88 ROOT::Detail::RDF::RLoopManager &, void *, std::shared_ptr<void> *);
89std::unordered_map<std::size_t, JitHelperFunc_t> &GetJitHelperFuncMap()
90{
91 static std::unordered_map<std::size_t, JitHelperFunc_t> map;
92 return map;
93}
94std::unordered_map<std::size_t, std::size_t> &GetJitFuncBodyToFuncIdMap()
95{
96 static std::unordered_map<std::size_t, std::size_t> map;
97 return map;
98}
99
100void DeclareAndRetrieveDeferredJitCalls(const std::string &codeToDeclare)
101{
102 // This function uses the interpreter and writes to the caches.
104
105 // Step 1: Declare the DeferredJitCall functions to the interpreter
106 // We use ProcessLine to ensure meta functionality (e.g. autoloading) is
107 // processed when needed.
108 // If instead we used Declare, builds with runtime_cxxmodules=OFF would fail
109 // in jitted actions with custom helpers with errors like:
110 // error: 'MyHelperType' is an incomplete type
111 // return std::make_unique<Action_t>(Helper_t(std::move(*h)), bl, std::move(prevNode), colRegister);
112 // ^
114 gInterpreter->ProcessLine(codeToDeclare.c_str(), &interpErrorCode);
115 if (interpErrorCode != TInterpreter::kNoError) {
116 throw std::runtime_error(
117 "\nAn error occurred during just-in-time compilation in RLoopManager::Run. The lines above might "
118 "indicate the cause of the error.\nAll RDF objects that have not run their event loop yet should be "
119 "considered in an invalid state.\n");
120 }
121
122 // Step 2: Retrieve the declared functions as function pointers, cache them
123 // for later use in RunDeferredCalls
124 auto &funcIdToFuncPointersMap = GetJitHelperFuncMap();
125 auto &funcBodyToFuncIdMap = GetJitFuncBodyToFuncIdMap();
126 auto clinfo = gInterpreter->ClassInfo_Factory("R_rdf");
127 assert(gInterpreter->ClassInfo_IsValid(clinfo));
128
129 for (auto &codeAndId : funcBodyToFuncIdMap) {
130 if (auto it = funcIdToFuncPointersMap.find(codeAndId.second); it == funcIdToFuncPointersMap.end()) {
131 // fast fetch of the address via gInterpreter
132 // (faster than gInterpreter->Evaluate(function name, ret), ret->GetAsPointer())
133 // Retrieve the JIT helper function we registered via RegisterJitHelperCall
134 const std::string funcName = "jitNodeRegistrator_" + std::to_string(codeAndId.second);
135 auto declid = gInterpreter->GetFunction(clinfo, funcName.c_str());
136 if (!declid) {
137 // The interpreter failed to compile the helper. Without this check
138 // we would later dereference a null function pointer and crash.
139 gInterpreter->ClassInfo_Delete(clinfo);
140 throw std::runtime_error(
141 "\nAn error occurred during just-in-time compilation in RLoopManager::Run: failed to retrieve "
142 "the JIT helper function '" +
143 funcName +
144 "'. The lines above might indicate the cause of the error.\nAll RDF objects that have not run "
145 "their event loop yet should be considered in an invalid state.\n");
146 }
147 auto minfo = gInterpreter->MethodInfo_Factory(declid);
148 assert(gInterpreter->MethodInfo_IsValid(minfo));
149 auto mname = gInterpreter->MethodInfo_GetMangledName(minfo);
150 [[maybe_unused]] auto res = funcIdToFuncPointersMap.insert(
151 {codeAndId.second, reinterpret_cast<JitHelperFunc_t>(gInterpreter->FindSym(mname))});
152 assert(res.second);
153 gInterpreter->MethodInfo_Delete(minfo);
154 }
155 }
156 gInterpreter->ClassInfo_Delete(clinfo);
157}
158
159void ThrowIfNSlotsChanged(unsigned int nSlots)
160{
161 const auto currentSlots = RDFInternal::GetNSlots();
162 if (currentSlots != nSlots) {
163 std::string msg = "RLoopManager::Run: when the RDataFrame was constructed the number of slots required was " +
164 std::to_string(nSlots) + ", but when starting the event loop it was " +
165 std::to_string(currentSlots) + ".";
166 if (currentSlots > nSlots)
167 msg += " Maybe EnableImplicitMT() was called after the RDataFrame was constructed?";
168 else
169 msg += " Maybe DisableImplicitMT() was called after the RDataFrame was constructed?";
170 throw std::runtime_error(msg);
171 }
172}
173
174/**
175\struct MaxTreeSizeRAII
176\brief Scope-bound change of `TTree::fgMaxTreeSize`.
177
178This RAII object stores the current value result of `TTree::GetMaxTreeSize`,
179changes it to maximum at construction time and restores it back at destruction
180time. Needed for issue #6523 and should be reverted when #6640 will be solved.
181*/
182struct MaxTreeSizeRAII {
183 Long64_t fOldMaxTreeSize;
184
185 MaxTreeSizeRAII() : fOldMaxTreeSize(TTree::GetMaxTreeSize())
186 {
187 TTree::SetMaxTreeSize(std::numeric_limits<Long64_t>::max());
188 }
189
190 ~MaxTreeSizeRAII() { TTree::SetMaxTreeSize(fOldMaxTreeSize); }
191};
192
193struct DatasetLogInfo {
194 std::string fDataSet;
195 ULong64_t fRangeStart;
196 ULong64_t fRangeEnd;
197 unsigned int fSlot;
198};
199
200std::string LogRangeProcessing(const DatasetLogInfo &info)
201{
202 std::stringstream msg;
203 msg << "Processing " << info.fDataSet << ": entry range [" << info.fRangeStart << "," << info.fRangeEnd - 1
204 << "], using slot " << info.fSlot << " in thread " << std::this_thread::get_id() << '.';
205 return msg.str();
206}
207
208auto MakeDatasetColReadersKey(std::string_view colName, const std::type_info &ti)
209{
210 // We use a combination of column name and column type name as the key because in some cases we might end up
211 // with concrete readers that use different types for the same column, e.g. std::vector and RVec here:
212 // df.Sum<vector<int>>("stdVectorBranch");
213 // df.Sum<RVecI>("stdVectorBranch");
214 return std::string(colName) + ':' + ti.name();
215}
216
217/// \brief Check if object of a certain type is in the directory
218///
219/// Attempts to read an object of the specified type via TDirectory::Get, wraps
220/// it in a std::unique_ptr to avoid leaking the object.
221template <typename T>
222bool IsObjectInDir(std::string_view objName, TDirectory &dir)
223{
224 std::unique_ptr<T> o{dir.Get<T>(objName.data())};
225 return o.get();
226}
227
228/// \brief Check if a generic object is in the directory
229///
230/// Checks if a generic object is in the directory, uses TDirectory::GetKey
231/// to avoid having to deal with memory management of the object being read
232/// without having its type.
233template <>
234bool IsObjectInDir<void>(std::string_view objName, TDirectory &dir)
235{
236 return dir.GetKey(objName.data());
237}
238} // anonymous namespace
239
240/**
241 * \brief Helper function to open a file (or the first file from a glob).
242 * This function is used at construction time of an RDataFrame, to check the
243 * concrete type of the dataset stored inside the file.
244 */
245std::unique_ptr<TFile> OpenFileWithSanityChecks(std::string_view fileNameGlob)
246{
247 // Follow same logic in TChain::Add to find the correct string to look for globbing:
248 // - If the extension ".root" is present in the file name, pass along the basename.
249 // - If not, use the "?" token to delimit the part of the string which represents the basename.
250 // - Otherwise, pass the full filename.
251 auto &&baseNameAndQuery = [&fileNameGlob]() {
252 constexpr std::string_view delim{".root"};
253 if (auto &&it = std::find_end(fileNameGlob.begin(), fileNameGlob.end(), delim.begin(), delim.end());
254 it != fileNameGlob.end()) {
255 auto &&distanceToEndOfDelim = std::distance(fileNameGlob.begin(), it + delim.length());
256 return std::make_pair(fileNameGlob.substr(0, distanceToEndOfDelim), fileNameGlob.substr(distanceToEndOfDelim));
257 } else if (auto &&lastQuestionMark = fileNameGlob.find_last_of('?'); lastQuestionMark != std::string_view::npos)
258 return std::make_pair(fileNameGlob.substr(0, lastQuestionMark), fileNameGlob.substr(lastQuestionMark));
259 else
260 return std::make_pair(fileNameGlob, std::string_view{});
261 }();
262 // Captured structured bindings variable are only valid since C++20
263 auto &&baseName = baseNameAndQuery.first;
264 auto &&query = baseNameAndQuery.second;
265
266 std::string fileToOpen{fileNameGlob};
267 if (baseName.find_first_of("[]*?") != std::string_view::npos) { // Wildcards accepted by TChain::Add
268 const auto expanded = ROOT::Internal::TreeUtils::ExpandGlob(std::string{baseName});
269 if (expanded.empty())
270 throw std::invalid_argument{"RDataFrame: The glob expression '" + std::string{baseName} +
271 "' did not match any files."};
272
273 fileToOpen = expanded.front() + std::string{query};
274 }
275
276 ::TDirectory::TContext ctxt; // Avoid changing gDirectory;
277 std::unique_ptr<TFile> inFile{TFile::Open(fileToOpen.c_str(), "READ_WITHOUT_GLOBALREGISTRATION")};
278 if (!inFile || inFile->IsZombie())
279 throw std::invalid_argument("RDataFrame: could not open file \"" + fileToOpen + "\".");
280
281 return inFile;
282}
283
284// These are three functions that release and take back the GIL (see SetEventLoopHooks());
285// they stay null in C++ programs.
286//
287// who owns what:
288// - these pointers are shared by the whole process and only the Python bindings write them, always with the same three
289// functions of the Python C API, so every thread sees the same values
290// - whether a thread holds the GIL is not stored here, it's kept separately for each thread
291// - what gRelease returns is kept in REventLoopHooksRAII as a local variable of Run() and RunGraphs(), each call has
292// its own so threads can't overwrite each other's
293//
294// gIsHeld only answers for the thread that calls it, and only one thread can hold the GIL at a time
295// so a thread only releases the GIL if it holds it, and takes it back itself when it is done;
296// a C++ thread that never held the GIL gets "no" to the check and does nothing
297//
298// limits: once a sub-interpreter is created gIsHeld says "yes" on every
299// thread, an event loop on a C++ thread then calls gRelease without the GIL and crashes
300// (see https://github.com/python/cpython/issues/59956)
301namespace {
302int (*gIsHeld)() = nullptr;
303void *(*gRelease)() = nullptr;
304void (*gRestore)(void *) = nullptr;
305} // namespace
306
307void ROOT::Internal::RDF::SetEventLoopHooks(int (*isHeld)(), void *(*release)(), void (*restore)(void *))
308{
309 gIsHeld = isHeld;
310 gRelease = release;
311 gRestore = restore;
312}
313
315{
316 if (gIsHeld && gIsHeld())
317 fState = gRelease();
318}
319
321{
322 if (fState)
323 gRestore(fState);
324}
325
326namespace ROOT {
327namespace Detail {
328namespace RDF {
329
330/// A RAII object that calls RLoopManager::CleanUpTask at destruction
333 unsigned int fArg;
335
336 RCallCleanUpTask(RLoopManager &lm, unsigned int arg = 0u, TTreeReader *reader = nullptr)
337 : fLoopManager(lm), fArg(arg), fReader(reader)
338 {
339 }
340 ~RCallCleanUpTask() { fLoopManager.CleanUpTask(fReader, fArg); }
341};
342
343} // namespace RDF
344} // namespace Detail
345} // namespace ROOT
346
347ROOT::Detail::RDF::RLoopManager::RLoopManager(const ROOT::Detail::RDF::ColumnNames_t &defaultColumns)
348 : fDefaultColumns(defaultColumns),
349 fNSlots(RDFInternal::GetNSlots()),
350 fNewSampleNotifier(fNSlots),
351 fSampleInfos(fNSlots),
352 fDatasetColumnReaders(fNSlots)
353{
354}
355
356RLoopManager::RLoopManager(TTree *tree, const ColumnNames_t &defaultBranches)
357 : fDefaultColumns(defaultBranches),
358 fNSlots(RDFInternal::GetNSlots()),
359 fLoopType(ROOT::IsImplicitMTEnabled() ? ELoopType::kDataSourceMT : ELoopType::kDataSource),
360 fDataSource(std::make_unique<ROOT::Internal::RDF::RTTreeDS>(ROOT::Internal::RDF::MakeAliasedSharedPtr(tree))),
361 fNewSampleNotifier(fNSlots),
362 fSampleInfos(fNSlots),
363 fDatasetColumnReaders(fNSlots)
364{
365 fDataSource->SetNSlots(fNSlots);
366}
367
369 : fEmptyEntryRange(0, nEmptyEntries),
370 fNSlots(RDFInternal::GetNSlots()),
371 fLoopType(ROOT::IsImplicitMTEnabled() ? ELoopType::kNoFilesMT : ELoopType::kNoFiles),
372 fNewSampleNotifier(fNSlots),
373 fSampleInfos(fNSlots),
374 fDatasetColumnReaders(fNSlots)
375{
376}
377
378RLoopManager::RLoopManager(std::unique_ptr<RDataSource> ds, const ColumnNames_t &defaultBranches)
379 : fDefaultColumns(defaultBranches),
380 fNSlots(RDFInternal::GetNSlots()),
381 fLoopType(ROOT::IsImplicitMTEnabled() ? ELoopType::kDataSourceMT : ELoopType::kDataSource),
382 fDataSource(std::move(ds)),
383 fNewSampleNotifier(fNSlots),
384 fSampleInfos(fNSlots),
385 fDatasetColumnReaders(fNSlots)
386{
387 fDataSource->SetNSlots(fNSlots);
388}
389
391 : fNSlots(RDFInternal::GetNSlots()),
392 fLoopType(ROOT::IsImplicitMTEnabled() ? ELoopType::kDataSourceMT : ELoopType::kDataSource),
393 fNewSampleNotifier(fNSlots),
394 fSampleInfos(fNSlots),
395 fDatasetColumnReaders(fNSlots)
396{
397 ChangeSpec(std::move(spec));
398}
399
400namespace {
401std::optional<std::string> GetRedirectedSampleId(std::string_view path, std::string_view datasetName)
402{
403 // Mimick the redirection done in TFile::Open to see if the path points to a FUSE-mounted EOS path.
404 // If so, we create a redirected sample ID with the full xroot URL.
405 TString expandedUrl(path.data());
406 gSystem->ExpandPathName(expandedUrl);
407 TUrl fileurl(expandedUrl, /* default is file */ kTRUE);
408 if (strcmp(fileurl.GetProtocol(), "file") == 0) {
409 if (auto xurl = ROOT::Internal::GetEOSRedirectedXRootURL(fileurl.GetFile()))
410 return *xurl + '/' + datasetName.data();
411 }
412 return std::nullopt;
413}
414} // namespace
415
416/**
417 * @brief Changes the internal TTree held by the RLoopManager.
418 *
419 * @warning This method may lead to potentially dangerous interactions if used
420 * after the construction of the RDataFrame. Changing the specification makes
421 * sense *if and only if* the schema of the dataset is *unchanged*, i.e. the
422 * new specification refers to exactly the same number of columns, with the
423 * same names and types. The actual use case of this method is moving the
424 * processing of the same RDataFrame to a different range of entries of the
425 * same dataset (which may be stored in a different set of files).
426 *
427 * @param spec The specification of the dataset to be adopted.
428 */
430{
431 auto filesVec = spec.GetFileNameGlobs();
432 auto inFile = OpenFileWithSanityChecks(
433 filesVec[0]); // we only need the first file, we assume all files are either TTree or RNTuple
434 auto datasetName = spec.GetTreeNames();
435
436 // Change the range of entries to be processed
437 fBeginEntry = spec.GetEntryRangeBegin();
438 fEndEntry = spec.GetEntryRangeEnd();
439
440 // Store the samples
441 fSamples = spec.MoveOutSamples();
442 fSampleMap.clear();
443
444 const bool isTTree = IsObjectInDir<TTree>(datasetName[0], *inFile);
445 const bool isRNTuple = IsObjectInDir<ROOT::RNTuple>(datasetName[0], *inFile);
446
447 if (isTTree || isRNTuple) {
448
449 if (isTTree) {
450 // Create the internal main chain
452 for (auto &sample : fSamples) {
453 const auto &trees = sample.GetTreeNames();
454 const auto &files = sample.GetFileNameGlobs();
455 for (std::size_t i = 0ul; i < files.size(); ++i) {
456 // We need to use `<filename>?#<treename>` as an argument to TChain::Add
457 // (see https://github.com/root-project/root/pull/8820 for why)
458 const auto fullpath = files[i] + "?#" + trees[i];
459 chain->Add(fullpath.c_str());
460 // ...but instead we use `<filename>/<treename>` as a sample ID (cannot
461 // change this easily because of backward compatibility: the sample ID
462 // is exposed to users via RSampleInfo and DefinePerSample).
463 const auto sampleId = files[i] + '/' + trees[i];
464 fSampleMap.insert({sampleId, &sample});
465
466 // Also add redirected EOS xroot URL when available
467 if (auto redirectedSampleId = GetRedirectedSampleId(files[i], trees[i]))
468 fSampleMap.insert({redirectedSampleId.value(), &sample});
469 }
470 }
471 fDataSource = std::make_unique<ROOT::Internal::RDF::RTTreeDS>(std::move(chain), spec.GetFriendInfo());
472 } else if (isRNTuple) {
473
474 std::vector<std::string> fileNames;
475 std::set<std::string> rntupleNames;
476
477 for (auto &sample : fSamples) {
478 const auto &trees = sample.GetTreeNames();
479 const auto &files = sample.GetFileNameGlobs();
480 for (std::size_t i = 0ul; i < files.size(); ++i) {
481 const auto sampleId = files[i] + '/' + trees[i];
482 fSampleMap.insert({sampleId, &sample});
483 fileNames.push_back(files[i]);
484 rntupleNames.insert(trees[i]);
485
486 // Also add redirected EOS xroot URL when available
487 if (auto redirectedSampleId = GetRedirectedSampleId(files[i], trees[i]))
488 fSampleMap.insert({redirectedSampleId.value(), &sample});
489 }
490 }
491
492 if (rntupleNames.size() == 1) {
493 fDataSource = std::make_unique<ROOT::RDF::RNTupleDS>(*rntupleNames.begin(), fileNames);
494
495 } else {
496 throw std::runtime_error(
497 "More than one RNTuple name was found, please make sure to use RNTuples with the same name.");
498 }
499 }
500
501 fDataSource->SetNSlots(fNSlots);
502
503 for (unsigned int slot{}; slot < fNSlots; slot++) {
504 for (auto &v : fDatasetColumnReaders[slot])
505 v.second.reset();
506 }
507 } else {
508 std::string errMsg =
509 IsObjectInDir<void>(datasetName[0].data(), *inFile) ? "unsupported data format for" : "cannot find";
510 throw std::invalid_argument("RDataFrame: " + errMsg + " dataset \"" + std::string(datasetName[0]) + "\" in file \"" +
511 inFile->GetName() + "\".");
512 }
513}
514
515/// Run event loop with no source files, in parallel.
517{
518#ifdef R__USE_IMT
519 std::shared_ptr<ROOT::Internal::RSlotStack> slotStack = SlotStack();
520 // Working with an empty tree.
521 // Evenly partition the entries according to fNSlots. Produce around 2 tasks per slot.
522 const auto nEmptyEntries = GetNEmptyEntries();
523 const auto nEntriesPerSlot = nEmptyEntries / (fNSlots * 2);
524 auto remainder = nEmptyEntries % (fNSlots * 2);
525 std::vector<std::pair<ULong64_t, ULong64_t>> entryRanges;
526 ULong64_t begin = fEmptyEntryRange.first;
527 while (begin < fEmptyEntryRange.second) {
528 ULong64_t end = begin + nEntriesPerSlot;
529 if (remainder > 0) {
530 ++end;
531 --remainder;
532 }
533 entryRanges.emplace_back(begin, end);
534 begin = end;
535 }
536
537 // Each task will generate a subrange of entries
538 auto genFunction = [this, &slotStack](const std::pair<ULong64_t, ULong64_t> &range) {
539 ROOT::Internal::RSlotStackRAII slotRAII(*slotStack);
540 auto slot = slotRAII.fSlot;
541 RCallCleanUpTask cleanup(*this, slot);
542 InitNodeSlots(nullptr, slot);
543 R__LOG_DEBUG(0, RDFLogChannel()) << LogRangeProcessing({"an empty source", range.first, range.second, slot});
544 try {
545 UpdateSampleInfo(slot, range);
546 for (auto currEntry = range.first; currEntry < range.second; ++currEntry) {
547 RunAndCheckFilters(slot, currEntry);
548 }
549 } catch (...) {
550 // Error might throw in experiment frameworks like CMSSW
551 std::cerr << "RDataFrame::Run: event loop was interrupted\n";
552 throw;
553 }
554 };
555
557 pool.Foreach(genFunction, entryRanges);
558
559#endif // not implemented otherwise
560}
561
562/// Run event loop with no source files, in sequence.
564{
565 InitNodeSlots(nullptr, 0);
566 R__LOG_DEBUG(0, RDFLogChannel()) << LogRangeProcessing(
567 {"an empty source", fEmptyEntryRange.first, fEmptyEntryRange.second, 0u});
568 RCallCleanUpTask cleanup(*this);
569 try {
571 for (ULong64_t currEntry = fEmptyEntryRange.first;
572 currEntry < fEmptyEntryRange.second && fNStopsReceived < fNChildren; ++currEntry) {
573 RunAndCheckFilters(0, currEntry);
574 }
575 } catch (...) {
576 std::cerr << "RDataFrame::Run: event loop was interrupted\n";
577 throw;
578 }
579}
580
581#ifdef R__USE_IMT
582namespace {
583/// Return true on succesful entry read.
584///
585/// TTreeReader encodes successful reads in the `kEntryValid` enum value, but
586/// there can be other situations where the read is still valid. For now, these
587/// are:
588/// - If there was no match of the current entry in one or more friend trees
589/// according to their respective indexes.
590/// - If there was a missing branch at the start of a new tree in the dataset.
591///
592/// In such situations, although the entry is not complete, the processing
593/// should not be aborted and nodes of the computation graph will take action
594/// accordingly.
595bool validTTreeReaderRead(TTreeReader &treeReader)
596{
597 treeReader.Next();
598 switch (treeReader.GetEntryStatus()) {
599 case TTreeReader::kEntryValid: return true;
600 case TTreeReader::kIndexedFriendNoMatch: return true;
602 default: return false;
603 }
604}
605} // namespace
606#endif
607
608namespace {
609struct DSRunRAII {
611 DSRunRAII(ROOT::RDF::RDataSource &ds, const std::set<std::string> &suppressErrorsForMissingColumns) : fDS(ds)
612 {
613 ROOT::Internal::RDF::CallInitializeWithOpts(fDS, suppressErrorsForMissingColumns);
614 }
615 ~DSRunRAII() { fDS.Finalize(); }
616};
617} // namespace
618
621 unsigned int fSlot;
623 RDSRangeRAII(ROOT::Detail::RDF::RLoopManager &lm, unsigned int slot, ULong64_t firstEntry,
624 TTreeReader *treeReader = nullptr)
625 : fLM(lm), fSlot(slot), fTreeReader(treeReader)
626 {
627 fLM.InitNodeSlots(fTreeReader, fSlot);
628 fLM.GetDataSource()->InitSlot(fSlot, firstEntry);
629 }
631};
632
633/// Run event loop over data accessed through a DataSource, in sequence.
635{
636 assert(fDataSource != nullptr);
637 // Shortcut if the entry range would result in not reading anything
638 if (fBeginEntry == fEndEntry)
639 return;
640 // Apply global entry range if necessary
641 if (fBeginEntry != 0 || fEndEntry != std::numeric_limits<Long64_t>::max())
642 fDataSource->SetGlobalEntryRange(std::make_pair<std::uint64_t, std::uint64_t>(fBeginEntry, fEndEntry));
643 // Initialize data source and book finalization
645 // Ensure cleanup task is always called at the end. Notably, this also resets the column readers for those data
646 // sources that need it (currently only TTree).
647 RCallCleanUpTask cleanup(*this);
648
649 // Main event loop. We start with an empty vector of ranges because we need to initialize the nodes and the data
650 // source before the first call to GetEntryRanges, since it could trigger reading (currently only happens with
651 // TTree).
652 std::uint64_t processedEntries{};
653 std::vector<std::pair<ULong64_t, ULong64_t>> ranges{};
654 do {
655
656 ROOT::Internal::RDF::RDSRangeRAII __{*this, 0u, 0ull};
657
658 ranges = fDataSource->GetEntryRanges();
659
661
662 try {
663 for (const auto &range : ranges) {
664 const auto start = range.first;
665 const auto end = range.second;
666 R__LOG_DEBUG(0, RDFLogChannel()) << LogRangeProcessing({fDataSource->GetLabel(), start, end, 0u});
667 for (auto entry = start; entry < end && fNStopsReceived < fNChildren; ++entry) {
668 if (fDataSource->SetEntry(0u, entry)) {
669 RunAndCheckFilters(0u, entry);
670 }
671 processedEntries++;
672 }
673 }
674 } catch (...) {
675 std::cerr << "RDataFrame::Run: event loop was interrupted\n";
676 throw;
677 }
678
679 } while (!ranges.empty() && fNStopsReceived < fNChildren);
680
682
683 if (fEndEntry != std::numeric_limits<Long64_t>::max() &&
684 static_cast<std::uint64_t>(fEndEntry - fBeginEntry) > processedEntries) {
685 std::ostringstream buf{};
686 buf << "RDataFrame stopped processing after ";
687 buf << processedEntries;
688 buf << " entries, whereas an entry range (begin=";
689 buf << fBeginEntry;
690 buf << ",end=";
691 buf << fEndEntry;
692 buf << ") was requested. Consider adjusting the end value of the entry range to a maximum of ";
693 buf << (fBeginEntry + processedEntries);
694 buf << ".";
695 Warning("RDataFrame::Run", "%s", buf.str().c_str());
696 }
697}
698
699/// Run event loop over data accessed through a DataSource, in parallel.
701{
702#ifdef R__USE_IMT
703 assert(fDataSource != nullptr);
704 // Shortcut if the entry range would result in not reading anything
705 if (fBeginEntry == fEndEntry)
706 return;
707 // Apply global entry range if necessary
708 if (fBeginEntry != 0 || fEndEntry != std::numeric_limits<Long64_t>::max())
709 fDataSource->SetGlobalEntryRange(std::make_pair<std::uint64_t, std::uint64_t>(fBeginEntry, fEndEntry));
710
712
714
715#endif // not implemented otherwise (never called)
716}
717
718/// Execute actions and make sure named filters are called for each event.
719/// Named filters must be called even if the analysis logic would not require it, lest they report confusing results.
720void RLoopManager::RunAndCheckFilters(unsigned int slot, Long64_t entry)
721{
722 // data-block callbacks run before the rest of the graph
723 if (fNewSampleNotifier.CheckFlag(slot)) {
724 for (auto &callback : fSampleCallbacks)
725 callback.second(slot, fSampleInfos[slot]);
727 }
728
729 for (auto *actionPtr : fBookedActions)
730 actionPtr->Run(slot, entry);
731 for (auto *namedFilterPtr : fBookedNamedFilters)
732 namedFilterPtr->CheckFilters(slot, entry);
733 for (auto &callback : fCallbacksEveryNEvents)
734 callback(slot);
735}
736
737/// Build TTreeReaderValues for all nodes
738/// This method loops over all filters, actions and other booked objects and
739/// calls their `InitSlot` method, to get them ready for running a task.
741{
742 SetupSampleCallbacks(r, slot);
743 for (auto *ptr : fBookedActions)
744 ptr->InitSlot(r, slot);
745 for (auto *ptr : fBookedFilters)
746 ptr->InitSlot(r, slot);
747 for (auto *ptr : fBookedDefines)
748 ptr->InitSlot(r, slot);
749 for (auto *ptr : fBookedVariations)
750 ptr->InitSlot(r, slot);
751
752 for (auto &callback : fCallbacksOnce)
753 callback(slot);
754}
755
757 if (r != nullptr) {
758 // we need to set a notifier so that we run the callbacks every time we switch to a new TTree
759 // `PrependLink` inserts this notifier into the TTree/TChain's linked list of notifiers
760 fNewSampleNotifier.GetChainNotifyLink(slot).PrependLink(*r->GetTree());
761 }
762 // Whatever the data source, initially set the "new data block" flag:
763 // - for TChains, this ensures that we don't skip the first data block because
764 // the correct tree is already loaded
765 // - for RDataSources and empty sources, which currently don't have data blocks, this
766 // ensures that we run once per task
768}
769
770void RLoopManager::UpdateSampleInfo(unsigned int slot, const std::pair<ULong64_t, ULong64_t> &range) {
772 "Empty source, range: {" + std::to_string(range.first) + ", " + std::to_string(range.second) + "}", range);
773}
774
776 // one GetTree to retrieve the TChain, another to retrieve the underlying TTree
777 auto *tree = r.GetTree()->GetTree();
778 R__ASSERT(tree != nullptr);
779 const std::string treename = ROOT::Internal::TreeUtils::GetTreeFullPaths(*tree)[0];
780 auto *file = tree->GetCurrentFile();
781 const std::string fname = file != nullptr ? file->GetName() : "#inmemorytree#";
782
783 std::pair<Long64_t, Long64_t> range = r.GetEntriesRange();
784 R__ASSERT(range.first >= 0);
785 if (range.second == -1) {
786 range.second = tree->GetEntries(); // convert '-1', i.e. 'until the end', to the actual entry number
787 }
788 // If the tree is stored in a subdirectory, treename will be the full path to it starting with the root directory '/'
789 const std::string &id = fname + (treename.rfind('/', 0) == 0 ? "" : "/") + treename;
790 if (fSampleMap.empty()) {
791 fSampleInfos[slot] = RSampleInfo(id, range, nullptr, tree->GetEntries());
792 } else {
793 if (fSampleMap.find(id) == fSampleMap.end())
794 throw std::runtime_error("Full sample identifier '" + id + "' cannot be found in the available samples.");
795 fSampleInfos[slot] = RSampleInfo(id, range, fSampleMap[id], tree->GetEntries());
796 }
797}
798
799/// Create a slot stack with the desired number of slots or reuse a shared instance.
800/// When a LoopManager runs in isolation, it will create its own slot stack from the
801/// number of slots. When it runs as part of RunGraphs(), each loop manager will be
802/// assigned a shared slot stack, so dataframe helpers can be shared in a thread-safe
803/// manner.
804std::shared_ptr<ROOT::Internal::RSlotStack> RLoopManager::SlotStack() const
805{
806#ifdef R__USE_IMT
807 if (auto shared = fSlotStack.lock(); shared) {
808 return shared;
809 } else {
810 return std::make_shared<ROOT::Internal::RSlotStack>(fNSlots);
811 }
812#else
813 return nullptr;
814#endif
815}
816
817/// Initialize all nodes of the functional graph before running the event loop.
818/// This method is called once per event-loop and performs generic initialization
819/// operations that do not depend on the specific processing slot (i.e. operations
820/// that are common for all threads).
822{
824 for (auto *filter : fBookedFilters)
825 filter->InitNode();
826 for (auto *range : fBookedRanges)
827 range->InitNode();
828 for (auto *ptr : fBookedActions)
829 ptr->Initialize();
830}
831
832/// Perform clean-up operations. To be called at the end of each event loop.
834{
835 fMustRunNamedFilters = false;
836
837 // forget RActions and detach TResultProxies
838 for (auto *ptr : fBookedActions)
839 ptr->Finalize();
840
841 fRunActions.insert(fRunActions.begin(), fBookedActions.begin(), fBookedActions.end());
842 fBookedActions.clear();
843
844 // reset children counts
845 fNChildren = 0;
846 fNStopsReceived = 0;
847 for (auto *ptr : fBookedFilters)
848 ptr->ResetChildrenCount();
849 for (auto *ptr : fBookedRanges)
850 ptr->ResetChildrenCount();
851
853 fCallbacksOnce.clear();
854}
855
856/// Perform clean-up operations. To be called at the end of each task execution.
857void RLoopManager::CleanUpTask(TTreeReader *r, unsigned int slot)
858{
859 if (r != nullptr)
860 fNewSampleNotifier.GetChainNotifyLink(slot).RemoveLink(*r->GetTree());
861 for (auto *ptr : fBookedActions)
862 ptr->FinalizeSlot(slot);
863 for (auto *ptr : fBookedFilters)
864 ptr->FinalizeSlot(slot);
865 for (auto *ptr : fBookedDefines)
866 ptr->FinalizeSlot(slot);
867
868 if (auto ds = GetDataSource(); ds && ds->GetLabel() == "TTreeDS") {
869 // we are reading from a tree/chain and we need to re-create the RTreeColumnReaders at every task
870 // because the TTreeReader object changes at every task
871 for (auto &v : fDatasetColumnReaders[slot])
872 v.second.reset();
873 }
874}
875
876/// Add RDF nodes that require just-in-time compilation to the computation graph.
877/// This method also clears the contents of GetCodeToJit().
879{
881 if (GetCodeToJit().empty() && GetCodeToDeclare().empty()) {
883 R__LOG_INFO(RDFLogChannel()) << "Nothing to jit and execute.";
884 return;
885 }
886
888 // Check again if another thread has already cleared the global string
889 // with the code to JIT. Without this check, we could end up calling
890 // InterpreterCalc with an empty string, which would raise an exception.
891 if (GetCodeToJit().empty() && GetCodeToDeclare().empty()) {
893 R__LOG_INFO(RDFLogChannel()) << "Nothing to jit and execute.";
894 return;
895 }
896 const std::string codeToDeclare = std::move(GetCodeToDeclare());
897 const std::string code = std::move(GetCodeToJit());
898
899 TStopwatch s;
900 s.Start();
901 if (!codeToDeclare.empty()) {
902 DeclareAndRetrieveDeferredJitCalls(codeToDeclare);
903 }
904 if (!code.empty()) {
905 RDFInternal::InterpreterCalc(code, "RLoopManager::Run");
906 }
907 s.Stop();
908 R__LOG_INFO(RDFLogChannel()) << "Just-in-time compilation phase completed"
909 << (s.RealTime() > 1e-3 ? " in " + std::to_string(s.RealTime()) + " seconds."
910 : " in less than 1ms.");
911
913}
914
916{
917 if (!fJitHelperCalls.empty()) {
918 // funcMap is not thread-safe
920 TStopwatch s;
921 s.Start();
922 const auto &funcMap = GetJitHelperFuncMap();
923 for (auto &call : fJitHelperCalls) {
924 funcMap.at(call.fFunctionId)(call.fColNames, *call.fColRegister, *this, call.fJittedNode.get(),
925 &call.fExtraArgs);
926 }
927 s.Stop();
928 const auto realTime = s.RealTime();
929 R__LOG_INFO(RDFLogChannel()) << fJitHelperCalls.size() << " deferred calls completed"
930 << (realTime > 1e-3 ? " in " + std::to_string(realTime) + " seconds."
931 : " in less than 1ms.");
932 // Promoting to write lock to clear the vector
934 fJitHelperCalls.clear();
935 }
936}
937
938/// Trigger counting of number of children nodes for each node of the functional graph.
939/// This is done once before starting the event loop. Each action sends an `increase children count` signal
940/// upstream, which is propagated until RLoopManager. Each time a node receives the signal, in increments its
941/// children counter. Each node only propagates the signal once, even if it receives it multiple times.
942/// Named filters also send an `increase children count` signal, just like actions, as they always execute during
943/// the event loop so the graph branch they belong to must count as active even if it does not end in an action.
945{
946 for (auto *actionPtr : fBookedActions)
947 actionPtr->TriggerChildrenCount();
948 for (auto *namedFilterPtr : fBookedNamedFilters)
949 namedFilterPtr->TriggerChildrenCount();
950}
951
952/// Start the event loop with a different mechanism depending on IMT/no IMT, data source/no data source.
953/// Also perform a few setup and clean-up operations (jit actions if necessary, clear booked actions after the loop...).
954/// The jitting phase is skipped if the `jit` parameter is `false` (unsafe, use with care).
955void RLoopManager::Run(bool jit)
956{
957 // Change value of TTree::GetMaxTreeSize only for this scope. Revert when #6640 will be solved.
958 MaxTreeSizeRAII ctxtmts;
959
960 R__LOG_INFO(RDFLogChannel()) << "Starting event loop number " << fNRuns << '.';
961
962 ThrowIfNSlotsChanged(GetNSlots());
963
964 if (jit)
965 Jit();
966
967 // Called here since in a RunGraphs run, multiple RLoopManager runs could be
968 // triggered from different threads.
970
971 InitNodes();
972
973 // Exceptions can occur during the event loop. In order to ensure proper cleanup of nodes
974 // we use RAII: even in case of an exception, the destructor of the object is invoked and
975 // all the cleanup takes place.
976 class NodesCleanerRAII {
977 RLoopManager &fRLM;
978
979 public:
980 NodesCleanerRAII(RLoopManager &thisRLM) : fRLM(thisRLM) {}
981 ~NodesCleanerRAII() { fRLM.CleanUpNodes(); }
982 };
983
984 NodesCleanerRAII runKeeper(*this);
985
986 // If called from Python, release the GIL until this function returns: other threads need it to run Python code
988
989 TStopwatch s;
990 s.Start();
991
992 switch (fLoopType) {
994 throw std::runtime_error("RDataFrame: executing the computation graph without a data source, aborting.");
995 break;
998 case ELoopType::kNoFiles: RunEmptySource(); break;
1000 }
1001 s.Stop();
1002
1003 fNRuns++;
1004
1005 R__LOG_INFO(RDFLogChannel()) << "Finished event loop number " << fNRuns - 1 << " (" << s.CpuTime() << "s CPU, "
1006 << s.RealTime() << "s elapsed).";
1007}
1008
1009/// Return the list of default columns -- empty if none was provided when constructing the RDataFrame
1014
1016{
1017 fBookedActions.emplace_back(actionPtr);
1018 AddSampleCallback(actionPtr, actionPtr->GetSampleCallback());
1019}
1020
1022{
1023 RDFInternal::Erase(actionPtr, fRunActions);
1025 fSampleCallbacks.erase(actionPtr);
1026}
1027
1029{
1030 fBookedFilters.emplace_back(filterPtr);
1031 if (filterPtr->HasName()) {
1032 fBookedNamedFilters.emplace_back(filterPtr);
1033 fMustRunNamedFilters = true;
1034 }
1035}
1036
1042
1044{
1045 fBookedRanges.emplace_back(rangePtr);
1046}
1047
1049{
1051}
1052
1054{
1055 fBookedDefines.emplace_back(ptr);
1056}
1057
1063
1068
1073
1074// dummy call, end of recursive chain of calls
1076{
1077 return true;
1078}
1079
1080/// Call `FillReport` on all booked filters
1082{
1083 for (const auto *fPtr : fBookedNamedFilters)
1084 fPtr->FillReport(rep);
1085}
1086
1087void RLoopManager::ToJitExec(const std::string &code) const
1088{
1090 GetCodeToJit().append(code);
1091}
1092
1093void RLoopManager::RegisterJitHelperCall(const std::string &funcBody,
1094 std::unique_ptr<ROOT::Internal::RDF::RColumnRegister> colRegister,
1095 const std::vector<std::string> &colNames, std::shared_ptr<void> jittedNode,
1096 std::shared_ptr<void> argument)
1097{
1098 auto &funcBodyToFuncIdMap = GetJitFuncBodyToFuncIdMap();
1099 {
1101 auto match = funcBodyToFuncIdMap.find(fStringHasher(funcBody));
1102 if (match != funcBodyToFuncIdMap.end()) {
1103 R__WRITE_LOCKGUARD(ROOT::gCoreMutex); // modifying fJitHelperCalls
1104 std::string funcName = "jitNodeRegistrator_" + std::to_string(match->second);
1105 R__LOG_DEBUG(0, RDFLogChannel()) << "JIT helper " << funcName << " was already registered.";
1106 fJitHelperCalls.emplace_back(match->second, std::move(colRegister), colNames, jittedNode, argument);
1107 return;
1108 }
1109 }
1110
1111 {
1112 // Register lazily a JIT helper
1114 auto registratorId = funcBodyToFuncIdMap.size();
1115 std::string funcName = "jitNodeRegistrator_" + std::to_string(registratorId);
1116 [[maybe_unused]] auto res = funcBodyToFuncIdMap.insert({fStringHasher(funcBody), registratorId});
1117 assert(res.second);
1118
1119 std::string toDeclare = "namespace R_rdf {\n void " + funcName + funcBody + "\n}\n";
1120 R__LOG_DEBUG(0, RDFLogChannel()) << "Registering deferred JIT helper:\n" << toDeclare;
1121
1122 GetCodeToDeclare().append(toDeclare);
1123 fJitHelperCalls.emplace_back(registratorId, std::move(colRegister), colNames, jittedNode, argument);
1124 }
1125}
1126
1127void RLoopManager::RegisterCallback(ULong64_t everyNEvents, std::function<void(unsigned int)> &&f)
1128{
1129 if (everyNEvents == 0ull)
1130 fCallbacksOnce.emplace_back(std::move(f), fNSlots);
1131 else
1132 fCallbacksEveryNEvents.emplace_back(everyNEvents, std::move(f), fNSlots);
1133}
1134
1135std::vector<std::string> RLoopManager::GetFiltersNames()
1136{
1137 std::vector<std::string> filters;
1138 for (auto *filter : fBookedFilters) {
1139 auto name = (filter->HasName() ? filter->GetName() : "Unnamed Filter");
1140 filters.push_back(name);
1141 }
1142 return filters;
1143}
1144
1145std::vector<RNodeBase *> RLoopManager::GetGraphEdges() const
1146{
1147 std::vector<RNodeBase *> nodes(fBookedFilters.size() + fBookedRanges.size());
1148 auto it = std::copy(fBookedFilters.begin(), fBookedFilters.end(), nodes.begin());
1149 std::copy(fBookedRanges.begin(), fBookedRanges.end(), it);
1150 return nodes;
1151}
1152
1153std::vector<RDFInternal::RActionBase *> RLoopManager::GetAllActions() const
1154{
1155 std::vector<RDFInternal::RActionBase *> actions(fBookedActions.size() + fRunActions.size());
1156 auto it = std::copy(fBookedActions.begin(), fBookedActions.end(), actions.begin());
1157 std::copy(fRunActions.begin(), fRunActions.end(), it);
1158 return actions;
1159}
1160
1161std::shared_ptr<ROOT::Internal::RDF::GraphDrawing::GraphNode> RLoopManager::GetGraph(
1162 std::unordered_map<void *, std::shared_ptr<ROOT::Internal::RDF::GraphDrawing::GraphNode>> &visitedMap)
1163{
1164 // If there is already a node for the RLoopManager return it. If there is not, return a new one.
1165 auto duplicateRLoopManagerIt = visitedMap.find((void *)this);
1166 if (duplicateRLoopManagerIt != visitedMap.end())
1167 return duplicateRLoopManagerIt->second;
1168
1169 std::string name;
1170 if (fDataSource) {
1171 name = fDataSource->GetLabel();
1172 } else {
1173 name = "Empty source\\nEntries: " + std::to_string(GetNEmptyEntries());
1174 }
1175 auto thisNode = std::make_shared<ROOT::Internal::RDF::GraphDrawing::GraphNode>(
1177 visitedMap[(void *)this] = thisNode;
1178 return thisNode;
1179}
1180
1181/// Return true if AddDataSourceColumnReaders was called for column name col.
1182bool RLoopManager::HasDataSourceColumnReaders(std::string_view col, const std::type_info &ti) const
1183{
1184 const auto key = MakeDatasetColReadersKey(col, ti);
1185 assert(fDataSource != nullptr);
1186 // since data source column readers are always added for all slots at the same time,
1187 // if the reader is present for slot 0 we have it for all other slots as well.
1188 auto it = fDatasetColumnReaders[0].find(key);
1189 return (it != fDatasetColumnReaders[0].end() && it->second);
1190}
1191
1193 std::vector<std::unique_ptr<RColumnReaderBase>> &&readers,
1194 const std::type_info &ti)
1195{
1196 const auto key = MakeDatasetColReadersKey(col, ti);
1197 assert(fDataSource != nullptr && !HasDataSourceColumnReaders(col, ti));
1198 assert(readers.size() == fNSlots);
1199
1200 for (auto slot = 0u; slot < fNSlots; ++slot) {
1201 fDatasetColumnReaders[slot][key] = std::move(readers[slot]);
1202 }
1203}
1204
1205RColumnReaderBase *RLoopManager::AddDataSourceColumnReader(unsigned int slot, std::string_view col,
1206 const std::type_info &ti, TTreeReader *treeReader)
1207{
1208 auto &readers = fDatasetColumnReaders[slot];
1209 const auto key = MakeDatasetColReadersKey(col, ti);
1210 // if a reader for this column and this slot was already there, we are doing something wrong
1211 assert(readers.find(key) == readers.end() || readers[key] == nullptr);
1212 assert(fDataSource && "Missing RDataSource to add column reader.");
1213
1214 readers[key] = ROOT::Internal::RDF::CreateColumnReader(*fDataSource, slot, col, ti, treeReader);
1215
1216 return readers[key].get();
1217}
1218
1220RLoopManager::GetDatasetColumnReader(unsigned int slot, std::string_view col, const std::type_info &ti) const
1221{
1222 const auto key = MakeDatasetColReadersKey(col, ti);
1223 if (auto it = fDatasetColumnReaders[slot].find(key); it != fDatasetColumnReaders[slot].end() && it->second)
1224 return it->second.get();
1225 else
1226 return nullptr;
1227}
1228
1230{
1231 if (callback)
1232 fSampleCallbacks.insert({nodePtr, std::move(callback)});
1233}
1234
1235void RLoopManager::SetEmptyEntryRange(std::pair<ULong64_t, ULong64_t> &&newRange)
1236{
1237 fEmptyEntryRange = std::move(newRange);
1238}
1239
1241{
1242 fBeginEntry = begin;
1243 fEndEntry = end;
1244}
1245
1247{
1248 fTTreeLifeline = std::move(lifeline);
1249}
1250
1251std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1252ROOT::Detail::RDF::CreateLMFromTTree(std::string_view datasetName, std::string_view fileNameGlob,
1253 const ROOT::RDF::ColumnNames_t &defaultColumns, bool checkFile)
1254{
1255 // Introduce the same behaviour as in CreateLMFromFile for consistency.
1256 // Creating an RDataFrame with a non-existing file will throw early rather
1257 // than wait for the start of the graph execution.
1258 if (checkFile) {
1259 OpenFileWithSanityChecks(fileNameGlob);
1260 }
1261
1262 auto dataSource = std::make_unique<ROOT::Internal::RDF::RTTreeDS>(datasetName, fileNameGlob);
1263 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1264 return lm;
1265}
1266
1267std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1268ROOT::Detail::RDF::CreateLMFromTTree(std::string_view datasetName, const std::vector<std::string> &fileNameGlobs,
1269 const std::vector<std::string> &defaultColumns, bool checkFile)
1270{
1271 if (fileNameGlobs.size() == 0)
1272 throw std::invalid_argument("RDataFrame: empty list of input files.");
1273 // Introduce the same behaviour as in CreateLMFromFile for consistency.
1274 // Creating an RDataFrame with a non-existing file will throw early rather
1275 // than wait for the start of the graph execution.
1276 if (checkFile) {
1277 OpenFileWithSanityChecks(fileNameGlobs[0]);
1278 }
1279 auto dataSource = std::make_unique<ROOT::Internal::RDF::RTTreeDS>(datasetName, fileNameGlobs);
1280 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1281 return lm;
1282}
1283
1284std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1285ROOT::Detail::RDF::CreateLMFromRNTuple(std::string_view datasetName, std::string_view fileNameGlob,
1286 const ROOT::RDF::ColumnNames_t &defaultColumns)
1287{
1288 auto dataSource = std::make_unique<ROOT::RDF::RNTupleDS>(datasetName, fileNameGlob);
1289 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1290 return lm;
1291}
1292
1293std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1294ROOT::Detail::RDF::CreateLMFromRNTuple(std::string_view datasetName, const std::vector<std::string> &fileNameGlobs,
1295 const ROOT::RDF::ColumnNames_t &defaultColumns)
1296{
1297 auto dataSource = std::make_unique<ROOT::RDF::RNTupleDS>(datasetName, fileNameGlobs);
1298 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1299 return lm;
1300}
1301
1302std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1303ROOT::Detail::RDF::CreateLMFromFile(std::string_view datasetName, std::string_view fileNameGlob,
1304 const ROOT::RDF::ColumnNames_t &defaultColumns)
1305{
1306
1307 auto inFile = OpenFileWithSanityChecks(fileNameGlob);
1308
1309 if (IsObjectInDir<TTree>(datasetName, *inFile)) {
1310 return CreateLMFromTTree(datasetName, fileNameGlob, defaultColumns, /*checkFile=*/false);
1311 } else if (IsObjectInDir<ROOT::RNTuple>(datasetName, *inFile)) {
1312 return CreateLMFromRNTuple(datasetName, fileNameGlob, defaultColumns);
1313 }
1314
1315 std::string errMsg = IsObjectInDir<void>(datasetName, *inFile) ? "unsupported data format for" : "cannot find";
1316
1317 throw std::invalid_argument("RDataFrame: " + errMsg + " dataset \"" + std::string(datasetName) + "\" in file \"" +
1318 inFile->GetName() + "\".");
1319}
1320
1321std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1322ROOT::Detail::RDF::CreateLMFromFile(std::string_view datasetName, const std::vector<std::string> &fileNameGlobs,
1323 const ROOT::RDF::ColumnNames_t &defaultColumns)
1324{
1325
1326 if (fileNameGlobs.size() == 0)
1327 throw std::invalid_argument("RDataFrame: empty list of input files.");
1328
1329 auto inFile = OpenFileWithSanityChecks(fileNameGlobs[0]);
1330
1331 if (IsObjectInDir<TTree>(datasetName, *inFile)) {
1332 return CreateLMFromTTree(datasetName, fileNameGlobs, defaultColumns, /*checkFile=*/false);
1333 } else if (IsObjectInDir<ROOT::RNTuple>(datasetName, *inFile)) {
1334 return CreateLMFromRNTuple(datasetName, fileNameGlobs, defaultColumns);
1335 }
1336
1337 std::string errMsg = IsObjectInDir<void>(datasetName, *inFile) ? "unsupported data format for" : "cannot find";
1338
1339 throw std::invalid_argument("RDataFrame: " + errMsg + " dataset \"" + std::string(datasetName) + "\" in file \"" +
1340 inFile->GetName() + "\".");
1341}
1342
1343// outlined to pin virtual table
1345
1346void ROOT::Detail::RDF::RLoopManager::SetDataSource(std::unique_ptr<ROOT::RDF::RDataSource> dataSource)
1347{
1348 if (dataSource) {
1349 fDataSource = std::move(dataSource);
1350 fDataSource->SetNSlots(fNSlots);
1351 fLoopType = ROOT::IsImplicitMTEnabled() ? ELoopType::kDataSourceMT : ELoopType::kDataSource;
1352 }
1353}
1354
1355void ROOT::Detail::RDF::RLoopManager::DataSourceThreadTask(const std::pair<ULong64_t, ULong64_t> &entryRange,
1356 ROOT::Internal::RSlotStack &slotStack,
1357 std::atomic<ULong64_t> &entryCount)
1358{
1359#ifdef R__USE_IMT
1360 ROOT::Internal::RSlotStackRAII slotRAII(slotStack);
1361 const auto &slot = slotRAII.fSlot;
1362
1363 const auto &[start, end] = entryRange;
1364 const auto nEntries = end - start;
1365 entryCount.fetch_add(nEntries);
1366
1367 RDSRangeRAII _{*this, slot, start};
1368 RCallCleanUpTask cleanup(*this, slot);
1369
1370 fSampleInfos[slot] = ROOT::Internal::RDF::CreateSampleInfo(*fDataSource, slot, fSampleMap);
1371
1372 R__LOG_DEBUG(0, RDFLogChannel()) << LogRangeProcessing({fDataSource->GetLabel(), start, end, slot});
1373
1374 try {
1375 for (auto entry = start; entry < end; ++entry) {
1376 if (fDataSource->SetEntry(slot, entry)) {
1377 RunAndCheckFilters(slot, entry);
1378 }
1379 }
1380 } catch (...) {
1381 std::cerr << "RDataFrame::Run: event loop was interrupted\n";
1382 throw;
1383 }
1384#else
1385 (void)entryRange;
1386 (void)slotStack;
1387 (void)entryCount;
1388#endif
1389}
1390
1391void ROOT::Detail::RDF::RLoopManager::RNTupleThreadTask(const std::pair<ULong64_t, ULong64_t> &entryRange,
1392 unsigned int slot, std::uint64_t columnReaderOffset)
1393{
1394#ifdef R__USE_IMT
1395 // These are begin and end entries of the current cluster in the file currently opened by the thread who owns the
1396 // slot, offset by a global atomic counter with the value passed via columnReaderOffset. The only reason we use it
1397 // for the moment is to provide a seed for a unique rdfentry_ sequence in the current slot task.
1398 const auto [begin, end] = entryRange;
1399
1400 RDSRangeRAII _{*this, slot, columnReaderOffset};
1401 RCallCleanUpTask cleanup(*this, slot);
1402
1403 fSampleInfos[slot] = ROOT::Internal::RDF::CreateSampleInfo(*fDataSource, slot, fSampleMap);
1404
1405 R__LOG_DEBUG(0, RDFLogChannel()) << LogRangeProcessing(
1406 {fDataSource->GetLabel(), begin - columnReaderOffset, end - columnReaderOffset, slot});
1407
1408 try {
1409 for (auto entry = begin; entry < end; ++entry) {
1410 if (fDataSource->SetEntry(slot, entry)) {
1411 RunAndCheckFilters(slot, entry);
1412 }
1413 }
1414 } catch (...) {
1415 std::cerr << "RDataFrame::Run: event loop was interrupted\n";
1416 throw;
1417 }
1418#else
1419 (void)entryRange;
1420 (void)slot;
1421 (void)columnReaderOffset;
1422#endif
1423}
1424
1426 std::atomic<ULong64_t> &entryCount)
1427{
1428#ifdef R__USE_IMT
1429 ROOT::Internal::RSlotStackRAII slotRAII(slotStack);
1430 const auto &slot = slotRAII.fSlot;
1431
1432 const auto entryRange = treeReader.GetEntriesRange(); // we trust TTreeProcessorMT to call SetEntriesRange
1433 const auto &[start, end] = entryRange;
1434 const auto nEntries = end - start;
1435 auto count = entryCount.fetch_add(nEntries);
1436
1437 RDSRangeRAII _{*this, slot, static_cast<ULong64_t>(start), &treeReader};
1438 RCallCleanUpTask cleanup(*this, slot, &treeReader);
1439
1440 R__LOG_DEBUG(0, RDFLogChannel()) << LogRangeProcessing(
1441 {fDataSource->GetLabel(), static_cast<ULong64_t>(start), static_cast<ULong64_t>(end), slot});
1442 try {
1443 // recursive call to check filters and conditionally execute actions
1444 while (validTTreeReaderRead(treeReader)) {
1445 if (fNewSampleNotifier.CheckFlag(slot)) {
1446 UpdateSampleInfo(slot, treeReader);
1447 }
1448 RunAndCheckFilters(slot, count++);
1449 }
1450 } catch (...) {
1451 std::cerr << "RDataFrame::Run: event loop was interrupted\n";
1452 throw;
1453 }
1454 // fNStopsReceived < fNChildren is always true at the moment as we don't support event loop early quitting in
1455 // multi-thread runs, but it costs nothing to be safe and future-proof in case we add support for that later.
1456 if (treeReader.GetEntryStatus() != TTreeReader::kEntryBeyondEnd && fNStopsReceived < fNChildren) {
1457 // something went wrong in the TTreeReader event loop
1458 throw std::runtime_error("An error was encountered while processing the data. TTreeReader status code is: " +
1459 std::to_string(treeReader.GetEntryStatus()));
1460 }
1461#else
1462 (void)treeReader;
1463 (void)slotStack;
1464 (void)entryCount;
1465#endif
1466}
1467
1470
1471ROOT::Detail::RDF::RLoopManager::DeferredJitCall &ROOT::Detail::RDF::RLoopManager::DeferredJitCall::operator=(
1472 ROOT::Detail::RDF::RLoopManager::DeferredJitCall &&) noexcept = default;
1473
1474ROOT::Detail::RDF::RLoopManager::DeferredJitCall::~DeferredJitCall() = default;
1475
1477 std::size_t id, std::unique_ptr<ROOT::Internal::RDF::RColumnRegister> colRegisterPtr,
1478 const std::vector<std::string> &colNamesArg, std::shared_ptr<void> jittedNode, std::shared_ptr<void> argPtr)
1479 : fFunctionId(id),
1480 fColRegister(std::move(colRegisterPtr)),
1481 fColNames(colNamesArg),
1482 fJittedNode(jittedNode),
1483 fExtraArgs(argPtr)
1484{
1485 assert(fJittedNode != nullptr);
1486}
#define R__LOG_DEBUG(DEBUGLEVEL,...)
Definition RLogger.hxx:359
#define R__LOG_INFO(...)
Definition RLogger.hxx:358
std::unique_ptr< TFile > OpenFileWithSanityChecks(std::string_view fileNameGlob)
Helper function to open a file (or the first file from a glob).
#define f(i)
Definition RSha256.hxx:104
#define e(i)
Definition RSha256.hxx:103
Basic types used by ROOT and required by TInterpreter.
long long Long64_t
Portable signed long integer 8 bytes.
Definition RtypesCore.h:84
unsigned long long ULong64_t
Portable unsigned long integer 8 bytes.
Definition RtypesCore.h:85
constexpr Bool_t kTRUE
Definition RtypesCore.h:108
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:139
void Warning(const char *location, const char *msgfmt,...)
Use this function in warning situations.
Definition TError.cxx:252
const char * filters[]
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
char name[80]
Definition TGX11.cxx:142
#define gInterpreter
R__EXTERN TSystem * gSystem
Definition TSystem.h:582
#define R__WRITE_LOCKGUARD(mutex)
#define R__READ_LOCKGUARD(mutex)
#define _(A, B)
Definition cfortran.h:112
The head node of a RDF computation graph.
RColumnReaderBase * AddDataSourceColumnReader(unsigned int slot, std::string_view col, const std::type_info &ti, TTreeReader *treeReader)
void UpdateSampleInfo(unsigned int slot, const std::pair< ULong64_t, ULong64_t > &range)
unsigned int fNRuns
Number of event loops run.
bool CheckFilters(unsigned int, Long64_t) final
void RegisterJitHelperCall(const std::string &funcBody, std::unique_ptr< ROOT::Internal::RDF::RColumnRegister > colRegister, const std::vector< std::string > &colnames, std::shared_ptr< void > jittedNode, std::shared_ptr< void > argument=nullptr)
void EvalChildrenCounts()
Trigger counting of number of children nodes for each node of the functional graph.
void RNTupleThreadTask(const std::pair< ULong64_t, ULong64_t > &entryRange, unsigned int slot, std::uint64_t columnReaderOffset)
The task run by every thread on the input entry range, for the RNTuple data source.
void CleanUpNodes()
Perform clean-up operations. To be called at the end of each event loop.
void RunEmptySource()
Run event loop with no source files, in sequence.
void SetEmptyEntryRange(std::pair< ULong64_t, ULong64_t > &&newRange)
void Report(ROOT::RDF::RCutFlowReport &rep) const final
Call FillReport on all booked filters.
void AddSampleCallback(void *nodePtr, ROOT::RDF::SampleCallback_t &&callback)
std::vector< RFilterBase * > fBookedNamedFilters
Contains a subset of fBookedFilters, i.e. only the named filters.
void RunEmptySourceMT()
Run event loop with no source files, in parallel.
std::hash< std::string > fStringHasher
std::unordered_map< std::string, ROOT::RDF::Experimental::RSample * > fSampleMap
Keys are fname + "/" + treename as RSampleInfo::fID; Values are pointers to the corresponding sample.
void AddDataSourceColumnReaders(std::string_view col, std::vector< std::unique_ptr< RColumnReaderBase > > &&readers, const std::type_info &ti)
std::shared_ptr< ROOT::Internal::RDF::GraphDrawing::GraphNode > GetGraph(std::unordered_map< void *, std::shared_ptr< ROOT::Internal::RDF::GraphDrawing::GraphNode > > &visitedMap) final
void ToJitExec(const std::string &) const
std::vector< RDFInternal::RActionBase * > GetAllActions() const
Return all actions, either booked or already run.
std::vector< ROOT::RDF::RSampleInfo > fSampleInfos
std::set< std::string > fSuppressErrorsForMissingBranches
void ChangeSpec(ROOT::RDF::Experimental::RDatasetSpec &&spec)
Changes the internal TTree held by the RLoopManager.
std::weak_ptr< ROOT::Internal::RSlotStack > fSlotStack
Pointer to a shared slot stack in case this instance runs concurrently with others:
std::vector< RDefineBase * > fBookedDefines
void TTreeThreadTask(TTreeReader &treeReader, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &entryCount)
The task run by every thread on an entry range (known by the input TTreeReader), for the TTree data s...
std::vector< RDFInternal::RActionBase * > fRunActions
Non-owning pointers to actions already run.
RLoopManager(const ColumnNames_t &defaultColumns={})
std::vector< RRangeBase * > fBookedRanges
std::vector< ROOT::RDF::Experimental::RSample > fSamples
Samples need to survive throughout the whole event loop, hence stored as an attribute.
std::vector< std::string > ColumnNames_t
void RunAndCheckFilters(unsigned int slot, Long64_t entry)
Execute actions and make sure named filters are called for each event.
void ChangeBeginAndEndEntries(Long64_t begin, Long64_t end)
std::vector< RFilterBase * > fBookedFilters
void Run(bool jit=true)
Start the event loop with a different mechanism depending on IMT/no IMT, data source/no data source.
std::unordered_map< void *, ROOT::RDF::SampleCallback_t > fSampleCallbacks
Registered callbacks to call at the beginning of each "data block".
std::vector< RDFInternal::RActionBase * > fBookedActions
Non-owning pointers to actions to be run.
void SetupSampleCallbacks(TTreeReader *r, unsigned int slot)
void CleanUpTask(TTreeReader *r, unsigned int slot)
Perform clean-up operations. To be called at the end of each task execution.
std::vector< RDFInternal::RCallback > fCallbacksEveryNEvents
Registered callbacks to be executed every N events.
std::vector< std::unordered_map< std::string, std::unique_ptr< RColumnReaderBase > > > fDatasetColumnReaders
Readers for TTree/RDataSource columns (one per slot), shared by all nodes in the computation graph.
void Register(RDFInternal::RActionBase *actionPtr)
std::vector< DeferredJitCall > fJitHelperCalls
const ColumnNames_t & GetDefaultColumnNames() const
Return the list of default columns – empty if none was provided when constructing the RDataFrame.
std::vector< RDFInternal::RVariationBase * > fBookedVariations
std::vector< RNodeBase * > GetGraphEdges() const
Return all graph edges known to RLoopManager This includes Filters and Ranges but not Defines.
RDataSource * GetDataSource() const
void RunDataSourceMT()
Run event loop over data accessed through a DataSource, in parallel.
std::vector< std::string > GetFiltersNames()
For each booked filter, returns either the name or "Unnamed Filter".
RDFInternal::RNewSampleNotifier fNewSampleNotifier
std::pair< ULong64_t, ULong64_t > fEmptyEntryRange
Range of entries created when no data source is specified.
std::unique_ptr< RDataSource > fDataSource
Owning pointer to a data-source object.
void DataSourceThreadTask(const std::pair< ULong64_t, ULong64_t > &entryRange, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &entryCount)
The task run by every thread on the input entry range, for the generic RDataSource.
void InitNodeSlots(TTreeReader *r, unsigned int slot)
Build TTreeReaderValues for all nodes This method loops over all filters, actions and other booked ob...
std::vector< RDFInternal::ROneTimeCallback > fCallbacksOnce
Registered callbacks to invoke just once before running the loop.
void SetDataSource(std::unique_ptr< ROOT::RDF::RDataSource > dataSource)
void RegisterCallback(ULong64_t everyNEvents, std::function< void(unsigned int)> &&f)
void SetTTreeLifeline(std::any lifeline)
void RunDataSource()
Run event loop over data accessed through a DataSource, in sequence.
void Jit()
Add RDF nodes that require just-in-time compilation to the computation graph.
RColumnReaderBase * GetDatasetColumnReader(unsigned int slot, std::string_view col, const std::type_info &ti) const
std::shared_ptr< ROOT::Internal::RSlotStack > SlotStack() const
Create a slot stack with the desired number of slots or reuse a shared instance.
void Deregister(RDFInternal::RActionBase *actionPtr)
ELoopType fLoopType
The kind of event loop that is going to be run (e.g. on ROOT files, on no files)
void InitNodes()
Initialize all nodes of the functional graph before running the event loop.
bool HasDataSourceColumnReaders(std::string_view col, const std::type_info &ti) const
Return true if AddDataSourceColumnReaders was called for column name col.
unsigned int fNStopsReceived
Number of times that a children node signaled to stop processing entries.
Definition RNodeBase.hxx:47
unsigned int fNChildren
Number of nodes of the functional graph hanging from this object.
Definition RNodeBase.hxx:46
virtual ROOT::RDF::SampleCallback_t GetSampleCallback()=0
A binder for user-defined columns, variations and aliases.
Calls release when created, if isHeld says so, and restore when destroyed.
bool CheckFlag(unsigned int slot) const
TNotifyLink< RNewSampleFlag > & GetChainNotifyLink(unsigned int slot)
This type includes all parts of RVariation that do not depend on the callable signature.
A thread-safe list of N indexes (0 to size - 1).
The dataset specification for RDataFrame.
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual void Finalize()
Convenience method called after concluding an event-loop.
virtual void InitSlot(unsigned int, ULong64_t)
Convenience method called at the start of the data processing associated to a slot.
virtual void FinalizeSlot(unsigned int)
Convenience method called at the end of the data processing associated to a slot.
virtual std::string GetLabel()
Return a string representation of the datasource type.
This type represents a sample identifier, to be used in conjunction with RDataFrame features such as ...
This class provides a simple interface to execute the same task multiple times in parallel threads,...
void Foreach(F func, unsigned nTimes, unsigned nChunks=0)
Execute a function without arguments several times in parallel, dividing the execution in nChunks.
TDirectory::TContext keeps track and restore the current directory.
Definition TDirectory.h:89
Describe directory structure in memory.
Definition TDirectory.h:45
virtual TObject * Get(const char *namecycle)
Return pointer to object identified by namecycle.
virtual TKey * GetKey(const char *, Short_t=9999) const
Definition TDirectory.h:222
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Definition TFile.cxx:3802
Stopwatch class.
Definition TStopwatch.h:28
Double_t RealTime()
Stop the stopwatch (if it is running) and return the realtime (in seconds) passed between the start a...
void Start(Bool_t reset=kTRUE)
Start the stopwatch.
Double_t CpuTime()
Stop the stopwatch (if it is running) and return the cputime (in seconds) passed between the start an...
void Stop()
Stop the stopwatch.
Basic string class.
Definition TString.h:137
virtual Bool_t ExpandPathName(TString &path)
Expand a pathname getting rid of special shell characters like ~.
Definition TSystem.cxx:1296
A simple, robust and fast interface to read values from ROOT columnar datasets such as TTree,...
Definition TTreeReader.h:46
std::pair< Long64_t, Long64_t > GetEntriesRange() const
Get the begin and end entry numbers.
@ kIndexedFriendNoMatch
A friend with TTreeIndex doesn't have an entry for this index.
@ kMissingBranchWhenSwitchingTree
A branch was not found when switching to the next TTree in the chain.
@ kEntryBeyondEnd
last entry loop has reached its end
@ kEntryValid
data read okay
EEntryStatus GetEntryStatus() const
bool Next()
Move to the next entry (or index of the TEntryList if that is set).
A TTree represents a columnar dataset.
Definition TTree.h:95
static void SetMaxTreeSize(Long64_t maxsize=100000000000LL)
Set the maximum size in bytes of a Tree file (static function).
Definition TTree.cxx:9622
This class represents a WWW compatible URL.
Definition TUrl.h:33
std::shared_ptr< ROOT::Detail::RDF::RLoopManager > CreateLMFromTTree(std::string_view datasetName, std::string_view fileNameGlob, const std::vector< std::string > &defaultColumns, bool checkFile=true)
Create an RLoopManager that reads a TChain.
ROOT::RLogChannel & RDFLogChannel()
Definition RDFUtils.cxx:43
std::shared_ptr< ROOT::Detail::RDF::RLoopManager > CreateLMFromFile(std::string_view datasetName, std::string_view fileNameGlob, const std::vector< std::string > &defaultColumns)
Create an RLoopManager opening a file and checking the data format of the dataset.
std::shared_ptr< ROOT::Detail::RDF::RLoopManager > CreateLMFromRNTuple(std::string_view datasetName, std::string_view fileNameGlob, const std::vector< std::string > &defaultColumns)
Create an RLoopManager that reads an RNTuple.
void RunFinalChecks(const ROOT::RDF::RDataSource &ds, bool nodesLeftNotRun)
Definition RDFUtils.cxx:709
ROOT::RDF::RSampleInfo CreateSampleInfo(const ROOT::RDF::RDataSource &ds, unsigned int slot, const std::unordered_map< std::string, ROOT::RDF::Experimental::RSample * > &sampleMap)
Definition RDFUtils.cxx:702
unsigned int GetNSlots()
Definition RDFUtils.cxx:423
void CallInitializeWithOpts(ROOT::RDF::RDataSource &ds, const std::set< std::string > &suppressErrorsForMissingColumns)
Definition RDFUtils.cxx:691
void Erase(const T &that, std::vector< T > &v)
Erase that element from vector v
Definition Utils.hxx:206
std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase > CreateColumnReader(ROOT::RDF::RDataSource &ds, unsigned int slot, std::string_view col, const std::type_info &tid, TTreeReader *treeReader)
Definition RDFUtils.cxx:720
void InterpreterCalc(const std::string &code, const std::string &context="")
Jit code in the interpreter with TInterpreter::Calc, throw in case of errors.
Definition RDFUtils.cxx:467
void ProcessMT(ROOT::RDF::RDataSource &ds, ROOT::Detail::RDF::RLoopManager &lm)
Definition RDFUtils.cxx:714
void SetEventLoopHooks(int(*isHeld)(), void *(*release)(), void(*restore)(void *))
Lets RDataFrame release the Python GIL while a thread waits for an event loop to finish,...
std::vector< std::string > GetTreeFullPaths(const TTree &tree)
std::unique_ptr< TChain > MakeChainForMT(const std::string &name="", const std::string &title="")
Create a TChain object with options that avoid common causes of thread contention.
std::vector< std::string > ExpandGlob(const std::string &glob)
Expands input glob into a collection of full paths to files.
auto MakeAliasedSharedPtr(T *rawPtr)
std::function< void(unsigned int, const ROOT::RDF::RSampleInfo &)> SampleCallback_t
The type of a data-block callback, registered with an RDataFrame computation graph via e....
std::vector< std::string > ColumnNames_t
Bool_t IsImplicitMTEnabled()
Returns true if the implicit multi-threading in ROOT is enabled.
Definition TROOT.cxx:673
R__EXTERN TVirtualRWMutex * gCoreMutex
A RAII object that calls RLoopManager::CleanUpTask at destruction.
RCallCleanUpTask(RLoopManager &lm, unsigned int arg=0u, TTreeReader *reader=nullptr)
DeferredJitCall(std::size_t id, std::unique_ptr< ROOT::Internal::RDF::RColumnRegister > cols, const std::vector< std::string > &colNamesArg, std::shared_ptr< void > jittedNode, std::shared_ptr< void > arg)
ROOT::Detail::RDF::RLoopManager & fLM
RDSRangeRAII(ROOT::Detail::RDF::RLoopManager &lm, unsigned int slot, ULong64_t firstEntry, TTreeReader *treeReader=nullptr)
A RAII object to pop and push slot numbers from a RSlotStack object.