46#include "ROOT/InternalIOUtils.hxx"
59#include <unordered_map>
73std::string &GetCodeToJit()
75 static std::string code;
79std::string &GetCodeToDeclare()
81 static std::string code;
89std::unordered_map<std::size_t, JitHelperFunc_t> &GetJitHelperFuncMap()
91 static std::unordered_map<std::size_t, JitHelperFunc_t> map;
94std::unordered_map<std::size_t, std::size_t> &GetJitFuncBodyToFuncIdMap()
96 static std::unordered_map<std::size_t, std::size_t> map;
100void DeclareAndRetrieveDeferredJitCalls(
const std::string &codeToDeclare)
114 gInterpreter->ProcessLine(codeToDeclare.c_str(), &interpErrorCode);
116 throw std::runtime_error(
117 "\nAn error occurred during just-in-time compilation in RLoopManager::Run. The lines above might "
118 "indicate the cause of the error.\nAll RDF objects that have not run their event loop yet should be "
119 "considered in an invalid state.\n");
124 auto &funcIdToFuncPointersMap = GetJitHelperFuncMap();
125 auto &funcBodyToFuncIdMap = GetJitFuncBodyToFuncIdMap();
129 for (
auto &codeAndId : funcBodyToFuncIdMap) {
130 if (
auto it = funcIdToFuncPointersMap.find(codeAndId.second); it == funcIdToFuncPointersMap.end()) {
134 const std::string funcName =
"jitNodeRegistrator_" + std::to_string(codeAndId.second);
135 auto declid =
gInterpreter->GetFunction(clinfo, funcName.c_str());
140 throw std::runtime_error(
141 "\nAn error occurred during just-in-time compilation in RLoopManager::Run: failed to retrieve "
142 "the JIT helper function '" +
144 "'. The lines above might indicate the cause of the error.\nAll RDF objects that have not run "
145 "their event loop yet should be considered in an invalid state.\n");
149 auto mname =
gInterpreter->MethodInfo_GetMangledName(minfo);
150 [[maybe_unused]]
auto res = funcIdToFuncPointersMap.insert(
151 {codeAndId.second,
reinterpret_cast<JitHelperFunc_t
>(
gInterpreter->FindSym(mname))});
159void ThrowIfNSlotsChanged(
unsigned int nSlots)
162 if (currentSlots != nSlots) {
163 std::string msg =
"RLoopManager::Run: when the RDataFrame was constructed the number of slots required was " +
164 std::to_string(nSlots) +
", but when starting the event loop it was " +
165 std::to_string(currentSlots) +
".";
166 if (currentSlots > nSlots)
167 msg +=
" Maybe EnableImplicitMT() was called after the RDataFrame was constructed?";
169 msg +=
" Maybe DisableImplicitMT() was called after the RDataFrame was constructed?";
170 throw std::runtime_error(msg);
182struct MaxTreeSizeRAII {
185 MaxTreeSizeRAII() : fOldMaxTreeSize(
TTree::GetMaxTreeSize())
193struct DatasetLogInfo {
194 std::string fDataSet;
200std::string LogRangeProcessing(
const DatasetLogInfo &info)
202 std::stringstream msg;
203 msg <<
"Processing " << info.fDataSet <<
": entry range [" << info.fRangeStart <<
"," << info.fRangeEnd - 1
204 <<
"], using slot " << info.fSlot <<
" in thread " << std::this_thread::get_id() <<
'.';
208auto MakeDatasetColReadersKey(std::string_view colName,
const std::type_info &ti)
214 return std::string(colName) +
':' + ti.name();
222bool IsObjectInDir(std::string_view objName,
TDirectory &dir)
224 std::unique_ptr<T> o{dir.
Get<T>(objName.data())};
234bool IsObjectInDir<void>(std::string_view objName,
TDirectory &dir)
236 return dir.
GetKey(objName.data());
251 auto &&baseNameAndQuery = [&fileNameGlob]() {
252 constexpr std::string_view delim{
".root"};
253 if (
auto &&it = std::find_end(fileNameGlob.begin(), fileNameGlob.end(), delim.begin(), delim.end());
254 it != fileNameGlob.end()) {
255 auto &&distanceToEndOfDelim = std::distance(fileNameGlob.begin(), it + delim.length());
256 return std::make_pair(fileNameGlob.substr(0, distanceToEndOfDelim), fileNameGlob.substr(distanceToEndOfDelim));
257 }
else if (
auto &&lastQuestionMark = fileNameGlob.find_last_of(
'?'); lastQuestionMark != std::string_view::npos)
258 return std::make_pair(fileNameGlob.substr(0, lastQuestionMark), fileNameGlob.substr(lastQuestionMark));
260 return std::make_pair(fileNameGlob, std::string_view{});
263 auto &&baseName = baseNameAndQuery.first;
264 auto &&query = baseNameAndQuery.second;
266 std::string fileToOpen{fileNameGlob};
267 if (baseName.find_first_of(
"[]*?") != std::string_view::npos) {
269 if (expanded.empty())
270 throw std::invalid_argument{
"RDataFrame: The glob expression '" + std::string{baseName} +
271 "' did not match any files."};
273 fileToOpen = expanded.front() + std::string{query};
277 std::unique_ptr<TFile> inFile{
TFile::Open(fileToOpen.c_str(),
"READ_WITHOUT_GLOBALREGISTRATION")};
278 if (!inFile || inFile->IsZombie())
279 throw std::invalid_argument(
"RDataFrame: could not open file \"" + fileToOpen +
"\".");
302int (*gIsHeld)() =
nullptr;
303void *(*gRelease)() =
nullptr;
304void (*gRestore)(
void *) =
nullptr;
316 if (gIsHeld && gIsHeld())
337 : fLoopManager(lm), fArg(arg), fReader(reader)
348 : fDefaultColumns(defaultColumns),
350 fNewSampleNotifier(fNSlots),
351 fSampleInfos(fNSlots),
352 fDatasetColumnReaders(fNSlots)
357 : fDefaultColumns(defaultBranches),
361 fNewSampleNotifier(fNSlots),
362 fSampleInfos(fNSlots),
363 fDatasetColumnReaders(fNSlots)
369 : fEmptyEntryRange(0, nEmptyEntries),
372 fNewSampleNotifier(fNSlots),
373 fSampleInfos(fNSlots),
374 fDatasetColumnReaders(fNSlots)
379 : fDefaultColumns(defaultBranches),
382 fDataSource(std::move(ds)),
383 fNewSampleNotifier(fNSlots),
384 fSampleInfos(fNSlots),
385 fDatasetColumnReaders(fNSlots)
393 fNewSampleNotifier(fNSlots),
394 fSampleInfos(fNSlots),
395 fDatasetColumnReaders(fNSlots)
401std::optional<std::string> GetRedirectedSampleId(std::string_view path, std::string_view datasetName)
405 TString expandedUrl(path.data());
408 if (strcmp(fileurl.GetProtocol(),
"file") == 0) {
409 if (
auto xurl = ROOT::Internal::GetEOSRedirectedXRootURL(fileurl.GetFile()))
410 return *xurl +
'/' + datasetName.data();
431 auto filesVec = spec.GetFileNameGlobs();
434 auto datasetName = spec.GetTreeNames();
444 const bool isTTree = IsObjectInDir<TTree>(datasetName[0], *inFile);
445 const bool isRNTuple = IsObjectInDir<ROOT::RNTuple>(datasetName[0], *inFile);
447 if (isTTree || isRNTuple) {
453 const auto &trees = sample.GetTreeNames();
454 const auto &files = sample.GetFileNameGlobs();
455 for (std::size_t i = 0ul; i < files.size(); ++i) {
458 const auto fullpath = files[i] +
"?#" + trees[i];
459 chain->Add(fullpath.c_str());
463 const auto sampleId = files[i] +
'/' + trees[i];
467 if (
auto redirectedSampleId = GetRedirectedSampleId(files[i], trees[i]))
468 fSampleMap.insert({redirectedSampleId.value(), &sample});
471 fDataSource = std::make_unique<ROOT::Internal::RDF::RTTreeDS>(std::move(chain), spec.GetFriendInfo());
472 }
else if (isRNTuple) {
474 std::vector<std::string> fileNames;
475 std::set<std::string> rntupleNames;
478 const auto &trees = sample.GetTreeNames();
479 const auto &files = sample.GetFileNameGlobs();
480 for (std::size_t i = 0ul; i < files.size(); ++i) {
481 const auto sampleId = files[i] +
'/' + trees[i];
483 fileNames.push_back(files[i]);
484 rntupleNames.insert(trees[i]);
487 if (
auto redirectedSampleId = GetRedirectedSampleId(files[i], trees[i]))
488 fSampleMap.insert({redirectedSampleId.value(), &sample});
492 if (rntupleNames.size() == 1) {
493 fDataSource = std::make_unique<ROOT::RDF::RNTupleDS>(*rntupleNames.begin(), fileNames);
496 throw std::runtime_error(
497 "More than one RNTuple name was found, please make sure to use RNTuples with the same name.");
501 fDataSource->SetNSlots(fNSlots);
503 for (
unsigned int slot{}; slot < fNSlots; slot++) {
504 for (
auto &
v : fDatasetColumnReaders[slot])
509 IsObjectInDir<void>(datasetName[0].
data(), *inFile) ?
"unsupported data format for" :
"cannot find";
510 throw std::invalid_argument(
"RDataFrame: " + errMsg +
" dataset \"" + std::string(datasetName[0]) +
"\" in file \"" +
511 inFile->GetName() +
"\".");
519 std::shared_ptr<ROOT::Internal::RSlotStack> slotStack =
SlotStack();
523 const auto nEntriesPerSlot = nEmptyEntries / (
fNSlots * 2);
524 auto remainder = nEmptyEntries % (
fNSlots * 2);
525 std::vector<std::pair<ULong64_t, ULong64_t>> entryRanges;
533 entryRanges.emplace_back(begin, end);
538 auto genFunction = [
this, &slotStack](
const std::pair<ULong64_t, ULong64_t> &range) {
540 auto slot = slotRAII.
fSlot;
546 for (
auto currEntry = range.first; currEntry < range.second; ++currEntry) {
551 std::cerr <<
"RDataFrame::Run: event loop was interrupted\n";
557 pool.
Foreach(genFunction, entryRanges);
576 std::cerr <<
"RDataFrame::Run: event loop was interrupted\n";
602 default:
return false;
611 DSRunRAII(
ROOT::RDF::RDataSource &ds,
const std::set<std::string> &suppressErrorsForMissingColumns) : fDS(ds)
625 : fLM(lm), fSlot(slot), fTreeReader(treeReader)
652 std::uint64_t processedEntries{};
653 std::vector<std::pair<ULong64_t, ULong64_t>> ranges{};
663 for (
const auto &range : ranges) {
664 const auto start = range.first;
665 const auto end = range.second;
675 std::cerr <<
"RDataFrame::Run: event loop was interrupted\n";
683 if (
fEndEntry != std::numeric_limits<Long64_t>::max() &&
685 std::ostringstream buf{};
686 buf <<
"RDataFrame stopped processing after ";
687 buf << processedEntries;
688 buf <<
" entries, whereas an entry range (begin=";
692 buf <<
") was requested. Consider adjusting the end value of the entry range to a maximum of ";
695 Warning(
"RDataFrame::Run",
"%s", buf.str().c_str());
730 actionPtr->Run(slot, entry);
732 namedFilterPtr->CheckFilters(slot, entry);
744 ptr->InitSlot(
r, slot);
746 ptr->InitSlot(
r, slot);
748 ptr->InitSlot(
r, slot);
750 ptr->InitSlot(
r, slot);
772 "Empty source, range: {" + std::to_string(range.first) +
", " + std::to_string(range.second) +
"}", range);
777 auto *tree =
r.GetTree()->GetTree();
780 auto *file = tree->GetCurrentFile();
781 const std::string fname = file !=
nullptr ? file->GetName() :
"#inmemorytree#";
783 std::pair<Long64_t, Long64_t> range =
r.GetEntriesRange();
785 if (range.second == -1) {
786 range.second = tree->GetEntries();
789 const std::string &
id = fname + (treename.rfind(
'/', 0) == 0 ?
"" :
"/") + treename;
794 throw std::runtime_error(
"Full sample identifier '" +
id +
"' cannot be found in the available samples.");
807 if (
auto shared =
fSlotStack.lock(); shared) {
810 return std::make_shared<ROOT::Internal::RSlotStack>(
fNSlots);
848 ptr->ResetChildrenCount();
850 ptr->ResetChildrenCount();
862 ptr->FinalizeSlot(slot);
864 ptr->FinalizeSlot(slot);
866 ptr->FinalizeSlot(slot);
881 if (GetCodeToJit().empty() && GetCodeToDeclare().empty()) {
891 if (GetCodeToJit().empty() && GetCodeToDeclare().empty()) {
896 const std::string codeToDeclare = std::move(GetCodeToDeclare());
897 const std::string code = std::move(GetCodeToJit());
901 if (!codeToDeclare.empty()) {
902 DeclareAndRetrieveDeferredJitCalls(codeToDeclare);
910 :
" in less than 1ms.");
922 const auto &funcMap = GetJitHelperFuncMap();
924 funcMap.at(call.fFunctionId)(call.fColNames, *call.fColRegister, *
this, call.fJittedNode.get(),
930 << (realTime > 1
e-3 ?
" in " + std::to_string(realTime) +
" seconds."
931 :
" in less than 1ms.");
947 actionPtr->TriggerChildrenCount();
949 namedFilterPtr->TriggerChildrenCount();
958 MaxTreeSizeRAII ctxtmts;
976 class NodesCleanerRAII {
980 NodesCleanerRAII(
RLoopManager &thisRLM) : fRLM(thisRLM) {}
984 NodesCleanerRAII runKeeper(*
this);
994 throw std::runtime_error(
"RDataFrame: executing the computation graph without a data source, aborting.");
1084 fPtr->FillReport(rep);
1090 GetCodeToJit().append(code);
1094 std::unique_ptr<ROOT::Internal::RDF::RColumnRegister> colRegister,
1095 const std::vector<std::string> &colNames, std::shared_ptr<void> jittedNode,
1096 std::shared_ptr<void> argument)
1098 auto &funcBodyToFuncIdMap = GetJitFuncBodyToFuncIdMap();
1101 auto match = funcBodyToFuncIdMap.find(
fStringHasher(funcBody));
1102 if (match != funcBodyToFuncIdMap.end()) {
1104 std::string funcName =
"jitNodeRegistrator_" + std::to_string(match->second);
1106 fJitHelperCalls.emplace_back(match->second, std::move(colRegister), colNames, jittedNode, argument);
1114 auto registratorId = funcBodyToFuncIdMap.size();
1115 std::string funcName =
"jitNodeRegistrator_" + std::to_string(registratorId);
1116 [[maybe_unused]]
auto res = funcBodyToFuncIdMap.insert({
fStringHasher(funcBody), registratorId});
1119 std::string toDeclare =
"namespace R_rdf {\n void " + funcName + funcBody +
"\n}\n";
1122 GetCodeToDeclare().append(toDeclare);
1123 fJitHelperCalls.emplace_back(registratorId, std::move(colRegister), colNames, jittedNode, argument);
1129 if (everyNEvents == 0ull)
1137 std::vector<std::string>
filters;
1139 auto name = (filter->HasName() ? filter->GetName() :
"Unnamed Filter");
1162 std::unordered_map<
void *, std::shared_ptr<ROOT::Internal::RDF::GraphDrawing::GraphNode>> &visitedMap)
1165 auto duplicateRLoopManagerIt = visitedMap.find((
void *)
this);
1166 if (duplicateRLoopManagerIt != visitedMap.end())
1167 return duplicateRLoopManagerIt->second;
1175 auto thisNode = std::make_shared<ROOT::Internal::RDF::GraphDrawing::GraphNode>(
1177 visitedMap[(
void *)
this] = thisNode;
1184 const auto key = MakeDatasetColReadersKey(col, ti);
1193 std::vector<std::unique_ptr<RColumnReaderBase>> &&readers,
1194 const std::type_info &ti)
1196 const auto key = MakeDatasetColReadersKey(col, ti);
1198 assert(readers.size() ==
fNSlots);
1200 for (
auto slot = 0u; slot <
fNSlots; ++slot) {
1206 const std::type_info &ti,
TTreeReader *treeReader)
1209 const auto key = MakeDatasetColReadersKey(col, ti);
1211 assert(readers.find(key) == readers.end() || readers[key] ==
nullptr);
1212 assert(
fDataSource &&
"Missing RDataSource to add column reader.");
1216 return readers[key].get();
1222 const auto key = MakeDatasetColReadersKey(col, ti);
1224 return it->second.get();
1248 fTTreeLifeline = std::move(lifeline);
1251std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1262 auto dataSource = std::make_unique<ROOT::Internal::RDF::RTTreeDS>(datasetName, fileNameGlob);
1263 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1267std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1269 const std::vector<std::string> &defaultColumns,
bool checkFile)
1271 if (fileNameGlobs.size() == 0)
1272 throw std::invalid_argument(
"RDataFrame: empty list of input files.");
1279 auto dataSource = std::make_unique<ROOT::Internal::RDF::RTTreeDS>(datasetName, fileNameGlobs);
1280 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1284std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1288 auto dataSource = std::make_unique<ROOT::RDF::RNTupleDS>(datasetName, fileNameGlob);
1289 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1293std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1297 auto dataSource = std::make_unique<ROOT::RDF::RNTupleDS>(datasetName, fileNameGlobs);
1298 auto lm = std::make_shared<ROOT::Detail::RDF::RLoopManager>(std::move(dataSource), defaultColumns);
1302std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1309 if (IsObjectInDir<TTree>(datasetName, *inFile)) {
1311 }
else if (IsObjectInDir<ROOT::RNTuple>(datasetName, *inFile)) {
1315 std::string errMsg = IsObjectInDir<void>(datasetName, *inFile) ?
"unsupported data format for" :
"cannot find";
1317 throw std::invalid_argument(
"RDataFrame: " + errMsg +
" dataset \"" + std::string(datasetName) +
"\" in file \"" +
1318 inFile->GetName() +
"\".");
1321std::shared_ptr<ROOT::Detail::RDF::RLoopManager>
1326 if (fileNameGlobs.size() == 0)
1327 throw std::invalid_argument(
"RDataFrame: empty list of input files.");
1331 if (IsObjectInDir<TTree>(datasetName, *inFile)) {
1333 }
else if (IsObjectInDir<ROOT::RNTuple>(datasetName, *inFile)) {
1337 std::string errMsg = IsObjectInDir<void>(datasetName, *inFile) ?
"unsupported data format for" :
"cannot find";
1339 throw std::invalid_argument(
"RDataFrame: " + errMsg +
" dataset \"" + std::string(datasetName) +
"\" in file \"" +
1340 inFile->GetName() +
"\".");
1349 fDataSource = std::move(dataSource);
1350 fDataSource->SetNSlots(fNSlots);
1357 std::atomic<ULong64_t> &entryCount)
1361 const auto &slot = slotRAII.
fSlot;
1363 const auto &[start, end] = entryRange;
1364 const auto nEntries = end - start;
1365 entryCount.fetch_add(nEntries);
1375 for (
auto entry = start; entry < end; ++entry) {
1376 if (fDataSource->SetEntry(slot, entry)) {
1377 RunAndCheckFilters(slot, entry);
1381 std::cerr <<
"RDataFrame::Run: event loop was interrupted\n";
1392 unsigned int slot, std::uint64_t columnReaderOffset)
1398 const auto [begin, end] = entryRange;
1406 {fDataSource->GetLabel(), begin - columnReaderOffset, end - columnReaderOffset, slot});
1409 for (
auto entry = begin; entry < end; ++entry) {
1410 if (fDataSource->SetEntry(slot, entry)) {
1411 RunAndCheckFilters(slot, entry);
1415 std::cerr <<
"RDataFrame::Run: event loop was interrupted\n";
1421 (void)columnReaderOffset;
1426 std::atomic<ULong64_t> &entryCount)
1430 const auto &slot = slotRAII.
fSlot;
1433 const auto &[start, end] = entryRange;
1434 const auto nEntries = end - start;
1435 auto count = entryCount.fetch_add(nEntries);
1441 {fDataSource->GetLabel(),
static_cast<ULong64_t>(start),
static_cast<ULong64_t>(end), slot});
1444 while (validTTreeReaderRead(treeReader)) {
1445 if (fNewSampleNotifier.CheckFlag(slot)) {
1446 UpdateSampleInfo(slot, treeReader);
1448 RunAndCheckFilters(slot, count++);
1451 std::cerr <<
"RDataFrame::Run: event loop was interrupted\n";
1458 throw std::runtime_error(
"An error was encountered while processing the data. TTreeReader status code is: " +
1474ROOT::Detail::RDF::
RLoopManager::DeferredJitCall::~DeferredJitCall() = default;
1478 const std::vector<std::
string> &colNamesArg, std::shared_ptr<
void> jittedNode, std::shared_ptr<
void> argPtr)
1480 fColRegister(std::move(colRegisterPtr)),
1481 fColNames(colNamesArg),
1482 fJittedNode(jittedNode),
1485 assert(fJittedNode !=
nullptr);
#define R__LOG_DEBUG(DEBUGLEVEL,...)
std::unique_ptr< TFile > OpenFileWithSanityChecks(std::string_view fileNameGlob)
Helper function to open a file (or the first file from a glob).
Basic types used by ROOT and required by TInterpreter.
long long Long64_t
Portable signed long integer 8 bytes.
unsigned long long ULong64_t
Portable unsigned long integer 8 bytes.
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
void Warning(const char *location, const char *msgfmt,...)
Use this function in warning situations.
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
R__EXTERN TSystem * gSystem
#define R__WRITE_LOCKGUARD(mutex)
#define R__READ_LOCKGUARD(mutex)
The head node of a RDF computation graph.
RColumnReaderBase * AddDataSourceColumnReader(unsigned int slot, std::string_view col, const std::type_info &ti, TTreeReader *treeReader)
void UpdateSampleInfo(unsigned int slot, const std::pair< ULong64_t, ULong64_t > &range)
unsigned int fNRuns
Number of event loops run.
bool CheckFilters(unsigned int, Long64_t) final
void RegisterJitHelperCall(const std::string &funcBody, std::unique_ptr< ROOT::Internal::RDF::RColumnRegister > colRegister, const std::vector< std::string > &colnames, std::shared_ptr< void > jittedNode, std::shared_ptr< void > argument=nullptr)
void EvalChildrenCounts()
Trigger counting of number of children nodes for each node of the functional graph.
void RNTupleThreadTask(const std::pair< ULong64_t, ULong64_t > &entryRange, unsigned int slot, std::uint64_t columnReaderOffset)
The task run by every thread on the input entry range, for the RNTuple data source.
void CleanUpNodes()
Perform clean-up operations. To be called at the end of each event loop.
void RunEmptySource()
Run event loop with no source files, in sequence.
void SetEmptyEntryRange(std::pair< ULong64_t, ULong64_t > &&newRange)
void Report(ROOT::RDF::RCutFlowReport &rep) const final
Call FillReport on all booked filters.
void AddSampleCallback(void *nodePtr, ROOT::RDF::SampleCallback_t &&callback)
std::vector< RFilterBase * > fBookedNamedFilters
Contains a subset of fBookedFilters, i.e. only the named filters.
void RunEmptySourceMT()
Run event loop with no source files, in parallel.
ULong64_t GetNEmptyEntries() const
std::hash< std::string > fStringHasher
std::unordered_map< std::string, ROOT::RDF::Experimental::RSample * > fSampleMap
Keys are fname + "/" + treename as RSampleInfo::fID; Values are pointers to the corresponding sample.
void AddDataSourceColumnReaders(std::string_view col, std::vector< std::unique_ptr< RColumnReaderBase > > &&readers, const std::type_info &ti)
std::shared_ptr< ROOT::Internal::RDF::GraphDrawing::GraphNode > GetGraph(std::unordered_map< void *, std::shared_ptr< ROOT::Internal::RDF::GraphDrawing::GraphNode > > &visitedMap) final
void ToJitExec(const std::string &) const
std::vector< RDFInternal::RActionBase * > GetAllActions() const
Return all actions, either booked or already run.
std::vector< ROOT::RDF::RSampleInfo > fSampleInfos
std::set< std::string > fSuppressErrorsForMissingBranches
bool fMustRunNamedFilters
void ChangeSpec(ROOT::RDF::Experimental::RDatasetSpec &&spec)
Changes the internal TTree held by the RLoopManager.
std::weak_ptr< ROOT::Internal::RSlotStack > fSlotStack
Pointer to a shared slot stack in case this instance runs concurrently with others:
std::vector< RDefineBase * > fBookedDefines
void TTreeThreadTask(TTreeReader &treeReader, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &entryCount)
The task run by every thread on an entry range (known by the input TTreeReader), for the TTree data s...
std::vector< RDFInternal::RActionBase * > fRunActions
Non-owning pointers to actions already run.
RLoopManager(const ColumnNames_t &defaultColumns={})
std::vector< RRangeBase * > fBookedRanges
std::vector< ROOT::RDF::Experimental::RSample > fSamples
Samples need to survive throughout the whole event loop, hence stored as an attribute.
std::vector< std::string > ColumnNames_t
void RunAndCheckFilters(unsigned int slot, Long64_t entry)
Execute actions and make sure named filters are called for each event.
void ChangeBeginAndEndEntries(Long64_t begin, Long64_t end)
std::vector< RFilterBase * > fBookedFilters
void Run(bool jit=true)
Start the event loop with a different mechanism depending on IMT/no IMT, data source/no data source.
std::unordered_map< void *, ROOT::RDF::SampleCallback_t > fSampleCallbacks
Registered callbacks to call at the beginning of each "data block".
std::vector< RDFInternal::RActionBase * > fBookedActions
Non-owning pointers to actions to be run.
ColumnNames_t fDefaultColumns
void SetupSampleCallbacks(TTreeReader *r, unsigned int slot)
void CleanUpTask(TTreeReader *r, unsigned int slot)
Perform clean-up operations. To be called at the end of each task execution.
std::vector< RDFInternal::RCallback > fCallbacksEveryNEvents
Registered callbacks to be executed every N events.
std::vector< std::unordered_map< std::string, std::unique_ptr< RColumnReaderBase > > > fDatasetColumnReaders
Readers for TTree/RDataSource columns (one per slot), shared by all nodes in the computation graph.
void Register(RDFInternal::RActionBase *actionPtr)
std::vector< DeferredJitCall > fJitHelperCalls
const ColumnNames_t & GetDefaultColumnNames() const
Return the list of default columns – empty if none was provided when constructing the RDataFrame.
std::vector< RDFInternal::RVariationBase * > fBookedVariations
std::vector< RNodeBase * > GetGraphEdges() const
Return all graph edges known to RLoopManager This includes Filters and Ranges but not Defines.
RDataSource * GetDataSource() const
unsigned int GetNSlots() const
void RunDataSourceMT()
Run event loop over data accessed through a DataSource, in parallel.
std::vector< std::string > GetFiltersNames()
For each booked filter, returns either the name or "Unnamed Filter".
RDFInternal::RNewSampleNotifier fNewSampleNotifier
std::pair< ULong64_t, ULong64_t > fEmptyEntryRange
Range of entries created when no data source is specified.
std::unique_ptr< RDataSource > fDataSource
Owning pointer to a data-source object.
void DataSourceThreadTask(const std::pair< ULong64_t, ULong64_t > &entryRange, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &entryCount)
The task run by every thread on the input entry range, for the generic RDataSource.
void InitNodeSlots(TTreeReader *r, unsigned int slot)
Build TTreeReaderValues for all nodes This method loops over all filters, actions and other booked ob...
std::vector< RDFInternal::ROneTimeCallback > fCallbacksOnce
Registered callbacks to invoke just once before running the loop.
void SetDataSource(std::unique_ptr< ROOT::RDF::RDataSource > dataSource)
void RegisterCallback(ULong64_t everyNEvents, std::function< void(unsigned int)> &&f)
void SetTTreeLifeline(std::any lifeline)
void RunDataSource()
Run event loop over data accessed through a DataSource, in sequence.
void Jit()
Add RDF nodes that require just-in-time compilation to the computation graph.
RColumnReaderBase * GetDatasetColumnReader(unsigned int slot, std::string_view col, const std::type_info &ti) const
std::shared_ptr< ROOT::Internal::RSlotStack > SlotStack() const
Create a slot stack with the desired number of slots or reuse a shared instance.
void Deregister(RDFInternal::RActionBase *actionPtr)
ELoopType fLoopType
The kind of event loop that is going to be run (e.g. on ROOT files, on no files)
void InitNodes()
Initialize all nodes of the functional graph before running the event loop.
bool HasDataSourceColumnReaders(std::string_view col, const std::type_info &ti) const
Return true if AddDataSourceColumnReaders was called for column name col.
unsigned int fNStopsReceived
Number of times that a children node signaled to stop processing entries.
unsigned int fNChildren
Number of nodes of the functional graph hanging from this object.
virtual ROOT::RDF::SampleCallback_t GetSampleCallback()=0
A binder for user-defined columns, variations and aliases.
Calls release when created, if isHeld says so, and restore when destroyed.
void SetFlag(unsigned int slot)
bool CheckFlag(unsigned int slot) const
void UnsetFlag(unsigned int slot)
TNotifyLink< RNewSampleFlag > & GetChainNotifyLink(unsigned int slot)
This type includes all parts of RVariation that do not depend on the callable signature.
A thread-safe list of N indexes (0 to size - 1).
The dataset specification for RDataFrame.
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual void Finalize()
Convenience method called after concluding an event-loop.
virtual void InitSlot(unsigned int, ULong64_t)
Convenience method called at the start of the data processing associated to a slot.
virtual void FinalizeSlot(unsigned int)
Convenience method called at the end of the data processing associated to a slot.
virtual std::string GetLabel()
Return a string representation of the datasource type.
This type represents a sample identifier, to be used in conjunction with RDataFrame features such as ...
This class provides a simple interface to execute the same task multiple times in parallel threads,...
void Foreach(F func, unsigned nTimes, unsigned nChunks=0)
Execute a function without arguments several times in parallel, dividing the execution in nChunks.
TDirectory::TContext keeps track and restore the current directory.
Describe directory structure in memory.
virtual TObject * Get(const char *namecycle)
Return pointer to object identified by namecycle.
virtual TKey * GetKey(const char *, Short_t=9999) const
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Double_t RealTime()
Stop the stopwatch (if it is running) and return the realtime (in seconds) passed between the start a...
void Start(Bool_t reset=kTRUE)
Start the stopwatch.
Double_t CpuTime()
Stop the stopwatch (if it is running) and return the cputime (in seconds) passed between the start an...
void Stop()
Stop the stopwatch.
virtual Bool_t ExpandPathName(TString &path)
Expand a pathname getting rid of special shell characters like ~.
A simple, robust and fast interface to read values from ROOT columnar datasets such as TTree,...
std::pair< Long64_t, Long64_t > GetEntriesRange() const
Get the begin and end entry numbers.
@ kIndexedFriendNoMatch
A friend with TTreeIndex doesn't have an entry for this index.
@ kMissingBranchWhenSwitchingTree
A branch was not found when switching to the next TTree in the chain.
@ kEntryBeyondEnd
last entry loop has reached its end
@ kEntryValid
data read okay
EEntryStatus GetEntryStatus() const
bool Next()
Move to the next entry (or index of the TEntryList if that is set).
A TTree represents a columnar dataset.
static void SetMaxTreeSize(Long64_t maxsize=100000000000LL)
Set the maximum size in bytes of a Tree file (static function).
This class represents a WWW compatible URL.
std::shared_ptr< ROOT::Detail::RDF::RLoopManager > CreateLMFromTTree(std::string_view datasetName, std::string_view fileNameGlob, const std::vector< std::string > &defaultColumns, bool checkFile=true)
Create an RLoopManager that reads a TChain.
ROOT::RLogChannel & RDFLogChannel()
std::shared_ptr< ROOT::Detail::RDF::RLoopManager > CreateLMFromFile(std::string_view datasetName, std::string_view fileNameGlob, const std::vector< std::string > &defaultColumns)
Create an RLoopManager opening a file and checking the data format of the dataset.
std::shared_ptr< ROOT::Detail::RDF::RLoopManager > CreateLMFromRNTuple(std::string_view datasetName, std::string_view fileNameGlob, const std::vector< std::string > &defaultColumns)
Create an RLoopManager that reads an RNTuple.
void RunFinalChecks(const ROOT::RDF::RDataSource &ds, bool nodesLeftNotRun)
ROOT::RDF::RSampleInfo CreateSampleInfo(const ROOT::RDF::RDataSource &ds, unsigned int slot, const std::unordered_map< std::string, ROOT::RDF::Experimental::RSample * > &sampleMap)
void CallInitializeWithOpts(ROOT::RDF::RDataSource &ds, const std::set< std::string > &suppressErrorsForMissingColumns)
void Erase(const T &that, std::vector< T > &v)
Erase that element from vector v
std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase > CreateColumnReader(ROOT::RDF::RDataSource &ds, unsigned int slot, std::string_view col, const std::type_info &tid, TTreeReader *treeReader)
void InterpreterCalc(const std::string &code, const std::string &context="")
Jit code in the interpreter with TInterpreter::Calc, throw in case of errors.
void ProcessMT(ROOT::RDF::RDataSource &ds, ROOT::Detail::RDF::RLoopManager &lm)
void SetEventLoopHooks(int(*isHeld)(), void *(*release)(), void(*restore)(void *))
Lets RDataFrame release the Python GIL while a thread waits for an event loop to finish,...
std::vector< std::string > GetTreeFullPaths(const TTree &tree)
std::unique_ptr< TChain > MakeChainForMT(const std::string &name="", const std::string &title="")
Create a TChain object with options that avoid common causes of thread contention.
std::vector< std::string > ExpandGlob(const std::string &glob)
Expands input glob into a collection of full paths to files.
auto MakeAliasedSharedPtr(T *rawPtr)
std::function< void(unsigned int, const ROOT::RDF::RSampleInfo &)> SampleCallback_t
The type of a data-block callback, registered with an RDataFrame computation graph via e....
std::vector< std::string > ColumnNames_t
Bool_t IsImplicitMTEnabled()
Returns true if the implicit multi-threading in ROOT is enabled.
R__EXTERN TVirtualRWMutex * gCoreMutex
A RAII object that calls RLoopManager::CleanUpTask at destruction.
RCallCleanUpTask(RLoopManager &lm, unsigned int arg=0u, TTreeReader *reader=nullptr)
RLoopManager & fLoopManager
DeferredJitCall(std::size_t id, std::unique_ptr< ROOT::Internal::RDF::RColumnRegister > cols, const std::vector< std::string > &colNamesArg, std::shared_ptr< void > jittedNode, std::shared_ptr< void > arg)
ROOT::Detail::RDF::RLoopManager & fLM
RDSRangeRAII(ROOT::Detail::RDF::RLoopManager &lm, unsigned int slot, ULong64_t firstEntry, TTreeReader *treeReader=nullptr)
TTreeReader * fTreeReader
A RAII object to pop and push slot numbers from a RSlotStack object.