The RDataSource implementation for RNTuple.

An RDataFrame that reads RNTuple data can be constructed using FromRNTuple().

For each column containing an array or a collection, a corresponding column #colname is available to access colname.size() without reading and deserializing the collection values.

Definition at line 100 of file RNTupleDS.hxx.

Classes
struct	REntryRangeDS
	Page source information consumed while running the computation graph. More...

struct	RFieldInfo
	Holds useful information about fields added to the RNTupleDS. More...

Public Member Functions
	RNTupleDS (const RNTupleDS &)=delete

	RNTupleDS (RNTupleDS &&)=delete

	RNTupleDS (std::string_view ntupleName, const std::vector< std::string > &fileNames)

	RNTupleDS (std::string_view ntupleName, std::string_view fileName)

	~RNTupleDS () final

ROOT::RDF::RSampleInfo	CreateSampleInfo (unsigned int, const std::unordered_map< std::string, ROOT::RDF::Experimental::RSample * > &) const final

void	Finalize () final
	Convenience method called after concluding an event-loop.

void	FinalizeSlot (unsigned int slot) final
	Convenience method called at the end of the data processing associated to a slot.

const std::vector< std::string > &	GetColumnNames () const final
	Returns a reference to the collection of the dataset's column names.

template<typename T >
std::vector< T ** >	GetColumnReaders (std::string_view columnName)
	Called at most once per column by RDF.

std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase >	GetColumnReaders (unsigned int, std::string_view, const std::type_info &) final
	If the other GetColumnReaders overload returns an empty vector, this overload will be called instead.

std::vector< std::pair< ULong64_t, ULong64_t > >	GetEntryRanges () final
	Return ranges of entries to distribute to tasks.

std::string	GetLabel () final
	Return a string representation of the datasource type.

std::size_t	GetNFiles () const final
	Returns the number of files from which the dataset is constructed.

const std::vector< std::string > &	GetTopLevelFieldNames () const final

std::string	GetTypeName (std::string_view colName) const final
	Type of a column as a string, e.g.

bool	HasColumn (std::string_view colName) const final
	Checks if the dataset has a certain column.

void	Initialize () final
	Convenience method called before starting an event-loop.

void	InitSlot (unsigned int slot, ULong64_t firstEntry) final
	Convenience method called at the start of the data processing associated to a slot.

RNTupleDS &	operator= (const RNTupleDS &)=delete

RNTupleDS &	operator= (RNTupleDS &&)=delete

bool	SetEntry (unsigned int, ULong64_t) final
	Advance the "cursors" returned by GetColumnReaders to the selected entry for a particular slot.

virtual void	SetGlobalEntryRange (std::pair< ULong64_t, ULong64_t > entryRange)
	Restrict processing to a [begin, end) range of entries.

void	SetNSlots (unsigned int nSlots) final
	Inform RDataSource of the number of processing slots (i.e.

Protected Types
using	Record_t = std::vector<void *>

Protected Member Functions
virtual std::string	AsString ()

virtual std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase >	CreateColumnReader (unsigned int slot, std::string_view col, const std::type_info &tid, TTreeReader *)
	Creates a column reader for the requested column.

virtual std::string	DescribeDataset ()

virtual const std::vector< std::string > &	GetColumnNamesNoDuplicates () const

Record_t	GetColumnReadersImpl (std::string_view name, const std::type_info &) final
	type-erased vector of pointers to pointers to column values - one per slot

virtual std::string	GetTypeNameWithOpts (std::string_view colName, bool) const

virtual void	InitializeWithOpts (const std::set< std::string > &)

virtual void	RunFinalChecks (bool) const

Protected Attributes
std::optional< std::pair< ULong64_t, ULong64_t > >	fGlobalEntryRange {}

unsigned int	fNSlots {}

Private Member Functions
	RNTupleDS (std::string_view ntupleName, const std::vector< std::string > &fileNames, const std::pair< ULong64_t, ULong64_t > &range)

	RNTupleDS (std::unique_ptr< ROOT::Internal::RPageSource > pageSource)

void	AddField (const ROOT::RNTupleDescriptor &desc, std::string_view colName, ROOT::DescriptorId_t fieldId, std::vector< RFieldInfo > fieldInfos, bool convertToRVec=true)
	Provides the RDF column "colName" given the field identified by fieldID.

std::vector< std::uint64_t >	ComputeGlobalEntryOffsets (ROOT::TThreadExecutor &pool) const
	Computes global entry offsets for the dataset.

void	ExecStaging ()
	The main function of the fThreadStaging background thread.

ROOT::RFieldBase *	GetFieldWithTypeChecks (std::string_view fieldName, const std::type_info &tid)

void	InsertActiveEntryRange (unsigned int slot, std::string_view fileName, std::shared_ptr< ROOT::Internal::RPageSource > pageSource, std::uint64_t beginEntry, std::uint64_t endEntry, std::uint64_t offset)

void	ParallelForBody (std::shared_ptr< ROOT::Internal::RPageSource > pageSource, const ROOT::Internal::IMTUtils::RAbstractTBBRangeImpl &rangeImpl, ROOT::Detail::RDF::RLoopManager &lm, std::string_view fileName, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &processedEntries, ULong64_t columnReaderOffset)
	Apply the computation graph to an input range.

void	PrepareNextRanges ()
	Populates fNextRanges with the next set of entry ranges.

void	ProcessMT (ROOT::Detail::RDF::RLoopManager &lm) final
	Run the computation graph by scheduling MT tasks.

void	ScheduleTaskForPageSource (ROOT::TThreadExecutor &pool, std::unique_ptr< ROOT::Internal::RPageSource > pageSource, std::string_view fileName, std::span< const ROOT::Internal::RNTupleClusterBoundaries > clusterBoundaries, std::uint64_t nEntries, ROOT::Detail::RDF::RLoopManager &lm, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &processedEntries, std::atomic< ULong64_t > &globalEntries)
	Call parallel_for to schedule a splittable task for a page source.

void	ScheduleTasksForFile (std::size_t fileIdx, ROOT::TThreadExecutor &pool, ROOT::Detail::RDF::RLoopManager &lm, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &processedEntries, std::atomic< ULong64_t > &globalEntries)
	Launch the task(s) for the input file index.

void	ScheduleTasksForFileWithGlobalOffset (std::size_t fileIdx, const std::vector< std::uint64_t > &globalFileOffsets, ROOT::TThreadExecutor &pool, ROOT::Detail::RDF::RLoopManager &lm, ROOT::Internal::RSlotStack &slotStack, std::atomic< ULong64_t > &processedEntries, std::atomic< ULong64_t > &globalEntries)
	Launch the task(s) for the input file index, taking into account dataset global entry offsets.

void	StageNextSources ()
	Starting from `fNextFileIndex`, opens the next `fNSlots` files.

Private Attributes
std::vector< std::vector< ROOT::Internal::RDF::RNTupleColumnReader * > >	fActiveColumnReaders
	List of column readers returned by GetColumnReaders() organized by slot.

std::vector< REntryRangeDS >	fActiveRangesPerSlot
	The page source and entry window seen by an active thread task in multi-threaded runs.

std::unordered_map< std::size_t, std::vector< std::unique_ptr< ROOT::RFieldBase > > >	fAlternativeProtoFields
	Columns may be requested with types other than with which they were initially added as proto fields.

std::vector< std::string >	fColumnNames

std::vector< std::string >	fColumnTypes

std::vector< REntryRangeDS >	fCurrentRanges
	Basis for the ranges returned by the last GetEntryRanges() call.

std::condition_variable	fCvStaging
	Signal for the state information of fIsReadyForStaging and fHasNextSources.

std::unordered_map< ROOT::DescriptorId_t, std::string >	fFieldId2QualifiedName
	Connects the IDs of active proto fields and their subfields to their fully qualified name (a.b.c.d).

std::vector< std::string >	fFileNames

std::unordered_map< ULong64_t, std::size_t >	fFirstEntry2RangeIdx
	Maps the first entries from the ranges of the last GetEntryRanges() call to their corresponding index in the fCurrentRanges vectors.

bool	fHasNextSources = false
	Is true when the staging thread has populated the next batch of files to fStagingArea.

bool	fIsReadyForStaging = false
	Is true when the staging thread should start working.

std::mutex	fMutexStaging
	Protects the shared state between the main thread and the I/O thread.

std::size_t	fNextFileIndex = 0
	Index into fFileNames to the next file to process.

std::vector< REntryRangeDS >	fNextRanges
	Basis for the ranges populated by the PrepareNextRanges() call.

std::string	fNTupleName
	The data source may be constructed with an ntuple name and a list of files.

std::vector< std::pair< ULong64_t, ULong64_t > >	fOriginalRanges

ROOT::RNTupleDescriptor	fPrincipalDescriptor
	A clone of the first pages source's descriptor.

std::vector< std::unique_ptr< ROOT::RFieldBase > >	fProtoFields
	We prepare a prototype field for every column.

ULong64_t	fSeenEntriesNoGlobalRange = 0
	The number of entries seen so far in GetEntryRanges()

std::vector< std::shared_ptr< ROOT::Internal::RPageSource > >	fStagingArea
	The staging area is relevant for chains of files, i.e.

bool	fStagingThreadShouldTerminate = false
	Is true when the I/O thread should quit.

std::thread	fThreadStaging
	The background thread that runs StageNextSources()

std::vector< std::string >	fTopLevelFieldNames

Friends
ROOT::RDataFrame	ROOT::Internal::RDF::FromRNTuple (std::string_view ntupleName, const std::vector< std::string > &fileNames, const std::pair< ULong64_t, ULong64_t > &range)

std::vector< std::pair< std::uint64_t, std::uint64_t > >	ROOT::Internal::RDF::GetDatasetGlobalClusterBoundaries (const ROOT::RDF::RInterface< ROOT::Detail::RDF::RNodeBase > &node)

class	ROOT::Internal::RDF::RNTupleColumnReader

#include <ROOT/RNTupleDS.hxx>

Inheritance diagram for ROOT::RDF::RNTupleDS:

[legend]

Member Typedef Documentation

◆ Record_t

using ROOT::RDF::RDataSource::Record_t = std::vector<void *>

protectedinherited

Definition at line 142 of file RDataSource.hxx.

Constructor & Destructor Documentation

◆ RNTupleDS() [1/6]

ROOT::RDF::RNTupleDS::RNTupleDS ( std::unique_ptr< ROOT::Internal::RPageSource > pageSource )

explicitprivate

Definition at line 310 of file RNTupleDS.cxx.

◆ RNTupleDS() [2/6]

ROOT::RDF::RNTupleDS::RNTupleDS	(	std::string_view	ntupleName,
		const std::vector< std::string > &	fileNames,
		const std::pair< ULong64_t, ULong64_t > &	range )

explicitprivate

Definition at line 369 of file RNTupleDS.cxx.

◆ RNTupleDS() [3/6]

ROOT::RDF::RNTupleDS::RNTupleDS	(	std::string_view	ntupleName,
		std::string_view	fileName )

Definition at line 354 of file RNTupleDS.cxx.

◆ RNTupleDS() [4/6]

ROOT::RDF::RNTupleDS::RNTupleDS	(	std::string_view	ntupleName,
		const std::vector< std::string > &	fileNames )

Definition at line 361 of file RNTupleDS.cxx.

◆ RNTupleDS() [5/6]

ROOT::RDF::RNTupleDS::RNTupleDS ( const RNTupleDS & )

delete

◆ RNTupleDS() [6/6]

ROOT::RDF::RNTupleDS::RNTupleDS ( RNTupleDS && )

delete

◆ ~RNTupleDS()

ROOT::RDF::RNTupleDS::~RNTupleDS ( )

finaldefault

Member Function Documentation

◆ AddField()

void ROOT::RDF::RNTupleDS::AddField	(	const ROOT::RNTupleDescriptor &	desc,
		std::string_view	colName,
		ROOT::DescriptorId_t	fieldId,
		std::vector< RFieldInfo >	fieldInfos,
		bool	convertToRVec = true )

private

Provides the RDF column "colName" given the field identified by fieldID.

For records and collections, AddField recurses into the sub fields. The fieldInfos argument is a list of objects holding info about the fields of the outer collection(s) (w.r.t. fieldId). For instance, if fieldId refers to an std::vector<Jet>, with

struct Jet {
   float pt;
   float eta;
};

AddField will recurse into Jet.pt and Jet.eta and provide the two inner fields as ROOT::VecOps::RVec<float> each.

In case the field is a collection of type ROOT::VecOps::RVec, std::vector or std::array, its corresponding column is added as a ROOT::VecOps::RVec. Otherwise, the collection field's on-disk type is used. Note, however, that inner record members of such collections will still be added as ROOT::VecOps::RVec (e.g., std::set<Jet> will be added as astd::set, butJet.[pt|eta] will be added as `ROOT::VecOps::RVec<float>).

Definition at line 153 of file RNTupleDS.cxx.

◆ AsString()

virtual std::string ROOT::RDF::RDataSource::AsString ( )

inlineprotectedvirtualinherited

Reimplemented in ROOT::Internal::RDF::RVecDS< ColumnTypes >, ROOT::RDF::RCsvDS, ROOT::RDF::RLazyDS< ColumnTypes >, ROOT::Internal::RDF::RRootDS, ROOT::RDF::RTrivialDS, and ROOT::Internal::RDF::RTTreeDS.

Definition at line 145 of file RDataSource.hxx.

◆ ComputeGlobalEntryOffsets()

std::vector< std::uint64_t > ROOT::RDF::RNTupleDS::ComputeGlobalEntryOffsets ( ROOT::TThreadExecutor & pool ) const

private

Computes global entry offsets for the dataset.

Used in MT scheduling when the user selects a global entry range to restrict processing to.

Definition at line 1018 of file RNTupleDS.cxx.

◆ CreateColumnReader()

virtual std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase > ROOT::RDF::RDataSource::CreateColumnReader	(	unsigned int	slot,
		std::string_view	col,
		const std::type_info &	tid,
		TTreeReader *	)

inlineprotectedvirtualinherited

Creates a column reader for the requested column.

In the general case, this is just a redirect to the right GetColumnReaders overload. The signature notably also has a TTreeReader * parameter. This is currently necessary to still allow the TTree-based MT scheduling via TTreeProcessorMT. We use the TTreeProcessorMT::Process method to launch the same kernel across all threads. In each thread task, TTreeProcessorMT creates a thread-local instance of a TTreeReader which is going to read the range of events assigned to that task. That TTreeReader instance is what is passed to this method whenever a column reader needs to be created in a thread task. In the future this method might be removed by either allowing to request a handle to the thread-local TTreeReader instance programmatically from the TTreeProcessorMT, or refactoring the TTreeProcessorMT scheduling into RTTreeDS altogether.

Reimplemented in ROOT::Internal::RDF::RTTreeDS.

Definition at line 195 of file RDataSource.hxx.

◆ CreateSampleInfo()

ROOT::RDF::RSampleInfo ROOT::Internal::RDF::RNTupleDS::CreateSampleInfo	(	unsigned int	slot,
		const std::unordered_map< std::string, ROOT::RDF::Experimental::RSample * > &	sampleMap ) const

finalvirtual

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 795 of file RNTupleDS.cxx.

◆ DescribeDataset()

virtual std::string ROOT::RDF::RDataSource::DescribeDataset ( )

inlineprotectedvirtualinherited

Reimplemented in ROOT::Internal::RDF::RTTreeDS.

Definition at line 165 of file RDataSource.hxx.

◆ ExecStaging()

void ROOT::RDF::RNTupleDS::ExecStaging ( )

private

The main function of the fThreadStaging background thread.

Definition at line 478 of file RNTupleDS.cxx.

◆ Finalize()

void ROOT::RDF::RNTupleDS::Finalize ( )

finalvirtual

Convenience method called after concluding an event-loop.

See Initialize for more details.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 751 of file RNTupleDS.cxx.

◆ FinalizeSlot()

void ROOT::RDF::RNTupleDS::FinalizeSlot ( unsigned int )

finalvirtual

Convenience method called at the end of the data processing associated to a slot.

Parameters

[in] slot The data processing slot wihch needs to be finalized This method might be called multiple times per thread per event-loop.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 698 of file RNTupleDS.cxx.

◆ GetColumnNames()

const std::vector< std::string > & ROOT::RDF::RNTupleDS::GetColumnNames ( ) const

inlinefinalvirtual

Returns a reference to the collection of the dataset's column names.

Implements ROOT::RDF::RDataSource.

Definition at line 301 of file RNTupleDS.hxx.

◆ GetColumnNamesNoDuplicates()

virtual const std::vector< std::string > & ROOT::RDF::RDataSource::GetColumnNamesNoDuplicates ( ) const

inlineprotectedvirtualinherited

Reimplemented in ROOT::Internal::RDF::RTTreeDS.

Definition at line 159 of file RDataSource.hxx.

◆ GetColumnReaders() [1/2]

template<typename T >

std::vector< T ** > ROOT::RDF::RDataSource::GetColumnReaders ( std::string_view columnName )

inlineinherited

Called at most once per column by RDF.

Return vector of pointers to pointers to column values - one per slot.

Template Parameters

T	The type of the data stored in the column

Parameters

[in] columnName The name of the column

These pointers are veritable cursors: it's a responsibility of the RDataSource implementation that they point to the "right" memory region.

Definition at line 248 of file RDataSource.hxx.

◆ GetColumnReaders() [2/2]

std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase > ROOT::RDF::RNTupleDS::GetColumnReaders	(	unsigned int	,
		std::string_view	,
		const std::type_info &	)

finalvirtual

If the other GetColumnReaders overload returns an empty vector, this overload will be called instead.

Parameters

[in]	slot	The data processing slot that needs to be considered
[in]	name	The name of the column for which a column reader needs to be returned
[in]	tid	A type_info At least one of the two must return a non-empty/non-null value.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 460 of file RNTupleDS.cxx.

◆ GetColumnReadersImpl()

ROOT::RDF::RDataSource::Record_t ROOT::RDF::RNTupleDS::GetColumnReadersImpl	(	std::string_view	name,
		const std::type_info &	)

finalprotectedvirtual

type-erased vector of pointers to pointers to column values - one per slot

Implements ROOT::RDF::RDataSource.

Definition at line 377 of file RNTupleDS.cxx.

◆ GetEntryRanges()

std::vector< std::pair< ULong64_t, ULong64_t > > ROOT::RDF::RNTupleDS::GetEntryRanges ( )

finalvirtual

Return ranges of entries to distribute to tasks.

They are required to be contiguous intervals with no entries skipped. Supposing a dataset with nEntries, the intervals must start at 0 and end at nEntries, e.g. [0-5],[5-10] for 10 entries. This function will be invoked repeatedly by RDataFrame as it needs additional entries to process. The same entry range should not be returned more than once. Returning an empty collection of ranges signals to RDataFrame that the processing can stop.

Implements ROOT::RDF::RDataSource.

Definition at line 552 of file RNTupleDS.cxx.

◆ GetFieldWithTypeChecks()

ROOT::RFieldBase * ROOT::RDF::RNTupleDS::GetFieldWithTypeChecks	(	std::string_view	fieldName,
		const std::type_info &	tid )

private

Definition at line 383 of file RNTupleDS.cxx.

◆ GetLabel()

std::string ROOT::RDF::RNTupleDS::GetLabel ( )

inlinefinalvirtual

Return a string representation of the datasource type.

The returned string will be used by ROOT::RDF::SaveGraph() to represent the datasource in the visualization of the computation graph. Concrete datasources can override the default implementation.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 306 of file RNTupleDS.hxx.

◆ GetNFiles()

std::size_t ROOT::RDF::RNTupleDS::GetNFiles ( ) const

inlinefinalvirtual

Returns the number of files from which the dataset is constructed.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 300 of file RNTupleDS.hxx.

◆ GetTopLevelFieldNames()

const std::vector< std::string > & ROOT::RDF::RNTupleDS::GetTopLevelFieldNames ( ) const

inlinefinalvirtual

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 302 of file RNTupleDS.hxx.

◆ GetTypeName()

std::string ROOT::RDF::RNTupleDS::GetTypeName ( std::string_view colName ) const

finalvirtual

Type of a column as a string, e.g.

GetTypeName("x") == "double". Required for jitting e.g. df.Filter("x>0").

Parameters

[in] colName The name of the column

Implements ROOT::RDF::RDataSource.

Definition at line 709 of file RNTupleDS.cxx.

◆ GetTypeNameWithOpts()

virtual std::string ROOT::RDF::RDataSource::GetTypeNameWithOpts	(	std::string_view	colName,
		bool	) const

inlineprotectedvirtualinherited

Reimplemented in ROOT::Internal::RDF::RTTreeDS.

Definition at line 152 of file RDataSource.hxx.

◆ HasColumn()

bool ROOT::RDF::RNTupleDS::HasColumn ( std::string_view colName ) const

finalvirtual

Checks if the dataset has a certain column.

Parameters

[in] colName The name of the column

Implements ROOT::RDF::RDataSource.

Definition at line 722 of file RNTupleDS.cxx.

◆ Initialize()

void ROOT::RDF::RNTupleDS::Initialize ( )

finalvirtual

Convenience method called before starting an event-loop.

This method might be called multiple times over the lifetime of a RDataSource, since users can run multiple event-loops with the same RDataFrame. Ideally, Initialize should set the state of the RDataSource so that multiple identical event-loops will produce identical results.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 727 of file RNTupleDS.cxx.

◆ InitializeWithOpts()

virtual void ROOT::RDF::RDataSource::InitializeWithOpts ( const std::set< std::string > & )

inlineprotectedvirtualinherited

Reimplemented in ROOT::Internal::RDF::RTTreeDS.

Definition at line 162 of file RDataSource.hxx.

◆ InitSlot()

void ROOT::RDF::RNTupleDS::InitSlot	(	unsigned int	,
		ULong64_t	)

finalvirtual

Convenience method called at the start of the data processing associated to a slot.

Parameters

[in]	slot	The data processing slot wihch needs to be initialized
[in]	firstEntry	The first entry of the range that the task will process. This method might be called multiple times per thread per event-loop.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 684 of file RNTupleDS.cxx.

◆ InsertActiveEntryRange()

void ROOT::RDF::RNTupleDS::InsertActiveEntryRange	(	unsigned int	slot,
		std::string_view	fileName,
		std::shared_ptr< ROOT::Internal::RPageSource >	pageSource,
		std::uint64_t	beginEntry,
		std::uint64_t	endEntry,
		std::uint64_t	offset )

private

Definition at line 910 of file RNTupleDS.cxx.

◆ operator=() [1/2]

RNTupleDS & ROOT::RDF::RNTupleDS::operator= ( const RNTupleDS & )

delete

◆ operator=() [2/2]

RNTupleDS & ROOT::RDF::RNTupleDS::operator= ( RNTupleDS && )

delete

◆ ParallelForBody()

void ROOT::RDF::RNTupleDS::ParallelForBody	(	std::shared_ptr< ROOT::Internal::RPageSource >	pageSource,
		const ROOT::Internal::IMTUtils::RAbstractTBBRangeImpl &	rangeImpl,
		ROOT::Detail::RDF::RLoopManager &	lm,
		std::string_view	fileName,
		ROOT::Internal::RSlotStack &	slotStack,
		std::atomic< ULong64_t > &	processedEntries,
		ULong64_t	columnReaderOffset )

private

Apply the computation graph to an input range.

Parameters

pageSource	The page source to read data from.
rangeImpl	A Range type carrying information on the range of entries to process.
lm	The RLoopManager which will run the computation graph.
fileName	Path to the storage location of the page source.
slotStack	A stack of slots, each thread needs to request one before running the computation graph.
processedEntries	Cumulating number of processed entries by this computation graph run.
columnReaderOffset	Offset value to pass to column readers used in this task. The column reader will request entries from the page source receiving a global entry number, this offset is subtracted so that the column reader requests the local entry from the page source.

Definition at line 934 of file RNTupleDS.cxx.

◆ PrepareNextRanges()

void ROOT::RDF::RNTupleDS::PrepareNextRanges ( )

private

Populates fNextRanges with the next set of entry ranges.

Moves files from the staging area as necessary and aligns ranges with cluster boundaries for scheduling the tail of files. Upon return, the fNextRanges list is ordered. It has usually fNSlots elements; fewer if there is not enough work to give at least one cluster to every slot.

Definition at line 515 of file RNTupleDS.cxx.

◆ ProcessMT()

void ROOT::RDF::RNTupleDS::ProcessMT ( ROOT::Detail::RDF::RLoopManager & lm )

finalprivatevirtual

Run the computation graph by scheduling MT tasks.

The data source interacts with the loop manager and with a pool of threads (via TThreadExecutor) at the same time. The function schedules one MT task per input file, eventually calling the parallel_for overload taking two input arguments respectively respecting the Range and Body requirements. This means that each of the file-based tasks can be split into various sub-tasks, each processing a set of consecutive clusters of the same file. Before launching the file-based tasks, the function checks if the user has selected a global range of entries to restrict processing to. If so, the function first computes the global entry offsets of all the input files, so they can be matched against the user-provided global range. These will be used to adjust the list of clusters of the file before spawning the corresponding task.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 1145 of file RNTupleDS.cxx.

◆ RunFinalChecks()

virtual void ROOT::RDF::RDataSource::RunFinalChecks ( bool ) const

inlineprotectedvirtualinherited

Reimplemented in ROOT::Internal::RDF::RTTreeDS.

Definition at line 174 of file RDataSource.hxx.

◆ ScheduleTaskForPageSource()

void ROOT::RDF::RNTupleDS::ScheduleTaskForPageSource	(	ROOT::TThreadExecutor &	pool,
		std::unique_ptr< ROOT::Internal::RPageSource >	pageSource,
		std::string_view	fileName,
		std::span< const ROOT::Internal::RNTupleClusterBoundaries >	clusterBoundaries,
		std::uint64_t	nEntries,
		ROOT::Detail::RDF::RLoopManager &	lm,
		ROOT::Internal::RSlotStack &	slotStack,
		std::atomic< ULong64_t > &	processedEntries,
		std::atomic< ULong64_t > &	globalEntries )

private

Call parallel_for to schedule a splittable task for a page source.

Parameters

pool	The pool of threads used to schedule the task.
pageSource	The RNTuple page source for the task.
fileName	Path to the storage location of the page source.
clusterBoundaries	A list of cluster boundaries valid for the page source. This may be computed differently, see ProcessMT.
nEntries	Number of entries in the page source.
lm	The RLoopManager which will run the computation graph.
slotStack	A stack of slots, each thread needs to request one before running the computation graph.
processedEntries	Cumulating number of processed entries by this computation graph run.
globalEntries	Cumulating total number of entries in the input dataset.

This function calls the parallel_for overload that takes two arguments, one respecting the Range and the other respecting the Body requirements. The Range argument is a RSplittableRangeOfClusterBoundaries, a struct with a range of N cluster boundaries to process (initialised to the whole list of clusters). Immediately when calling parallel_for, TBB will recursively ask the RSplittableRangeOfClusterBoundaries to split itself into two roughly equal sub-tasks, until there are only leaf tasks that cannot be split further. This helps balancing cases where some tasks may end up running much longer than others.

Definition at line 992 of file RNTupleDS.cxx.

◆ ScheduleTasksForFile()

void ROOT::RDF::RNTupleDS::ScheduleTasksForFile	(	std::size_t	fileIdx,
		ROOT::TThreadExecutor &	pool,
		ROOT::Detail::RDF::RLoopManager &	lm,
		ROOT::Internal::RSlotStack &	slotStack,
		std::atomic< ULong64_t > &	processedEntries,
		std::atomic< ULong64_t > &	globalEntries )

private

Launch the task(s) for the input file index.

This function is used in MT scheduling to launch a task processing one input file. It is selected when there is no global entry range. The task may be split into smaller tasks based on entry ranges aligned to cluster boundaries, see ScheduleTaskForPageSource.

Definition at line 1111 of file RNTupleDS.cxx.

◆ ScheduleTasksForFileWithGlobalOffset()

void ROOT::RDF::RNTupleDS::ScheduleTasksForFileWithGlobalOffset	(	std::size_t	fileIdx,
		const std::vector< std::uint64_t > &	globalFileOffsets,
		ROOT::TThreadExecutor &	pool,
		ROOT::Detail::RDF::RLoopManager &	lm,
		ROOT::Internal::RSlotStack &	slotStack,
		std::atomic< ULong64_t > &	processedEntries,
		std::atomic< ULong64_t > &	globalEntries )

private

Launch the task(s) for the input file index, taking into account dataset global entry offsets.

This function is used in MT scheduling to launch a task processing one input file. It is selected when the user specifies a global entry range to restrict processing to. In this scenario we have precomputed the global entry offsets of each file upfront so we can then adjust the range of entries in each file accordingly. The task may be split into smaller tasks based on entry ranges aligned to cluster boundaries, see ScheduleTaskForPageSource.

Definition at line 1050 of file RNTupleDS.cxx.

◆ SetEntry()

bool ROOT::RDF::RNTupleDS::SetEntry	(	unsigned int	slot,
		ULong64_t	entry )

inlinefinalvirtual

Advance the "cursors" returned by GetColumnReaders to the selected entry for a particular slot.

Parameters

[in]	slot	The data processing slot that needs to be considered
[in]	entry	The entry which needs to be pointed to by the reader pointers Slots are adopted to accommodate parallel data processing. Different workers will loop over different ranges and will be labelled by different "slot" values. Returns true if the entry has to be processed, false otherwise.

Implements ROOT::RDF::RDataSource.

Definition at line 321 of file RNTupleDS.hxx.

◆ SetGlobalEntryRange()

virtual void ROOT::RDF::RDataSource::SetGlobalEntryRange ( std::pair< ULong64_t, ULong64_t > entryRange )

inlinevirtualinherited

Restrict processing to a [begin, end) range of entries.

Parameters

entryRange The range of entries to process.

Definition at line 327 of file RDataSource.hxx.

◆ SetNSlots()

void ROOT::RDF::RNTupleDS::SetNSlots ( unsigned int nSlots )

finalvirtual

Inform RDataSource of the number of processing slots (i.e.

worker threads) used by the associated RDataFrame. Slots numbers are used to simplify parallel execution: RDataFrame guarantees that different threads will always pass different slot values when calling methods concurrently.

Reimplemented from ROOT::RDF::RDataSource.

Definition at line 776 of file RNTupleDS.cxx.

◆ StageNextSources()

void ROOT::RDF::RNTupleDS::StageNextSources ( )

private

Starting from fNextFileIndex, opens the next fNSlots files.

Calls LoadStructure() on the opened files. The very first file is already available from the constructor.

Definition at line 496 of file RNTupleDS.cxx.

Friends And Related Symbol Documentation

◆ ROOT::Internal::RDF::FromRNTuple

ROOT::RDataFrame ROOT::Internal::RDF::FromRNTuple	(	std::string_view	ntupleName,
		const std::vector< std::string > &	fileNames,
		const std::pair< ULong64_t, ULong64_t > &	range )

friend

◆ ROOT::Internal::RDF::GetDatasetGlobalClusterBoundaries

std::vector< std::pair< std::uint64_t, std::uint64_t > > ROOT::Internal::RDF::GetDatasetGlobalClusterBoundaries ( const ROOT::RDF::RInterface< ROOT::Detail::RDF::RNodeBase > & node )

friend

◆ ROOT::Internal::RDF::RNTupleColumnReader

friend class ROOT::Internal::RDF::RNTupleColumnReader

friend

Definition at line 101 of file RNTupleDS.hxx.

Member Data Documentation

◆ fActiveColumnReaders

std::vector<std::vector<ROOT::Internal::RDF::RNTupleColumnReader *> > ROOT::RDF::RNTupleDS::fActiveColumnReaders

private

List of column readers returned by GetColumnReaders() organized by slot.

Used to reconnect readers to new page sources when the files in the chain change.

Definition at line 174 of file RNTupleDS.hxx.

◆ fActiveRangesPerSlot

std::vector<REntryRangeDS> ROOT::RDF::RNTupleDS::fActiveRangesPerSlot

private

The page source and entry window seen by an active thread task in multi-threaded runs.

Definition at line 182 of file RNTupleDS.hxx.

◆ fAlternativeProtoFields

std::unordered_map<std::size_t, std::vector<std::unique_ptr<ROOT::RFieldBase> > > ROOT::RDF::RNTupleDS::fAlternativeProtoFields

private

Columns may be requested with types other than with which they were initially added as proto fields.

For example, a column with a ROOT::RVec<float> proto field may instead be requested as a std::vector<float>. In case this happens, we create an alternative proto field and store it here, with the original index in fProtoFields as key. A single column can have more than one alternative proto fields.

Definition at line 164 of file RNTupleDS.hxx.

◆ fColumnNames

std::vector<std::string> ROOT::RDF::RNTupleDS::fColumnNames

private

Definition at line 169 of file RNTupleDS.hxx.

◆ fColumnTypes

std::vector<std::string> ROOT::RDF::RNTupleDS::fColumnTypes

private

Definition at line 170 of file RNTupleDS.hxx.

◆ fCurrentRanges

std::vector<REntryRangeDS> ROOT::RDF::RNTupleDS::fCurrentRanges

private

Basis for the ranges returned by the last GetEntryRanges() call.

Definition at line 178 of file RNTupleDS.hxx.

◆ fCvStaging

std::condition_variable ROOT::RDF::RNTupleDS::fCvStaging

private

Signal for the state information of fIsReadyForStaging and fHasNextSources.

Definition at line 195 of file RNTupleDS.hxx.

◆ fFieldId2QualifiedName

std::unordered_map<ROOT::DescriptorId_t, std::string> ROOT::RDF::RNTupleDS::fFieldId2QualifiedName

private

Connects the IDs of active proto fields and their subfields to their fully qualified name (a.b.c.d).

This enables the column reader to rewire the field IDs when the file changes (chain), using the fully qualified name as a search key in the descriptor of the other page sources.

Definition at line 168 of file RNTupleDS.hxx.

◆ fFileNames

std::vector<std::string> ROOT::RDF::RNTupleDS::fFileNames

private

Definition at line 135 of file RNTupleDS.hxx.

◆ fFirstEntry2RangeIdx

std::unordered_map<ULong64_t, std::size_t> ROOT::RDF::RNTupleDS::fFirstEntry2RangeIdx

private

Maps the first entries from the ranges of the last GetEntryRanges() call to their corresponding index in the fCurrentRanges vectors.

This is necessary because the returned ranges get distributed arbitrarily onto slots. In the InitSlot method, the column readers use this map to find the correct range to connect to.

Definition at line 186 of file RNTupleDS.hxx.

◆ fGlobalEntryRange

std::optional<std::pair<ULong64_t, ULong64_t> > ROOT::RDF::RDataSource::fGlobalEntryRange {}

protectedinherited

Definition at line 149 of file RDataSource.hxx.

◆ fHasNextSources

bool ROOT::RDF::RNTupleDS::fHasNextSources = false

private

Is true when the staging thread has populated the next batch of files to fStagingArea.

Definition at line 199 of file RNTupleDS.hxx.

◆ fIsReadyForStaging

bool ROOT::RDF::RNTupleDS::fIsReadyForStaging = false

private

Is true when the staging thread should start working.

Definition at line 197 of file RNTupleDS.hxx.

◆ fMutexStaging

std::mutex ROOT::RDF::RNTupleDS::fMutexStaging

private

Protects the shared state between the main thread and the I/O thread.

Definition at line 193 of file RNTupleDS.hxx.

◆ fNextFileIndex

std::size_t ROOT::RDF::RNTupleDS::fNextFileIndex = 0

private

Index into fFileNames to the next file to process.

Definition at line 153 of file RNTupleDS.hxx.

◆ fNextRanges

std::vector<REntryRangeDS> ROOT::RDF::RNTupleDS::fNextRanges

private

Basis for the ranges populated by the PrepareNextRanges() call.

Definition at line 179 of file RNTupleDS.hxx.

◆ fNSlots

unsigned int ROOT::RDF::RDataSource::fNSlots {}

protectedinherited

Definition at line 147 of file RDataSource.hxx.

◆ fNTupleName

std::string ROOT::RDF::RNTupleDS::fNTupleName

private

The data source may be constructed with an ntuple name and a list of files.

Definition at line 134 of file RNTupleDS.hxx.

◆ fOriginalRanges

std::vector<std::pair<ULong64_t, ULong64_t> > ROOT::RDF::RNTupleDS::fOriginalRanges

private

Definition at line 188 of file RNTupleDS.hxx.

◆ fPrincipalDescriptor

ROOT::RNTupleDescriptor ROOT::RDF::RNTupleDS::fPrincipalDescriptor

private

A clone of the first pages source's descriptor.

Definition at line 131 of file RNTupleDS.hxx.

◆ fProtoFields

std::vector<std::unique_ptr<ROOT::RFieldBase> > ROOT::RDF::RNTupleDS::fProtoFields

private

We prepare a prototype field for every column.

If a column reader is actually requested in GetColumnReaders(), we move a clone of the field into a new column reader for RDataFrame. Only the clone connects to the backing page store and acquires I/O resources. The field IDs are set in the context of the first source and used as keys in fFieldId2QualifiedName.

Definition at line 159 of file RNTupleDS.hxx.

◆ fSeenEntriesNoGlobalRange

ULong64_t ROOT::RDF::RNTupleDS::fSeenEntriesNoGlobalRange = 0

private

The number of entries seen so far in GetEntryRanges()

Definition at line 176 of file RNTupleDS.hxx.

◆ fStagingArea

std::vector<std::shared_ptr<ROOT::Internal::RPageSource> > ROOT::RDF::RNTupleDS::fStagingArea

private

The staging area is relevant for chains of files, i.e.

when fFileNames is not empty. In this case, files are opened in the background in batches of size fNSlots and kept in the staging area. The first file (chains or no chains) is always opened on construction in order to process the schema. For all subsequent files, the corresponding page sources in the staging area only executed LoadStructure(), i.e. they should have a compressed buffer of the meta-data available. Concretely:

We open the first file on construction to read the schema and then move the corresponding page source in the staging area.
On Initialize(), we start the I/O background thread, which in turn opens the first batch of files.
At the beginning of GetEntryRanges(), we a) wait for the I/O thread to finish, b) call PrepareNextRanges() in the main thread to move the page sources from the staging area into fNextRanges; this will also call Attach() on the page sources (i.e., deserialize the meta-data), and c) trigger staging of the next batch of files in the I/O background thread.
On Finalize(), the I/O background thread is stopped.

Definition at line 152 of file RNTupleDS.hxx.

◆ fStagingThreadShouldTerminate

bool ROOT::RDF::RNTupleDS::fStagingThreadShouldTerminate = false

private

Is true when the I/O thread should quit.

Definition at line 201 of file RNTupleDS.hxx.

◆ fThreadStaging

std::thread ROOT::RDF::RNTupleDS::fThreadStaging

private

The background thread that runs StageNextSources()

Definition at line 191 of file RNTupleDS.hxx.

◆ fTopLevelFieldNames

std::vector<std::string> ROOT::RDF::RNTupleDS::fTopLevelFieldNames

private

Definition at line 171 of file RNTupleDS.hxx.

Libraries for ROOT::RDF::RNTupleDS:

[legend]

The documentation for this class was generated from the following files:

tree/dataframe/inc/ROOT/RNTupleDS.hxx
tree/dataframe/src/RNTupleDS.cxx

Classes

Public Member Functions

Protected Types

Protected Member Functions

Protected Attributes

Private Member Functions

Private Attributes

Friends

Member Typedef Documentation

◆ Record_t

Constructor & Destructor Documentation

◆ RNTupleDS() [1/6]

◆ RNTupleDS() [2/6]

◆ RNTupleDS() [3/6]

◆ RNTupleDS() [4/6]

◆ RNTupleDS() [5/6]

◆ RNTupleDS() [6/6]

◆ ~RNTupleDS()

Member Function Documentation

◆ AddField()

◆ AsString()

◆ ComputeGlobalEntryOffsets()

◆ CreateColumnReader()

◆ CreateSampleInfo()

◆ DescribeDataset()

◆ ExecStaging()

◆ Finalize()

◆ FinalizeSlot()

◆ GetColumnNames()

◆ GetColumnNamesNoDuplicates()

◆ GetColumnReaders() [1/2]

◆ GetColumnReaders() [2/2]

◆ GetColumnReadersImpl()

◆ GetEntryRanges()

◆ GetFieldWithTypeChecks()

◆ GetLabel()

◆ GetNFiles()

◆ GetTopLevelFieldNames()

◆ GetTypeName()

◆ GetTypeNameWithOpts()

◆ HasColumn()

◆ Initialize()

◆ InitializeWithOpts()

◆ InitSlot()

◆ InsertActiveEntryRange()

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ ParallelForBody()

◆ PrepareNextRanges()

◆ ProcessMT()

◆ RunFinalChecks()

◆ ScheduleTaskForPageSource()

◆ ScheduleTasksForFile()

◆ ScheduleTasksForFileWithGlobalOffset()

◆ SetEntry()

◆ SetGlobalEntryRange()

◆ SetNSlots()

◆ StageNextSources()

Friends And Related Symbol Documentation

◆ ROOT::Internal::RDF::FromRNTuple

◆ ROOT::Internal::RDF::GetDatasetGlobalClusterBoundaries

◆ ROOT::Internal::RDF::RNTupleColumnReader

Member Data Documentation

◆ fActiveColumnReaders

◆ fActiveRangesPerSlot

◆ fAlternativeProtoFields

◆ fColumnNames

◆ fColumnTypes

◆ fCurrentRanges

◆ fCvStaging

◆ fFieldId2QualifiedName

◆ fFileNames

◆ fFirstEntry2RangeIdx

◆ fGlobalEntryRange

◆ fHasNextSources

◆ fIsReadyForStaging

◆ fMutexStaging

◆ fNextFileIndex

◆ fNextRanges

◆ fNSlots