Storage provider that reads ntuple pages from a file.

Definition at line 126 of file RPageStorageFile.hxx.

Classes
struct	RFileCounters
	File-specific I/O performance counters. More...

Public Types
using	ColumnHandle_t = RColumnHandle
	The column handle identifies a column with the current open page storage.

using	SealedPageSequence_t = std::deque<RSealedPage>

Public Member Functions
	RPageSourceFile (const RPageSourceFile &)=delete

	RPageSourceFile (RPageSourceFile &&)=delete

	RPageSourceFile (std::string_view ntupleName, std::string_view path, const ROOT::RNTupleReadOptions &options)

	RPageSourceFile (std::string_view ntupleName, std::unique_ptr< RRawFile > file, const ROOT::RNTupleReadOptions &options)

	~RPageSourceFile () override

ColumnHandle_t	AddColumn (ROOT::DescriptorId_t fieldId, ROOT::Internal::RColumn &column) override
	Register a new column.

void	Attach (ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode mode=ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode::kForReading)
	Open the physical storage container and deserialize header and footer.

std::unique_ptr< RPageSource >	Clone () const
	Open the same storage multiple time, e.g.

void	DropColumn (ColumnHandle_t columnHandle) override
	Unregisters a column.

RAnyDescriptorGuard	FindClusterId (ROOT::DescriptorId_t physicalColumnId, ROOT::NTupleSize_t index, ROOT::DescriptorId_t &cid)
	Returns a shared descriptor guard to ensure that the returned cluster id is useable, i.e.

RAnyDescriptorGuard	FindClusterId (ROOT::NTupleSize_t entryIdx, ROOT::DescriptorId_t &cid)
	An overload of FindClusterId that searches using a certain column element index.

RAnyDescriptorGuard	FindNextClusterId (ROOT::DescriptorId_t clusterId, ROOT::DescriptorId_t &nextId)
	Uses FindClusterId to search for the cluster with the entry index following the last entry index of the given cluster.

ROOT::DescriptorId_t	GetColumnId (ColumnHandle_t columnHandle) const

REntryRange	GetEntryRange () const

virtual ROOT::Experimental::Detail::RNTupleMetrics &	GetMetrics ()
	Returns the default metrics object.

ROOT::NTupleSize_t	GetNElements (ROOT::DescriptorId_t physicalColumnId)

ROOT::NTupleSize_t	GetNEntries ()

const std::string &	GetNTupleName () const
	Returns the NTuple name.

const std::unordered_set< ROOT::DescriptorId_t > &	GetPinnedClusters () const

const ROOT::RNTupleReadOptions &	GetReadOptions () const

RSharedDescriptorGuard	GetSharedDescriptorGuard () const
	Takes the read lock for the descriptor.

EPageStorageType	GetType () final
	Whether the concrete implementation is a sink or a source.

void	LoadAllPageLists ()
	Load all remaining cluster details.

std::vector< std::unique_ptr< ROOT::Internal::RCluster > >	LoadClusters (std::span< ROOT::Internal::RCluster::RKey > clusterKeys) final
	Populates all the pages of the given cluster ids and columns; it is possible that some columns do not contain any pages.

virtual ROOT::Internal::RPageRef	LoadPage (ColumnHandle_t columnHandle, RNTupleLocalIndex localIndex)
	Another version of `LoadPage` that allows to specify cluster-relative indexes.

virtual ROOT::Internal::RPageRef	LoadPage (ColumnHandle_t columnHandle, ROOT::NTupleSize_t globalIndex)
	Allocates and fills a page that contains the index-th element.

void	LoadSealedPage (ROOT::DescriptorId_t physicalColumnId, RNTupleLocalIndex localIndex, RSealedPage &sealedPage)
	Read the packed and compressed bytes of a page into the memory buffer provided by `sealedPage`.

void	LoadStructure ()
	Loads header and footer without decompressing or deserializing them.

std::unique_ptr< RPageSource >	OpenWithDifferentAnchor (const ROOT::Internal::RNTupleLink &anchorLink, const ROOT::RNTupleReadOptions &options={}) final
	Creates a new PageSource using the same underlying file as this but referring to a different RNTuple, described by `anchorLink`.

RPageSourceFile &	operator= (const RPageSourceFile &)=delete

RPageSourceFile &	operator= (RPageSourceFile &&)=delete

void	PinCluster (ROOT::DescriptorId_t clusterId)
	Instructs the cluster pool and page pool to consider the given cluster as active (should stay cached).

void	RegisterStreamerInfos ()
	Builds the streamer info records from the descriptor's extra type info section.

void	SetEntryRange (const REntryRange &range)
	Promise to only read from the given entry range.

void	SetTaskScheduler (RTaskScheduler *taskScheduler)

void	UnpinCluster (ROOT::DescriptorId_t clusterId)
	Allows the given cluster to be evicted from the cluster pool and page pool.

RResult< ROOT::Internal::RPage >	UnsealPage (const RSealedPage &sealedPage, const ROOT::Internal::RColumnElementBase &element)

void	UnzipCluster (ROOT::Internal::RCluster *cluster)
	Parallel decompression and unpacking of the pages in the given cluster.

Static Public Member Functions
static std::unique_ptr< RPageSource >	Create (std::string_view ntupleName, std::string_view location, const ROOT::RNTupleReadOptions &options=ROOT::RNTupleReadOptions())
	Guess the concrete derived page source from the file name (location)

static std::unique_ptr< RPageSourceFile >	CreateFromAnchor (const RNTuple &anchor, const ROOT::RNTupleReadOptions &options=ROOT::RNTupleReadOptions())
	Used from the RNTuple class to build a datasource if the anchor is already available.

static RResult< ROOT::Internal::RPage >	UnsealPage (const RSealedPage &sealedPage, const ROOT::Internal::RColumnElementBase &element, ROOT::Internal::RPageAllocator &pageAlloc)
	Helper for unstreaming a page.

Static Public Attributes
static constexpr std::size_t	kNBytesPageChecksum = sizeof(std::uint64_t)
	The page checksum is a 64bit xxhash3.

Protected Member Functions
ROOT::RNTupleDescriptor	AttachImpl () final
	`LoadStructureImpl()` has been called before `AttachImpl()` is called

std::unique_ptr< RPageSource >	CloneImpl () const final
	The cloned page source creates a new raw file and reader and opens its own file descriptor to the data.

void	EnableDefaultMetrics (const std::string &prefix)
	Enables the default set of metrics provided by RPageSource.

RExclDescriptorGuard	GetExclDescriptorGuard ()
	Note that the underlying lock is not recursive. See `GetSharedDescriptorGuard()` for further information.

void	LoadPageListImpl (const RNTupleLocator &locator, unsigned char *buffer) final

void	LoadSealedPageImpl (const RNTupleLocator &locator, RSealedPage &sealedPage) final

void	LoadStructureImpl () final
	Fills fStructureBuffer with the compressed header and footer.

void	PrepareLoadCluster (const ROOT::Internal::RCluster::RKey &clusterKey, ROOT::Internal::ROnDiskPageMap &pageZeroMap, const std::function< void(ROOT::DescriptorId_t, ROOT::NTupleSize_t, const ROOT::RClusterDescriptor::RPageInfo &)> &perPageFunc)
	Prepare a page range read for the column set in `clusterKey`.

void	StopClusterPoolBackgroundThread ()

void	WaitForAllTasks ()

Protected Attributes
std::unique_ptr< RCounters >	fCounters

ROOT::Experimental::Detail::RNTupleMetrics	fMetrics

std::string	fNTupleName

ROOT::RNTupleReadOptions	fOptions

std::unique_ptr< ROOT::Internal::RPageAllocator >	fPageAllocator
	For the time being, we will use the heap allocator for all sources and sinks. This may change in the future.

RStructureBuffer	fStructureBuffer
	Populated by LoadStructureImpl(), reset at the end of Attach()

RTaskScheduler *	fTaskScheduler = nullptr

Private Member Functions
	RPageSourceFile (std::string_view ntupleName, const ROOT::RNTupleReadOptions &options)

RAnyDescriptorGuard	EnsureClusterDetails (DescriptorId_t cgId, RAnyDescriptorGuard descGuard)
	Ensures that fDescriptor has cluster details loaded for the given cluster group ID.

DescriptorId_t	FindClusterGroupId (DescriptorId_t clusterId) const
	Uses binary search in fCumulativeClusterCounts to determine the cluster group that the cluster ID belongs in.

ROOT::Internal::RPageRef	LoadPageFromSummary (ColumnHandle_t columnHandle, const RPageSummary &pageSummary)

void	LoadPageList (DescriptorId_t clusterGroupId, const RExclDescriptorGuard &exclGuard)

ROOT::Internal::RPageRef	LoadZeroPage (ColumnHandle_t columnHandle, const RPageSummary &pageSummary)

std::unique_ptr< ROOT::Internal::RCluster >	PrepareSingleCluster (const ROOT::Internal::RCluster::RKey &clusterKey, std::vector< RRawFile::RIOVec > &readRequests)
	Helper function for LoadClusters: it prepares the memory buffer (page map) and the read requests for a given cluster and columns.

void	UpdateLastUsedCluster (ROOT::DescriptorId_t clusterId)
	Does nothing if fLastUsedCluster == clusterId.

void	UpdateSkipCounter (std::uint64_t offset)

Private Attributes
RActivePhysicalColumns	fActivePhysicalColumns
	The active columns are implicitly defined by the model fields or views.

std::optional< RNTuple >	fAnchor
	Either provided by CreateFromAnchor, or read from the ROOT file given the ntuple name.

ROOT::Internal::RClusterPool	fClusterPool
	The cluster pool asynchronously preloads the next few clusters.

std::vector< NTupleSize_t >	fCumulativeClusterCounts
	This vector is aligned with the cluster groups in the descriptor.

ROOT::Internal::RCluster *	fCurrentCluster = nullptr
	The last cluster from which a page got loaded. Points into fClusterPool->fPool.

ROOT::RNTupleDescriptor	fDescriptor

RNTupleDescriptorBuilder	fDescriptorBuilder
	The descriptor is created from the header and footer either in AttachImpl or in CreateFromAnchor.

std::shared_mutex	fDescriptorLock

Internal::RNTupleSerializer::EDescriptorDeserializeMode	fDeserializationMode
	The interpretation of page lists, set in Attach()

REntryRange	fEntryRange
	Used by the cluster pool to prevent reading beyond the given range.

std::unique_ptr< RRawFile >	fFile
	An RRawFile is used to request the necessary byte ranges from a local or a remote file.

std::unique_ptr< RFileCounters >	fFileCounters

std::int64_t	fFileSize = 0
	Total file size, set once in AttachImpl()

bool	fHasStreamerInfosRegistered = false
	Set to true when RegisterStreamerInfos() is called.

bool	fHasStructure = false
	Set to true once `LoadStructure()` is called.

bool	fIsAttached = false
	Set to true once `Attach()` is called.

std::uint64_t	fLastOffset = 0
	Tracks the last read offset for seek distance calculation.

ROOT::DescriptorId_t	fLastUsedCluster = ROOT::kInvalidDescriptorId
	Remembers the last cluster id from which a page was requested.

ROOT::Internal::RPagePool	fPagePool
	Pages that are unzipped with IMT are staged into the page pool.

std::unordered_set< ROOT::DescriptorId_t >	fPinnedClusters
	Pinned clusters and their $2 * (cluster bunch size) - 1$ successors will not be evicted from the cluster pool.

std::map< ROOT::NTupleSize_t, ROOT::DescriptorId_t >	fPreloadedClusters
	Clusters from where pages got preloaded in UnzipCluster(), ordered by first entry number of the clusters.

ROOT::Internal::RMiniFileReader	fReader
	Takes the fFile to read ntuple blobs from it.

std::mutex	fReaderLock
	Protects the raw file, the reader, and fLastOffset.

Static Private Attributes
static constexpr std::size_t	kInlineDecompressionThreshold = 4096

Friends
const ROOT::RNTuple *	ROOT::Internal::GetAnchorFromFile (const RPageSourceFile &)

class	ROOT::RNTuple

#include <ROOT/RPageStorageFile.hxx>

Inheritance diagram for ROOT::Internal::RPageSourceFile:

[legend]

Member Typedef Documentation

◆ ColumnHandle_t

using ROOT::Internal::RPageStorage::ColumnHandle_t = RColumnHandle

inherited

The column handle identifies a column with the current open page storage.

Definition at line 181 of file RPageStorage.hxx.

◆ SealedPageSequence_t

using ROOT::Internal::RPageStorage::SealedPageSequence_t = std::deque<RSealedPage>

inherited

Definition at line 131 of file RPageStorage.hxx.

Constructor & Destructor Documentation

◆ RPageSourceFile() [1/5]

ROOT::Internal::RPageSourceFile::RPageSourceFile	(	std::string_view	ntupleName,
		const ROOT::RNTupleReadOptions &	options )

private

Definition at line 310 of file RPageStorageFile.cxx.

◆ RPageSourceFile() [2/5]

ROOT::Internal::RPageSourceFile::RPageSourceFile	(	std::string_view	ntupleName,
		std::string_view	path,
		const ROOT::RNTupleReadOptions &	options )

Definition at line 366 of file RPageStorageFile.cxx.

◆ RPageSourceFile() [3/5]

ROOT::Internal::RPageSourceFile::RPageSourceFile	(	std::string_view	ntupleName,
		std::unique_ptr< RRawFile >	file,
		const ROOT::RNTupleReadOptions &	options )

Definition at line 356 of file RPageStorageFile.cxx.

◆ RPageSourceFile() [4/5]

ROOT::Internal::RPageSourceFile::RPageSourceFile ( const RPageSourceFile & )

delete

◆ RPageSourceFile() [5/5]

ROOT::Internal::RPageSourceFile::RPageSourceFile ( RPageSourceFile && )

delete

◆ ~RPageSourceFile()

ROOT::Internal::RPageSourceFile::~RPageSourceFile ( )

override

Definition at line 398 of file RPageStorageFile.cxx.

Member Function Documentation

◆ AddColumn()

ROOT::Internal::RPageStorage::ColumnHandle_t ROOT::Internal::RPageSource::AddColumn	(	ROOT::DescriptorId_t	fieldId,
		ROOT::Internal::RColumn &	column )

overridevirtualinherited

Register a new column.

When reading, the column must exist in the ntuple on disk corresponding to the metadata. When writing, every column can only be attached once.

Implements ROOT::Internal::RPageStorage.

Definition at line 206 of file RPageStorage.cxx.

◆ Attach()

void ROOT::Internal::RPageSource::Attach ( ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode mode = ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode::kForReading )

inherited

Open the physical storage container and deserialize header and footer.

Definition at line 236 of file RPageStorage.cxx.

◆ AttachImpl()

ROOT::RNTupleDescriptor ROOT::Internal::RPageSourceFile::AttachImpl ( )

finalprotectedvirtual

LoadStructureImpl() has been called before AttachImpl() is called

Implements ROOT::Internal::RPageSource.

Definition at line 467 of file RPageStorageFile.cxx.

◆ Clone()

std::unique_ptr< ROOT::Internal::RPageSource > ROOT::Internal::RPageSource::Clone ( ) const

inherited

Open the same storage multiple time, e.g.

for reading in multiple threads. If the source is already attached, the clone will be attached, too. The clone will use, however, it's own connection to the underlying storage (e.g., file descriptor, XRootD handle, etc.)

Definition at line 289 of file RPageStorage.cxx.

◆ CloneImpl()

std::unique_ptr< ROOT::Internal::RPageSource > ROOT::Internal::RPageSourceFile::CloneImpl ( ) const

finalprotectedvirtual

The cloned page source creates a new raw file and reader and opens its own file descriptor to the data.

Implements ROOT::Internal::RPageSource.

Definition at line 525 of file RPageStorageFile.cxx.

◆ Create()

std::unique_ptr< ROOT::Internal::RPageSource > ROOT::Internal::RPageSource::Create	(	std::string_view	ntupleName,
		std::string_view	location,
		const ROOT::RNTupleReadOptions &	options = ROOT::RNTupleReadOptions() )

staticinherited

Guess the concrete derived page source from the file name (location)

Definition at line 177 of file RPageStorage.cxx.

◆ CreateFromAnchor()

std::unique_ptr< ROOT::Internal::RPageSourceFile > ROOT::Internal::RPageSourceFile::CreateFromAnchor	(	const RNTuple &	anchor,
		const ROOT::RNTupleReadOptions &	options = ROOT::RNTupleReadOptions() )

static

Used from the RNTuple class to build a datasource if the anchor is already available.

Requires the RNTuple object to be streamed from a file.

Definition at line 373 of file RPageStorageFile.cxx.

◆ DropColumn()

void ROOT::Internal::RPageSource::DropColumn ( ColumnHandle_t columnHandle )

overridevirtualinherited

Unregisters a column.

A page source decreases the reference counter for the corresponding active column. For a page sink, dropping columns is currently a no-op.

Implements ROOT::Internal::RPageStorage.

Definition at line 216 of file RPageStorage.cxx.

◆ EnableDefaultMetrics()

void ROOT::Internal::RPageSource::EnableDefaultMetrics ( const std::string & prefix )

protectedinherited

Enables the default set of metrics provided by RPageSource.

prefix will be used as the prefix for the counters registered in the internal RNTupleMetrics object. A subclass using the default set of metrics is responsible for updating the counters appropriately, e.g. fCounters->fNRead.Inc() Alternatively, a subclass might provide its own RNTupleMetrics object by overriding the GetMetrics() member function.

Definition at line 832 of file RPageStorage.cxx.

◆ EnsureClusterDetails()

ROOT::Internal::RPageSource::RAnyDescriptorGuard ROOT::Internal::RPageSource::EnsureClusterDetails	(	DescriptorId_t	cgId,
		RAnyDescriptorGuard	descGuard )

privateinherited

Ensures that fDescriptor has cluster details loaded for the given cluster group ID.

This method is expected to be called while holding the lock passed by descGuard. The returned descriptor guard is either the passed one or a new, exclusive guard if cluster details needed to be loaded.

Definition at line 303 of file RPageStorage.cxx.

◆ FindClusterGroupId()

ROOT::DescriptorId_t ROOT::Internal::RPageSource::FindClusterGroupId ( DescriptorId_t clusterId ) const

privateinherited

Uses binary search in fCumulativeClusterCounts to determine the cluster group that the cluster ID belongs in.

Definition at line 324 of file RPageStorage.cxx.

◆ FindClusterId() [1/2]

ROOT::Internal::RPageSource::RAnyDescriptorGuard ROOT::Internal::RPageSource::FindClusterId	(	ROOT::DescriptorId_t	physicalColumnId,
		ROOT::NTupleSize_t	index,
		ROOT::DescriptorId_t &	cid )

inherited

Returns a shared descriptor guard to ensure that the returned cluster id is useable, i.e.

that the corresponding cluster was not meanwhile evicted from the set of active clusters.

Definition at line 433 of file RPageStorage.cxx.

◆ FindClusterId() [2/2]

ROOT::Internal::RPageSource::RAnyDescriptorGuard ROOT::Internal::RPageSource::FindClusterId	(	ROOT::NTupleSize_t	entryIdx,
		ROOT::DescriptorId_t &	cid )

inherited

An overload of FindClusterId that searches using a certain column element index.

Definition at line 373 of file RPageStorage.cxx.

◆ FindNextClusterId()

ROOT::Internal::RPageSource::RAnyDescriptorGuard ROOT::Internal::RPageSource::FindNextClusterId	(	ROOT::DescriptorId_t	clusterId,
		ROOT::DescriptorId_t &	nextId )

inherited

Uses FindClusterId to search for the cluster with the entry index following the last entry index of the given cluster.

Definition at line 357 of file RPageStorage.cxx.

◆ GetColumnId()

ROOT::DescriptorId_t ROOT::Internal::RPageStorage::GetColumnId ( ColumnHandle_t columnHandle ) const

inlineinherited

Definition at line 189 of file RPageStorage.hxx.

◆ GetEntryRange()

REntryRange ROOT::Internal::RPageSource::GetEntryRange ( ) const

inlineinherited

Definition at line 978 of file RPageStorage.hxx.

◆ GetExclDescriptorGuard()

RExclDescriptorGuard ROOT::Internal::RPageSource::GetExclDescriptorGuard ( )

inlineprotectedinherited

Note that the underlying lock is not recursive. See GetSharedDescriptorGuard() for further information.

Definition at line 908 of file RPageStorage.hxx.

◆ GetMetrics()

virtual ROOT::Experimental::Detail::RNTupleMetrics & ROOT::Internal::RPageStorage::GetMetrics ( )

inlinevirtualinherited

Returns the default metrics object.

Subclasses might alternatively provide their own metrics object by overriding this.

Definition at line 193 of file RPageStorage.hxx.

◆ GetNElements()

ROOT::NTupleSize_t ROOT::Internal::RPageSource::GetNElements ( ROOT::DescriptorId_t physicalColumnId )

inherited

Definition at line 339 of file RPageStorage.cxx.

◆ GetNEntries()

ROOT::NTupleSize_t ROOT::Internal::RPageSource::GetNEntries ( )

inherited

Definition at line 334 of file RPageStorage.cxx.

◆ GetNTupleName()

const std::string & ROOT::Internal::RPageStorage::GetNTupleName ( ) const

inlineinherited

Returns the NTuple name.

Definition at line 196 of file RPageStorage.hxx.

◆ GetPinnedClusters()

const std::unordered_set< ROOT::DescriptorId_t > & ROOT::Internal::RPageSource::GetPinnedClusters ( ) const

inlineinherited

Definition at line 1016 of file RPageStorage.hxx.

◆ GetReadOptions()

const ROOT::RNTupleReadOptions & ROOT::Internal::RPageSource::GetReadOptions ( ) const

inlineinherited

Definition at line 936 of file RPageStorage.hxx.

◆ GetSharedDescriptorGuard()

RSharedDescriptorGuard ROOT::Internal::RPageSource::GetSharedDescriptorGuard ( ) const

inlineinherited

Takes the read lock for the descriptor.

Multiple threads can take the lock concurrently. The underlying std::shared_mutex, however, is neither read nor write recursive: within one thread, only one lock (shared or exclusive) must be acquired at the same time. This requires special care in sections protected by GetSharedDescriptorGuard() and GetExclDescriptorGuard() especially to avoid that the locks are acquired indirectly. As a general guideline, no other method of the page source should be called (directly or indirectly) in a guarded section.

Definition at line 944 of file RPageStorage.hxx.

◆ GetType()

EPageStorageType ROOT::Internal::RPageSource::GetType ( )

inlinefinalvirtualinherited

Whether the concrete implementation is a sink or a source.

Implements ROOT::Internal::RPageStorage.

Definition at line 935 of file RPageStorage.hxx.

◆ LoadAllPageLists()

void ROOT::Internal::RPageSource::LoadAllPageLists ( )

inherited

Load all remaining cluster details.

Definition at line 264 of file RPageStorage.cxx.

◆ LoadClusters()

std::vector< std::unique_ptr< ROOT::Internal::RCluster > > ROOT::Internal::RPageSourceFile::LoadClusters ( std::span< ROOT::Internal::RCluster::RKey > clusterKeys )

finalvirtual

Populates all the pages of the given cluster ids and columns; it is possible that some columns do not contain any pages.

The page source may load more columns than the minimal necessary set from columns. To indicate which columns have been loaded, LoadClusters()must mark them withSetColumnAvailable(). That includes the ones from thecolumnsthat don't have pages; otherwise subsequent requests for the cluster would assume an incomplete cluster and trigger loading again. LoadClusters()` is typically called from the I/O thread of a cluster pool, i.e. the method runs concurrently to other methods of the page source.

Implements ROOT::Internal::RPageSource.

Definition at line 659 of file RPageStorageFile.cxx.

◆ LoadPage() [1/2]

ROOT::Internal::RPageRef ROOT::Internal::RPageSource::LoadPage	(	ColumnHandle_t	columnHandle,
		RNTupleLocalIndex	localIndex )

virtualinherited

Another version of LoadPage that allows to specify cluster-relative indexes.

Returns a default-constructed RPage for suppressed columns.

Definition at line 801 of file RPageStorage.cxx.

◆ LoadPage() [2/2]

ROOT::Internal::RPageRef ROOT::Internal::RPageSource::LoadPage	(	ColumnHandle_t	columnHandle,
		ROOT::NTupleSize_t	globalIndex )

virtualinherited

Allocates and fills a page that contains the index-th element.

Calls into the concrete page source for loading the corresponding sealed page of cluster where necessary. Returns a default-constructed RPage for suppressed columns.

Definition at line 769 of file RPageStorage.cxx.

◆ LoadPageFromSummary()

ROOT::Internal::RPageRef ROOT::Internal::RPageSource::LoadPageFromSummary	(	ColumnHandle_t	columnHandle,
		const RPageSummary &	pageSummary )

privateinherited

Definition at line 705 of file RPageStorage.cxx.

◆ LoadPageList()

void ROOT::Internal::RPageSource::LoadPageList	(	DescriptorId_t	clusterGroupId,
		const RExclDescriptorGuard &	exclGuard )

privateinherited

Definition at line 273 of file RPageStorage.cxx.

◆ LoadPageListImpl()

void ROOT::Internal::RPageSourceFile::LoadPageListImpl	(	const RNTupleLocator &	locator,
		unsigned char *	buffer )

finalprotectedvirtual

Implements ROOT::Internal::RPageSource.

Definition at line 505 of file RPageStorageFile.cxx.

◆ LoadSealedPage()

void ROOT::Internal::RPageSource::LoadSealedPage	(	ROOT::DescriptorId_t	physicalColumnId,
		RNTupleLocalIndex	localIndex,
		RSealedPage &	sealedPage )

inherited

Read the packed and compressed bytes of a page into the memory buffer provided by sealedPage.

The sealed page can be used subsequently in a call to RPageSink::CommitSealedPage. The fSize and fNElements member of the sealedPage parameters are always set. If sealedPage.fBuffer is nullptr, no data will be copied but the returned size information can be used by the caller to allocate a large enough buffer and call LoadSealedPage again.

Definition at line 661 of file RPageStorage.cxx.

◆ LoadSealedPageImpl()

void ROOT::Internal::RPageSourceFile::LoadSealedPageImpl	(	const RNTupleLocator &	locator,
		RSealedPage &	sealedPage )

finalprotectedvirtual

Implements ROOT::Internal::RPageSource.

Definition at line 515 of file RPageStorageFile.cxx.

◆ LoadStructure()

void ROOT::Internal::RPageSource::LoadStructure ( )

inherited

Loads header and footer without decompressing or deserializing them.

This can be used to asynchronously open a file in the background. The method is idempotent and it is called as a first step in Attach(). Pages sources may or may not make use of splitting loading and processing metadata. Therefore, LoadStructure() may do nothing and defer loading the metadata to Attach().

Definition at line 229 of file RPageStorage.cxx.

◆ LoadStructureImpl()

void ROOT::Internal::RPageSourceFile::LoadStructureImpl ( )

finalprotectedvirtual

Fills fStructureBuffer with the compressed header and footer.

Implements ROOT::Internal::RPageSource.

Definition at line 418 of file RPageStorageFile.cxx.

◆ LoadZeroPage()

ROOT::Internal::RPageRef ROOT::Internal::RPageSource::LoadZeroPage	(	ColumnHandle_t	columnHandle,
		const RPageSummary &	pageSummary )

privateinherited

Definition at line 687 of file RPageStorage.cxx.

◆ OpenWithDifferentAnchor()

std::unique_ptr< ROOT::Internal::RPageSource > ROOT::Internal::RPageSourceFile::OpenWithDifferentAnchor	(	const ROOT::Internal::RNTupleLink &	anchorLink,
		const ROOT::RNTupleReadOptions &	options = {} )

finalvirtual

Creates a new PageSource using the same underlying file as this but referring to a different RNTuple, described by anchorLink.

Implements ROOT::Internal::RPageSource.

Definition at line 404 of file RPageStorageFile.cxx.

◆ operator=() [1/2]

RPageSourceFile & ROOT::Internal::RPageSourceFile::operator= ( const RPageSourceFile & )

delete

◆ operator=() [2/2]

RPageSourceFile & ROOT::Internal::RPageSourceFile::operator= ( RPageSourceFile && )

delete

◆ PinCluster()

void ROOT::Internal::RPageSource::PinCluster ( ROOT::DescriptorId_t clusterId )

inlineinherited

Instructs the cluster pool and page pool to consider the given cluster as active (should stay cached).

Definition at line 1013 of file RPageStorage.hxx.

◆ PrepareLoadCluster()

void ROOT::Internal::RPageSource::PrepareLoadCluster	(	const ROOT::Internal::RCluster::RKey &	clusterKey,
		ROOT::Internal::ROnDiskPageMap &	pageZeroMap,
		const std::function< void(ROOT::DescriptorId_t, ROOT::NTupleSize_t, const ROOT::RClusterDescriptor::RPageInfo &)> &	perPageFunc )

protectedinherited

Prepare a page range read for the column set in clusterKey.

Specifically, pages referencing the kTypePageZero locator are filled in pageZeroMap; otherwise, perPageFunc is called for each page. This is commonly used as part of LoadClusters() in derived classes.

Definition at line 597 of file RPageStorage.cxx.

◆ PrepareSingleCluster()

std::unique_ptr< ROOT::Internal::RCluster > ROOT::Internal::RPageSourceFile::PrepareSingleCluster	(	const ROOT::Internal::RCluster::RKey &	clusterKey,
		std::vector< RRawFile::RIOVec > &	readRequests )

private

Helper function for LoadClusters: it prepares the memory buffer (page map) and the read requests for a given cluster and columns.

The reead requests are appended to the provided vector. This way, requests can be collected for multiple clusters before sending them to RRawFile::ReadV().

Definition at line 534 of file RPageStorageFile.cxx.

◆ RegisterStreamerInfos()

void ROOT::Internal::RPageSource::RegisterStreamerInfos ( )

inherited

Builds the streamer info records from the descriptor's extra type info section.

This is necessary when connecting streamer fields so that emulated classes can be read.

Definition at line 969 of file RPageStorage.cxx.

◆ SetEntryRange()

void ROOT::Internal::RPageSource::SetEntryRange ( const REntryRange & range )

inherited

Promise to only read from the given entry range.

If set, prevents the cluster pool from reading-ahead beyond the given range. The range needs to be within [0, GetNEntries()).

Definition at line 221 of file RPageStorage.cxx.

◆ SetTaskScheduler()

void ROOT::Internal::RPageStorage::SetTaskScheduler ( RTaskScheduler * taskScheduler )

inlineinherited

Definition at line 198 of file RPageStorage.hxx.

◆ StopClusterPoolBackgroundThread()

void ROOT::Internal::RPageSource::StopClusterPoolBackgroundThread ( )

inlineprotectedinherited

Definition at line 911 of file RPageStorage.hxx.

◆ UnpinCluster()

void ROOT::Internal::RPageSource::UnpinCluster ( ROOT::DescriptorId_t clusterId )

inlineinherited

Allows the given cluster to be evicted from the cluster pool and page pool.

Definition at line 1015 of file RPageStorage.hxx.

◆ UnsealPage() [1/2]

ROOT::RResult< ROOT::Internal::RPage > ROOT::Internal::RPageSource::UnsealPage	(	const RSealedPage &	sealedPage,
		const ROOT::Internal::RColumnElementBase &	element )

inherited

Definition at line 925 of file RPageStorage.cxx.

◆ UnsealPage() [2/2]

ROOT::RResult< ROOT::Internal::RPage > ROOT::Internal::RPageSource::UnsealPage	(	const RSealedPage &	sealedPage,
		const ROOT::Internal::RColumnElementBase &	element,
		ROOT::Internal::RPageAllocator &	pageAlloc )

staticinherited

Helper for unstreaming a page.

This is commonly used in derived, concrete page sources. The implementation currently always makes a memory copy, even if the sealed page is uncompressed and in the final memory layout. The optimization of directly mapping pages is left to the concrete page source implementations.

Definition at line 930 of file RPageStorage.cxx.

◆ UnzipCluster()

void ROOT::Internal::RPageSource::UnzipCluster ( ROOT::Internal::RCluster * cluster )

inherited

Parallel decompression and unpacking of the pages in the given cluster.

The unzipped pages are supposed to be preloaded in a page pool attached to the source. The method is triggered by the cluster pool's unzip thread. It is an optional optimization, the method can safely do nothing. In particular, the unzipping will only run if a task scheduler is set. In practice, a task scheduler is set if implicit multi-threading is turned on.

Definition at line 499 of file RPageStorage.cxx.

◆ UpdateLastUsedCluster()

void ROOT::Internal::RPageSource::UpdateLastUsedCluster ( ROOT::DescriptorId_t clusterId )

privateinherited

Does nothing if fLastUsedCluster == clusterId.

Otherwise, updated fLastUsedCluster and evict unused paged from the page pool of all previous clusters. Must not be called when the descriptor guard is taken.

Definition at line 624 of file RPageStorage.cxx.

◆ UpdateSkipCounter()

void ROOT::Internal::RPageSourceFile::UpdateSkipCounter ( std::uint64_t offset )

private

Definition at line 493 of file RPageStorageFile.cxx.

◆ WaitForAllTasks()

void ROOT::Internal::RPageStorage::WaitForAllTasks ( )

inlineprotectedinherited

Definition at line 154 of file RPageStorage.hxx.

Friends And Related Symbol Documentation

◆ ROOT::Internal::GetAnchorFromFile

const ROOT::RNTuple * ROOT::Internal::GetAnchorFromFile ( const RPageSourceFile & )

friend

◆ ROOT::RNTuple

friend class ROOT::RNTuple

friend

Definition at line 127 of file RPageStorageFile.hxx.

Member Data Documentation

◆ fActivePhysicalColumns

RActivePhysicalColumns ROOT::Internal::RPageSource::fActivePhysicalColumns

privateinherited

The active columns are implicitly defined by the model fields or views.

Definition at line 790 of file RPageStorage.hxx.

◆ fAnchor

std::optional<RNTuple> ROOT::Internal::RPageSourceFile::fAnchor

private

Either provided by CreateFromAnchor, or read from the ROOT file given the ntuple name.

Definition at line 132 of file RPageStorageFile.hxx.

◆ fClusterPool

ROOT::Internal::RClusterPool ROOT::Internal::RPageSource::fClusterPool

privateinherited

The cluster pool asynchronously preloads the next few clusters.

Note that derived classes should call StopClusterPoolBackgroundThread() in their destructor so that the I/O background thread does not call methods from the destructed derived class.

Definition at line 795 of file RPageStorage.hxx.

◆ fCounters

std::unique_ptr<RCounters> ROOT::Internal::RPageSource::fCounters

protectedinherited

Definition at line 870 of file RPageStorage.hxx.

◆ fCumulativeClusterCounts

std::vector<NTupleSize_t> ROOT::Internal::RPageSource::fCumulativeClusterCounts

privateinherited

This vector is aligned with the cluster groups in the descriptor.

It stores the cumulative number of clusters in the cluster groups. Given a cluster ID, we can thus quickly determine the cluster group that the cluster comes from. While descriptor IDs in a descriptor in general are arbitrary, for the descriptor in the page source, that was created from a serialized on-disk representation, we know that cluster and cluster group IDs are issued consecutively.

Definition at line 787 of file RPageStorage.hxx.

◆ fCurrentCluster

ROOT::Internal::RCluster* ROOT::Internal::RPageSourceFile::fCurrentCluster = nullptr

private

The last cluster from which a page got loaded. Points into fClusterPool->fPool.

Definition at line 134 of file RPageStorageFile.hxx.

◆ fDescriptor

ROOT::RNTupleDescriptor ROOT::Internal::RPageSource::fDescriptor

privateinherited

Definition at line 771 of file RPageStorage.hxx.

◆ fDescriptorBuilder

RNTupleDescriptorBuilder ROOT::Internal::RPageSourceFile::fDescriptorBuilder

private

The descriptor is created from the header and footer either in AttachImpl or in CreateFromAnchor.

Definition at line 140 of file RPageStorageFile.hxx.

◆ fDescriptorLock

std::shared_mutex ROOT::Internal::RPageSource::fDescriptorLock

mutableprivateinherited

Definition at line 772 of file RPageStorage.hxx.

◆ fDeserializationMode

Internal::RNTupleSerializer::EDescriptorDeserializeMode ROOT::Internal::RPageSource::fDeserializationMode

privateinherited

Initial value:

=

Internal::RNTupleSerializer::EDescriptorDeserializeMode::kRaw

ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode::kRaw

@ kRaw

Deserializes the descriptor as-is without performing any additional fixup.

The interpretation of page lists, set in Attach()

Definition at line 779 of file RPageStorage.hxx.

◆ fEntryRange

REntryRange ROOT::Internal::RPageSource::fEntryRange

privateinherited

Used by the cluster pool to prevent reading beyond the given range.

Definition at line 773 of file RPageStorage.hxx.

◆ fFile

std::unique_ptr<RRawFile> ROOT::Internal::RPageSourceFile::fFile

private

An RRawFile is used to request the necessary byte ranges from a local or a remote file.

Definition at line 136 of file RPageStorageFile.hxx.

◆ fFileCounters

std::unique_ptr<RFileCounters> ROOT::Internal::RPageSourceFile::fFileCounters

private

Definition at line 153 of file RPageStorageFile.hxx.

◆ fFileSize

std::int64_t ROOT::Internal::RPageSourceFile::fFileSize = 0

private

Total file size, set once in AttachImpl()

Definition at line 155 of file RPageStorageFile.hxx.

◆ fHasStreamerInfosRegistered

bool ROOT::Internal::RPageSource::fHasStreamerInfosRegistered = false

privateinherited

Set to true when RegisterStreamerInfos() is called.

Definition at line 776 of file RPageStorage.hxx.

◆ fHasStructure

bool ROOT::Internal::RPageSource::fHasStructure = false

privateinherited

Set to true once LoadStructure() is called.

Definition at line 774 of file RPageStorage.hxx.

◆ fIsAttached

bool ROOT::Internal::RPageSource::fIsAttached = false

privateinherited

Set to true once Attach() is called.

Definition at line 775 of file RPageStorage.hxx.

◆ fLastOffset

std::uint64_t ROOT::Internal::RPageSourceFile::fLastOffset = 0

private

Tracks the last read offset for seek distance calculation.

Definition at line 142 of file RPageStorageFile.hxx.

◆ fLastUsedCluster

ROOT::DescriptorId_t ROOT::Internal::RPageSource::fLastUsedCluster = ROOT::kInvalidDescriptorId

privateinherited

Remembers the last cluster id from which a page was requested.

Definition at line 802 of file RPageStorage.hxx.

◆ fMetrics

ROOT::Experimental::Detail::RNTupleMetrics ROOT::Internal::RPageStorage::fMetrics

protectedinherited

Definition at line 147 of file RPageStorage.hxx.

◆ fNTupleName

std::string ROOT::Internal::RPageStorage::fNTupleName

protectedinherited

Definition at line 152 of file RPageStorage.hxx.

◆ fOptions

ROOT::RNTupleReadOptions ROOT::Internal::RPageSource::fOptions

protectedinherited

Definition at line 873 of file RPageStorage.hxx.

◆ fPageAllocator

std::unique_ptr<ROOT::Internal::RPageAllocator> ROOT::Internal::RPageStorage::fPageAllocator

protectedinherited

For the time being, we will use the heap allocator for all sources and sinks. This may change in the future.

Definition at line 150 of file RPageStorage.hxx.

◆ fPagePool

ROOT::Internal::RPagePool ROOT::Internal::RPageSource::fPagePool

privateinherited

Pages that are unzipped with IMT are staged into the page pool.

Definition at line 799 of file RPageStorage.hxx.

◆ fPinnedClusters

std::unordered_set<ROOT::DescriptorId_t> ROOT::Internal::RPageSource::fPinnedClusters

privateinherited

Pinned clusters and their $2 * (cluster bunch size) - 1$ successors will not be evicted from the cluster pool.

Pages of pinned clusters won't be evicted from the page pool.

Definition at line 810 of file RPageStorage.hxx.

◆ fPreloadedClusters

std::map<ROOT::NTupleSize_t, ROOT::DescriptorId_t> ROOT::Internal::RPageSource::fPreloadedClusters

privateinherited

Clusters from where pages got preloaded in UnzipCluster(), ordered by first entry number of the clusters.

If the last used cluster changes in LoadPage(), all unused pages from previous clusters are evicted from the page pool. Pinned clusters won't be evicted.

Definition at line 806 of file RPageStorage.hxx.

◆ fReader

ROOT::Internal::RMiniFileReader ROOT::Internal::RPageSourceFile::fReader

private

Takes the fFile to read ntuple blobs from it.

Definition at line 138 of file RPageStorageFile.hxx.

◆ fReaderLock

std::mutex ROOT::Internal::RPageSourceFile::fReaderLock

private

Protects the raw file, the reader, and fLastOffset.

Definition at line 144 of file RPageStorageFile.hxx.

◆ fStructureBuffer

RStructureBuffer ROOT::Internal::RPageSource::fStructureBuffer

protectedinherited

Populated by LoadStructureImpl(), reset at the end of Attach()

Definition at line 871 of file RPageStorage.hxx.

◆ fTaskScheduler

RTaskScheduler* ROOT::Internal::RPageStorage::fTaskScheduler = nullptr

protectedinherited

Definition at line 153 of file RPageStorage.hxx.

◆ kInlineDecompressionThreshold

constexpr std::size_t ROOT::Internal::RPageSource::kInlineDecompressionThreshold = 4096

staticconstexprprivateinherited

Definition at line 596 of file RPageStorage.hxx.

◆ kNBytesPageChecksum

constexpr std::size_t ROOT::Internal::RPageStorage::kNBytesPageChecksum = sizeof(std::uint64_t)

staticconstexprinherited

The page checksum is a 64bit xxhash3.

Definition at line 74 of file RPageStorage.hxx.

Libraries for ROOT::Internal::RPageSourceFile:

[legend]

The documentation for this class was generated from the following files:

tree/ntuple/inc/ROOT/RPageStorageFile.hxx
tree/ntuple/src/RPageStorageFile.cxx

Classes

Public Types

Public Member Functions

Static Public Member Functions

Static Public Attributes

Protected Member Functions

Protected Attributes

Private Member Functions

Private Attributes

Static Private Attributes

Friends

Member Typedef Documentation

◆ ColumnHandle_t

◆ SealedPageSequence_t

Constructor & Destructor Documentation

◆ RPageSourceFile() [1/5]

◆ RPageSourceFile() [2/5]

◆ RPageSourceFile() [3/5]

◆ RPageSourceFile() [4/5]

◆ RPageSourceFile() [5/5]

◆ ~RPageSourceFile()

Member Function Documentation

◆ AddColumn()

◆ Attach()

◆ AttachImpl()

◆ Clone()

◆ CloneImpl()

◆ Create()

◆ CreateFromAnchor()

◆ DropColumn()

◆ EnableDefaultMetrics()

◆ EnsureClusterDetails()

◆ FindClusterGroupId()

◆ FindClusterId() [1/2]

◆ FindClusterId() [2/2]

◆ FindNextClusterId()

◆ GetColumnId()

◆ GetEntryRange()

◆ GetExclDescriptorGuard()

◆ GetMetrics()

◆ GetNElements()

◆ GetNEntries()

◆ GetNTupleName()

◆ GetPinnedClusters()

◆ GetReadOptions()

◆ GetSharedDescriptorGuard()

◆ GetType()

◆ LoadAllPageLists()

◆ LoadClusters()

◆ LoadPage() [1/2]

◆ LoadPage() [2/2]

◆ LoadPageFromSummary()

◆ LoadPageList()

◆ LoadPageListImpl()

◆ LoadSealedPage()

◆ LoadSealedPageImpl()

◆ LoadStructure()

◆ LoadStructureImpl()

◆ LoadZeroPage()

◆ OpenWithDifferentAnchor()

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ PinCluster()

◆ PrepareLoadCluster()

◆ PrepareSingleCluster()

◆ RegisterStreamerInfos()

◆ SetEntryRange()

◆ SetTaskScheduler()

◆ StopClusterPoolBackgroundThread()

◆ UnpinCluster()

◆ UnsealPage() [1/2]

◆ UnsealPage() [2/2]

◆ UnzipCluster()

◆ UpdateLastUsedCluster()

◆ UpdateSkipCounter()

◆ WaitForAllTasks()

Friends And Related Symbol Documentation

◆ ROOT::Internal::GetAnchorFromFile

◆ ROOT::RNTuple

Member Data Documentation