Base class for a sink with a physical storage backend.

Definition at line 435 of file RPageStorage.hxx.

Classes
struct	RCounters
	Default I/O performance counters that get registered in fMetrics. More...

struct	RFeatures
	Set of optional features supported by the persistent sink. More...

Public Member Functions
	RPagePersistentSink (const RPagePersistentSink &)=delete

	RPagePersistentSink (RPagePersistentSink &&)=default

	RPagePersistentSink (std::string_view ntupleName, const RNTupleWriteOptions &options)

	~RPagePersistentSink () override

ColumnHandle_t	AddColumn (DescriptorId_t fieldId, RColumn &column) final
	Register a new column.

void	CommitClusterGroup () final
	Write out the page locations (page list envelope) for all the committed clusters since the last call of CommitClusterGroup (or the beginning of writing).

void	CommitDatasetImpl () final

void	CommitPage (ColumnHandle_t columnHandle, const RPage &page) final
	Write a page to the storage. The column must have been added before.

void	CommitSealedPage (DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
	Write a preprocessed page to storage. The column must have been added before.

void	CommitSealedPageV (std::span< RPageStorage::RSealedPageGroup > ranges) final
	Write a vector of preprocessed pages to storage. The corresponding columns must have been added before.

void	CommitStagedClusters (std::span< RStagedCluster > clusters) final
	Commit staged clusters, logically appending them to the ntuple descriptor.

void	CommitSuppressedColumn (ColumnHandle_t columnHandle) final
	Commits a suppressed column for the current cluster.

const RNTupleDescriptor &	GetDescriptor () const final
	Return the RNTupleDescriptor being constructed.

void	InitFromDescriptor (const RNTupleDescriptor &descriptor)
	Initialize sink based on an existing descriptor and fill into the descriptor builder.

void	InitImpl (RNTupleModel &model) final
	Updates the descriptor and calls InitImpl() that handles the backend-specific details (file, DAOS, etc.)

RPagePersistentSink &	operator= (const RPagePersistentSink &)=delete

RPagePersistentSink &	operator= (RPagePersistentSink &&)=default

RStagedCluster	StageCluster (NTupleSize_t nNewEntries) final
	Stage the current cluster and create a new one for the following data.

void	UpdateExtraTypeInfo (const RExtraTypeInfoDescriptor &extraTypeInfo) final
	Adds an extra type information record to schema.

void	UpdateSchema (const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry) final
	Incorporate incremental changes to the model into the ntuple descriptor.

Public Member Functions inherited from ROOT::Experimental::Internal::RPageSink
	RPageSink (const RPageSink &)=delete

	RPageSink (RPageSink &&)=default

	RPageSink (std::string_view ntupleName, const RNTupleWriteOptions &options)

	~RPageSink () override

virtual std::uint64_t	CommitCluster (NTupleSize_t nNewEntries)
	Finalize the current cluster and create a new one for the following data.

void	CommitDataset ()
	Run the registered callbacks and finalize the current cluster and the entrire data set.

void	DropColumn (ColumnHandle_t) final
	Unregisters a column.

virtual RSinkGuard	GetSinkGuard ()

EPageStorageType	GetType () final
	Whether the concrete implementation is a sink or a source.

const RNTupleWriteOptions &	GetWriteOptions () const
	Returns the sink's write options.

void	Init (RNTupleModel &model)
	Physically creates the storage container to hold the ntuple (e.g., a keys a TFile or an S3 bucket) Init() associates column handles to the columns referenced by the model.

bool	IsInitialized () const

RPageSink &	operator= (const RPageSink &)=delete

RPageSink &	operator= (RPageSink &&)=default

void	RegisterOnCommitDatasetCallback (Callback_t callback)
	The registered callback is executed at the beginning of CommitDataset();.

virtual RPage	ReservePage (ColumnHandle_t columnHandle, std::size_t nElements)
	Get a new, empty page for the given column that can be filled with up to nElements; nElements must be larger than zero.

Public Member Functions inherited from ROOT::Experimental::Internal::RPageStorage
	RPageStorage (const RPageStorage &other)=delete

	RPageStorage (RPageStorage &&other)=default

	RPageStorage (std::string_view name)

virtual	~RPageStorage ()

ColumnId_t	GetColumnId (ColumnHandle_t columnHandle) const

virtual Detail::RNTupleMetrics &	GetMetrics ()
	Returns the default metrics object.

const std::string &	GetNTupleName () const
	Returns the NTuple name.

RPageStorage &	operator= (const RPageStorage &other)=delete

RPageStorage &	operator= (RPageStorage &&other)=default

void	SetTaskScheduler (RTaskScheduler *taskScheduler)

Static Public Member Functions
static std::unique_ptr< RPageSink >	Create (std::string_view ntupleName, std::string_view location, const RNTupleWriteOptions &options=RNTupleWriteOptions())
	Guess the concrete derived page source from the location.

Static Public Member Functions inherited from ROOT::Experimental::Internal::RPageSink
static RSealedPage	SealPage (const RSealPageConfig &config)
	Seal a page using the provided info.

Protected Member Functions
virtual RNTupleLocator	CommitClusterGroupImpl (unsigned char *serializedPageList, std::uint32_t length)=0
	Returns the locator of the page list envelope of the given buffer that contains the serialized page list.

virtual void	CommitDatasetImpl (unsigned char *serializedFooter, std::uint32_t length)=0

virtual RNTupleLocator	CommitPageImpl (ColumnHandle_t columnHandle, const RPage &page)=0

virtual RNTupleLocator	CommitSealedPageImpl (DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage)=0

virtual std::vector< RNTupleLocator >	CommitSealedPageVImpl (std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask)
	Vector commit of preprocessed pages.

void	EnableDefaultMetrics (const std::string &prefix)
	Enables the default set of metrics provided by RPageSink.

virtual void	InitImpl (unsigned char *serializedHeader, std::uint32_t length)=0

virtual std::uint64_t	StageClusterImpl ()=0
	Returns the number of bytes written to storage (excluding metadata)

Protected Member Functions inherited from ROOT::Experimental::Internal::RPageSink
RSealedPage	SealPage (const RPage &page, const RColumnElementBase &element)
	Helper for streaming a page.

Protected Member Functions inherited from ROOT::Experimental::Internal::RPageStorage
void	WaitForAllTasks ()

Protected Attributes
std::unique_ptr< RCounters >	fCounters

Internal::RNTupleDescriptorBuilder	fDescriptorBuilder

RFeatures	fFeatures

Protected Attributes inherited from ROOT::Experimental::Internal::RPageSink
std::unique_ptr< RNTupleCompressor >	fCompressor
	Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.

std::unique_ptr< RNTupleWriteOptions >	fOptions

Protected Attributes inherited from ROOT::Experimental::Internal::RPageStorage
Detail::RNTupleMetrics	fMetrics

std::string	fNTupleName

std::unique_ptr< RPageAllocator >	fPageAllocator
	For the time being, we will use the heap allocator for all sources and sinks. This may change in the future.

RTaskScheduler *	fTaskScheduler = nullptr

Private Attributes
std::uint64_t	fNextClusterInGroup = 0
	Remembers the starting cluster id for the next cluster group.

std::vector< RClusterDescriptor::RColumnRange >	fOpenColumnRanges
	Keeps track of the number of elements in the currently open cluster. Indexed by column id.

std::vector< RClusterDescriptor::RPageRange >	fOpenPageRanges
	Keeps track of the written pages in the currently open cluster. Indexed by column id.

NTupleSize_t	fPrevClusterNEntries = 0
	Used to calculate the number of entries in the current cluster.

RNTupleSerializer::RContext	fSerializationContext
	Used to map the IDs of the descriptor to the physical IDs issued during header/footer serialization.

RNTupleSerializer::StreamerInfoMap_t	fStreamerInfos
	Union of the streamer info records that are sent from unsplit fields to the sink before committing the dataset.

Additional Inherited Members
Public Types inherited from ROOT::Experimental::Internal::RPageSink
using	Callback_t = std::function< void(RPageSink &)>

Public Types inherited from ROOT::Experimental::Internal::RPageStorage
using	ColumnHandle_t = RColumnHandle
	The column handle identifies a column with the current open page storage.

using	SealedPageSequence_t = std::deque< RSealedPage >

Static Public Attributes inherited from ROOT::Experimental::Internal::RPageStorage
static constexpr std::size_t	kNBytesPageChecksum = sizeof(std::uint64_t)
	The page checksum is a 64bit xxhash3.

#include <ROOT/RPageStorage.hxx>

Inheritance diagram for ROOT::Experimental::Internal::RPagePersistentSink:

[legend]

Constructor & Destructor Documentation

◆ RPagePersistentSink() [1/3]

ROOT::Experimental::Internal::RPagePersistentSink::RPagePersistentSink	(	std::string_view	ntupleName,
		const RNTupleWriteOptions &	options
	)

Definition at line 709 of file RPageStorage.cxx.

◆ RPagePersistentSink() [2/3]

ROOT::Experimental::Internal::RPagePersistentSink::RPagePersistentSink ( const RPagePersistentSink & )

delete

◆ RPagePersistentSink() [3/3]

ROOT::Experimental::Internal::RPagePersistentSink::RPagePersistentSink ( RPagePersistentSink && )

default

◆ ~RPagePersistentSink()

ROOT::Experimental::Internal::RPagePersistentSink::~RPagePersistentSink ( )

override

Definition at line 715 of file RPageStorage.cxx.

Member Function Documentation

◆ AddColumn()

ROOT::Experimental::Internal::RPageStorage::ColumnHandle_t ROOT::Experimental::Internal::RPagePersistentSink::AddColumn	(	DescriptorId_t	fieldId,
		RColumn &	column
	)

finalvirtual

Register a new column.

When reading, the column must exist in the ntuple on disk corresponding to the meta-data. When writing, every column can only be attached once.

Implements ROOT::Experimental::Internal::RPageStorage.

Definition at line 718 of file RPageStorage.cxx.

◆ CommitClusterGroup()

void ROOT::Experimental::Internal::RPagePersistentSink::CommitClusterGroup ( )

finalvirtual

Write out the page locations (page list envelope) for all the committed clusters since the last call of CommitClusterGroup (or the beginning of writing).

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 1055 of file RPageStorage.cxx.

◆ CommitClusterGroupImpl()

virtual RNTupleLocator ROOT::Experimental::Internal::RPagePersistentSink::CommitClusterGroupImpl	(	unsigned char *	serializedPageList,
		std::uint32_t	length
	)

protectedpure virtual

Returns the locator of the page list envelope of the given buffer that contains the serialized page list.

Typically, the implementation takes care of compressing and writing the provided buffer.

Implemented in ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ CommitDatasetImpl() [1/2]

void ROOT::Experimental::Internal::RPagePersistentSink::CommitDatasetImpl ( )

finalvirtual

Implements ROOT::Experimental::Internal::RPageSink.

Reimplemented in ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

Definition at line 1094 of file RPageStorage.cxx.

◆ CommitDatasetImpl() [2/2]

virtual void ROOT::Experimental::Internal::RPagePersistentSink::CommitDatasetImpl	(	unsigned char *	serializedFooter,
		std::uint32_t	length
	)

protectedpure virtual

Implemented in ROOT::Experimental::Internal::RPageSinkDaos, ROOT::Experimental::Internal::RPageSinkFile, ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ CommitPage()

void ROOT::Experimental::Internal::RPagePersistentSink::CommitPage	(	ColumnHandle_t	columnHandle,
		const RPage &	page
	)

finalvirtual

Write a page to the storage. The column must have been added before.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 881 of file RPageStorage.cxx.

◆ CommitPageImpl()

virtual RNTupleLocator ROOT::Experimental::Internal::RPagePersistentSink::CommitPageImpl	(	ColumnHandle_t	columnHandle,
		const RPage &	page
	)

protectedpure virtual

Implemented in ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ CommitSealedPage()

void ROOT::Experimental::Internal::RPagePersistentSink::CommitSealedPage	(	DescriptorId_t	physicalColumnId,
		const RPageStorage::RSealedPage &	sealedPage
	)

finalvirtual

Write a preprocessed page to storage. The column must have been added before.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 892 of file RPageStorage.cxx.

◆ CommitSealedPageImpl()

virtual RNTupleLocator ROOT::Experimental::Internal::RPagePersistentSink::CommitSealedPageImpl	(	DescriptorId_t	physicalColumnId,
		const RPageStorage::RSealedPage &	sealedPage
	)

protectedpure virtual

Implemented in ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ CommitSealedPageV()

void ROOT::Experimental::Internal::RPagePersistentSink::CommitSealedPageV ( std::span< RPageStorage::RSealedPageGroup > ranges )

finalvirtual

Write a vector of preprocessed pages to storage. The corresponding columns must have been added before.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 921 of file RPageStorage.cxx.

◆ CommitSealedPageVImpl()

std::vector< ROOT::Experimental::RNTupleLocator > ROOT::Experimental::Internal::RPagePersistentSink::CommitSealedPageVImpl	(	std::span< RPageStorage::RSealedPageGroup >	ranges,
		const std::vector< bool > &	mask
	)

protectedvirtual

Vector commit of preprocessed pages.

The ranges array specifies a range of sealed pages to be committed for each column. The returned vector contains, in order, the RNTupleLocator for each page on each range in ranges, i.e. the first N entries refer to the N pages in ranges[0], followed by M entries that refer to the M pages in ranges[1], etc. The mask allows to skip writing out certain pages. The vector has the size of all the pages. For every false value in the mask, the corresponding locator is skipped (missing) in the output vector. The default is to call CommitSealedPageImpl for each page; derived classes may provide an optimized implementation though.

Reimplemented in ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

Definition at line 905 of file RPageStorage.cxx.

◆ CommitStagedClusters()

void ROOT::Experimental::Internal::RPagePersistentSink::CommitStagedClusters ( std::span< RStagedCluster > clusters )

finalvirtual

Commit staged clusters, logically appending them to the ntuple descriptor.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 1018 of file RPageStorage.cxx.

◆ CommitSuppressedColumn()

void ROOT::Experimental::Internal::RPagePersistentSink::CommitSuppressedColumn ( ColumnHandle_t columnHandle )

finalvirtual

Commits a suppressed column for the current cluster.

Can be called anytime before CommitCluster(). For any given column and cluster, there must be no calls to both CommitSuppressedColumn() and page commits.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 876 of file RPageStorage.cxx.

◆ Create()

std::unique_ptr< ROOT::Experimental::Internal::RPageSink > ROOT::Experimental::Internal::RPagePersistentSink::Create	(	std::string_view	ntupleName,
		std::string_view	location,
		const RNTupleWriteOptions &	options = `RNTupleWriteOptions()`
	)

static

Guess the concrete derived page source from the location.

Definition at line 688 of file RPageStorage.cxx.

◆ EnableDefaultMetrics()

void ROOT::Experimental::Internal::RPagePersistentSink::EnableDefaultMetrics ( const std::string & prefix )

protected

Enables the default set of metrics provided by RPageSink.

prefix will be used as the prefix for the counters registered in the internal RNTupleMetrics object. This set of counters can be extended by a subclass by calling fMetrics.MakeCounter<...>().

A subclass using the default set of metrics is always responsible for updating the counters appropriately, e.g. fCounters->fNPageCommited.Inc()

Definition at line 1112 of file RPageStorage.cxx.

◆ GetDescriptor()

const RNTupleDescriptor & ROOT::Experimental::Internal::RPagePersistentSink::GetDescriptor ( ) const

inlinefinalvirtual

Return the RNTupleDescriptor being constructed.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 518 of file RPageStorage.hxx.

◆ InitFromDescriptor()

void ROOT::Experimental::Internal::RPagePersistentSink::InitFromDescriptor ( const RNTupleDescriptor & descriptor )

Initialize sink based on an existing descriptor and fill into the descriptor builder.

Definition at line 841 of file RPageStorage.cxx.

◆ InitImpl() [1/2]

void ROOT::Experimental::Internal::RPagePersistentSink::InitImpl ( RNTupleModel & model )

finalvirtual

Updates the descriptor and calls InitImpl() that handles the backend-specific details (file, DAOS, etc.)

Implements ROOT::Experimental::Internal::RPageSink.

Reimplemented in ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

Definition at line 816 of file RPageStorage.cxx.

◆ InitImpl() [2/2]

virtual void ROOT::Experimental::Internal::RPagePersistentSink::InitImpl	(	unsigned char *	serializedHeader,
		std::uint32_t	length
	)

protectedpure virtual

Implemented in ROOT::Experimental::Internal::RPageSinkDaos, ROOT::Experimental::Internal::RPageSinkFile, ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ operator=() [1/2]

RPagePersistentSink & ROOT::Experimental::Internal::RPagePersistentSink::operator= ( const RPagePersistentSink & )

delete

◆ operator=() [2/2]

RPagePersistentSink & ROOT::Experimental::Internal::RPagePersistentSink::operator= ( RPagePersistentSink && )

default

◆ StageCluster()

ROOT::Experimental::Internal::RPageSink::RStagedCluster ROOT::Experimental::Internal::RPagePersistentSink::StageCluster ( NTupleSize_t nNewEntries )

finalvirtual

Stage the current cluster and create a new one for the following data.

Returns the object that must be passed to CommitStagedClusters to logically append the staged cluster to the ntuple descriptor.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 990 of file RPageStorage.cxx.

◆ StageClusterImpl()

virtual std::uint64_t ROOT::Experimental::Internal::RPagePersistentSink::StageClusterImpl ( )

protectedpure virtual

Returns the number of bytes written to storage (excluding metadata)

Implemented in ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ UpdateExtraTypeInfo()

void ROOT::Experimental::Internal::RPagePersistentSink::UpdateExtraTypeInfo ( const RExtraTypeInfoDescriptor & extraTypeInfo )

finalvirtual

Adds an extra type information record to schema.

The extra type information will be written to the extension header. The information in the record will be merged with the existing information, e.g. duplicate streamer info records will be removed. This method is called by the "on commit dataset" callback registered by specific fields (e.g., unsplit field) and during merging.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 807 of file RPageStorage.cxx.

◆ UpdateSchema()

void ROOT::Experimental::Internal::RPagePersistentSink::UpdateSchema	(	const RNTupleModelChangeset &	changeset,
		NTupleSize_t	firstEntry
	)

finalvirtual

Incorporate incremental changes to the model into the ntuple descriptor.

This happens, e.g. if new fields were added after the initial call to RPageSink::Init(RNTupleModel &). firstEntry specifies the global index for the first stored element in the added columns.

Implements ROOT::Experimental::Internal::RPageSink.

Definition at line 739 of file RPageStorage.cxx.

Member Data Documentation

◆ fCounters

std::unique_ptr<RCounters> ROOT::Experimental::Internal::RPagePersistentSink::fCounters

protected

Definition at line 471 of file RPageStorage.hxx.

◆ fDescriptorBuilder

Internal::RNTupleDescriptorBuilder ROOT::Experimental::Internal::RPagePersistentSink::fDescriptorBuilder

protected

Definition at line 459 of file RPageStorage.hxx.

◆ fFeatures

RFeatures ROOT::Experimental::Internal::RPagePersistentSink::fFeatures

protected

Definition at line 458 of file RPageStorage.hxx.

◆ fNextClusterInGroup

std::uint64_t ROOT::Experimental::Internal::RPagePersistentSink::fNextClusterInGroup = 0

private

Remembers the starting cluster id for the next cluster group.

Definition at line 441 of file RPageStorage.hxx.

◆ fOpenColumnRanges

std::vector<RClusterDescriptor::RColumnRange> ROOT::Experimental::Internal::RPagePersistentSink::fOpenColumnRanges

private

Keeps track of the number of elements in the currently open cluster. Indexed by column id.

Definition at line 445 of file RPageStorage.hxx.

◆ fOpenPageRanges

std::vector<RClusterDescriptor::RPageRange> ROOT::Experimental::Internal::RPagePersistentSink::fOpenPageRanges

private

Keeps track of the written pages in the currently open cluster. Indexed by column id.

Definition at line 447 of file RPageStorage.hxx.

◆ fPrevClusterNEntries

NTupleSize_t ROOT::Experimental::Internal::RPagePersistentSink::fPrevClusterNEntries = 0

private

Used to calculate the number of entries in the current cluster.

Definition at line 443 of file RPageStorage.hxx.

◆ fSerializationContext

RNTupleSerializer::RContext ROOT::Experimental::Internal::RPagePersistentSink::fSerializationContext

private

Used to map the IDs of the descriptor to the physical IDs issued during header/footer serialization.

Definition at line 438 of file RPageStorage.hxx.

◆ fStreamerInfos

RNTupleSerializer::StreamerInfoMap_t ROOT::Experimental::Internal::RPagePersistentSink::fStreamerInfos

private

Union of the streamer info records that are sent from unsplit fields to the sink before committing the dataset.

Definition at line 450 of file RPageStorage.hxx.

Libraries for ROOT::Experimental::Internal::RPagePersistentSink:

[legend]

The documentation for this class was generated from the following files:

tree/ntuple/v7/inc/ROOT/RPageStorage.hxx
tree/ntuple/v7/src/RPageStorage.cxx

Classes

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Private Attributes

Additional Inherited Members

Constructor & Destructor Documentation

◆ RPagePersistentSink() [1/3]

◆ RPagePersistentSink() [2/3]

◆ RPagePersistentSink() [3/3]

◆ ~RPagePersistentSink()

Member Function Documentation

◆ AddColumn()

◆ CommitClusterGroup()

◆ CommitClusterGroupImpl()

◆ CommitDatasetImpl() [1/2]

◆ CommitDatasetImpl() [2/2]

◆ CommitPage()

◆ CommitPageImpl()

◆ CommitSealedPage()

◆ CommitSealedPageImpl()

◆ CommitSealedPageV()

◆ CommitSealedPageVImpl()

◆ CommitStagedClusters()

◆ CommitSuppressedColumn()

◆ Create()

◆ EnableDefaultMetrics()

◆ GetDescriptor()

◆ InitFromDescriptor()

◆ InitImpl() [1/2]

◆ InitImpl() [2/2]

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ StageCluster()

◆ StageClusterImpl()

◆ UpdateExtraTypeInfo()

◆ UpdateSchema()

Member Data Documentation

◆ fCounters

◆ fDescriptorBuilder

◆ fFeatures

◆ fNextClusterInGroup

◆ fOpenColumnRanges

◆ fOpenPageRanges

◆ fPrevClusterNEntries

◆ fSerializationContext

◆ fStreamerInfos