Abstract interface to write data into an ntuple.

The page sink takes the list of columns and afterwards a series of page commits and cluster commits. The user is responsible to commit clusters at a consistent point, i.e. when all pages corresponding to data up to the given entry number are committed.

An object of this class may either be a wrapper (for example a RPageSinkBuf) or a "persistent" sink, inheriting from RPagePersistentSink.

Definition at line 257 of file RPageStorage.hxx.

Classes
struct	RSealPageConfig
	Parameters for the SealPage() method. More...

class	RSinkGuard
	An RAII wrapper used to synchronize a page sink. See GetSinkGuard(). More...

struct	RStagedCluster
	Cluster that was staged, but not yet logically appended to the RNTuple. More...

Public Types
using	Callback_t = std::function< void(RPageSink &)>

Public Types inherited from ROOT::Experimental::Internal::RPageStorage
using	ColumnHandle_t = RColumnHandle
	The column handle identifies a column with the current open page storage.

using	SealedPageSequence_t = std::deque< RSealedPage >

Public Member Functions
	RPageSink (const RPageSink &)=delete

	RPageSink (RPageSink &&)=default

	RPageSink (std::string_view ntupleName, const RNTupleWriteOptions &options)

	~RPageSink () override

virtual std::uint64_t	CommitCluster (NTupleSize_t nNewEntries)
	Finalize the current cluster and create a new one for the following data.

virtual void	CommitClusterGroup ()=0
	Write out the page locations (page list envelope) for all the committed clusters since the last call of CommitClusterGroup (or the beginning of writing).

void	CommitDataset ()
	Run the registered callbacks and finalize the current cluster and the entrire data set.

virtual void	CommitPage (ColumnHandle_t columnHandle, const RPage &page)=0
	Write a page to the storage. The column must have been added before.

virtual void	CommitSealedPage (DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage)=0
	Write a preprocessed page to storage. The column must have been added before.

virtual void	CommitSealedPageV (std::span< RPageStorage::RSealedPageGroup > ranges)=0
	Write a vector of preprocessed pages to storage. The corresponding columns must have been added before.

virtual void	CommitStagedClusters (std::span< RStagedCluster > clusters)=0
	Commit staged clusters, logically appending them to the ntuple descriptor.

virtual void	CommitSuppressedColumn (ColumnHandle_t columnHandle)=0
	Commits a suppressed column for the current cluster.

void	DropColumn (ColumnHandle_t) final
	Unregisters a column.

virtual const RNTupleDescriptor &	GetDescriptor () const =0
	Return the RNTupleDescriptor being constructed.

virtual RSinkGuard	GetSinkGuard ()

EPageStorageType	GetType () final
	Whether the concrete implementation is a sink or a source.

const RNTupleWriteOptions &	GetWriteOptions () const
	Returns the sink's write options.

void	Init (RNTupleModel &model)
	Physically creates the storage container to hold the ntuple (e.g., a keys a TFile or an S3 bucket) Init() associates column handles to the columns referenced by the model.

bool	IsInitialized () const

RPageSink &	operator= (const RPageSink &)=delete

RPageSink &	operator= (RPageSink &&)=default

void	RegisterOnCommitDatasetCallback (Callback_t callback)
	The registered callback is executed at the beginning of CommitDataset();.

virtual RPage	ReservePage (ColumnHandle_t columnHandle, std::size_t nElements)
	Get a new, empty page for the given column that can be filled with up to nElements; nElements must be larger than zero.

virtual RStagedCluster	StageCluster (NTupleSize_t nNewEntries)=0
	Stage the current cluster and create a new one for the following data.

virtual void	UpdateExtraTypeInfo (const RExtraTypeInfoDescriptor &extraTypeInfo)=0
	Adds an extra type information record to schema.

virtual void	UpdateSchema (const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry)=0
	Incorporate incremental changes to the model into the ntuple descriptor.

Public Member Functions inherited from ROOT::Experimental::Internal::RPageStorage
	RPageStorage (const RPageStorage &other)=delete

	RPageStorage (RPageStorage &&other)=default

	RPageStorage (std::string_view name)

virtual	~RPageStorage ()

virtual ColumnHandle_t	AddColumn (DescriptorId_t fieldId, RColumn &column)=0
	Register a new column.

ColumnId_t	GetColumnId (ColumnHandle_t columnHandle) const

virtual Detail::RNTupleMetrics &	GetMetrics ()
	Returns the default metrics object.

const std::string &	GetNTupleName () const
	Returns the NTuple name.

RPageStorage &	operator= (const RPageStorage &other)=delete

RPageStorage &	operator= (RPageStorage &&other)=default

void	SetTaskScheduler (RTaskScheduler *taskScheduler)

Static Public Member Functions
static RSealedPage	SealPage (const RSealPageConfig &config)
	Seal a page using the provided info.

Protected Member Functions
virtual void	CommitDatasetImpl ()=0

virtual void	InitImpl (RNTupleModel &model)=0

RSealedPage	SealPage (const RPage &page, const RColumnElementBase &element)
	Helper for streaming a page.

Protected Member Functions inherited from ROOT::Experimental::Internal::RPageStorage
void	WaitForAllTasks ()

Protected Attributes
std::unique_ptr< RNTupleCompressor >	fCompressor
	Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.

std::unique_ptr< RNTupleWriteOptions >	fOptions

Protected Attributes inherited from ROOT::Experimental::Internal::RPageStorage
Detail::RNTupleMetrics	fMetrics

std::string	fNTupleName

std::unique_ptr< RPageAllocator >	fPageAllocator
	For the time being, we will use the heap allocator for all sources and sinks. This may change in the future.

RTaskScheduler *	fTaskScheduler = nullptr

Private Attributes
bool	fIsInitialized = false
	Flag if sink was initialized.

std::vector< Callback_t >	fOnDatasetCommitCallbacks

std::vector< unsigned char >	fSealPageBuffer
	Used as destination buffer in the simple SealPage overload.

RWritePageMemoryManager	fWritePageMemoryManager
	Used in ReservePage to maintain the page buffer budget.

Additional Inherited Members
Static Public Attributes inherited from ROOT::Experimental::Internal::RPageStorage
static constexpr std::size_t	kNBytesPageChecksum = sizeof(std::uint64_t)
	The page checksum is a 64bit xxhash3.

#include <ROOT/RPageStorage.hxx>

Inheritance diagram for ROOT::Experimental::Internal::RPageSink:

[legend]

Member Typedef Documentation

◆ Callback_t

using ROOT::Experimental::Internal::RPageSink::Callback_t = std::function<void(RPageSink &)>

Definition at line 259 of file RPageStorage.hxx.

Constructor & Destructor Documentation

◆ RPageSink() [1/3]

ROOT::Experimental::Internal::RPageSink::RPageSink	(	std::string_view	ntupleName,
		const RNTupleWriteOptions &	options
	)

Definition at line 627 of file RPageStorage.cxx.

◆ RPageSink() [2/3]

ROOT::Experimental::Internal::RPageSink::RPageSink ( const RPageSink & )

delete

◆ RPageSink() [3/3]

ROOT::Experimental::Internal::RPageSink::RPageSink ( RPageSink && )

default

◆ ~RPageSink()

ROOT::Experimental::Internal::RPageSink::~RPageSink ( )

override

Definition at line 633 of file RPageStorage.cxx.

Member Function Documentation

◆ CommitCluster()

virtual std::uint64_t ROOT::Experimental::Internal::RPageSink::CommitCluster ( NTupleSize_t nNewEntries )

inlinevirtual

Finalize the current cluster and create a new one for the following data.

Returns the number of bytes written to storage (excluding meta-data).

Reimplemented in ROOT::Experimental::Internal::RPageSinkBuf.

Definition at line 380 of file RPageStorage.hxx.

◆ CommitClusterGroup()

virtual void ROOT::Experimental::Internal::RPageSink::CommitClusterGroup ( )

pure virtual

Write out the page locations (page list envelope) for all the committed clusters since the last call of CommitClusterGroup (or the beginning of writing).

Implemented in ROOT::Experimental::Internal::RPageNullSink, ROOT::Experimental::Internal::RPageSinkBuf, and ROOT::Experimental::Internal::RPagePersistentSink.

◆ CommitDataset()

void ROOT::Experimental::Internal::RPageSink::CommitDataset ( )

Run the registered callbacks and finalize the current cluster and the entrire data set.

Definition at line 690 of file RPageStorage.cxx.

◆ CommitDatasetImpl()

virtual void ROOT::Experimental::Internal::RPageSink::CommitDatasetImpl ( )

protectedpure virtual

Implemented in ROOT::Experimental::Internal::RPageNullSink, ROOT::Experimental::Internal::RPageSinkBuf, ROOT::Experimental::Internal::RPagePersistentSink, ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ CommitPage()

virtual void ROOT::Experimental::Internal::RPageSink::CommitPage	(	ColumnHandle_t	columnHandle,
		const RPage &	page
	)

pure virtual

Write a page to the storage. The column must have been added before.

Implemented in ROOT::Experimental::Internal::RPageSinkBuf, ROOT::Experimental::Internal::RPagePersistentSink, and ROOT::Experimental::Internal::RPageNullSink.

◆ CommitSealedPage()

virtual void ROOT::Experimental::Internal::RPageSink::CommitSealedPage	(	DescriptorId_t	physicalColumnId,
		const RPageStorage::RSealedPage &	sealedPage
	)

pure virtual

Write a preprocessed page to storage. The column must have been added before.

Implemented in ROOT::Experimental::Internal::RPagePersistentSink, ROOT::Experimental::Internal::RPageSinkBuf, and ROOT::Experimental::Internal::RPageNullSink.

◆ CommitSealedPageV()

virtual void ROOT::Experimental::Internal::RPageSink::CommitSealedPageV ( std::span< RPageStorage::RSealedPageGroup > ranges )

pure virtual

Write a vector of preprocessed pages to storage. The corresponding columns must have been added before.

Implemented in ROOT::Experimental::Internal::RPageSinkBuf, ROOT::Experimental::Internal::RPagePersistentSink, and ROOT::Experimental::Internal::RPageNullSink.

◆ CommitStagedClusters()

virtual void ROOT::Experimental::Internal::RPageSink::CommitStagedClusters ( std::span< RStagedCluster > clusters )

pure virtual

Commit staged clusters, logically appending them to the ntuple descriptor.

Implemented in ROOT::Experimental::Internal::RPageSinkBuf, ROOT::Experimental::Internal::RPagePersistentSink, and ROOT::Experimental::Internal::RPageNullSink.

◆ CommitSuppressedColumn()

virtual void ROOT::Experimental::Internal::RPageSink::CommitSuppressedColumn ( ColumnHandle_t columnHandle )

pure virtual

Commits a suppressed column for the current cluster.

Can be called anytime before CommitCluster(). For any given column and cluster, there must be no calls to both CommitSuppressedColumn() and page commits.

Implemented in ROOT::Experimental::Internal::RPageSinkBuf, ROOT::Experimental::Internal::RPagePersistentSink, and ROOT::Experimental::Internal::RPageNullSink.

◆ DropColumn()

void ROOT::Experimental::Internal::RPageSink::DropColumn ( ColumnHandle_t columnHandle )

inlinefinalvirtual

Unregisters a column.

A page source decreases the reference counter for the corresponding active column. For a page sink, dropping columns is currently a no-op.

Implements ROOT::Experimental::Internal::RPageStorage.

Definition at line 312 of file RPageStorage.hxx.

◆ GetDescriptor()

virtual const RNTupleDescriptor & ROOT::Experimental::Internal::RPageSink::GetDescriptor ( ) const

pure virtual

Return the RNTupleDescriptor being constructed.

Implemented in ROOT::Experimental::Internal::RPageNullSink, ROOT::Experimental::Internal::RPageSinkBuf, and ROOT::Experimental::Internal::RPagePersistentSink.

◆ GetSinkGuard()

virtual RSinkGuard ROOT::Experimental::Internal::RPageSink::GetSinkGuard ( )

inlinevirtual

Definition at line 422 of file RPageStorage.hxx.

◆ GetType()

EPageStorageType ROOT::Experimental::Internal::RPageSink::GetType ( )

inlinefinalvirtual

Whether the concrete implementation is a sink or a source.

Implements ROOT::Experimental::Internal::RPageStorage.

Definition at line 308 of file RPageStorage.hxx.

◆ GetWriteOptions()

const RNTupleWriteOptions & ROOT::Experimental::Internal::RPageSink::GetWriteOptions ( ) const

inline

Returns the sink's write options.

Definition at line 310 of file RPageStorage.hxx.

◆ Init()

void ROOT::Experimental::Internal::RPageSink::Init ( RNTupleModel & model )

inline

Physically creates the storage container to hold the ntuple (e.g., a keys a TFile or an S3 bucket) Init() associates column handles to the columns referenced by the model.

Definition at line 321 of file RPageStorage.hxx.

◆ InitImpl()

virtual void ROOT::Experimental::Internal::RPageSink::InitImpl ( RNTupleModel & model )

protectedpure virtual

Implemented in ROOT::Experimental::Internal::RPageNullSink, ROOT::Experimental::Internal::RPageSinkBuf, ROOT::Experimental::Internal::RPagePersistentSink, ROOT::Experimental::Internal::RPageSinkDaos, and ROOT::Experimental::Internal::RPageSinkFile.

◆ IsInitialized()

bool ROOT::Experimental::Internal::RPageSink::IsInitialized ( ) const

inline

Definition at line 314 of file RPageStorage.hxx.

◆ operator=() [1/2]

RPageSink & ROOT::Experimental::Internal::RPageSink::operator= ( const RPageSink & )

delete

◆ operator=() [2/2]

RPageSink & ROOT::Experimental::Internal::RPageSink::operator= ( RPageSink && )

default

◆ RegisterOnCommitDatasetCallback()

void ROOT::Experimental::Internal::RPageSink::RegisterOnCommitDatasetCallback ( Callback_t callback )

inline

The registered callback is executed at the beginning of CommitDataset();.

Definition at line 391 of file RPageStorage.hxx.

◆ ReservePage()

ROOT::Experimental::Internal::RPage ROOT::Experimental::Internal::RPageSink::ReservePage	(	ColumnHandle_t	columnHandle,
		std::size_t	nElements
	)

virtual

Get a new, empty page for the given column that can be filled with up to nElements; nElements must be larger than zero.

Reimplemented in ROOT::Experimental::Internal::RPageSinkBuf.

Definition at line 698 of file RPageStorage.cxx.

◆ SealPage() [1/2]

ROOT::Experimental::Internal::RPageStorage::RSealedPage ROOT::Experimental::Internal::RPageSink::SealPage	(	const RPage &	page,
		const RColumnElementBase &	element
	)

protected

Helper for streaming a page.

This is commonly used in derived, concrete page sinks. Note that if compressionSetting is 0 (uncompressed) and the page is mappable and not checksummed, the returned sealed page will point directly to the input page buffer. Otherwise, the sealed page references an internal buffer of fCompressor. Thus, the buffer pointed to by the RSealedPage should never be freed. Usage of this method requires construction of fCompressor.

Definition at line 673 of file RPageStorage.cxx.

◆ SealPage() [2/2]

ROOT::Experimental::Internal::RPageStorage::RSealedPage ROOT::Experimental::Internal::RPageSink::SealPage ( const RSealPageConfig & config )

static

Seal a page using the provided info.

Definition at line 636 of file RPageStorage.cxx.

◆ StageCluster()

virtual RStagedCluster ROOT::Experimental::Internal::RPageSink::StageCluster ( NTupleSize_t nNewEntries )

pure virtual

Stage the current cluster and create a new one for the following data.

Returns the object that must be passed to CommitStagedClusters to logically append the staged cluster to the ntuple descriptor.

Implemented in ROOT::Experimental::Internal::RPageSinkBuf, ROOT::Experimental::Internal::RPagePersistentSink, and ROOT::Experimental::Internal::RPageNullSink.

◆ UpdateExtraTypeInfo()

virtual void ROOT::Experimental::Internal::RPageSink::UpdateExtraTypeInfo ( const RExtraTypeInfoDescriptor & extraTypeInfo )

pure virtual

Adds an extra type information record to schema.

The extra type information will be written to the extension header. The information in the record will be merged with the existing information, e.g. duplicate streamer info records will be removed. This method is called by the "on commit dataset" callback registered by specific fields (e.g., streamer field) and during merging.

Implemented in ROOT::Experimental::Internal::RPageNullSink, ROOT::Experimental::Internal::RPageSinkBuf, and ROOT::Experimental::Internal::RPagePersistentSink.

◆ UpdateSchema()

virtual void ROOT::Experimental::Internal::RPageSink::UpdateSchema	(	const RNTupleModelChangeset &	changeset,
		NTupleSize_t	firstEntry
	)

pure virtual

Incorporate incremental changes to the model into the ntuple descriptor.

This happens, e.g. if new fields were added after the initial call to RPageSink::Init(RNTupleModel &). firstEntry specifies the global index for the first stored element in the added columns.

Implemented in ROOT::Experimental::Internal::RPageNullSink, ROOT::Experimental::Internal::RPageSinkBuf, and ROOT::Experimental::Internal::RPagePersistentSink.

Member Data Documentation

◆ fCompressor

std::unique_ptr<RNTupleCompressor> ROOT::Experimental::Internal::RPageSink::fCompressor

protected

Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.

There could be concrete page sinks that don't need a compressor. Therefore, and in order to stay consistent with the page source, we leave it up to the derived class whether or not the compressor gets constructed.

Definition at line 281 of file RPageStorage.hxx.

◆ fIsInitialized

bool ROOT::Experimental::Internal::RPageSink::fIsInitialized = false

private

Flag if sink was initialized.

Definition at line 292 of file RPageStorage.hxx.

◆ fOnDatasetCommitCallbacks

std::vector<Callback_t> ROOT::Experimental::Internal::RPageSink::fOnDatasetCommitCallbacks

private

Definition at line 293 of file RPageStorage.hxx.

◆ fOptions

std::unique_ptr<RNTupleWriteOptions> ROOT::Experimental::Internal::RPageSink::fOptions

protected

Definition at line 276 of file RPageStorage.hxx.

◆ fSealPageBuffer

std::vector<unsigned char> ROOT::Experimental::Internal::RPageSink::fSealPageBuffer

private

Used as destination buffer in the simple SealPage overload.

Definition at line 294 of file RPageStorage.hxx.

◆ fWritePageMemoryManager

RWritePageMemoryManager ROOT::Experimental::Internal::RPageSink::fWritePageMemoryManager

private

Used in ReservePage to maintain the page buffer budget.

Definition at line 297 of file RPageStorage.hxx.

Libraries for ROOT::Experimental::Internal::RPageSink:

[legend]

The documentation for this class was generated from the following files:

tree/ntuple/v7/inc/ROOT/RPageStorage.hxx
tree/ntuple/v7/src/RPageStorage.cxx

Classes

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Protected Attributes

Private Attributes

Additional Inherited Members

Member Typedef Documentation

◆ Callback_t

Constructor & Destructor Documentation

◆ RPageSink() [1/3]

◆ RPageSink() [2/3]

◆ RPageSink() [3/3]

◆ ~RPageSink()

Member Function Documentation

◆ CommitCluster()

◆ CommitClusterGroup()

◆ CommitDataset()

◆ CommitDatasetImpl()

◆ CommitPage()

◆ CommitSealedPage()

◆ CommitSealedPageV()

◆ CommitStagedClusters()

◆ CommitSuppressedColumn()

◆ DropColumn()

◆ GetDescriptor()

◆ GetSinkGuard()

◆ GetType()

◆ GetWriteOptions()

◆ Init()

◆ InitImpl()

◆ IsInitialized()

◆ operator=() [1/2]

◆ operator=() [2/2]

◆ RegisterOnCommitDatasetCallback()

◆ ReservePage()

◆ SealPage() [1/2]

◆ SealPage() [2/2]

◆ StageCluster()

◆ UpdateExtraTypeInfo()

◆ UpdateSchema()

Member Data Documentation

◆ fCompressor

◆ fIsInitialized

◆ fOnDatasetCommitCallbacks

◆ fOptions

◆ fSealPageBuffer

◆ fWritePageMemoryManager