41#include <unordered_map>
89 return R__FAIL(
"page checksum verification failed, data corruption detected");
96 return R__FAIL(
"invalid attempt to extract non-existing page checksum");
98 assert(fBufferSize >= kNBytesPageChecksum);
101 reinterpret_cast<const unsigned char *
>(
fBuffer) + fBufferSize - kNBytesPageChecksum,
checksum);
125 for (std::size_t i = 0; i <
itr->second.size(); ++i) {
129 itr->second[i].fRefCounter--;
130 if (
itr->second[i].fRefCounter == 0) {
132 if (
itr->second.empty()) {
133 fColumnInfos.erase(
itr);
159 if (
clusterDesc.GetFirstEntryIndex() >= (fFirstEntry + fNEntries))
171std::unique_ptr<ROOT::Internal::RPageSource>
178 if (location.empty()) {
181 if (location.find(
"daos://") == 0)
183 return std::make_unique<ROOT::Experimental::Internal::RPageSourceDaos>(
ntupleName, location, options);
188 return std::make_unique<ROOT::Internal::RPageSourceFile>(
ntupleName, location, options);
209 if ((
range.fFirstEntry +
range.fNEntries) > GetNEntries()) {
219 fHasStructure =
true;
226 GetExclDescriptorGuard().MoveIn(AttachImpl(
mode));
232 auto clone = CloneImpl();
234 clone->GetExclDescriptorGuard().MoveIn(GetSharedDescriptorGuard()->Clone());
235 clone->fHasStructure =
true;
236 clone->fIsAttached =
true;
243 return GetSharedDescriptorGuard()->GetNEntries();
248 return GetSharedDescriptorGuard()->GetNElements(
columnHandle.fPhysicalId);
259 RNTupleAtomicTimer
timer(fCounters->fTimeWallUnzip, fCounters->fTimeCpuUnzip);
269 std::vector<std::unique_ptr<RColumnElementBase>>
allElements;
275 if (!fActivePhysicalColumns.HasColumnInfos(
columnId))
286 for (
const auto &pi :
pageRange.GetPageInfos()) {
291 sealedPage.SetBufferSize(pi.GetLocator().GetNBytesOnStorage() + pi.HasChecksum() * kNBytesPageChecksum);
318 fCounters->fNPageUnsealed.Add(
pageNo);
322 fTaskScheduler->Wait();
325 throw RException(
R__FAIL(
"page checksum verification failed, data corruption detected"));
347 pageInfo.GetLocator().GetNBytesOnStorage()));
362 GetSharedDescriptorGuard()->GetClusterDescriptor(
clusterId).GetFirstEntryIndex();
363 auto itr = fPreloadedClusters.
begin();
365 fPagePool.Evict(
itr->second);
366 itr = fPreloadedClusters.erase(
itr);
369 while ((
itr != fPreloadedClusters.
end()) &&
374 while (
itr != fPreloadedClusters.
end()) {
375 fPagePool.Evict(
itr->second);
376 itr = fPreloadedClusters.erase(
itr);
390 UpdateLastUsedCluster(
cachedPageRef.Get().GetClusterInfo().GetId());
461 fCounters = std::make_unique<RCounters>(
RCounters{
464 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"szReadPayload",
"B",
"volume read from storage (required)"),
465 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"szReadOverhead",
"B",
"volume read from storage (overhead)"),
468 "number of partial clusters preloaded from storage"),
469 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"nPageRead",
"",
"number of pages read from storage"),
470 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"nPageUnsealed",
"",
"number of pages unzipped and decoded"),
471 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"timeWallRead",
"ns",
"wall clock time spent reading"),
472 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"timeWallUnzip",
"ns",
"wall clock time spent decompressing"),
475 "CPU time spent decompressing"),
477 "bwRead",
"MB/s",
"bandwidth compressed bytes read per second", fMetrics,
494 "bwReadUnzip",
"MB/s",
"bandwidth uncompressed bytes read per second", fMetrics,
508 "bwUnzip",
"MB/s",
"decompression bandwidth of uncompressed bytes per second", fMetrics,
522 "rtReadEfficiency",
"",
"ratio of payload over all bytes read", fMetrics,
528 return {
true, 1. / (1. + (1. *
szReadOverhead->GetValueAsInt()) / payload)};
534 *fMetrics.MakeCounter<
RNTupleCalcPerf *>(
"rtCompression",
"",
"ratio of compressed bytes / uncompressed bytes",
537 metrics.GetLocalCounter(
"szReadPayload")) {
586 page = std::move(tmp);
595 if (fHasStreamerInfosRegistered)
598 for (
const auto &
extraTypeInfo : fDescriptor.GetExtraTypeInfoIterable()) {
606 fHasStreamerInfosRegistered =
true;
614 if (fCurrentPageSize ==
other.fCurrentPageSize)
615 return fColumn->GetOnDiskId() >
other.fColumn->GetOnDiskId();
616 return fCurrentPageSize >
other.fCurrentPageSize;
624 auto itr = fColumnsSortedByPageSize.
begin();
625 while (
itr != fColumnsSortedByPageSize.
end()) {
628 if (
itr->fCurrentPageSize ==
itr->fInitialPageSize) {
637 if (
itr != fColumnsSortedByPageSize.
end())
646 itr = fColumnsSortedByPageSize.find(next);
655 auto itr = fColumnsSortedByPageSize.find(key);
656 if (
itr == fColumnsSortedByPageSize.
end()) {
670 fColumnsSortedByPageSize.erase(
itr);
676 fColumnsSortedByPageSize.insert(
elem);
685 fColumnsSortedByPageSize.insert(
elem);
690 fColumnsSortedByPageSize.insert(
elem);
697 :
RPageStorage(
name), fOptions(options.Clone()), fWritePageMemoryManager(options.GetPageBufferBudget())
710 unsigned char *
pageBuf =
reinterpret_cast<unsigned char *
>(config.
fPage->GetBuffer());
715 if (!config.
fElement->IsMappable()) {
744 const auto nBytes =
page.GetNBytes() + GetWriteOptions().GetEnablePageChecksums() * kNBytesPageChecksum;
745 if (fSealPageBuffer.size() <
nBytes)
746 fSealPageBuffer.resize(
nBytes);
752 config.
fWriteChecksum = GetWriteOptions().GetEnablePageChecksums();
754 config.
fBuffer = fSealPageBuffer.data();
756 return SealPage(config);
761 for (
const auto &cb : fOnDatasetCommitCallbacks)
778std::unique_ptr<ROOT::Internal::RPageSink>
785 if (location.empty()) {
788 if (location.find(
"daos://") == 0) {
790 return std::make_unique<ROOT::Experimental::Internal::RPageSinkDaos>(
ntupleName, location, options);
797 return std::make_unique<ROOT::Internal::RPageSinkFile>(
ntupleName, location, options);
811 auto columnId = fDescriptorBuilder.GetDescriptor().GetNPhysicalColumns();
826 fDescriptorBuilder.AddColumn(
columnBuilder.MakeDescriptor().Unwrap());
838 const auto &
descriptor = fDescriptorBuilder.GetDescriptor();
844 const auto &
reps =
f.GetColumnRepresentatives();
847 return reps.size() *
reps[0].size();
861 fDescriptorBuilder.AddFieldLink(
f.GetParent()->GetOnDiskId(),
fieldId);
870 fDescriptorBuilder.AddFieldLink(
f.GetParent()->GetOnDiskId(),
fieldId);
877 .PhysicalColumnId(
source.GetLogicalId())
879 .BitsOnStorage(
source.GetBitsOnStorage())
880 .ValueRange(
source.GetValueRange())
883 .RepresentationIndex(
source.GetRepresentationIndex());
884 fDescriptorBuilder.AddColumn(
columnBuilder.MakeDescriptor().Unwrap());
895 for (
auto f :
changeset.fAddedProjectedFields) {
912 columnRange.SetCompressionSettings(GetWriteOptions().GetCompression());
916 fOpenPageRanges.emplace_back(std::move(
pageRange));
921 if (fSerializationContext.GetHeaderSize() > 0)
922 fSerializationContext.MapSchema(
descriptor,
true);
928 throw RException(
R__FAIL(
"ROOT bug: unexpected type extra info in UpdateExtraTypeInfo()"));
935 fDescriptorBuilder.SetNTuple(fNTupleName, model.
GetDescription());
936 const auto &
descriptor = fDescriptorBuilder.GetDescriptor();
946 for (
auto f :
fieldZero.GetMutableSubfields())
956 InitImpl(buffer.get(), fSerializationContext.GetHeaderSize());
958 fDescriptorBuilder.BeginHeaderExtension();
961std::unique_ptr<ROOT::RNTupleModel>
966 const auto &
descriptor = fDescriptorBuilder.GetDescriptor();
970 R__ASSERT(fOpenColumnRanges.empty() && fOpenPageRanges.empty());
971 fOpenColumnRanges.reserve(
nColumns);
974 const auto &column =
descriptor.GetColumnDescriptor(i);
977 columnRange.SetFirstElementIndex(column.GetFirstElementIndex());
979 columnRange.SetCompressionSettings(GetWriteOptions().GetCompression());
983 fOpenPageRanges.emplace_back(std::move(
pageRange));
992 for (
unsigned int i = 0; i < fOpenColumnRanges.size(); ++i) {
993 R__ASSERT(fOpenColumnRanges[i].GetPhysicalColumnId() == i);
994 if (!
cluster.ContainsColumn(i))
999 fOpenColumnRanges[i].IncrementFirstElementIndex(
columnRange.GetNElements());
1001 fDescriptorBuilder.AddCluster(
cluster.Clone());
1010 modelOpts.SetReconstructProjections(
true);
1014 projectedFields.GetFieldZero().SetOnDiskId(model->GetConstFieldZero().GetOnDiskId());
1021 InitImpl(buffer.get(), fSerializationContext.GetHeaderSize());
1023 fDescriptorBuilder.BeginHeaderExtension();
1026 fIsInitialized =
true;
1033 fOpenColumnRanges.at(
columnHandle.fPhysicalId).SetIsSuppressed(
true);
1038 fOpenColumnRanges.at(
columnHandle.fPhysicalId).IncrementNElements(
page.GetNElements());
1043 pageInfo.SetHasChecksum(GetWriteOptions().GetEnablePageChecksums());
1059std::vector<ROOT::RNTupleLocator>
1061 const std::vector<bool> &
mask)
1063 std::vector<ROOT::RNTupleLocator>
locators;
1066 for (
auto &
range : ranges) {
1084 std::vector<bool>
mask;
1088 std::unordered_map<std::uint64_t, RSealedPageLink>
originalPages;
1090 for (
auto &
range : ranges) {
1096 if (!fFeatures.fCanMergePages || !fOptions->GetEnableSamePageMerging()) {
1097 mask.emplace_back(
true);
1108 mask.emplace_back(
true);
1113 const auto *
p =
itr->second.fSealedPage;
1116 mask.emplace_back(
true);
1121 mask.emplace_back(
false);
1125 mask.shrink_to_fit();
1132 for (
auto &
range : ranges) {
1134 fOpenColumnRanges.at(
range.fPhysicalColumnId).IncrementNElements(
sealedPageIt->GetNElements());
1140 fOpenPageRanges.at(
range.fPhysicalColumnId).GetPageInfos().emplace_back(
pageInfo);
1152 for (
unsigned int i = 0; i < fOpenColumnRanges.size(); ++i) {
1154 columnInfo.fCompressionSettings = fOpenColumnRanges[i].GetCompressionSettings().value();
1155 if (fOpenColumnRanges[i].IsSuppressed()) {
1156 assert(fOpenPageRanges[i].GetPageInfos().empty());
1157 columnInfo.fPageRange.SetPhysicalColumnId(i);
1160 fOpenColumnRanges[i].SetNElements(0);
1161 fOpenColumnRanges[i].SetIsSuppressed(
false);
1163 std::swap(
columnInfo.fPageRange, fOpenPageRanges[i]);
1164 fOpenPageRanges[i].SetPhysicalColumnId(i);
1166 columnInfo.fNElements = fOpenColumnRanges[i].GetNElements();
1167 fOpenColumnRanges[i].SetNElements(0);
1179 clusterBuilder.ClusterId(fDescriptorBuilder.GetDescriptor().GetNActiveClusters())
1180 .FirstEntryIndex(fPrevClusterNEntries)
1190 fOpenColumnRanges[
colId].IncrementFirstElementIndex(
columnInfo.fNElements);
1194 clusterBuilder.CommitSuppressedColumnRanges(fDescriptorBuilder.GetDescriptor()).ThrowOnError();
1207 fDescriptorBuilder.AddCluster(
clusterBuilder.MoveDescriptor().Unwrap());
1208 fPrevClusterNEntries +=
cluster.fNEntries;
1214 const auto &
descriptor = fDescriptorBuilder.GetDescriptor();
1219 for (
auto i = fNextClusterInGroup; i <
nClusters; ++i) {
1220 physClusterIDs.emplace_back(fSerializationContext.MapClusterId(i));
1233 cgBuilder.MinEntry(0).EntrySpan(0).NClusters(0);
1240 .NClusters(
nClusters - fNextClusterInGroup);
1242 std::vector<ROOT::DescriptorId_t>
clusterIds;
1244 for (
auto i = fNextClusterInGroup; i <
nClusters; ++i) {
1248 fDescriptorBuilder.AddClusterGroup(
cgBuilder.MoveDescriptor().Unwrap());
1256 if (!fStreamerInfos.empty()) {
1259 for (
const auto &
etDesc : fDescriptorBuilder.GetDescriptor().GetExtraTypeInfoIterable()) {
1266 fStreamerInfos.merge(
etInfo);
1273 fDescriptorBuilder.ReplaceExtraTypeInfo(
extraInfoBuilder.MoveDescriptor().Unwrap());
1276 const auto &
descriptor = fDescriptorBuilder.GetDescriptor();
1288 fCounters = std::make_unique<RCounters>(
RCounters{
1289 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"nPageCommitted",
"",
"number of pages committed to storage"),
1290 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"szWritePayload",
"B",
"volume written for committed pages"),
1292 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"timeWallWrite",
"ns",
"wall clock time spent writing"),
1293 *fMetrics.MakeCounter<
RNTupleAtomicCounter *>(
"timeWallZip",
"ns",
"wall clock time spent compressing"),
1296 "CPU time spent compressing")});
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t mask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char mode
A thread-safe integral performance counter.
A metric element that computes its floating point value from other counters.
A collection of Counter objects with a name, a unit, and a description.
A helper class for piece-wise construction of an RClusterDescriptor.
A helper class for piece-wise construction of an RClusterGroupDescriptor.
An in-memory subset of the packed and compressed pages of a cluster.
std::unordered_set< ROOT::DescriptorId_t > ColumnSet_t
A helper class for piece-wise construction of an RColumnDescriptor.
A column element encapsulates the translation between basic C++ types and their column representation...
virtual RIdentifier GetIdentifier() const =0
A column is a storage-backed array of a simple, fixed-size type, from which pages can be mapped into ...
std::optional< std::pair< double, double > > GetValueRange() const
std::uint16_t GetRepresentationIndex() const
ROOT::Internal::RColumnElementBase * GetElement() const
ROOT::ENTupleColumnType GetType() const
ROOT::NTupleSize_t GetFirstElementIndex() const
std::size_t GetWritePageCapacity() const
std::uint16_t GetBitsOnStorage() const
std::uint32_t GetIndex() const
A helper class for piece-wise construction of an RFieldDescriptor.
static RFieldDescriptorBuilder FromField(const ROOT::RFieldBase &field)
Make a new RFieldDescriptorBuilder based off a live RNTuple field.
static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to)
Returns the size of the compressed data, written into the provided output buffer.
static void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
static unsigned int GetClusterBunchSize(const RNTupleReadOptions &options)
A helper class for serializing and deserialization of the RNTuple binary format.
static std::uint32_t SerializeXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3, void *buffer)
Writes a XxHash-3 64bit checksum of the byte range given by data and length.
static RResult< StreamerInfoMap_t > DeserializeStreamerInfos(const std::string &extraTypeInfoContent)
static RResult< void > VerifyXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3)
Expects an xxhash3 checksum in the 8 bytes following data + length and verifies it.
EDescriptorDeserializeMode
static RResult< std::uint32_t > SerializePageList(void *buffer, const RNTupleDescriptor &desc, std::span< ROOT::DescriptorId_t > physClusterIDs, const RContext &context)
static RResult< std::uint32_t > SerializeFooter(void *buffer, const RNTupleDescriptor &desc, const RContext &context)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static RResult< RContext > SerializeHeader(void *buffer, const RNTupleDescriptor &desc)
static std::string SerializeStreamerInfos(const StreamerInfoMap_t &infos)
A memory region that contains packed and compressed pages.
A page as being stored on disk, that is packed and compressed.
Uses standard C++ memory allocation for the column data pages.
Abstract interface to allocate and release pages.
RStagedCluster StageCluster(ROOT::NTupleSize_t nNewEntries) final
Stage the current cluster and create a new one for the following data.
void CommitSealedPage(ROOT::DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
Write a preprocessed page to storage. The column must have been added before.
std::unique_ptr< RNTupleModel > InitFromDescriptor(const ROOT::RNTupleDescriptor &descriptor, bool copyClusters)
Initialize sink based on an existing descriptor and fill into the descriptor builder,...
void UpdateExtraTypeInfo(const ROOT::RExtraTypeInfoDescriptor &extraTypeInfo) final
Adds an extra type information record to schema.
ColumnHandle_t AddColumn(ROOT::DescriptorId_t fieldId, ROOT::Internal::RColumn &column) final
Register a new column.
virtual std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask)
Vector commit of preprocessed pages.
RPagePersistentSink(std::string_view ntupleName, const ROOT::RNTupleWriteOptions &options)
void CommitDatasetImpl() final
void CommitSuppressedColumn(ColumnHandle_t columnHandle) final
Commits a suppressed column for the current cluster.
void UpdateSchema(const ROOT::Internal::RNTupleModelChangeset &changeset, ROOT::NTupleSize_t firstEntry) final
Incorporate incremental changes to the model into the ntuple descriptor.
void CommitStagedClusters(std::span< RStagedCluster > clusters) final
Commit staged clusters, logically appending them to the ntuple descriptor.
static std::unique_ptr< RPageSink > Create(std::string_view ntupleName, std::string_view location, const ROOT::RNTupleWriteOptions &options=ROOT::RNTupleWriteOptions())
Guess the concrete derived page source from the location.
void CommitPage(ColumnHandle_t columnHandle, const ROOT::Internal::RPage &page) final
Write a page to the storage. The column must have been added before.
~RPagePersistentSink() override
virtual void InitImpl(unsigned char *serializedHeader, std::uint32_t length)=0
void CommitClusterGroup() final
Write out the page locations (page list envelope) for all the committed clusters since the last call ...
void CommitSealedPageV(std::span< RPageStorage::RSealedPageGroup > ranges) final
Write a vector of preprocessed pages to storage. The corresponding columns must have been added befor...
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSink.
Abstract interface to write data into an ntuple.
virtual ROOT::Internal::RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements)
Get a new, empty page for the given column that can be filled with up to nElements; nElements must be...
RSealedPage SealPage(const ROOT::Internal::RPage &page, const ROOT::Internal::RColumnElementBase &element)
Helper for streaming a page.
RPageSink(std::string_view ntupleName, const ROOT::RNTupleWriteOptions &options)
void CommitDataset()
Run the registered callbacks and finalize the current cluster and the entrire data set.
void Insert(ROOT::DescriptorId_t physicalColumnId, ROOT::Internal::RColumnElementBase::RIdentifier elementId)
ROOT::Internal::RCluster::ColumnSet_t ToColumnSet() const
void Erase(ROOT::DescriptorId_t physicalColumnId, ROOT::Internal::RColumnElementBase::RIdentifier elementId)
void LoadStructure()
Loads header and footer without decompressing or deserializing them.
virtual ROOT::Internal::RPageRef LoadPage(ColumnHandle_t columnHandle, ROOT::NTupleSize_t globalIndex)
Allocates and fills a page that contains the index-th element.
void RegisterStreamerInfos()
Builds the streamer info records from the descriptor's extra type info section.
void Attach(ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode mode=ROOT::Internal::RNTupleSerializer::EDescriptorDeserializeMode::kForReading)
Open the physical storage container and deserialize header and footer.
ColumnHandle_t AddColumn(ROOT::DescriptorId_t fieldId, ROOT::Internal::RColumn &column) override
Register a new column.
void UnzipCluster(ROOT::Internal::RCluster *cluster)
Parallel decompression and unpacking of the pages in the given cluster.
void PrepareLoadCluster(const ROOT::Internal::RCluster::RKey &clusterKey, ROOT::Internal::ROnDiskPageMap &pageZeroMap, std::function< void(ROOT::DescriptorId_t, ROOT::NTupleSize_t, const ROOT::RClusterDescriptor::RPageInfo &)> perPageFunc)
Prepare a page range read for the column set in clusterKey.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSource.
ROOT::NTupleSize_t GetNEntries()
void UpdateLastUsedCluster(ROOT::DescriptorId_t clusterId)
Does nothing if fLastUsedCluster == clusterId.
ROOT::NTupleSize_t GetNElements(ColumnHandle_t columnHandle)
void DropColumn(ColumnHandle_t columnHandle) override
Unregisters a column.
virtual void UnzipClusterImpl(ROOT::Internal::RCluster *cluster)
RPageSource(std::string_view ntupleName, const ROOT::RNTupleReadOptions &fOptions)
void SetEntryRange(const REntryRange &range)
Promise to only read from the given entry range.
std::unique_ptr< RPageSource > Clone() const
Open the same storage multiple time, e.g.
static std::unique_ptr< RPageSource > Create(std::string_view ntupleName, std::string_view location, const ROOT::RNTupleReadOptions &options=ROOT::RNTupleReadOptions())
Guess the concrete derived page source from the file name (location)
static RResult< ROOT::Internal::RPage > UnsealPage(const RSealedPage &sealedPage, const ROOT::Internal::RColumnElementBase &element, ROOT::Internal::RPageAllocator &pageAlloc)
Helper for unstreaming a page.
Common functionality of an ntuple storage for both reading and writing.
RPageStorage(std::string_view name)
Stores information about the cluster in which this page resides.
A page is a slice of a column that is mapped into memory.
static const void * GetPageZeroBuffer()
Return a pointer to the page zero buffer used if there is no on-disk data for a particular deferred c...
const ROOT::RFieldBase * GetSourceField(const ROOT::RFieldBase *target) const
bool TryEvict(std::size_t targetAvailableSize, std::size_t pageSizeLimit)
Flush columns in order of allocated write page size until the sum of all write page allocations leave...
bool TryUpdate(ROOT::Internal::RColumn &column, std::size_t newWritePageSize)
Try to register the new write page size for the given column.
The window of element indexes of a particular column in a particular cluster.
Metadata for RNTuple clusters.
Base class for all ROOT issued exceptions.
A field translates read and write calls from/to underlying columns to/from tree values.
The on-storage metadata of an RNTuple.
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
The RNTupleModel encapulates the schema of an RNTuple.
const std::string & GetDescription() const
Common user-tunable settings for reading RNTuples.
Common user-tunable settings for storing RNTuples.
const_iterator begin() const
const_iterator end() const
void ThrowOnError()
Short-hand method to throw an exception in the case of errors.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
ROOT::RFieldZero & GetFieldZeroOfModel(RNTupleModel &model)
RResult< void > EnsureValidNameForRNTuple(std::string_view name, std::string_view where)
Check whether a given string is a valid name according to the RNTuple specification.
std::unique_ptr< T[]> MakeUninitArray(std::size_t size)
Make an array of default-initialized elements.
RProjectedFields & GetProjectedFieldsOfModel(RNTupleModel &model)
std::unique_ptr< RColumnElementBase > GenerateColumnElement(std::type_index inMemoryType, ROOT::ENTupleColumnType onDiskType)
void CallConnectPageSinkOnField(RFieldBase &, ROOT::Internal::RPageSink &, ROOT::NTupleSize_t firstEntry=0)
Namespace for new ROOT classes and functions.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
constexpr NTupleSize_t kInvalidNTupleIndex
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
constexpr DescriptorId_t kInvalidDescriptorId
The identifiers that specifies the content of a (partial) cluster.
Every concrete RColumnElement type is identified by its on-disk type (column type) and the in-memory ...
The incremental changes to a RNTupleModel
On-disk pages within a page source are identified by the column and page number.
Default I/O performance counters that get registered in fMetrics.
Parameters for the SealPage() method.
bool fWriteChecksum
Adds a 8 byte little-endian xxhash3 checksum to the page payload.
std::uint32_t fCompressionSettings
Compression algorithm and level to apply.
void * fBuffer
Location for sealed output. The memory buffer has to be large enough.
const ROOT::Internal::RPage * fPage
Input page to be sealed.
bool fAllowAlias
If false, the output buffer must not point to the input page buffer, which would otherwise be an opti...
const ROOT::Internal::RColumnElementBase * fElement
Corresponds to the page's elements, for size calculation etc.
Cluster that was staged, but not yet logically appended to the RNTuple.
Summarizes cluster-level information that are necessary to load a certain page.
Default I/O performance counters that get registered in fMetrics
Used in SetEntryRange / GetEntryRange.
bool IntersectsWith(const ROOT::RClusterDescriptor &clusterDesc) const
Returns true if the given cluster has entries within the entry range.
A sealed page contains the bytes of a page as written to storage (packed & compressed).
RResult< void > VerifyChecksumIfEnabled() const
RResult< std::uint64_t > GetChecksum() const
Returns a failure if the sealed page has no checksum.
ROOT::Internal::RColumn * fColumn
bool operator>(const RColumnInfo &other) const
Information about a single page in the context of a cluster's page range.
Modifiers passed to CreateModel()