Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RPageStorage.cxx
Go to the documentation of this file.
1/// \file RPageStorage.cxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2018-10-04
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#include <ROOT/RPageStorage.hxx>
18#include <ROOT/RColumn.hxx>
19#include <ROOT/RField.hxx>
22#include <ROOT/RNTupleModel.hxx>
23#include <ROOT/RPagePool.hxx>
24#include <ROOT/RPageSinkBuf.hxx>
26#include <ROOT/RStringView.hxx>
27#ifdef R__ENABLE_DAOS
29#endif
30
31#include <Compression.h>
32#include <TError.h>
33
34#include <utility>
35
36
38{
39}
40
42{
43}
44
45
46//------------------------------------------------------------------------------
47
48
50 : RPageStorage(name), fMetrics(""), fOptions(options)
51{
52}
53
55{
56}
57
58std::unique_ptr<ROOT::Experimental::Detail::RPageSource> ROOT::Experimental::Detail::RPageSource::Create(
59 std::string_view ntupleName, std::string_view location, const RNTupleReadOptions &options)
60{
61 if (ntupleName.empty()) {
62 throw RException(R__FAIL("empty RNTuple name"));
63 }
64 if (location.empty()) {
65 throw RException(R__FAIL("empty storage location"));
66 }
67 if (location.find("daos://") == 0)
68#ifdef R__ENABLE_DAOS
69 return std::make_unique<RPageSourceDaos>(ntupleName, location, options);
70#else
71 throw RException(R__FAIL("This RNTuple build does not support DAOS."));
72#endif
73
74 return std::make_unique<RPageSourceFile>(ntupleName, location, options);
75}
76
79{
81 auto columnId = GetSharedDescriptorGuard()->FindColumnId(fieldId, column.GetIndex());
83 fActiveColumns.emplace(columnId);
84 return ColumnHandle_t{columnId, &column};
85}
86
88{
89 fActiveColumns.erase(columnHandle.fId);
90}
91
93{
94 return GetSharedDescriptorGuard()->GetNEntries();
95}
96
98{
99 return GetSharedDescriptorGuard()->GetNElements(columnHandle.fId);
100}
101
103{
104 // TODO(jblomer) distinguish trees
105 return columnHandle.fId;
106}
107
109{
110 if (fTaskScheduler)
111 UnzipClusterImpl(cluster);
112}
113
114
116 const RSealedPage &sealedPage, const RColumnElementBase &element)
117{
118 const auto bytesPacked = element.GetPackedSize(sealedPage.fNElements);
119 const auto pageSize = element.GetSize() * sealedPage.fNElements;
120
121 // TODO(jblomer): We might be able to do better memory handling for unsealing pages than a new malloc for every
122 // new page.
123 auto pageBuffer = std::make_unique<unsigned char[]>(bytesPacked);
124 if (sealedPage.fSize != bytesPacked) {
125 fDecompressor->Unzip(sealedPage.fBuffer, sealedPage.fSize, bytesPacked, pageBuffer.get());
126 } else {
127 // We cannot simply map the sealed page as we don't know its life time. Specialized page sources
128 // may decide to implement to not use UnsealPage but to custom mapping / decompression code.
129 // Note that usually pages are compressed.
130 memcpy(pageBuffer.get(), sealedPage.fBuffer, bytesPacked);
131 }
132
133 if (!element.IsMappable()) {
134 auto unpackedBuffer = new unsigned char[pageSize];
135 element.Unpack(unpackedBuffer, pageBuffer.get(), sealedPage.fNElements);
136 pageBuffer = std::unique_ptr<unsigned char []>(unpackedBuffer);
137 }
138
139 return pageBuffer;
140}
141
143{
144 fMetrics = RNTupleMetrics(prefix);
145 fCounters = std::unique_ptr<RCounters>(new RCounters{
146 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nReadV", "", "number of vector read requests"),
147 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nRead", "", "number of byte ranges read"),
148 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("szReadPayload", "B", "volume read from storage (required)"),
149 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("szReadOverhead", "B", "volume read from storage (overhead)"),
150 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("szUnzip", "B", "volume after unzipping"),
151 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nClusterLoaded", "",
152 "number of partial clusters preloaded from storage"),
153 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nPageLoaded", "", "number of pages loaded from storage"),
154 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nPagePopulated", "", "number of populated pages"),
155 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("timeWallRead", "ns", "wall clock time spent reading"),
156 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("timeWallUnzip", "ns", "wall clock time spent decompressing"),
157 *fMetrics.MakeCounter<RNTupleTickCounter<RNTupleAtomicCounter>*>("timeCpuRead", "ns", "CPU time spent reading"),
158 *fMetrics.MakeCounter<RNTupleTickCounter<RNTupleAtomicCounter>*> ("timeCpuUnzip", "ns",
159 "CPU time spent decompressing"),
160 *fMetrics.MakeCounter<RNTupleCalcPerf*> ("bwRead", "MB/s", "bandwidth compressed bytes read per second",
161 fMetrics, [](const RNTupleMetrics &metrics) -> std::pair<bool, double> {
162 if (const auto szReadPayload = metrics.GetLocalCounter("szReadPayload")) {
163 if (const auto szReadOverhead = metrics.GetLocalCounter("szReadOverhead")) {
164 if (const auto timeWallRead = metrics.GetLocalCounter("timeWallRead")) {
165 if (auto walltime = timeWallRead->GetValueAsInt()) {
166 double payload = szReadPayload->GetValueAsInt();
167 double overhead = szReadOverhead->GetValueAsInt();
168 // unit: bytes / nanosecond = GB/s
169 return {true, (1000. * (payload + overhead) / walltime)};
170 }
171 }
172 }
173 }
174 return {false, -1.};
175 }
176 ),
177 *fMetrics.MakeCounter<RNTupleCalcPerf*> ("bwReadUnzip", "MB/s", "bandwidth uncompressed bytes read per second",
178 fMetrics, [](const RNTupleMetrics &metrics) -> std::pair<bool, double> {
179 if (const auto szUnzip = metrics.GetLocalCounter("szUnzip")) {
180 if (const auto timeWallRead = metrics.GetLocalCounter("timeWallRead")) {
181 if (auto walltime = timeWallRead->GetValueAsInt()) {
182 double unzip = szUnzip->GetValueAsInt();
183 // unit: bytes / nanosecond = GB/s
184 return {true, 1000. * unzip / walltime};
185 }
186 }
187 }
188 return {false, -1.};
189 }
190 ),
191 *fMetrics.MakeCounter<RNTupleCalcPerf*> ("bwUnzip", "MB/s", "decompression bandwidth of uncompressed bytes per second",
192 fMetrics, [](const RNTupleMetrics &metrics) -> std::pair<bool, double> {
193 if (const auto szUnzip = metrics.GetLocalCounter("szUnzip")) {
194 if (const auto timeWallUnzip = metrics.GetLocalCounter("timeWallUnzip")) {
195 if (auto walltime = timeWallUnzip->GetValueAsInt()) {
196 double unzip = szUnzip->GetValueAsInt();
197 // unit: bytes / nanosecond = GB/s
198 return {true, 1000. * unzip / walltime};
199 }
200 }
201 }
202 return {false, -1.};
203 }
204 ),
205 *fMetrics.MakeCounter<RNTupleCalcPerf*> ("rtReadEfficiency", "", "ratio of payload over all bytes read",
206 fMetrics, [](const RNTupleMetrics &metrics) -> std::pair<bool, double> {
207 if (const auto szReadPayload = metrics.GetLocalCounter("szReadPayload")) {
208 if (const auto szReadOverhead = metrics.GetLocalCounter("szReadOverhead")) {
209 if (auto payload = szReadPayload->GetValueAsInt()) {
210 // r/(r+o) = 1/((r+o)/r) = 1/(1 + o/r)
211 return {true, 1./(1. + (1. * szReadOverhead->GetValueAsInt()) / payload)};
212 }
213 }
214 }
215 return {false, -1.};
216 }
217 ),
218 *fMetrics.MakeCounter<RNTupleCalcPerf*> ("rtCompression", "", "ratio of compressed bytes / uncompressed bytes",
219 fMetrics, [](const RNTupleMetrics &metrics) -> std::pair<bool, double> {
220 if (const auto szReadPayload = metrics.GetLocalCounter("szReadPayload")) {
221 if (const auto szUnzip = metrics.GetLocalCounter("szUnzip")) {
222 if (auto unzip = szUnzip->GetValueAsInt()) {
223 return {true, (1. * szReadPayload->GetValueAsInt()) / unzip};
224 }
225 }
226 }
227 return {false, -1.};
228 }
229 )
230 });
231}
232
233
234//------------------------------------------------------------------------------
235
236
238 : RPageStorage(name), fMetrics(""), fOptions(options.Clone())
239{
240}
241
243{
244}
245
246std::unique_ptr<ROOT::Experimental::Detail::RPageSink> ROOT::Experimental::Detail::RPageSink::Create(
247 std::string_view ntupleName, std::string_view location, const RNTupleWriteOptions &options)
248{
249 if (ntupleName.empty()) {
250 throw RException(R__FAIL("empty RNTuple name"));
251 }
252 if (location.empty()) {
253 throw RException(R__FAIL("empty storage location"));
254 }
255 std::unique_ptr<ROOT::Experimental::Detail::RPageSink> realSink;
256 if (location.find("daos://") == 0) {
257#ifdef R__ENABLE_DAOS
258 realSink = std::make_unique<RPageSinkDaos>(ntupleName, location, options);
259#else
260 throw RException(R__FAIL("This RNTuple build does not support DAOS."));
261#endif
262 } else {
263 realSink = std::make_unique<RPageSinkFile>(ntupleName, location, options);
264 }
265
266 if (options.GetUseBufferedWrite())
267 return std::make_unique<RPageSinkBuf>(std::move(realSink));
268 return realSink;
269}
270
273{
274 auto columnId = fDescriptorBuilder.GetDescriptor().GetNColumns();
275 fDescriptorBuilder.AddColumn(columnId, fieldId, column.GetModel(), column.GetIndex());
276 return ColumnHandle_t{columnId, &column};
277}
278
279
281{
282 fDescriptorBuilder.SetNTuple(fNTupleName, model.GetDescription());
283 const auto &descriptor = fDescriptorBuilder.GetDescriptor();
284
285 auto &fieldZero = *model.GetFieldZero();
286 fDescriptorBuilder.AddField(RFieldDescriptorBuilder::FromField(fieldZero).FieldId(0).MakeDescriptor().Unwrap());
287 fieldZero.SetOnDiskId(0);
288 for (auto &f : fieldZero) {
289 auto fieldId = descriptor.GetNFields();
290 fDescriptorBuilder.AddField(RFieldDescriptorBuilder::FromField(f).FieldId(fieldId).MakeDescriptor().Unwrap());
291 fDescriptorBuilder.AddFieldLink(f.GetParent()->GetOnDiskId(), fieldId);
292 f.SetOnDiskId(fieldId);
293 f.ConnectPageSink(*this); // issues in turn one or several calls to AddColumn()
294 }
295
296 auto nColumns = descriptor.GetNColumns();
297 for (DescriptorId_t i = 0; i < nColumns; ++i) {
299 columnRange.fColumnId = i;
300 columnRange.fFirstElementIndex = 0;
301 columnRange.fNElements = 0;
302 columnRange.fCompressionSettings = GetWriteOptions().GetCompression();
303 fOpenColumnRanges.emplace_back(columnRange);
305 pageRange.fColumnId = i;
306 fOpenPageRanges.emplace_back(std::move(pageRange));
307 }
308
309 fSerializationContext = Internal::RNTupleSerializer::SerializeHeaderV1(nullptr, descriptor);
310 auto buffer = std::make_unique<unsigned char[]>(fSerializationContext.GetHeaderSize());
311 fSerializationContext = Internal::RNTupleSerializer::SerializeHeaderV1(buffer.get(), descriptor);
312
313 CreateImpl(model, buffer.get(), fSerializationContext.GetHeaderSize());
314}
315
316
318{
319 fOpenColumnRanges.at(columnHandle.fId).fNElements += page.GetNElements();
320
322 pageInfo.fNElements = page.GetNElements();
323 pageInfo.fLocator = CommitPageImpl(columnHandle, page);
324 fOpenPageRanges.at(columnHandle.fId).fPageInfos.emplace_back(pageInfo);
325}
326
327
331{
332 fOpenColumnRanges.at(columnId).fNElements += sealedPage.fNElements;
333
335 pageInfo.fNElements = sealedPage.fNElements;
336 pageInfo.fLocator = CommitSealedPageImpl(columnId, sealedPage);
337 fOpenPageRanges.at(columnId).fPageInfos.emplace_back(pageInfo);
338}
339
340std::vector<ROOT::Experimental::RNTupleLocator>
341ROOT::Experimental::Detail::RPageSink::CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges)
342{
343 std::vector<ROOT::Experimental::RNTupleLocator> locators;
344 for (auto &range : ranges) {
345 for (auto sealedPageIt = range.fFirst; sealedPageIt != range.fLast; ++sealedPageIt)
346 locators.push_back(CommitSealedPageImpl(range.fColumnId, *sealedPageIt));
347 }
348 return locators;
349}
350
351void ROOT::Experimental::Detail::RPageSink::CommitSealedPageV(std::span<RPageStorage::RSealedPageGroup> ranges)
352{
353 auto locators = CommitSealedPageVImpl(ranges);
354 unsigned i = 0;
355
356 for (auto &range : ranges) {
357 for (auto sealedPageIt = range.fFirst; sealedPageIt != range.fLast; ++sealedPageIt) {
358 fOpenColumnRanges.at(range.fColumnId).fNElements += sealedPageIt->fNElements;
359
361 pageInfo.fNElements = sealedPageIt->fNElements;
362 pageInfo.fLocator = locators[i++];
363 fOpenPageRanges.at(range.fColumnId).fPageInfos.emplace_back(pageInfo);
364 }
365 }
366}
367
369{
370 auto nbytes = CommitClusterImpl(nEntries);
371
372 R__ASSERT((nEntries - fPrevClusterNEntries) < ClusterSize_t(-1));
373 auto nEntriesInCluster = ClusterSize_t(nEntries - fPrevClusterNEntries);
374 RClusterDescriptorBuilder clusterBuilder(fDescriptorBuilder.GetDescriptor().GetNClusters(), fPrevClusterNEntries,
375 nEntriesInCluster);
376 for (unsigned int i = 0; i < fOpenColumnRanges.size(); ++i) {
378 fullRange.fColumnId = i;
379 std::swap(fullRange, fOpenPageRanges[i]);
380 clusterBuilder.CommitColumnRange(i, fOpenColumnRanges[i].fFirstElementIndex,
381 fOpenColumnRanges[i].fCompressionSettings, fullRange);
382 fOpenColumnRanges[i].fFirstElementIndex += fOpenColumnRanges[i].fNElements;
383 fOpenColumnRanges[i].fNElements = 0;
384 }
385 fDescriptorBuilder.AddClusterWithDetails(clusterBuilder.MoveDescriptor().Unwrap());
386 fPrevClusterNEntries = nEntries;
387 return nbytes;
388}
389
391{
392 const auto &descriptor = fDescriptorBuilder.GetDescriptor();
393
394 const auto nClusters = descriptor.GetNClusters();
395 std::vector<DescriptorId_t> physClusterIDs;
396 for (auto i = fNextClusterInGroup; i < nClusters; ++i) {
397 physClusterIDs.emplace_back(fSerializationContext.MapClusterId(i));
398 }
399
400 auto szPageList =
401 Internal::RNTupleSerializer::SerializePageListV1(nullptr, descriptor, physClusterIDs, fSerializationContext);
402 auto bufPageList = std::make_unique<unsigned char[]>(szPageList);
403 Internal::RNTupleSerializer::SerializePageListV1(bufPageList.get(), descriptor, physClusterIDs,
404 fSerializationContext);
405
406 const auto clusterGroupId = descriptor.GetNClusterGroups();
407 const auto locator = CommitClusterGroupImpl(bufPageList.get(), szPageList);
409 cgBuilder.ClusterGroupId(clusterGroupId).PageListLocator(locator).PageListLength(szPageList);
410 for (auto i = fNextClusterInGroup; i < nClusters; ++i) {
411 cgBuilder.AddCluster(i);
412 }
413 fDescriptorBuilder.AddClusterGroup(std::move(cgBuilder));
414 fSerializationContext.MapClusterGroupId(clusterGroupId);
415
416 fNextClusterInGroup = nClusters;
417}
418
420{
421 const auto &descriptor = fDescriptorBuilder.GetDescriptor();
422
423 auto szFooter = Internal::RNTupleSerializer::SerializeFooterV1(nullptr, descriptor, fSerializationContext);
424 auto bufFooter = std::make_unique<unsigned char[]>(szFooter);
425 Internal::RNTupleSerializer::SerializeFooterV1(bufFooter.get(), descriptor, fSerializationContext);
426
427 CommitDatasetImpl(bufFooter.get(), szFooter);
428}
429
432 const RColumnElementBase &element, int compressionSetting, void *buf)
433{
434 unsigned char *pageBuf = reinterpret_cast<unsigned char *>(page.GetBuffer());
435 bool isAdoptedBuffer = true;
436 auto packedBytes = page.GetNBytes();
437
438 if (!element.IsMappable()) {
439 packedBytes = element.GetPackedSize(page.GetNElements());
440 pageBuf = new unsigned char[packedBytes];
441 isAdoptedBuffer = false;
442 element.Pack(pageBuf, page.GetBuffer(), page.GetNElements());
443 }
444 auto zippedBytes = packedBytes;
445
446 if ((compressionSetting != 0) || !element.IsMappable()) {
447 zippedBytes = RNTupleCompressor::Zip(pageBuf, packedBytes, compressionSetting, buf);
448 if (!isAdoptedBuffer)
449 delete[] pageBuf;
450 pageBuf = reinterpret_cast<unsigned char *>(buf);
451 isAdoptedBuffer = true;
452 }
453
454 R__ASSERT(isAdoptedBuffer);
455
456 return RSealedPage{pageBuf, zippedBytes, page.GetNElements()};
457}
458
461 const RPage &page, const RColumnElementBase &element, int compressionSetting)
462{
463 R__ASSERT(fCompressor);
464 return SealPage(page, element, compressionSetting, fCompressor->GetZipBuffer());
465}
466
468{
469 fMetrics = RNTupleMetrics(prefix);
470 fCounters = std::unique_ptr<RCounters>(new RCounters{
471 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nPageCommitted", "", "number of pages committed to storage"),
472 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("szWritePayload", "B", "volume written for committed pages"),
473 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("szZip", "B", "volume before zipping"),
474 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("timeWallWrite", "ns", "wall clock time spent writing"),
475 *fMetrics.MakeCounter<RNTupleAtomicCounter*>("timeWallZip", "ns", "wall clock time spent compressing"),
476 *fMetrics.MakeCounter<RNTupleTickCounter<RNTupleAtomicCounter>*>("timeCpuWrite", "ns", "CPU time spent writing"),
477 *fMetrics.MakeCounter<RNTupleTickCounter<RNTupleAtomicCounter>*> ("timeCpuZip", "ns",
478 "CPU time spent compressing")
479 });
480}
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:303
#define f(i)
Definition RSha256.hxx:104
#define R__ASSERT(e)
Definition TError.h:117
char name[80]
Definition TGX11.cxx:110
An in-memory subset of the packed and compressed pages of a cluster.
Definition RCluster.hxx:154
virtual void Pack(void *destination, void *source, std::size_t count) const
If the on-storage layout and the in-memory layout differ, packing creates an on-disk page from an in-...
virtual bool IsMappable() const
Derived, typed classes tell whether the on-storage layout is bitwise identical to the memory layout.
virtual void Unpack(void *destination, void *source, std::size_t count) const
If the on-storage layout and the in-memory layout differ, unpacking creates a memory page from an on-...
std::size_t GetPackedSize(std::size_t nElements) const
const RColumnModel & GetModel() const
Definition RColumn.hxx:309
std::uint32_t GetIndex() const
Definition RColumn.hxx:310
NTupleSize_t GetNElements() const
Definition RColumn.hxx:307
void SetOnDiskId(DescriptorId_t id)
Definition RField.hxx:298
A thread-safe integral performance counter.
A metric element that computes its floating point value from other counters.
std::int64_t GetValueAsInt() const override
size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter)
Returns the size of the compressed data.
A collection of Counter objects with a name, a unit, and a description.
const RNTuplePerfCounter * GetLocalCounter(std::string_view name) const
Searches counters registered in this object only. Returns nullptr if name is not found.
An either thread-safe or non thread safe counter for CPU ticks.
void CommitDataset()
Finalize the current cluster and the entrire data set.
RSealedPage SealPage(const RPage &page, const RColumnElementBase &element, int compressionSetting)
Helper for streaming a page.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSink.
void CommitSealedPageV(std::span< RPageStorage::RSealedPageGroup > ranges)
Write a vector of preprocessed pages to storage. The corresponding columns must have been added befor...
RPageSink(std::string_view ntupleName, const RNTupleWriteOptions &options)
void CommitPage(ColumnHandle_t columnHandle, const RPage &page)
Write a page to the storage. The column must have been added before.
void CommitSealedPage(DescriptorId_t columnId, const RPageStorage::RSealedPage &sealedPage)
Write a preprocessed page to storage. The column must have been added before.
static std::unique_ptr< RPageSink > Create(std::string_view ntupleName, std::string_view location, const RNTupleWriteOptions &options=RNTupleWriteOptions())
Guess the concrete derived page source from the file name (location)
std::uint64_t CommitCluster(NTupleSize_t nEntries)
Finalize the current cluster and create a new one for the following data.
void CommitClusterGroup()
Write out the page locations (page list envelope) for all the committed clusters since the last call ...
virtual std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges)
Vector commit of preprocessed pages.
ColumnHandle_t AddColumn(DescriptorId_t fieldId, const RColumn &column) final
Register a new column.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSource.
std::unique_ptr< unsigned char[]> UnsealPage(const RSealedPage &sealedPage, const RColumnElementBase &element)
Helper for unstreaming a page.
void DropColumn(ColumnHandle_t columnHandle) override
Unregisters a column.
NTupleSize_t GetNElements(ColumnHandle_t columnHandle)
ColumnHandle_t AddColumn(DescriptorId_t fieldId, const RColumn &column) override
Register a new column.
static std::unique_ptr< RPageSource > Create(std::string_view ntupleName, std::string_view location, const RNTupleReadOptions &options=RNTupleReadOptions())
Guess the concrete derived page source from the file name (location)
RPageSource(std::string_view ntupleName, const RNTupleReadOptions &fOptions)
void UnzipCluster(RCluster *cluster)
Parallel decompression and unpacking of the pages in the given cluster.
ColumnId_t GetColumnId(ColumnHandle_t columnHandle)
Common functionality of an ntuple storage for both reading and writing.
A page is a slice of a column that is mapped into memory.
Definition RPage.hxx:41
ClusterSize_t::ValueType GetNElements() const
Definition RPage.hxx:83
ClusterSize_t::ValueType GetNBytes() const
The space taken by column elements in the buffer.
Definition RPage.hxx:81
static std::uint32_t SerializePageListV1(void *buffer, const RNTupleDescriptor &desc, std::span< DescriptorId_t > physClusterIDs, const RContext &context)
static RContext SerializeHeaderV1(void *buffer, const RNTupleDescriptor &desc)
static std::uint32_t SerializeFooterV1(void *buffer, const RNTupleDescriptor &desc, const RContext &context)
A helper class for piece-wise construction of an RClusterDescriptor.
RResult< void > CommitColumnRange(DescriptorId_t columnId, std::uint64_t firstElementIndex, std::uint32_t compressionSettings, const RClusterDescriptor::RPageRange &pageRange)
RResult< RClusterDescriptor > MoveDescriptor()
Move out the full cluster descriptor including page locations.
A helper class for piece-wise construction of an RClusterGroupDescriptor.
RClusterGroupDescriptorBuilder & ClusterGroupId(DescriptorId_t clusterGroupId)
RClusterGroupDescriptorBuilder & PageListLength(std::uint32_t pageListLength)
RClusterGroupDescriptorBuilder & PageListLocator(const RNTupleLocator &pageListLocator)
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
static RFieldDescriptorBuilder FromField(const Detail::RFieldBase &field)
Make a new RFieldDescriptorBuilder based off a live NTuple field.
RResult< RFieldDescriptor > MakeDescriptor() const
Attempt to make a field descriptor.
RFieldDescriptorBuilder & FieldId(DescriptorId_t fieldId)
The RNTupleModel encapulates the schema of an ntuple.
RFieldZero * GetFieldZero() const
Common user-tunable settings for reading ntuples.
Common user-tunable settings for storing ntuples.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
RClusterSize ClusterSize_t
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
std::int64_t ColumnId_t
Uniquely identifies a physical column within the scope of the current process, used to tag pages.
constexpr DescriptorId_t kInvalidDescriptorId
Default I/O performance counters that get registered in fMetrics.
Default I/O performance counters that get registered in fMetrics.
A sealed page contains the bytes of a page as written to storage (packed & compressed).
The window of element indexes of a particular column in a particular cluster.
std::int64_t fCompressionSettings
The usual format for ROOT compression settings (see Compression.h).
ClusterSize_t fNElements
A 32bit value for the number of column elements in the cluster.
We do not need to store the element size / uncompressed page size because we know to which column the...
RNTupleLocator fLocator
The meaning of fLocator depends on the storage backend.
ClusterSize_t fNElements
The sum of the elements of all the pages must match the corresponding fNElements field in fColumnRang...
Records the parition of data into pages for a particular column in a particular cluster.