Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RPageStorage.hxx
Go to the documentation of this file.
1/// \file ROOT/RPageStorage.hxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2018-07-19
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RPageStorage
17#define ROOT7_RPageStorage
18
19#include <ROOT/RError.hxx>
20#include <ROOT/RCluster.hxx>
27#include <ROOT/RNTupleUtil.hxx>
28#include <ROOT/RPage.hxx>
29#include <ROOT/RPagePool.hxx>
30#include <ROOT/RSpan.hxx>
31#include <string_view>
32
33#include <atomic>
34#include <cassert>
35#include <cstddef>
36#include <deque>
37#include <functional>
38#include <map>
39#include <memory>
40#include <mutex>
41#include <set>
42#include <shared_mutex>
43#include <unordered_map>
44#include <unordered_set>
45#include <vector>
46
47namespace ROOT {
48namespace Experimental {
49
50class RNTupleModel;
51
52namespace Internal {
53class RColumn;
54class RNTupleCompressor;
55struct RNTupleModelChangeset;
56class RPageAllocator;
57
58enum class EPageStorageType {
59 kSink,
60 kSource,
61};
62
63// clang-format off
64/**
65\class ROOT::Experimental::Internal::RPageStorage
66\ingroup NTuple
67\brief Common functionality of an ntuple storage for both reading and writing
68
69The RPageStore provides access to a storage container that keeps the bits of pages and clusters comprising
70an ntuple. Concrete implementations can use a TFile, a raw file, an object store, and so on.
71*/
72// clang-format on
74public:
75 /// The page checksum is a 64bit xxhash3
76 static constexpr std::size_t kNBytesPageChecksum = sizeof(std::uint64_t);
77
78 /// The interface of a task scheduler to schedule page (de)compression tasks
80 public:
81 virtual ~RTaskScheduler() = default;
82 /// Take a callable that represents a task
83 virtual void AddTask(const std::function<void(void)> &taskFunc) = 0;
84 /// Blocks until all scheduled tasks finished
85 virtual void Wait() = 0;
86 };
87
88 /// A sealed page contains the bytes of a page as written to storage (packed & compressed). It is used
89 /// as an input to UnsealPages() as well as to transfer pages between different storage media.
90 /// RSealedPage does _not_ own the buffer it is pointing to in order to not interfere with the memory management
91 /// of concrete page sink and page source implementations.
92 struct RSealedPage {
93 private:
94 const void *fBuffer = nullptr;
95 std::size_t fBufferSize = 0; ///< Size of the page payload and the trailing checksum (if available)
96 std::uint32_t fNElements = 0;
97 bool fHasChecksum = false; ///< If set, the last 8 bytes of the buffer are the xxhash of the rest of the buffer
98
99 public:
100 RSealedPage() = default;
101 RSealedPage(const void *buffer, std::size_t bufferSize, std::uint32_t nElements, bool hasChecksum = false)
102 : fBuffer(buffer), fBufferSize(bufferSize), fNElements(nElements), fHasChecksum(hasChecksum)
103 {
104 }
105 RSealedPage(const RSealedPage &other) = default;
106 RSealedPage &operator=(const RSealedPage &other) = default;
107 RSealedPage(RSealedPage &&other) = default;
108 RSealedPage &operator=(RSealedPage &&other) = default;
109
110 const void *GetBuffer() const { return fBuffer; }
111 void SetBuffer(const void *buffer) { fBuffer = buffer; }
112
113 std::size_t GetDataSize() const
114 {
117 }
118 std::size_t GetBufferSize() const { return fBufferSize; }
119 void SetBufferSize(std::size_t bufferSize) { fBufferSize = bufferSize; }
120
121 std::uint32_t GetNElements() const { return fNElements; }
122 void SetNElements(std::uint32_t nElements) { fNElements = nElements; }
123
124 bool GetHasChecksum() const { return fHasChecksum; }
125 void SetHasChecksum(bool hasChecksum) { fHasChecksum = hasChecksum; }
126
127 void ChecksumIfEnabled();
129 /// Returns a failure if the sealed page has no checksum
131 };
132
133 using SealedPageSequence_t = std::deque<RSealedPage>;
134 /// A range of sealed pages referring to the same column that can be used for vector commit
137 SealedPageSequence_t::const_iterator fFirst;
138 SealedPageSequence_t::const_iterator fLast;
139
140 RSealedPageGroup() = default;
141 RSealedPageGroup(DescriptorId_t d, SealedPageSequence_t::const_iterator b, SealedPageSequence_t::const_iterator e)
143 {
144 }
145 };
146
147protected:
149
150 /// For the time being, we will use the heap allocator for all sources and sinks. This may change in the future.
151 std::unique_ptr<RPageAllocator> fPageAllocator;
152
153 std::string fNTupleName;
156 {
157 if (!fTaskScheduler)
158 return;
160 }
161
162public:
163 explicit RPageStorage(std::string_view name);
164 RPageStorage(const RPageStorage &other) = delete;
165 RPageStorage &operator=(const RPageStorage &other) = delete;
166 RPageStorage(RPageStorage &&other) = default;
168 virtual ~RPageStorage();
169
170 /// Whether the concrete implementation is a sink or a source
172
175 RColumn *fColumn = nullptr;
176
177 /// Returns true for a valid column handle; fColumn and fPhysicalId should always either both
178 /// be valid or both be invalid.
179 explicit operator bool() const { return fPhysicalId != kInvalidDescriptorId && fColumn; }
180 };
181 /// The column handle identifies a column with the current open page storage
183
184 /// Register a new column. When reading, the column must exist in the ntuple on disk corresponding to the meta-data.
185 /// When writing, every column can only be attached once.
186 virtual ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) = 0;
187 /// Unregisters a column. A page source decreases the reference counter for the corresponding active column.
188 /// For a page sink, dropping columns is currently a no-op.
189 virtual void DropColumn(ColumnHandle_t columnHandle) = 0;
190 ColumnId_t GetColumnId(ColumnHandle_t columnHandle) const { return columnHandle.fPhysicalId; }
191
192 /// Returns the default metrics object. Subclasses might alternatively provide their own metrics object by
193 /// overriding this.
195
196 /// Returns the NTuple name.
197 const std::string &GetNTupleName() const { return fNTupleName; }
198
199 void SetTaskScheduler(RTaskScheduler *taskScheduler) { fTaskScheduler = taskScheduler; }
200}; // class RPageStorage
201
202// clang-format off
203/**
204\class ROOT::Experimental::Internal::RWritePageMemoryManager
205\ingroup NTuple
206\brief Helper to maintain a memory budget for the write pages of a set of columns
207
208The memory manager keeps track of the sum of bytes used by the write pages of a set of columns.
209It will flush (and shrink) large pages of other columns on the attempt to expand a page.
210*/
211// clang-format on
213private:
214 struct RColumnInfo {
215 RColumn *fColumn = nullptr;
216 std::size_t fCurrentPageSize = 0;
217 std::size_t fInitialPageSize = 0;
218
219 bool operator>(const RColumnInfo &other) const;
220 };
221
222 /// Sum of all the write page sizes (their capacity) of the columns in `fColumnsSortedByPageSize`
223 std::size_t fCurrentAllocatedBytes = 0;
224 /// Maximum allowed value for `fCurrentAllocatedBytes`, set from RNTupleWriteOptions::fPageBufferBudget
225 std::size_t fMaxAllocatedBytes = 0;
226 /// All columns that called `ReservePage()` (hence `TryUpdate()`) at least once,
227 /// sorted by their current write page size from large to small
228 std::set<RColumnInfo, std::greater<RColumnInfo>> fColumnsSortedByPageSize;
229
230 /// Flush columns in order of allocated write page size until the sum of all write page allocations
231 /// leaves space for at least targetAvailableSize bytes. Only use columns with a write page size larger
232 /// than pageSizeLimit.
233 bool TryEvict(std::size_t targetAvailableSize, std::size_t pageSizeLimit);
234
235public:
236 explicit RWritePageMemoryManager(std::size_t maxAllocatedBytes) : fMaxAllocatedBytes(maxAllocatedBytes) {}
237
238 /// Try to register the new write page size for the given column. Flush large columns to make space, if necessary.
239 /// If not enough space is available after all (sum of write pages would be larger than fMaxAllocatedBytes),
240 /// return false.
241 bool TryUpdate(RColumn &column, std::size_t newWritePageSize);
242};
243
244// clang-format off
245/**
246\class ROOT::Experimental::Internal::RPageSink
247\ingroup NTuple
248\brief Abstract interface to write data into an ntuple
249
250The page sink takes the list of columns and afterwards a series of page commits and cluster commits.
251The user is responsible to commit clusters at a consistent point, i.e. when all pages corresponding to data
252up to the given entry number are committed.
253
254An object of this class may either be a wrapper (for example a RPageSinkBuf) or a "persistent" sink,
255inheriting from RPagePersistentSink.
256*/
257// clang-format on
258class RPageSink : public RPageStorage {
259public:
260 using Callback_t = std::function<void(RPageSink &)>;
261
262 /// Cluster that was staged, but not yet logically appended to the RNTuple
264 std::uint64_t fNBytesWritten = 0;
266
267 struct RColumnInfo {
270 bool fIsSuppressed = false;
271 };
272
273 std::vector<RColumnInfo> fColumnInfos;
274 };
275
276protected:
277 std::unique_ptr<RNTupleWriteOptions> fOptions;
278
279 /// Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.
280 /// There could be concrete page sinks that don't need a compressor. Therefore, and in order to stay consistent
281 /// with the page source, we leave it up to the derived class whether or not the compressor gets constructed.
282 std::unique_ptr<RNTupleCompressor> fCompressor;
283
284 /// Helper for streaming a page. This is commonly used in derived, concrete page sinks. Note that if
285 /// compressionSetting is 0 (uncompressed) and the page is mappable and not checksummed, the returned sealed page
286 /// will point directly to the input page buffer. Otherwise, the sealed page references an internal buffer
287 /// of fCompressor. Thus, the buffer pointed to by the RSealedPage should never be freed.
288 /// Usage of this method requires construction of fCompressor.
289 RSealedPage SealPage(const RPage &page, const RColumnElementBase &element);
290
291private:
292 /// Flag if sink was initialized
293 bool fIsInitialized = false;
294 std::vector<Callback_t> fOnDatasetCommitCallbacks;
295 std::vector<unsigned char> fSealPageBuffer; ///< Used as destination buffer in the simple SealPage overload
296
297 /// Used in ReservePage to maintain the page buffer budget
299
300public:
301 RPageSink(std::string_view ntupleName, const RNTupleWriteOptions &options);
302
303 RPageSink(const RPageSink &) = delete;
304 RPageSink &operator=(const RPageSink &) = delete;
305 RPageSink(RPageSink &&) = default;
307 ~RPageSink() override;
308
310 /// Returns the sink's write options.
311 const RNTupleWriteOptions &GetWriteOptions() const { return *fOptions; }
312
313 void DropColumn(ColumnHandle_t /*columnHandle*/) final {}
314
315 bool IsInitialized() const { return fIsInitialized; }
316
317 /// Return the RNTupleDescriptor being constructed.
318 virtual const RNTupleDescriptor &GetDescriptor() const = 0;
319
320 /// Physically creates the storage container to hold the ntuple (e.g., a keys a TFile or an S3 bucket)
321 /// Init() associates column handles to the columns referenced by the model
322 void Init(RNTupleModel &model)
323 {
324 if (fIsInitialized) {
325 throw RException(R__FAIL("already initialized"));
326 }
327 fIsInitialized = true;
328 InitImpl(model);
329 }
330
331protected:
332 virtual void InitImpl(RNTupleModel &model) = 0;
333 virtual void CommitDatasetImpl() = 0;
334
335public:
336 /// Parameters for the SealPage() method
338 const RPage *fPage = nullptr; ///< Input page to be sealed
339 const RColumnElementBase *fElement = nullptr; ///< Corresponds to the page's elements, for size calculation etc.
340 int fCompressionSetting = 0; ///< Compression algorithm and level to apply
341 /// Adds a 8 byte little-endian xxhash3 checksum to the page payload. The buffer has to be large enough to
342 /// to store the additional 8 bytes.
343 bool fWriteChecksum = true;
344 /// If false, the output buffer must not point to the input page buffer, which would otherwise be an option
345 /// if the page is mappable and should not be compressed
346 bool fAllowAlias = false;
347 /// Location for sealed output. The memory buffer has to be large enough.
348 void *fBuffer = nullptr;
349 };
350
351 /// Seal a page using the provided info.
352 static RSealedPage SealPage(const RSealPageConfig &config);
353
354 /// Incorporate incremental changes to the model into the ntuple descriptor. This happens, e.g. if new fields were
355 /// added after the initial call to `RPageSink::Init(RNTupleModel &)`.
356 /// `firstEntry` specifies the global index for the first stored element in the added columns.
357 virtual void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry) = 0;
358 /// Adds an extra type information record to schema. The extra type information will be written to the
359 /// extension header. The information in the record will be merged with the existing information, e.g.
360 /// duplicate streamer info records will be removed. This method is called by the "on commit dataset" callback
361 /// registered by specific fields (e.g., streamer field) and during merging.
362 virtual void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo) = 0;
363
364 /// Commits a suppressed column for the current cluster. Can be called anytime before CommitCluster().
365 /// For any given column and cluster, there must be no calls to both CommitSuppressedColumn() and page commits.
366 virtual void CommitSuppressedColumn(ColumnHandle_t columnHandle) = 0;
367 /// Write a page to the storage. The column must have been added before.
368 virtual void CommitPage(ColumnHandle_t columnHandle, const RPage &page) = 0;
369 /// Write a preprocessed page to storage. The column must have been added before.
370 virtual void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) = 0;
371 /// Write a vector of preprocessed pages to storage. The corresponding columns must have been added before.
372 virtual void CommitSealedPageV(std::span<RPageStorage::RSealedPageGroup> ranges) = 0;
373 /// Stage the current cluster and create a new one for the following data.
374 /// Returns the object that must be passed to CommitStagedClusters to logically append the staged cluster to the
375 /// ntuple descriptor.
376 virtual RStagedCluster StageCluster(NTupleSize_t nNewEntries) = 0;
377 /// Commit staged clusters, logically appending them to the ntuple descriptor.
378 virtual void CommitStagedClusters(std::span<RStagedCluster> clusters) = 0;
379 /// Finalize the current cluster and create a new one for the following data.
380 /// Returns the number of bytes written to storage (excluding meta-data).
381 virtual std::uint64_t CommitCluster(NTupleSize_t nNewEntries)
382 {
383 RStagedCluster stagedClusters[] = {StageCluster(nNewEntries)};
384 CommitStagedClusters(stagedClusters);
385 return stagedClusters[0].fNBytesWritten;
386 }
387 /// Write out the page locations (page list envelope) for all the committed clusters since the last call of
388 /// CommitClusterGroup (or the beginning of writing).
389 virtual void CommitClusterGroup() = 0;
390
391 /// The registered callback is executed at the beginning of CommitDataset();
393 /// Run the registered callbacks and finalize the current cluster and the entrire data set.
394 void CommitDataset();
395
396 /// Get a new, empty page for the given column that can be filled with up to nElements;
397 /// nElements must be larger than zero.
398 virtual RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements);
399
400 /// An RAII wrapper used to synchronize a page sink. See GetSinkGuard().
402 std::mutex *fLock;
403
404 public:
405 explicit RSinkGuard(std::mutex *lock) : fLock(lock)
406 {
407 if (fLock != nullptr) {
408 fLock->lock();
409 }
410 }
411 RSinkGuard(const RSinkGuard &) = delete;
412 RSinkGuard &operator=(const RSinkGuard &) = delete;
413 RSinkGuard(RSinkGuard &&) = delete;
416 {
417 if (fLock != nullptr) {
418 fLock->unlock();
419 }
420 }
421 };
422
424 {
425 // By default, there is no lock and the guard does nothing.
426 return RSinkGuard(nullptr);
427 }
428}; // class RPageSink
429
430// clang-format off
431/**
432\class ROOT::Experimental::Internal::RPagePersistentSink
433\ingroup NTuple
434\brief Base class for a sink with a physical storage backend
435*/
436// clang-format on
438private:
439 /// Used to map the IDs of the descriptor to the physical IDs issued during header/footer serialization
441
442 /// Remembers the starting cluster id for the next cluster group
443 std::uint64_t fNextClusterInGroup = 0;
444 /// Used to calculate the number of entries in the current cluster
446 /// Keeps track of the number of elements in the currently open cluster. Indexed by column id.
447 std::vector<RClusterDescriptor::RColumnRange> fOpenColumnRanges;
448 /// Keeps track of the written pages in the currently open cluster. Indexed by column id.
449 std::vector<RClusterDescriptor::RPageRange> fOpenPageRanges;
450
451 /// Union of the streamer info records that are sent from streamer fields to the sink before committing the dataset.
453
454protected:
455 /// Set of optional features supported by the persistent sink
456 struct RFeatures {
457 bool fCanMergePages = false;
458 };
459
462
463 /// Default I/O performance counters that get registered in fMetrics
464 struct RCounters {
472 };
473 std::unique_ptr<RCounters> fCounters;
474
475 virtual void InitImpl(unsigned char *serializedHeader, std::uint32_t length) = 0;
476
477 virtual RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) = 0;
478 virtual RNTupleLocator
479 CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) = 0;
480 /// Vector commit of preprocessed pages. The `ranges` array specifies a range of sealed pages to be
481 /// committed for each column. The returned vector contains, in order, the RNTupleLocator for each
482 /// page on each range in `ranges`, i.e. the first N entries refer to the N pages in `ranges[0]`,
483 /// followed by M entries that refer to the M pages in `ranges[1]`, etc.
484 /// The mask allows to skip writing out certain pages. The vector has the size of all the pages.
485 /// For every `false` value in the mask, the corresponding locator is skipped (missing) in the output vector.
486 /// The default is to call `CommitSealedPageImpl` for each page; derived classes may provide an
487 /// optimized implementation though.
488 virtual std::vector<RNTupleLocator>
489 CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges, const std::vector<bool> &mask);
490 /// Returns the number of bytes written to storage (excluding metadata)
491 virtual std::uint64_t StageClusterImpl() = 0;
492 /// Returns the locator of the page list envelope of the given buffer that contains the serialized page list.
493 /// Typically, the implementation takes care of compressing and writing the provided buffer.
494 virtual RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) = 0;
495 virtual void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) = 0;
496
497 /// Enables the default set of metrics provided by RPageSink. `prefix` will be used as the prefix for
498 /// the counters registered in the internal RNTupleMetrics object.
499 /// This set of counters can be extended by a subclass by calling `fMetrics.MakeCounter<...>()`.
500 ///
501 /// A subclass using the default set of metrics is always responsible for updating the counters
502 /// appropriately, e.g. `fCounters->fNPageCommited.Inc()`
503 void EnableDefaultMetrics(const std::string &prefix);
504
505public:
506 RPagePersistentSink(std::string_view ntupleName, const RNTupleWriteOptions &options);
507
512 ~RPagePersistentSink() override;
513
514 /// Guess the concrete derived page source from the location
515 static std::unique_ptr<RPageSink> Create(std::string_view ntupleName, std::string_view location,
516 const RNTupleWriteOptions &options = RNTupleWriteOptions());
517
518 ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) final;
519
521
522 /// Updates the descriptor and calls InitImpl() that handles the backend-specific details (file, DAOS, etc.)
523 void InitImpl(RNTupleModel &model) final;
524 void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry) final;
525 void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo) final;
526
527 /// Initialize sink based on an existing descriptor and fill into the descriptor builder.
528 void InitFromDescriptor(const RNTupleDescriptor &descriptor);
529
530 void CommitSuppressedColumn(ColumnHandle_t columnHandle) final;
531 void CommitPage(ColumnHandle_t columnHandle, const RPage &page) final;
532 void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final;
533 void CommitSealedPageV(std::span<RPageStorage::RSealedPageGroup> ranges) final;
534 RStagedCluster StageCluster(NTupleSize_t nNewEntries) final;
535 void CommitStagedClusters(std::span<RStagedCluster> clusters) final;
536 void CommitClusterGroup() final;
537 void CommitDatasetImpl() final;
538}; // class RPagePersistentSink
539
540// clang-format off
541/**
542\class ROOT::Experimental::Internal::RPageSource
543\ingroup NTuple
544\brief Abstract interface to read data from an ntuple
545
546The page source is initialized with the columns of interest. Alias columns from projected fields are mapped to the
547corresponding physical columns. Pages from the columns of interest can then be mapped into memory.
548The page source also gives access to the ntuple's meta-data.
549*/
550// clang-format on
551class RPageSource : public RPageStorage {
552public:
553 /// Used in SetEntryRange / GetEntryRange
554 struct REntryRange {
556 NTupleSize_t fNEntries = 0;
557
558 /// Returns true if the given cluster has entries within the entry range
559 bool IntersectsWith(const RClusterDescriptor &clusterDesc) const;
560 };
561
562 /// An RAII wrapper used for the read-only access to `RPageSource::fDescriptor`. See `GetExclDescriptorGuard()``.
565 std::shared_mutex &fLock;
566
567 public:
568 RSharedDescriptorGuard(const RNTupleDescriptor &desc, std::shared_mutex &lock) : fDescriptor(desc), fLock(lock)
569 {
570 fLock.lock_shared();
571 }
576 ~RSharedDescriptorGuard() { fLock.unlock_shared(); }
577 const RNTupleDescriptor *operator->() const { return &fDescriptor; }
578 const RNTupleDescriptor &GetRef() const { return fDescriptor; }
579 };
580
581 /// An RAII wrapper used for the writable access to `RPageSource::fDescriptor`. See `GetSharedDescriptorGuard()`.
584 std::shared_mutex &fLock;
585
586 public:
587 RExclDescriptorGuard(RNTupleDescriptor &desc, std::shared_mutex &lock) : fDescriptor(desc), fLock(lock)
588 {
589 fLock.lock();
590 }
596 {
597 fDescriptor.IncGeneration();
598 fLock.unlock();
599 }
600 RNTupleDescriptor *operator->() const { return &fDescriptor; }
601 void MoveIn(RNTupleDescriptor &&desc) { fDescriptor = std::move(desc); }
602 };
603
604private:
606 mutable std::shared_mutex fDescriptorLock;
607 REntryRange fEntryRange; ///< Used by the cluster pool to prevent reading beyond the given range
608 bool fHasStructure = false; ///< Set to true once `LoadStructure()` is called
609 bool fIsAttached = false; ///< Set to true once `Attach()` is called
610
611 /// Remembers the last cluster id from which a page was requested
613 /// Clusters from where pages got preloaded in UnzipClusterImpl(), ordered by first entry number
614 /// of the clusters. If the last used cluster changes in LoadPage(), all unused pages from
615 /// previous clusters are evicted from the page pool.
616 std::map<NTupleSize_t, DescriptorId_t> fPreloadedClusters;
617
618 /// Does nothing if fLastUsedCluster == clusterId. Otherwise, updated fLastUsedCluster
619 /// and evict unused paged from the page pool of all previous clusters.
620 /// Must not be called when the descriptor guard is taken.
621 void UpdateLastUsedCluster(DescriptorId_t clusterId);
622
623protected:
624 /// Default I/O performance counters that get registered in `fMetrics`
625 struct RCounters {
643 };
644
645 /// Keeps track of the requested physical column IDs and their in-memory target type via a column element identifier.
646 /// When using alias columns (projected fields), physical columns may be requested multiple times.
648 public:
649 struct RColumnInfo {
651 std::size_t fRefCounter = 0;
652 };
653
654 private:
655 /// Maps physical column IDs to all the requested in-memory representations.
656 /// A pair of physical column ID and in-memory representation can be requested multiple times, which is
657 /// indicated by the reference counter.
658 /// We can only have a handful of possible in-memory representations for a given column,
659 /// so it is fine to search them linearly.
660 std::unordered_map<DescriptorId_t, std::vector<RColumnInfo>> fColumnInfos;
661
662 public:
663 void Insert(DescriptorId_t physicalColumnId, RColumnElementBase::RIdentifier elementId);
664 void Erase(DescriptorId_t physicalColumnId, RColumnElementBase::RIdentifier elementId);
665 RCluster::ColumnSet_t ToColumnSet() const;
666 bool HasColumnInfos(DescriptorId_t physicalColumnId) const { return fColumnInfos.count(physicalColumnId) > 0; }
667 const std::vector<RColumnInfo> &GetColumnInfos(DescriptorId_t physicalColumnId) const
668 {
669 return fColumnInfos.at(physicalColumnId);
670 }
671 };
672
673 /// Summarizes cluster-level information that are necessary to load a certain page.
674 /// Used by LoadPageImpl().
676 DescriptorId_t fClusterId = 0;
677 /// Location of the page on disk
679 /// The first element number of the page's column in the given cluster
680 std::uint64_t fColumnOffset = 0;
681 };
682
683 std::unique_ptr<RCounters> fCounters;
684
686 /// The active columns are implicitly defined by the model fields or views
688
689 /// Pages that are unzipped with IMT are staged into the page pool
691
692 virtual void LoadStructureImpl() = 0;
693 /// `LoadStructureImpl()` has been called before `AttachImpl()` is called
695 /// Returns a new, unattached page source for the same data set
696 virtual std::unique_ptr<RPageSource> CloneImpl() const = 0;
697 // Only called if a task scheduler is set. No-op be default.
698 virtual void UnzipClusterImpl(RCluster *cluster);
699 // Returns a page from storage if not found in the page pool. Should be able to handle zero page locators.
700 virtual RPageRef LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo,
701 ClusterSize_t::ValueType idxInCluster) = 0;
702
703 /// Prepare a page range read for the column set in `clusterKey`. Specifically, pages referencing the
704 /// `kTypePageZero` locator are filled in `pageZeroMap`; otherwise, `perPageFunc` is called for each page. This is
705 /// commonly used as part of `LoadClusters()` in derived classes.
706 void PrepareLoadCluster(
707 const RCluster::RKey &clusterKey, ROnDiskPageMap &pageZeroMap,
708 std::function<void(DescriptorId_t, NTupleSize_t, const RClusterDescriptor::RPageRange::RPageInfo &)> perPageFunc);
709
710 /// Enables the default set of metrics provided by RPageSource. `prefix` will be used as the prefix for
711 /// the counters registered in the internal RNTupleMetrics object.
712 /// A subclass using the default set of metrics is responsible for updating the counters
713 /// appropriately, e.g. `fCounters->fNRead.Inc()`
714 /// Alternatively, a subclass might provide its own RNTupleMetrics object by overriding the
715 /// `GetMetrics()` member function.
716 void EnableDefaultMetrics(const std::string &prefix);
717
718 /// Note that the underlying lock is not recursive. See `GetSharedDescriptorGuard()` for further information.
719 RExclDescriptorGuard GetExclDescriptorGuard() { return RExclDescriptorGuard(fDescriptor, fDescriptorLock); }
720
721public:
722 RPageSource(std::string_view ntupleName, const RNTupleReadOptions &fOptions);
723 RPageSource(const RPageSource &) = delete;
727 ~RPageSource() override;
728 /// Guess the concrete derived page source from the file name (location)
729 static std::unique_ptr<RPageSource> Create(std::string_view ntupleName, std::string_view location,
730 const RNTupleReadOptions &options = RNTupleReadOptions());
731 /// Open the same storage multiple time, e.g. for reading in multiple threads.
732 /// If the source is already attached, the clone will be attached, too. The clone will use, however,
733 /// it's own connection to the underlying storage (e.g., file descriptor, XRootD handle, etc.)
734 std::unique_ptr<RPageSource> Clone() const;
735
736 /// Helper for unstreaming a page. This is commonly used in derived, concrete page sources. The implementation
737 /// currently always makes a memory copy, even if the sealed page is uncompressed and in the final memory layout.
738 /// The optimization of directly mapping pages is left to the concrete page source implementations.
739 RResult<RPage> static UnsealPage(const RSealedPage &sealedPage, const RColumnElementBase &element,
740 RPageAllocator &pageAlloc);
741
743 const RNTupleReadOptions &GetReadOptions() const { return fOptions; }
744
745 /// Takes the read lock for the descriptor. Multiple threads can take the lock concurrently.
746 /// The underlying `std::shared_mutex`, however, is neither read nor write recursive:
747 /// within one thread, only one lock (shared or exclusive) must be acquired at the same time. This requires special
748 /// care in sections protected by `GetSharedDescriptorGuard()` and `GetExclDescriptorGuard()` especially to avoid
749 /// that the locks are acquired indirectly (e.g. by a call to `GetNEntries()`). As a general guideline, no other
750 /// method of the page source should be called (directly or indirectly) in a guarded section.
752 {
753 return RSharedDescriptorGuard(fDescriptor, fDescriptorLock);
754 }
755
756 ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) override;
757 void DropColumn(ColumnHandle_t columnHandle) override;
758
759 /// Loads header and footer without decompressing or deserializing them. This can be used to asynchronously open
760 /// a file in the background. The method is idempotent and it is called as a first step in `Attach()`.
761 /// Pages sources may or may not make use of splitting loading and processing meta-data.
762 /// Therefore, `LoadStructure()` may do nothing and defer loading the meta-data to `Attach()`.
763 void LoadStructure();
764 /// Open the physical storage container and deserialize header and footer
765 void Attach();
766 NTupleSize_t GetNEntries();
767 NTupleSize_t GetNElements(ColumnHandle_t columnHandle);
768
769 /// Promise to only read from the given entry range. If set, prevents the cluster pool from reading-ahead beyond
770 /// the given range. The range needs to be within `[0, GetNEntries())`.
771 void SetEntryRange(const REntryRange &range);
772 REntryRange GetEntryRange() const { return fEntryRange; }
773
774 /// Allocates and fills a page that contains the index-th element. The default implementation searches
775 /// the page and calls LoadPageImpl(). Returns a default-constructed RPage for suppressed columns.
776 virtual RPageRef LoadPage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex);
777 /// Another version of `LoadPage` that allows to specify cluster-relative indexes.
778 /// Returns a default-constructed RPage for suppressed columns.
779 virtual RPageRef LoadPage(ColumnHandle_t columnHandle, RClusterIndex clusterIndex);
780
781 /// Read the packed and compressed bytes of a page into the memory buffer provided by `sealedPage`. The sealed page
782 /// can be used subsequently in a call to `RPageSink::CommitSealedPage`.
783 /// The `fSize` and `fNElements` member of the sealedPage parameters are always set. If `sealedPage.fBuffer` is
784 /// `nullptr`, no data will be copied but the returned size information can be used by the caller to allocate a large
785 /// enough buffer and call `LoadSealedPage` again.
786 virtual void
787 LoadSealedPage(DescriptorId_t physicalColumnId, RClusterIndex clusterIndex, RSealedPage &sealedPage) = 0;
788
789 /// Populates all the pages of the given cluster ids and columns; it is possible that some columns do not
790 /// contain any pages. The page source may load more columns than the minimal necessary set from `columns`.
791 /// To indicate which columns have been loaded, `LoadClusters()`` must mark them with `SetColumnAvailable()`.
792 /// That includes the ones from the `columns` that don't have pages; otherwise subsequent requests
793 /// for the cluster would assume an incomplete cluster and trigger loading again.
794 /// `LoadClusters()` is typically called from the I/O thread of a cluster pool, i.e. the method runs
795 /// concurrently to other methods of the page source.
796 virtual std::vector<std::unique_ptr<RCluster>> LoadClusters(std::span<RCluster::RKey> clusterKeys) = 0;
797
798 /// Parallel decompression and unpacking of the pages in the given cluster. The unzipped pages are supposed
799 /// to be preloaded in a page pool attached to the source. The method is triggered by the cluster pool's
800 /// unzip thread. It is an optional optimization, the method can safely do nothing. In particular, the
801 /// actual implementation will only run if a task scheduler is set. In practice, a task scheduler is set
802 /// if implicit multi-threading is turned on.
803 void UnzipCluster(RCluster *cluster);
804
805 // TODO(gparolini): for symmetry with SealPage(), we should either make this private or SealPage() public.
806 RResult<RPage> UnsealPage(const RSealedPage &sealedPage, const RColumnElementBase &element);
807}; // class RPageSource
808
809} // namespace Internal
810
811} // namespace Experimental
812} // namespace ROOT
813
814#endif
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:290
#define d(i)
Definition RSha256.hxx:102
#define b(i)
Definition RSha256.hxx:100
#define e(i)
Definition RSha256.hxx:103
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t mask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
char name[80]
Definition TGX11.cxx:110
A thread-safe integral performance counter.
A metric element that computes its floating point value from other counters.
A collection of Counter objects with a name, a unit, and a description.
An either thread-safe or non thread safe counter for CPU ticks.
An in-memory subset of the packed and compressed pages of a cluster.
Definition RCluster.hxx:152
std::unordered_set< DescriptorId_t > ColumnSet_t
Definition RCluster.hxx:154
A column element encapsulates the translation between basic C++ types and their column representation...
A column is a storage-backed array of a simple, fixed-size type, from which pages can be mapped into ...
Definition RColumn.hxx:40
A helper class for piece-wise construction of an RNTupleDescriptor.
The serialization context is used for the piecewise serialization of a descriptor.
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
A memory region that contains packed and compressed pages.
Definition RCluster.hxx:103
Abstract interface to allocate and release pages.
Base class for a sink with a physical storage backend.
RPagePersistentSink(const RPagePersistentSink &)=delete
ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) final
Register a new column.
RPagePersistentSink(RPagePersistentSink &&)=default
RStagedCluster StageCluster(NTupleSize_t nNewEntries) final
Stage the current cluster and create a new one for the following data.
std::uint64_t fNextClusterInGroup
Remembers the starting cluster id for the next cluster group.
virtual std::uint64_t StageClusterImpl()=0
Returns the number of bytes written to storage (excluding metadata)
RPagePersistentSink & operator=(RPagePersistentSink &&)=default
virtual void InitImpl(unsigned char *serializedHeader, std::uint32_t length)=0
virtual std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask)
Vector commit of preprocessed pages.
RNTupleSerializer::RContext fSerializationContext
Used to map the IDs of the descriptor to the physical IDs issued during header/footer serialization.
virtual RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page)=0
void InitFromDescriptor(const RNTupleDescriptor &descriptor)
Initialize sink based on an existing descriptor and fill into the descriptor builder.
virtual RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length)=0
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
NTupleSize_t fPrevClusterNEntries
Used to calculate the number of entries in the current cluster.
std::vector< RClusterDescriptor::RPageRange > fOpenPageRanges
Keeps track of the written pages in the currently open cluster. Indexed by column id.
const RNTupleDescriptor & GetDescriptor() const final
Return the RNTupleDescriptor being constructed.
void CommitPage(ColumnHandle_t columnHandle, const RPage &page) final
Write a page to the storage. The column must have been added before.
virtual void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length)=0
RPagePersistentSink & operator=(const RPagePersistentSink &)=delete
static std::unique_ptr< RPageSink > Create(std::string_view ntupleName, std::string_view location, const RNTupleWriteOptions &options=RNTupleWriteOptions())
Guess the concrete derived page source from the location.
Internal::RNTupleDescriptorBuilder fDescriptorBuilder
RNTupleSerializer::StreamerInfoMap_t fStreamerInfos
Union of the streamer info records that are sent from streamer fields to the sink before committing t...
void CommitClusterGroup() final
Write out the page locations (page list envelope) for all the committed clusters since the last call ...
std::vector< RClusterDescriptor::RColumnRange > fOpenColumnRanges
Keeps track of the number of elements in the currently open cluster. Indexed by column id.
void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry) final
Incorporate incremental changes to the model into the ntuple descriptor.
void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
Write a preprocessed page to storage. The column must have been added before.
void CommitSealedPageV(std::span< RPageStorage::RSealedPageGroup > ranges) final
Write a vector of preprocessed pages to storage. The corresponding columns must have been added befor...
void CommitSuppressedColumn(ColumnHandle_t columnHandle) final
Commits a suppressed column for the current cluster.
void CommitStagedClusters(std::span< RStagedCluster > clusters) final
Commit staged clusters, logically appending them to the ntuple descriptor.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSink.
void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo) final
Adds an extra type information record to schema.
virtual RNTupleLocator CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage)=0
A thread-safe cache of pages loaded from the page source.
Definition RPagePool.hxx:48
Reference to a page stored in the page pool.
An RAII wrapper used to synchronize a page sink. See GetSinkGuard().
RSinkGuard & operator=(const RSinkGuard &)=delete
RSinkGuard & operator=(RSinkGuard &&)=delete
Abstract interface to write data into an ntuple.
std::vector< unsigned char > fSealPageBuffer
Used as destination buffer in the simple SealPage overload.
std::vector< Callback_t > fOnDatasetCommitCallbacks
RPageSink & operator=(RPageSink &&)=default
virtual RStagedCluster StageCluster(NTupleSize_t nNewEntries)=0
Stage the current cluster and create a new one for the following data.
bool fIsInitialized
Flag if sink was initialized.
virtual void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo)=0
Adds an extra type information record to schema.
void CommitDataset()
Run the registered callbacks and finalize the current cluster and the entrire data set.
virtual void CommitSuppressedColumn(ColumnHandle_t columnHandle)=0
Commits a suppressed column for the current cluster.
virtual const RNTupleDescriptor & GetDescriptor() const =0
Return the RNTupleDescriptor being constructed.
void Init(RNTupleModel &model)
Physically creates the storage container to hold the ntuple (e.g., a keys a TFile or an S3 bucket) In...
virtual RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements)
Get a new, empty page for the given column that can be filled with up to nElements; nElements must be...
RWritePageMemoryManager fWritePageMemoryManager
Used in ReservePage to maintain the page buffer budget.
virtual void CommitPage(ColumnHandle_t columnHandle, const RPage &page)=0
Write a page to the storage. The column must have been added before.
const RNTupleWriteOptions & GetWriteOptions() const
Returns the sink's write options.
RPageSink & operator=(const RPageSink &)=delete
virtual void CommitClusterGroup()=0
Write out the page locations (page list envelope) for all the committed clusters since the last call ...
virtual std::uint64_t CommitCluster(NTupleSize_t nNewEntries)
Finalize the current cluster and create a new one for the following data.
void RegisterOnCommitDatasetCallback(Callback_t callback)
The registered callback is executed at the beginning of CommitDataset();.
RPageSink(const RPageSink &)=delete
std::function< void(RPageSink &)> Callback_t
virtual void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage)=0
Write a preprocessed page to storage. The column must have been added before.
virtual void CommitStagedClusters(std::span< RStagedCluster > clusters)=0
Commit staged clusters, logically appending them to the ntuple descriptor.
virtual void InitImpl(RNTupleModel &model)=0
void DropColumn(ColumnHandle_t) final
Unregisters a column.
EPageStorageType GetType() final
Whether the concrete implementation is a sink or a source.
std::unique_ptr< RNTupleCompressor > fCompressor
Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.
virtual void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry)=0
Incorporate incremental changes to the model into the ntuple descriptor.
virtual void CommitSealedPageV(std::span< RPageStorage::RSealedPageGroup > ranges)=0
Write a vector of preprocessed pages to storage. The corresponding columns must have been added befor...
std::unique_ptr< RNTupleWriteOptions > fOptions
RSealedPage SealPage(const RPage &page, const RColumnElementBase &element)
Helper for streaming a page.
Keeps track of the requested physical column IDs and their in-memory target type via a column element...
bool HasColumnInfos(DescriptorId_t physicalColumnId) const
const std::vector< RColumnInfo > & GetColumnInfos(DescriptorId_t physicalColumnId) const
std::unordered_map< DescriptorId_t, std::vector< RColumnInfo > > fColumnInfos
Maps physical column IDs to all the requested in-memory representations.
An RAII wrapper used for the writable access to RPageSource::fDescriptor. See GetSharedDescriptorGuar...
RExclDescriptorGuard(RNTupleDescriptor &desc, std::shared_mutex &lock)
RExclDescriptorGuard(const RExclDescriptorGuard &)=delete
RExclDescriptorGuard & operator=(RExclDescriptorGuard &&)=delete
RExclDescriptorGuard & operator=(const RExclDescriptorGuard &)=delete
An RAII wrapper used for the read-only access to RPageSource::fDescriptor. See GetExclDescriptorGuard...
RSharedDescriptorGuard & operator=(RSharedDescriptorGuard &&)=delete
RSharedDescriptorGuard(const RSharedDescriptorGuard &)=delete
RSharedDescriptorGuard(const RNTupleDescriptor &desc, std::shared_mutex &lock)
RSharedDescriptorGuard & operator=(const RSharedDescriptorGuard &)=delete
Abstract interface to read data from an ntuple.
virtual void LoadSealedPage(DescriptorId_t physicalColumnId, RClusterIndex clusterIndex, RSealedPage &sealedPage)=0
Read the packed and compressed bytes of a page into the memory buffer provided by sealedPage.
RPagePool fPagePool
Pages that are unzipped with IMT are staged into the page pool.
EPageStorageType GetType() final
Whether the concrete implementation is a sink or a source.
RPageSource(const RPageSource &)=delete
RPageSource & operator=(RPageSource &&)=delete
std::unique_ptr< RCounters > fCounters
RExclDescriptorGuard GetExclDescriptorGuard()
Note that the underlying lock is not recursive. See GetSharedDescriptorGuard() for further informatio...
RActivePhysicalColumns fActivePhysicalColumns
The active columns are implicitly defined by the model fields or views.
RPageSource & operator=(const RPageSource &)=delete
virtual RNTupleDescriptor AttachImpl()=0
LoadStructureImpl() has been called before AttachImpl() is called
const RNTupleReadOptions & GetReadOptions() const
virtual std::vector< std::unique_ptr< RCluster > > LoadClusters(std::span< RCluster::RKey > clusterKeys)=0
Populates all the pages of the given cluster ids and columns; it is possible that some columns do not...
REntryRange fEntryRange
Used by the cluster pool to prevent reading beyond the given range.
virtual RPageRef LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ClusterSize_t::ValueType idxInCluster)=0
virtual std::unique_ptr< RPageSource > CloneImpl() const =0
Returns a new, unattached page source for the same data set.
std::map< NTupleSize_t, DescriptorId_t > fPreloadedClusters
Clusters from where pages got preloaded in UnzipClusterImpl(), ordered by first entry number of the c...
const RSharedDescriptorGuard GetSharedDescriptorGuard() const
Takes the read lock for the descriptor.
The interface of a task scheduler to schedule page (de)compression tasks.
virtual void Wait()=0
Blocks until all scheduled tasks finished.
virtual void AddTask(const std::function< void(void)> &taskFunc)=0
Take a callable that represents a task.
Common functionality of an ntuple storage for both reading and writing.
std::deque< RSealedPage > SealedPageSequence_t
std::unique_ptr< RPageAllocator > fPageAllocator
For the time being, we will use the heap allocator for all sources and sinks. This may change in the ...
virtual ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column)=0
Register a new column.
static constexpr std::size_t kNBytesPageChecksum
The page checksum is a 64bit xxhash3.
virtual void DropColumn(ColumnHandle_t columnHandle)=0
Unregisters a column.
virtual Detail::RNTupleMetrics & GetMetrics()
Returns the default metrics object.
const std::string & GetNTupleName() const
Returns the NTuple name.
virtual EPageStorageType GetType()=0
Whether the concrete implementation is a sink or a source.
RPageStorage & operator=(RPageStorage &&other)=default
RPageStorage & operator=(const RPageStorage &other)=delete
RPageStorage(const RPageStorage &other)=delete
RPageStorage(RPageStorage &&other)=default
void SetTaskScheduler(RTaskScheduler *taskScheduler)
RColumnHandle ColumnHandle_t
The column handle identifies a column with the current open page storage.
ColumnId_t GetColumnId(ColumnHandle_t columnHandle) const
A page is a slice of a column that is mapped into memory.
Definition RPage.hxx:47
Helper to maintain a memory budget for the write pages of a set of columns.
bool TryUpdate(RColumn &column, std::size_t newWritePageSize)
Try to register the new write page size for the given column.
std::size_t fCurrentAllocatedBytes
Sum of all the write page sizes (their capacity) of the columns in fColumnsSortedByPageSize
bool TryEvict(std::size_t targetAvailableSize, std::size_t pageSizeLimit)
Flush columns in order of allocated write page size until the sum of all write page allocations leave...
std::size_t fMaxAllocatedBytes
Maximum allowed value for fCurrentAllocatedBytes, set from RNTupleWriteOptions::fPageBufferBudget.
std::set< RColumnInfo, std::greater< RColumnInfo > > fColumnsSortedByPageSize
All columns that called ReservePage() (hence TryUpdate()) at least once, sorted by their current writ...
Records the partition of data into pages for a particular column in a particular cluster.
Meta-data for a set of ntuple clusters.
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
Field specific extra type information from the header / extenstion header.
The on-storage meta-data of an ntuple.
The RNTupleModel encapulates the schema of an ntuple.
Common user-tunable settings for reading ntuples.
Common user-tunable settings for storing ntuples.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:194
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
constexpr NTupleSize_t kInvalidNTupleIndex
std::int64_t ColumnId_t
Uniquely identifies a physical column within the scope of the current process, used to tag pages.
constexpr ClusterSize_t kInvalidClusterIndex(std::uint64_t(-1))
constexpr DescriptorId_t kInvalidDescriptorId
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
The identifiers that specifies the content of a (partial) cluster.
Definition RCluster.hxx:156
Every concrete RColumnElement type is identified by its on-disk type (column type) and the in-memory ...
The incremental changes to a RNTupleModel
Default I/O performance counters that get registered in fMetrics.
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuZip
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuWrite
Set of optional features supported by the persistent sink.
const RColumnElementBase * fElement
Corresponds to the page's elements, for size calculation etc.
void * fBuffer
Location for sealed output. The memory buffer has to be large enough.
bool fAllowAlias
If false, the output buffer must not point to the input page buffer, which would otherwise be an opti...
int fCompressionSetting
Compression algorithm and level to apply.
bool fWriteChecksum
Adds a 8 byte little-endian xxhash3 checksum to the page payload.
Cluster that was staged, but not yet logically appended to the RNTuple.
Summarizes cluster-level information that are necessary to load a certain page.
RClusterDescriptor::RPageRange::RPageInfoExtended fPageInfo
Location of the page on disk.
Default I/O performance counters that get registered in fMetrics
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuUnzip
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuRead
A range of sealed pages referring to the same column that can be used for vector commit.
RSealedPageGroup(DescriptorId_t d, SealedPageSequence_t::const_iterator b, SealedPageSequence_t::const_iterator e)
A sealed page contains the bytes of a page as written to storage (packed & compressed).
RResult< std::uint64_t > GetChecksum() const
Returns a failure if the sealed page has no checksum.
bool fHasChecksum
If set, the last 8 bytes of the buffer are the xxhash of the rest of the buffer.
std::size_t fBufferSize
Size of the page payload and the trailing checksum (if available)
RSealedPage(const void *buffer, std::size_t bufferSize, std::uint32_t nElements, bool hasChecksum=false)
RSealedPage & operator=(const RSealedPage &other)=default
RSealedPage & operator=(RSealedPage &&other)=default
We do not need to store the element size / uncompressed page size because we know to which column the...
Wrap the integer in a struct in order to avoid template specialization clash with std::uint64_t.
Generic information about the physical location of data.