Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RMiniFile.cxx
Go to the documentation of this file.
1/// \file RMiniFile.cxx
2/// \ingroup NTuple
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-12-22
5
6/*************************************************************************
7 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
8 * All rights reserved. *
9 * *
10 * For the licensing terms see $ROOTSYS/LICENSE. *
11 * For the list of contributors see $ROOTSYS/README/CREDITS. *
12 *************************************************************************/
13
14#include "Rtypes.h"
15#include <ROOT/RConfig.hxx>
16#include <ROOT/RError.hxx>
17#include <ROOT/RMiniFile.hxx>
18#include <ROOT/RRawFile.hxx>
19#include <ROOT/RNTupleUtils.hxx>
20#include <ROOT/RNTupleZip.hxx>
23#include <ROOT/RFile.hxx>
24
25#include <Byteswap.h>
26#include <TBufferFile.h>
27#include <TDirectory.h>
28#include <TError.h>
29#include <TFile.h>
30#include <TKey.h>
31#include <TObjString.h>
32#include <TUUID.h>
33#include <TStreamerInfo.h>
34
35#include <xxhash.h>
36
37#include <algorithm>
38#include <cassert>
39#include <cerrno>
40#include <cstdio>
41#include <cstring>
42#include <memory>
43#include <string>
44#include <chrono>
45
46#ifdef R__LINUX
47#include <fcntl.h>
48#endif
49
50#ifndef R__LITTLE_ENDIAN
51#ifdef R__BYTESWAP
52// `R__BYTESWAP` is defined in RConfig.hxx for little-endian architectures; undefined otherwise
53#define R__LITTLE_ENDIAN 1
54#else
55#define R__LITTLE_ENDIAN 0
56#endif
57#endif /* R__LITTLE_ENDIAN */
58
63
64namespace {
65
66// The following types are used to read and write the TFile binary format
67
68/// Big-endian 16-bit unsigned integer
69class RUInt16BE {
70private:
71 std::uint16_t fValBE = 0;
72 static std::uint16_t Swap(std::uint16_t val)
73 {
74#if R__LITTLE_ENDIAN == 1
75 return RByteSwap<sizeof(val)>::bswap(val);
76#else
77 return val;
78#endif
79 }
80
81public:
82 RUInt16BE() = default;
83 explicit RUInt16BE(const std::uint16_t val) : fValBE(Swap(val)) {}
84 operator std::uint16_t() const { return Swap(fValBE); }
85 RUInt16BE &operator=(const std::uint16_t val)
86 {
87 fValBE = Swap(val);
88 return *this;
89 }
90};
91
92/// Big-endian 32-bit unsigned integer
93class RUInt32BE {
94private:
95 std::uint32_t fValBE = 0;
96 static std::uint32_t Swap(std::uint32_t val)
97 {
98#if R__LITTLE_ENDIAN == 1
99 return RByteSwap<sizeof(val)>::bswap(val);
100#else
101 return val;
102#endif
103 }
104
105public:
106 RUInt32BE() = default;
107 explicit RUInt32BE(const std::uint32_t val) : fValBE(Swap(val)) {}
108 operator std::uint32_t() const { return Swap(fValBE); }
109 RUInt32BE &operator=(const std::uint32_t val)
110 {
111 fValBE = Swap(val);
112 return *this;
113 }
114};
115
116/// Big-endian 32-bit signed integer
117class RInt32BE {
118private:
119 std::int32_t fValBE = 0;
120 static std::int32_t Swap(std::int32_t val)
121 {
122#if R__LITTLE_ENDIAN == 1
123 return RByteSwap<sizeof(val)>::bswap(val);
124#else
125 return val;
126#endif
127 }
128
129public:
130 RInt32BE() = default;
131 explicit RInt32BE(const std::int32_t val) : fValBE(Swap(val)) {}
132 operator std::int32_t() const { return Swap(fValBE); }
133 RInt32BE &operator=(const std::int32_t val)
134 {
135 fValBE = Swap(val);
136 return *this;
137 }
138};
139
140/// Big-endian 64-bit unsigned integer
141class RUInt64BE {
142private:
143 std::uint64_t fValBE = 0;
144 static std::uint64_t Swap(std::uint64_t val)
145 {
146#if R__LITTLE_ENDIAN == 1
147 return RByteSwap<sizeof(val)>::bswap(val);
148#else
149 return val;
150#endif
151 }
152
153public:
154 RUInt64BE() = default;
155 explicit RUInt64BE(const std::uint64_t val) : fValBE(Swap(val)) {}
156 operator std::uint64_t() const { return Swap(fValBE); }
157 RUInt64BE &operator=(const std::uint64_t val)
158 {
159 fValBE = Swap(val);
160 return *this;
161 }
162};
163
164#pragma pack(push, 1)
165/// A name (type, identifies, ...) in the TFile binary format
166struct RTFString {
167 unsigned char fLName{0};
168 char fData[255];
169 RTFString() = default;
170 RTFString(const std::string &str)
171 {
172 // The length of strings with 255 characters and longer are encoded with a 32-bit integer following the first
173 // byte. This is currently not handled.
174 R__ASSERT(str.length() < 255);
175 fLName = static_cast<unsigned char>(str.length());
176 memcpy(fData, str.data(), fLName);
177 }
178 std::size_t GetSize() const
179 {
180 // A length of 255 is special and means that the first byte is followed by a 32-bit integer with the actual
181 // length.
182 R__ASSERT(fLName != 255);
183 return 1 + fLName;
184 }
185};
186
187/// The timestamp format used in TFile; the default constructor initializes with the current time
188struct RTFDatetime {
189 RUInt32BE fDatetime;
190 RTFDatetime()
191 {
192 auto now = std::chrono::system_clock::now();
193 auto tt = std::chrono::system_clock::to_time_t(now);
194 auto tm = *localtime(&tt);
195 fDatetime = (tm.tm_year + 1900 - 1995) << 26 | (tm.tm_mon + 1) << 22 | tm.tm_mday << 17 | tm.tm_hour << 12 |
196 tm.tm_min << 6 | tm.tm_sec;
197 }
198 explicit RTFDatetime(RUInt32BE val) : fDatetime(val) {}
199};
200
201/// The key part of a TFile record excluding the class, object, and title names
202struct RTFKey {
203 static constexpr unsigned kBigKeyVersion = 1000;
204
205 RInt32BE fNbytes{0};
206 RUInt16BE fVersion{4};
207 RUInt32BE fObjLen{0};
208 RTFDatetime fDatetime;
209 RUInt16BE fKeyLen{0};
210 RUInt16BE fCycle{1};
211 union {
212 struct {
213 RUInt32BE fSeekKey{0};
214 RUInt32BE fSeekPdir{0};
215 } fInfoShort;
216 struct {
217 RUInt64BE fSeekKey{0};
218 RUInt64BE fSeekPdir{0};
219 } fInfoLong;
220 };
221
222 RTFKey() : fInfoLong() {}
223 RTFKey(std::uint64_t seekKey, std::uint64_t seekPdir, const RTFString &clName, const RTFString &objName,
224 const RTFString &titleName, std::size_t szObjInMem, std::size_t szObjOnDisk = 0)
225 {
226 R__ASSERT(szObjInMem <= std::numeric_limits<std::uint32_t>::max());
227 R__ASSERT(szObjOnDisk <= std::numeric_limits<std::uint32_t>::max());
228 // For writing, we alywas produce "big" keys with 64-bit SeekKey and SeekPdir.
229 fVersion = fVersion + kBigKeyVersion;
230 fObjLen = szObjInMem;
231 fKeyLen = static_cast<RUInt16BE>(GetHeaderSize() + clName.GetSize() + objName.GetSize() + titleName.GetSize());
232 fInfoLong.fSeekKey = seekKey;
233 fInfoLong.fSeekPdir = seekPdir;
234 // Depends on fKeyLen being set
235 fNbytes = fKeyLen + ((szObjOnDisk == 0) ? szObjInMem : szObjOnDisk);
236 }
237
238 std::uint32_t GetSize() const
239 {
240 // Negative size indicates a gap in the file
241 if (fNbytes < 0)
242 return -fNbytes;
243 return fNbytes;
244 }
245
246 std::uint32_t GetHeaderSize() const
247 {
248 if (fVersion >= kBigKeyVersion)
249 return 18 + sizeof(fInfoLong);
250 return 18 + sizeof(fInfoShort);
251 }
252
253 std::uint64_t GetSeekKey() const
254 {
255 if (fVersion >= kBigKeyVersion)
256 return fInfoLong.fSeekKey;
257 return fInfoShort.fSeekKey;
258 }
259};
260
261/// The TFile global header
262struct RTFHeader {
263 static constexpr unsigned kBEGIN = 100;
264 static constexpr unsigned kBigHeaderVersion = 1000000;
265
266 char fMagic[4]{'r', 'o', 'o', 't'};
267 RUInt32BE fVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
268 (ROOT_VERSION_CODE & 0xFF)};
269 RUInt32BE fBEGIN{kBEGIN};
270 union {
271 struct {
272 RUInt32BE fEND{0};
273 RUInt32BE fSeekFree{0};
274 RUInt32BE fNbytesFree{0};
275 RUInt32BE fNfree{1};
276 RUInt32BE fNbytesName{0};
277 unsigned char fUnits{4};
278 RUInt32BE fCompress{0};
279 RUInt32BE fSeekInfo{0};
280 RUInt32BE fNbytesInfo{0};
281 } fInfoShort;
282 struct {
283 RUInt64BE fEND{0};
284 RUInt64BE fSeekFree{0};
285 RUInt32BE fNbytesFree{0};
286 RUInt32BE fNfree{1};
287 RUInt32BE fNbytesName{0};
288 unsigned char fUnits{8};
289 RUInt32BE fCompress{0};
290 RUInt64BE fSeekInfo{0};
291 RUInt32BE fNbytesInfo{0};
292 } fInfoLong;
293 };
294
295 RTFHeader() : fInfoShort() {}
296 RTFHeader(int compression) : fInfoShort() { fInfoShort.fCompress = compression; }
297
298 void SetBigFile()
299 {
300 if (fVersion >= kBigHeaderVersion)
301 return;
302
303 // clang-format off
304 std::uint32_t end = fInfoShort.fEND;
305 std::uint32_t seekFree = fInfoShort.fSeekFree;
306 std::uint32_t nbytesFree = fInfoShort.fNbytesFree;
307 std::uint32_t nFree = fInfoShort.fNfree;
308 std::uint32_t nbytesName = fInfoShort.fNbytesName;
309 std::uint32_t compress = fInfoShort.fCompress;
310 std::uint32_t seekInfo = fInfoShort.fSeekInfo;
311 std::uint32_t nbytesInfo = fInfoShort.fNbytesInfo;
312 fInfoLong.fEND = end;
313 fInfoLong.fSeekFree = seekFree;
314 fInfoLong.fNbytesFree = nbytesFree;
315 fInfoLong.fNfree = nFree;
316 fInfoLong.fNbytesName = nbytesName;
317 fInfoLong.fUnits = 8;
318 fInfoLong.fCompress = compress;
319 fInfoLong.fSeekInfo = seekInfo;
320 fInfoLong.fNbytesInfo = nbytesInfo;
321 fVersion = fVersion + kBigHeaderVersion;
322 // clang-format on
323 }
324
325 bool IsBigFile(std::uint64_t offset = 0) const
326 {
327 return (fVersion >= kBigHeaderVersion) ||
328 (offset > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max()));
329 }
330
331 std::uint32_t GetSize() const
332 {
333 std::uint32_t sizeHead = sizeof(fMagic) + sizeof(fVersion) + sizeof(fBEGIN);
334 if (IsBigFile())
335 return sizeHead + sizeof(fInfoLong);
336 return sizeHead + sizeof(fInfoShort);
337 }
338
339 std::uint64_t GetEnd() const
340 {
341 if (IsBigFile())
342 return fInfoLong.fEND;
343 return fInfoShort.fEND;
344 }
345
346 void SetEnd(std::uint64_t value)
347 {
348 if (IsBigFile(value)) {
349 SetBigFile();
350 fInfoLong.fEND = value;
351 } else {
352 fInfoShort.fEND = value;
353 }
354 }
355
356 std::uint64_t GetSeekFree() const
357 {
358 if (IsBigFile())
359 return fInfoLong.fSeekFree;
360 return fInfoShort.fSeekFree;
361 }
362
363 void SetSeekFree(std::uint64_t value)
364 {
365 if (IsBigFile(value)) {
366 SetBigFile();
367 fInfoLong.fSeekFree = value;
368 } else {
369 fInfoShort.fSeekFree = value;
370 }
371 }
372
373 void SetNbytesFree(std::uint32_t value)
374 {
375 if (IsBigFile()) {
376 fInfoLong.fNbytesFree = value;
377 } else {
378 fInfoShort.fNbytesFree = value;
379 }
380 }
381
382 void SetNbytesName(std::uint32_t value)
383 {
384 if (IsBigFile()) {
385 fInfoLong.fNbytesName = value;
386 } else {
387 fInfoShort.fNbytesName = value;
388 }
389 }
390
391 std::uint64_t GetSeekInfo() const
392 {
393 if (IsBigFile())
394 return fInfoLong.fSeekInfo;
395 return fInfoShort.fSeekInfo;
396 }
397
398 void SetSeekInfo(std::uint64_t value)
399 {
400 if (IsBigFile(value)) {
401 SetBigFile();
402 fInfoLong.fSeekInfo = value;
403 } else {
404 fInfoShort.fSeekInfo = value;
405 }
406 }
407
408 std::uint64_t GetNbytesInfo() const
409 {
410 if (IsBigFile())
411 return fInfoLong.fNbytesInfo;
412 return fInfoShort.fNbytesInfo;
413 }
414
415 void SetNbytesInfo(std::uint32_t value)
416 {
417 if (IsBigFile()) {
418 fInfoLong.fNbytesInfo = value;
419 } else {
420 fInfoShort.fNbytesInfo = value;
421 }
422 }
423
424 void SetCompression(std::uint32_t value)
425 {
426 if (IsBigFile()) {
427 fInfoLong.fCompress = value;
428 } else {
429 fInfoShort.fCompress = value;
430 }
431 }
432};
433
434/// A reference to an unused byte-range in a TFile
435struct RTFFreeEntry {
436 static constexpr unsigned kBigFreeEntryVersion = 1000;
437
438 RUInt16BE fVersion{1};
439 union {
440 struct {
441 RUInt32BE fFirst{0};
442 RUInt32BE fLast{0};
443 } fInfoShort;
444 struct {
445 RUInt64BE fFirst{0};
446 RUInt64BE fLast{0};
447 } fInfoLong;
448 };
449
450 RTFFreeEntry() : fInfoShort() {}
451 void Set(std::uint64_t first, std::uint64_t last)
452 {
453 if (last > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
454 fVersion = fVersion + kBigFreeEntryVersion;
455 fInfoLong.fFirst = first;
456 fInfoLong.fLast = last;
457 } else {
458 fInfoShort.fFirst = first;
459 fInfoShort.fLast = last;
460 }
461 }
462 std::uint32_t GetSize() { return (fVersion >= kBigFreeEntryVersion) ? 18 : 10; }
463};
464
465/// The header of the directory key index
466struct RTFKeyList {
467 RUInt32BE fNKeys;
468 std::uint32_t GetSize() const { return sizeof(RTFKeyList); }
469 explicit RTFKeyList(std::uint32_t nKeys) : fNKeys(nKeys) {}
470};
471
472/// A streamed TDirectory (TFile) object
473struct RTFDirectory {
474 static constexpr unsigned kBigFileVersion = 1000;
475
476 RUInt16BE fClassVersion{5};
477 RTFDatetime fDateC;
478 RTFDatetime fDateM;
479 RUInt32BE fNBytesKeys{0};
480 RUInt32BE fNBytesName{0};
481 // The version of the key has to tell whether offsets are 32bit or 64bit long
482 union {
483 struct {
484 RUInt32BE fSeekDir{RTFHeader::kBEGIN};
485 RUInt32BE fSeekParent{0};
486 RUInt32BE fSeekKeys{0};
487 } fInfoShort;
488 struct {
489 RUInt64BE fSeekDir{RTFHeader::kBEGIN};
490 RUInt64BE fSeekParent{0};
491 RUInt64BE fSeekKeys{0};
492 } fInfoLong;
493 };
494
495 RTFDirectory() : fInfoShort() {}
496
497 // In case of a short TFile record (<2G), 3 padding ints are written after the UUID
498 std::uint32_t GetSize() const
499 {
500 if (fClassVersion >= kBigFileVersion)
501 return sizeof(RTFDirectory);
502 return 18 + sizeof(fInfoShort);
503 }
504
505 std::uint64_t GetSeekKeys() const
506 {
507 if (fClassVersion >= kBigFileVersion)
508 return fInfoLong.fSeekKeys;
509 return fInfoShort.fSeekKeys;
510 }
511
512 void SetSeekKeys(std::uint64_t seekKeys)
513 {
514 if (seekKeys > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
515 std::uint32_t seekDir = fInfoShort.fSeekDir;
516 std::uint32_t seekParent = fInfoShort.fSeekParent;
517 fInfoLong.fSeekDir = seekDir;
518 fInfoLong.fSeekParent = seekParent;
519 fInfoLong.fSeekKeys = seekKeys;
520 fClassVersion = fClassVersion + kBigFileVersion;
521 } else {
522 fInfoShort.fSeekKeys = seekKeys;
523 }
524 }
525};
526
527/// A zero UUID stored at the end of the TFile record
528struct RTFUUID {
529 RUInt16BE fVersionClass{1};
530 unsigned char fUUID[16];
531
532 RTFUUID()
533 {
534 TUUID uuid{TUUID::UUIDv4()};
535 char *buffer = reinterpret_cast<char *>(this);
536 uuid.FillBuffer(buffer);
537 assert(reinterpret_cast<RTFUUID *>(buffer) <= (this + 1));
538 }
539 std::uint32_t GetSize() const { return sizeof(RTFUUID); }
540};
541
542/// A streamed RNTuple class
543///
544/// NOTE: this must be kept in sync with RNTuple.hxx.
545/// Aside ensuring consistency between the two classes' members, you need to make sure
546/// that fVersionClass matches the class version of RNTuple.
547struct RTFNTuple {
548 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFNTuple) - sizeof(fByteCount))};
549 RUInt16BE fVersionClass{2};
550 RUInt16BE fVersionEpoch{0};
551 RUInt16BE fVersionMajor{0};
552 RUInt16BE fVersionMinor{0};
553 RUInt16BE fVersionPatch{0};
554 RUInt64BE fSeekHeader{0};
555 RUInt64BE fNBytesHeader{0};
556 RUInt64BE fLenHeader{0};
557 RUInt64BE fSeekFooter{0};
558 RUInt64BE fNBytesFooter{0};
559 RUInt64BE fLenFooter{0};
560 RUInt64BE fMaxKeySize{0};
561
562 static constexpr std::uint32_t GetSizePlusChecksum() { return sizeof(RTFNTuple) + sizeof(std::uint64_t); }
563
564 RTFNTuple() = default;
565 explicit RTFNTuple(const ROOT::RNTuple &inMemoryAnchor)
566 {
567 fVersionEpoch = inMemoryAnchor.GetVersionEpoch();
568 fVersionMajor = inMemoryAnchor.GetVersionMajor();
569 fVersionMinor = inMemoryAnchor.GetVersionMinor();
570 fVersionPatch = inMemoryAnchor.GetVersionPatch();
571 fSeekHeader = inMemoryAnchor.GetSeekHeader();
572 fNBytesHeader = inMemoryAnchor.GetNBytesHeader();
573 fLenHeader = inMemoryAnchor.GetLenHeader();
574 fSeekFooter = inMemoryAnchor.GetSeekFooter();
575 fNBytesFooter = inMemoryAnchor.GetNBytesFooter();
576 fLenFooter = inMemoryAnchor.GetLenFooter();
577 fMaxKeySize = inMemoryAnchor.GetMaxKeySize();
578 }
579 std::uint32_t GetSize() const { return sizeof(RTFNTuple); }
580 // The byte count and class version members are not checksummed
581 std::uint32_t GetOffsetCkData() { return sizeof(fByteCount) + sizeof(fVersionClass); }
582 std::uint32_t GetSizeCkData() { return GetSize() - GetOffsetCkData(); }
583 unsigned char *GetPtrCkData() { return reinterpret_cast<unsigned char *>(this) + GetOffsetCkData(); }
584};
585
586/// The bare file global header
587struct RBareFileHeader {
588 char fMagic[7]{'r', 'n', 't', 'u', 'p', 'l', 'e'};
589 RUInt32BE fRootVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
590 (ROOT_VERSION_CODE & 0xFF)};
591 RUInt32BE fFormatVersion{1};
592 RUInt32BE fCompress{0};
593 RTFNTuple fNTuple;
594 // followed by the ntuple name
595};
596#pragma pack(pop)
597
598/// The artifical class name shown for opaque RNTuple keys (see TBasket)
599constexpr char const *kBlobClassName = "RBlob";
600/// The class name of the RNTuple anchor
601constexpr char const *kNTupleClassName = "ROOT::RNTuple";
602
603} // anonymous namespace
604
605namespace ROOT {
606namespace Internal {
607/// If a TFile container is written by a C stream (simple file), on dataset commit, the file header
608/// and the TFile record need to be updated
610 RTFHeader fHeader;
611 RTFDirectory fFileRecord;
612 std::uint64_t fSeekNTuple{0}; // Remember the offset for the keys list
613 std::uint64_t fSeekFileRecord{0};
614};
615
616/// The RKeyBlob writes an invisible key into a TFile. That is, a key that is not indexed in the list of keys,
617/// like a TBasket.
618/// NOTE: out of anonymous namespace because otherwise ClassDefInline fails to compile
619/// on some platforms.
620class RKeyBlob : public TKey {
621public:
622 RKeyBlob() = default;
623
624 explicit RKeyBlob(TFile *file) : TKey(file)
625 {
627 fVersion += RTFKey::kBigKeyVersion;
628 fKeylen = Sizeof();
629 }
630
631 /// Register a new key for a data record of size nbytes
632 void Reserve(size_t nbytes, std::uint64_t *seekKey)
633 {
634 Create(nbytes);
635 *seekKey = fSeekKey;
636 }
637
638 bool WasAllocatedInAFreeSlot() const { return fLeft > 0; }
639
641};
642
643} // namespace Internal
644} // namespace ROOT
645
646// Computes how many chunks do we need to fit `nbytes` of payload, considering that the
647// first chunk also needs to house the offsets of the other chunks and no chunk can
648// be bigger than `maxChunkSize`. When saved to a TFile, each chunk is part of a separate TKey.
649static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
650{
651 constexpr size_t kChunkOffsetSize = sizeof(std::uint64_t);
652
654 size_t nChunks = (nbytes + maxChunkSize - 1) / maxChunkSize;
655 assert(nChunks > 1);
656 size_t nbytesTail = nbytes % maxChunkSize;
657 size_t nbytesExtra = (nbytesTail > 0) * (maxChunkSize - nbytesTail);
660 ++nChunks;
662 }
663
664 // We don't support having more chunkOffsets than what fits in one chunk.
665 // For a reasonable-sized maxKeySize it looks very unlikely that we can have more chunks
666 // than we can fit in the first `maxKeySize` bytes. E.g. for maxKeySize = 1GiB we can fit
667 // 134217728 chunk offsets, making our multi-key blob's capacity exactly 128 PiB.
669
670 return nChunks;
671}
672
674
676{
677 char ident[4];
678 ReadBuffer(ident, 4, 0);
679 if (std::string(ident, 4) == "root")
680 return GetNTupleProper(ntupleName);
681 fIsBare = true;
682 return GetNTupleBare(ntupleName);
683}
684
685/// Searches for a key with the given name and type in the key index of the given directory.
686/// Return 0 if the key was not found.
687std::uint64_t ROOT::Internal::RMiniFileReader::SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName,
688 std::string_view typeName)
689{
690 RTFDirectory directory;
692
693 RTFKey key;
694 RUInt32BE nKeys;
695 std::uint64_t offset = directory.GetSeekKeys();
696 ReadBuffer(&key, sizeof(key), offset);
697 offset += key.fKeyLen;
698 ReadBuffer(&nKeys, sizeof(nKeys), offset);
699 offset += sizeof(nKeys);
700
701 for (unsigned int i = 0; i < nKeys; ++i) {
702 ReadBuffer(&key, sizeof(key), offset);
703 auto offsetNextKey = offset + key.fKeyLen;
704
705 offset += key.GetHeaderSize();
706 RTFString name;
707 ReadBuffer(&name, 1, offset);
708 ReadBuffer(&name, name.GetSize(), offset);
709 if (std::string_view(name.fData, name.fLName) != typeName) {
711 continue;
712 }
713 offset += name.GetSize();
714 ReadBuffer(&name, 1, offset);
715 ReadBuffer(&name, name.GetSize(), offset);
716 if (std::string_view(name.fData, name.fLName) == keyName) {
717 return key.GetSeekKey();
718 }
720 }
721
722 // Not found
723 return 0;
724}
725
727{
728 RTFHeader fileHeader;
729 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
730
731 const std::uint64_t seekKeyInfo = fileHeader.GetSeekInfo();
732
733 RTFKey key;
734 ReadBuffer(&key, sizeof(key), seekKeyInfo);
735
736 const std::uint64_t nbytesInfo = fileHeader.GetNbytesInfo() - key.fKeyLen;
737 const std::uint64_t seekInfo = seekKeyInfo + key.fKeyLen;
738 const std::uint32_t uncompLenInfo = key.fObjLen;
740 if (nbytesInfo == uncompLenInfo) {
741 // Uncompressed
743 } else {
745 ReadBuffer(buffer.get(), nbytesInfo, seekInfo);
747 }
748
750 // This is necessary to allow the "class tags" inside the StreamerInfo list to refer to the proper offset into
751 // the buffer. Normally TFile loads the StreamerInfo via TKey::ReadObjWithBuffer, whose buffer also includes the
752 // key itself. Since we dealt with the key above already, we are only passing the payload to TBufferFile so offsets
753 // need to be patched up.
754 buffer.SetBufferDisplacement(key.fKeyLen);
756 streamerInfoList.Streamer(buffer);
757 TObjLink *lnk = streamerInfoList.FirstLink();
758 while (lnk) {
759 auto obj = lnk->GetObject();
760 // NOTE: the last element of the streamer info list may be a TList with the IO customization rules, so we need
761 // to check before static casting.
762 if (obj->IsA() == TStreamerInfo::Class()) {
763 auto info = static_cast<TStreamerInfo *>(obj);
764 info->BuildCheck();
765 }
766 lnk = lnk->Next();
767 }
768}
769
771{
772 RTFHeader fileHeader;
773 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
774
775 RTFKey key;
776 RTFString name;
777 ReadBuffer(&key, sizeof(key), fileHeader.fBEGIN);
778 // Skip over the entire key length, including the class name, object name, and title stored in it.
779 std::uint64_t offset = fileHeader.fBEGIN + key.fKeyLen;
780 // Skip over the name and title of the TNamed preceding the TFile (root TDirectory) entry.
781 ReadBuffer(&name, 1, offset);
782 offset += name.GetSize();
783 ReadBuffer(&name, 1, offset);
784 offset += name.GetSize();
785
786 // split ntupleName by '/' character to open datasets in subdirectories.
787 std::string ntuplePathTail(ntuplePath);
788 if (!ntuplePathTail.empty() && ntuplePathTail[0] == '/')
789 ntuplePathTail = ntuplePathTail.substr(1);
790 auto pos = std::string::npos;
791 while ((pos = ntuplePathTail.find('/')) != std::string::npos) {
792 auto directoryName = ntuplePathTail.substr(0, pos);
793 ntuplePathTail.erase(0, pos + 1);
794
795 offset = SearchInDirectory(offset, directoryName, "TDirectory");
796 if (offset == 0) {
797 return R__FAIL("no directory named '" + std::string(directoryName) + "' in file '" + fRawFile->GetUrl() + "'");
798 }
799 ReadBuffer(&key, sizeof(key), offset);
800 offset = key.GetSeekKey() + key.fKeyLen;
801 }
802 // no more '/' delimiter in ntuplePath
804
805 offset = SearchInDirectory(offset, ntupleName, kNTupleClassName);
806 if (offset == 0) {
807 return R__FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + fRawFile->GetUrl() + "'");
808 }
809
810 ReadBuffer(&key, sizeof(key), offset);
811 offset = key.GetSeekKey() + key.fKeyLen;
812
813 // size of a RTFNTuple version 2 (min supported version); future anchor versions can grow.
814 constexpr size_t kMinNTupleSize = 78;
815 static_assert(kMinNTupleSize == RTFNTuple::GetSizePlusChecksum());
816 if (key.fObjLen < kMinNTupleSize) {
817 return R__FAIL("invalid anchor size: " + std::to_string(key.fObjLen) + " < " + std::to_string(sizeof(RTFNTuple)));
818 }
819
820 const auto objNbytes = key.GetSize() - key.fKeyLen;
821 auto res = GetNTupleProperAtOffset(offset, objNbytes, key.fObjLen);
822 return res;
823}
824
826 std::uint64_t compSize,
827 std::uint64_t uncompLen)
828{
829 if (compSize > uncompLen) {
830 return R__FAIL("Invalid size and length combination of anchor at offset " + std::to_string(payloadOffset) +
831 " (length: " + std::to_string(uncompLen) + ", bytes on disk: " + std::to_string(compSize) + ")");
832 }
833
834 // The object length can be smaller than the size of RTFNTuple if it comes from a past RNTuple class version,
835 // or larger than it if it comes from a future RNTuple class version.
836 auto bufAnchor = MakeUninitArray<unsigned char>(std::max<size_t>(uncompLen, sizeof(RTFNTuple)));
837 RTFNTuple *ntuple = new (bufAnchor.get()) RTFNTuple;
838
839 if (compSize < uncompLen) {
840 // Read into a temporary buffer
843 // Unzip into the final buffer
845 } else {
846 // We can directly decompress into ntuple because this RTFNTuple object has been constructed in a memory area
847 // of (max(uncompLen, sizeof(RTFNTuple))) bytes.
849 }
850
851 // We require that future class versions only append members and store the checksum in the last 8 bytes
852 // Checksum calculation: strip byte count, class version, fChecksum member
853 const auto lenCkData = uncompLen - ntuple->GetOffsetCkData() - sizeof(uint64_t);
854 const auto ckCalc = XXH3_64bits(ntuple->GetPtrCkData(), lenCkData);
855 uint64_t ckOnDisk;
856
857 RUInt64BE *ckOnDiskPtr = reinterpret_cast<RUInt64BE *>(bufAnchor.get() + uncompLen - sizeof(uint64_t));
858 ckOnDisk = static_cast<uint64_t>(*ckOnDiskPtr);
859 if (ckCalc != ckOnDisk) {
860 return R__FAIL("RNTuple anchor checksum mismatch");
861 }
862
863 return CreateAnchor(ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch,
864 ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter,
865 ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize);
866}
867
869{
870 RBareFileHeader fileHeader;
871 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
872 RTFString name;
873 auto offset = sizeof(fileHeader);
874 ReadBuffer(&name, 1, offset);
875 ReadBuffer(&name, name.GetSize(), offset);
876 std::string_view foundName(name.fData, name.fLName);
877 if (foundName != ntupleName) {
878 return R__FAIL("expected RNTuple named '" + std::string(ntupleName) + "' but instead found '" +
879 std::string(foundName) + "' in file '" + fRawFile->GetUrl() + "'");
880 }
881 offset += name.GetSize();
882
883 RTFNTuple ntuple;
884 ReadBuffer(&ntuple, sizeof(ntuple), offset);
885 std::uint64_t onDiskChecksum;
887 auto checksum = XXH3_64bits(ntuple.GetPtrCkData(), ntuple.GetSizeCkData());
888 if (checksum != static_cast<uint64_t>(onDiskChecksum))
889 return R__FAIL("RNTuple bare file: anchor checksum mismatch");
890
891 return CreateAnchor(ntuple.fVersionEpoch, ntuple.fVersionMajor, ntuple.fVersionMinor, ntuple.fVersionPatch,
892 ntuple.fSeekHeader, ntuple.fNBytesHeader, ntuple.fLenHeader, ntuple.fSeekFooter,
893 ntuple.fNBytesFooter, ntuple.fLenFooter, ntuple.fMaxKeySize);
894}
895
896void ROOT::Internal::RMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
897{
898 TryReadBuffer(buffer, nbytes, offset).ThrowOnError();
899}
900
902{
903 const auto ByteReadErr = [](std::size_t expected, std::size_t nread) {
904 return R__FAIL("invalid read (expected bytes: " + std::to_string(expected) + ", read: " + std::to_string(nread) +
905 ")");
906 };
907
908 size_t nread;
909 if (fMaxKeySize == 0 || nbytes <= fMaxKeySize) {
910 // Fast path: read single blob
911 nread = fRawFile->ReadAt(buffer, nbytes, offset);
912 } else {
913 // Read chunked blob. See RNTupleFileWriter::WriteBlob() for details.
914 const size_t nChunks = ComputeNumChunks(nbytes, fMaxKeySize);
915 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
916 const size_t nbytesFirstChunk = fMaxKeySize - nbytesChunkOffsets;
917 uint8_t *bufCur = reinterpret_cast<uint8_t *>(buffer);
918
919 // Read first chunk
920 nread = fRawFile->ReadAt(bufCur, fMaxKeySize, offset);
921 if (nread != fMaxKeySize)
922 return ByteReadErr(fMaxKeySize, nread);
923
924 // NOTE: we read the entire chunk in `bufCur`, but we only advance the pointer by `nbytesFirstChunk`,
925 // since the last part of `bufCur` will later be overwritten by the next chunk's payload.
926 // We do this to avoid a second ReadAt to read in the chunk offsets.
929
932
934 std::uint64_t *curChunkOffset = &chunkOffsets[0];
935
936 do {
937 std::uint64_t chunkOffset;
940
941 const size_t bytesToRead = std::min<size_t>(fMaxKeySize, remainingBytes);
942 // Ensure we don't read outside of the buffer
943 R__ASSERT(static_cast<size_t>(bufCur - reinterpret_cast<uint8_t *>(buffer)) <= nbytes - bytesToRead);
944
945 auto nbytesRead = fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
946 if (nbytesRead != bytesToRead)
948
952 } while (remainingBytes > 0);
953 }
954
955 if (nread != nbytes)
956 return ByteReadErr(nbytes, nread);
957
958 return RResult<void>::Success();
959}
960
961////////////////////////////////////////////////////////////////////////////////
962
963/// Prepare a blob key in the provided buffer, which must provide space for kBlobKeyLen bytes. Note that the array type
964/// is purely documentation, the argument is actually just a pointer.
966 unsigned char buffer[kBlobKeyLen])
967{
968 RTFString strClass{kBlobClassName};
969 RTFString strObject;
970 RTFString strTitle;
971 RTFKey keyHeader(offset, RTFHeader::kBEGIN, strClass, strObject, strTitle, len, nbytes);
972 R__ASSERT(keyHeader.fKeyLen == kBlobKeyLen);
973
974 // Copy structures into the buffer.
975 unsigned char *writeBuffer = buffer;
976 memcpy(writeBuffer, &keyHeader, keyHeader.GetHeaderSize());
977 writeBuffer += keyHeader.GetHeaderSize();
978 memcpy(writeBuffer, &strClass, strClass.GetSize());
979 writeBuffer += strClass.GetSize();
981 writeBuffer += strObject.GetSize();
982 memcpy(writeBuffer, &strTitle, strTitle.GetSize());
983 writeBuffer += strTitle.GetSize();
984 R__ASSERT(writeBuffer == buffer + kBlobKeyLen);
985}
986
987////////////////////////////////////////////////////////////////////////////////
988
990
992{
993 static_assert(kHeaderBlockSize % kBlockAlign == 0, "invalid header block size");
994 if (bufferSize % kBlockAlign != 0)
995 throw RException(R__FAIL("Buffer size not a multiple of alignment: " + std::to_string(bufferSize)));
996 fBlockSize = bufferSize;
997
998 std::align_val_t blockAlign{kBlockAlign};
999 fHeaderBlock = static_cast<unsigned char *>(::operator new[](kHeaderBlockSize, blockAlign));
1000 memset(fHeaderBlock, 0, kHeaderBlockSize);
1001 fBlock = static_cast<unsigned char *>(::operator new[](fBlockSize, blockAlign));
1002 memset(fBlock, 0, fBlockSize);
1003}
1004
1006{
1007 if (fFile)
1008 fclose(fFile);
1009
1010 std::align_val_t blockAlign{kBlockAlign};
1011 if (fHeaderBlock)
1012 ::operator delete[](fHeaderBlock, blockAlign);
1013 if (fBlock)
1014 ::operator delete[](fBlock, blockAlign);
1015}
1016
1017namespace {
1018int FSeek64(FILE *stream, std::int64_t offset, int origin)
1019{
1020#ifdef R__SEEK64
1021 return fseeko64(stream, offset, origin);
1022#else
1023 return fseek(stream, offset, origin);
1024#endif
1025}
1026} // namespace
1027
1029{
1030 // Write the last partially filled block, which may still need appropriate alignment for Direct I/O.
1031 // If it is the first block, get the updated header block.
1032 if (fBlockOffset == 0) {
1033 std::size_t headerBlockSize = kHeaderBlockSize;
1034 if (headerBlockSize > fFilePos) {
1035 headerBlockSize = fFilePos;
1036 }
1037 memcpy(fBlock, fHeaderBlock, headerBlockSize);
1038 }
1039
1040 std::size_t retval = FSeek64(fFile, fBlockOffset, SEEK_SET);
1041 if (retval)
1042 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1043
1044 std::size_t lastBlockSize = fFilePos - fBlockOffset;
1045 R__ASSERT(lastBlockSize <= fBlockSize);
1046 if (fDirectIO) {
1047 // Round up to a multiple of kBlockAlign.
1048 lastBlockSize += kBlockAlign - 1;
1049 lastBlockSize = (lastBlockSize / kBlockAlign) * kBlockAlign;
1050 R__ASSERT(lastBlockSize <= fBlockSize);
1051 }
1052 retval = fwrite(fBlock, 1, lastBlockSize, fFile);
1053 if (retval != lastBlockSize)
1054 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1055
1056 // Write the (updated) header block, unless it was part of the write above.
1057 if (fBlockOffset > 0) {
1058 retval = FSeek64(fFile, 0, SEEK_SET);
1059 if (retval)
1060 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1061
1062 retval = fwrite(fHeaderBlock, 1, kHeaderBlockSize, fFile);
1063 if (retval != RImplSimple::kHeaderBlockSize)
1064 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1065 }
1066
1067 retval = fflush(fFile);
1068 if (retval)
1069 throw RException(R__FAIL(std::string("Flush failed: ") + strerror(errno)));
1070}
1071
1072void ROOT::Internal::RNTupleFileWriter::RImplSimple::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1073{
1074 R__ASSERT(fFile);
1075 size_t retval;
1076 if ((offset >= 0) && (static_cast<std::uint64_t>(offset) != fFilePos)) {
1077 fFilePos = offset;
1078 }
1079
1080 // Keep header block to overwrite on commit.
1081 if (fFilePos < kHeaderBlockSize) {
1082 std::size_t headerBytes = nbytes;
1083 if (fFilePos + headerBytes > kHeaderBlockSize) {
1084 headerBytes = kHeaderBlockSize - fFilePos;
1085 }
1086 memcpy(fHeaderBlock + fFilePos, buffer, headerBytes);
1087 }
1088
1089 R__ASSERT(fFilePos >= fBlockOffset);
1090
1091 while (nbytes > 0) {
1092 std::uint64_t posInBlock = fFilePos % fBlockSize;
1093 std::uint64_t blockOffset = fFilePos - posInBlock;
1094 if (blockOffset != fBlockOffset) {
1095 // Write the block.
1096 retval = FSeek64(fFile, fBlockOffset, SEEK_SET);
1097 if (retval)
1098 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1099
1100 retval = fwrite(fBlock, 1, fBlockSize, fFile);
1101 if (retval != fBlockSize)
1102 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1103
1104 // Null the buffer contents for good measure.
1105 memset(fBlock, 0, fBlockSize);
1106 }
1107
1108 fBlockOffset = blockOffset;
1109 std::size_t blockSize = nbytes;
1110 if (blockSize > fBlockSize - posInBlock) {
1111 blockSize = fBlockSize - posInBlock;
1112 }
1113 memcpy(fBlock + posInBlock, buffer, blockSize);
1114 buffer = static_cast<const unsigned char *>(buffer) + blockSize;
1115 nbytes -= blockSize;
1116 fFilePos += blockSize;
1117 }
1118}
1119
1120std::uint64_t
1121ROOT::Internal::RNTupleFileWriter::RImplSimple::WriteKey(const void *buffer, std::size_t nbytes, std::size_t len,
1122 std::int64_t offset, std::uint64_t directoryOffset,
1123 const std::string &className, const std::string &objectName,
1124 const std::string &title)
1125{
1126 if (offset > 0)
1127 fKeyOffset = offset;
1128 RTFString strClass{className};
1129 RTFString strObject{objectName};
1130 RTFString strTitle{title};
1131
1132 RTFKey key(fKeyOffset, directoryOffset, strClass, strObject, strTitle, len, nbytes);
1133 Write(&key, key.GetHeaderSize(), fKeyOffset);
1134 Write(&strClass, strClass.GetSize());
1135 Write(&strObject, strObject.GetSize());
1136 Write(&strTitle, strTitle.GetSize());
1137 auto offsetData = fFilePos;
1138 // The next key starts after the data.
1139 fKeyOffset = offsetData + nbytes;
1140 if (buffer)
1141 Write(buffer, nbytes);
1142
1143 return offsetData;
1144}
1145
1147 unsigned char keyBuffer[kBlobKeyLen])
1148{
1149 if (keyBuffer) {
1150 PrepareBlobKey(fKeyOffset, nbytes, len, keyBuffer);
1151 } else {
1152 unsigned char localKeyBuffer[kBlobKeyLen];
1153 PrepareBlobKey(fKeyOffset, nbytes, len, localKeyBuffer);
1154 Write(localKeyBuffer, kBlobKeyLen, fKeyOffset);
1155 }
1156
1157 auto offsetData = fKeyOffset + kBlobKeyLen;
1158 // The next key starts after the data.
1159 fKeyOffset = offsetData + nbytes;
1160
1161 return offsetData;
1162}
1163
1164////////////////////////////////////////////////////////////////////////////////
1165
1166template <typename T>
1168 std::size_t len, unsigned char keyBuffer[kBlobKeyLen])
1169{
1170 std::uint64_t offsetKey;
1172 // Since it is unknown beforehand if offsetKey is beyond the 2GB limit or not,
1173 // RKeyBlob will always reserve space for a big key (version >= 1000)
1174 keyBlob.Reserve(nbytes, &offsetKey);
1175
1176 if (keyBuffer) {
1177 PrepareBlobKey(offsetKey, nbytes, len, keyBuffer);
1178 } else {
1179 unsigned char localKeyBuffer[kBlobKeyLen];
1180 PrepareBlobKey(offsetKey, nbytes, len, localKeyBuffer);
1181 caller.Write(localKeyBuffer, kBlobKeyLen, offsetKey);
1182 }
1183
1184 if (keyBlob.WasAllocatedInAFreeSlot()) {
1185 // If the key was allocated in a free slot, the last 4 bytes of its buffer contain the new size
1186 // of the remaining free slot and we need to write it to disk before the key gets destroyed at the end of the
1187 // function.
1188 caller.Write(keyBlob.GetBuffer() + nbytes, sizeof(Int_t), offsetKey + kBlobKeyLen + nbytes);
1189 }
1190
1191 auto offsetData = offsetKey + kBlobKeyLen;
1192
1193 return offsetData;
1194}
1195
1196void ROOT::Internal::RNTupleFileWriter::RImplTFile::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1197{
1198 fDirectory->GetFile()->Seek(offset);
1199 bool rv = fDirectory->GetFile()->WriteBuffer((char *)(buffer), nbytes);
1200 if (rv)
1201 throw RException(R__FAIL("WriteBuffer failed."));
1202}
1203
1205 unsigned char keyBuffer[kBlobKeyLen])
1206{
1207 auto offsetData = RNTupleFileWriter::ReserveBlobKey(*this, *fDirectory->GetFile(), nbytes, len, keyBuffer);
1208 return offsetData;
1209}
1210
1211////////////////////////////////////////////////////////////////////////////////
1212
1213void ROOT::Internal::RNTupleFileWriter::RImplRFile::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1214{
1216 file->Seek(offset);
1217 bool rv = file->WriteBuffer((char *)(buffer), nbytes);
1218 if (rv)
1219 throw RException(R__FAIL("WriteBuffer failed."));
1220}
1221
1223 unsigned char keyBuffer[kBlobKeyLen])
1224{
1227 return offsetData;
1228}
1229
1230////////////////////////////////////////////////////////////////////////////////
1231
1232ROOT::Internal::RNTupleFileWriter::RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize, bool hidden)
1233 : fIsHidden(hidden), fNTupleName(name)
1234{
1235 auto &fileSimple = fFile.emplace<RImplSimple>();
1236 fileSimple.fControlBlock = std::make_unique<ROOT::Internal::RTFileControlBlock>();
1238 auto infoRNTuple = RNTuple::Class()->GetStreamerInfo();
1240}
1241
1243
1244std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1245ROOT::Internal::RNTupleFileWriter::Recreate(std::string_view ntupleName, std::string_view path,
1247{
1248 std::string fileName(path);
1249 size_t idxDirSep = fileName.find_last_of("\\/");
1250 if (idxDirSep != std::string::npos) {
1251 fileName.erase(0, idxDirSep + 1);
1252 }
1253#ifdef R__LINUX
1254 int flags = O_WRONLY | O_CREAT | O_TRUNC;
1255#ifdef O_LARGEFILE
1256 // Add the equivalent flag that is passed by fopen64.
1257 flags |= O_LARGEFILE;
1258#endif
1259 if (options.GetUseDirectIO()) {
1260 flags |= O_DIRECT;
1261 }
1262 int fd = open(std::string(path).c_str(), flags, 0666);
1263 if (fd == -1) {
1264 throw RException(R__FAIL(std::string("open failed for file \"") + std::string(path) + "\": " + strerror(errno)));
1265 }
1266 FILE *fileStream = fdopen(fd, "wb");
1267#else
1268#ifdef R__SEEK64
1269 FILE *fileStream = fopen64(std::string(path.data(), path.size()).c_str(), "wb");
1270#else
1271 FILE *fileStream = fopen(std::string(path.data(), path.size()).c_str(), "wb");
1272#endif
1273#endif
1274 if (!fileStream) {
1275 throw RException(R__FAIL(std::string("open failed for file \"") + std::string(path) + "\": " + strerror(errno)));
1276 }
1277 // RNTupleFileWriter::RImplSimple does its own buffering, turn off additional buffering from C stdio.
1278 std::setvbuf(fileStream, nullptr, _IONBF, 0);
1279
1280 auto writer =
1281 std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, options.GetMaxKeySize(), /*hidden=*/false));
1282 RImplSimple &fileSimple = std::get<RImplSimple>(writer->fFile);
1283 fileSimple.fFile = fileStream;
1284 fileSimple.fDirectIO = options.GetUseDirectIO();
1285 fileSimple.AllocateBuffers(options.GetWriteBufferSize());
1286 writer->fFileName = fileName;
1287
1288 int defaultCompression = options.GetCompression();
1289 switch (containerFormat) {
1290 case EContainerFormat::kTFile: writer->WriteTFileSkeleton(defaultCompression); break;
1291 case EContainerFormat::kBare:
1292 writer->fIsBare = true;
1293 writer->WriteBareFileSkeleton(defaultCompression);
1294 break;
1295 default: R__ASSERT(false && "Internal error: unhandled container format");
1296 }
1297
1298 return writer;
1299}
1300
1301std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1303 std::uint64_t maxKeySize, bool hidden)
1304{
1305 TFile *file = fileOrDirectory.GetFile();
1306 if (!file)
1307 throw RException(R__FAIL("invalid attempt to add an RNTuple to a directory that is not backed by a file"));
1308 assert(file->IsBinary());
1309
1310 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize, hidden));
1311 auto &fileProper = writer->fFile.emplace<RImplTFile>();
1312 fileProper.fDirectory = &fileOrDirectory;
1313 return writer;
1314}
1315
1316std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1318 std::string_view ntupleDir, std::uint64_t maxKeySize)
1319{
1320 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize, /*hidden=*/false));
1321 auto &rfile = writer->fFile.emplace<RImplRFile>();
1322 rfile.fFile = &file;
1323 R__ASSERT(ntupleDir.empty() || ntupleDir[ntupleDir.size() - 1] == '/');
1324 rfile.fDir = ntupleDir;
1325 return writer;
1326}
1327
1328std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1330{
1331 if (auto *file = std::get_if<RImplTFile>(&fFile)) {
1332 return Append(ntupleName, *file->fDirectory, fNTupleAnchor.fMaxKeySize, /* hidden= */ true);
1333 }
1334 // TODO: support also non-TFile-based writers
1335 throw ROOT::RException(R__FAIL("cannot clone a non-TFile-based RNTupleFileWriter."));
1336}
1337
1339{
1340 RImplSimple *fileSimple = std::get_if<RImplSimple>(&fFile);
1341 if (!fileSimple)
1342 throw RException(R__FAIL("invalid attempt to seek non-simple writer"));
1343
1344 fileSimple->fFilePos = offset;
1345 fileSimple->fKeyOffset = offset;
1346 // The next Write() will Flush() if necessary.
1347}
1348
1353
1355{
1356 const auto WriteStreamerInfoToFile = [&](TFile *file) {
1357 // Make sure the streamer info records used in the RNTuple are written to the file
1359 buf.SetParent(file);
1360 for (auto [_, info] : fStreamerInfoMap)
1361 buf.TagStreamerInfo(info);
1362 };
1363
1365 // NOTE: checksum length is included in the uncompressed len
1366 anchorInfo.fLength = RTFNTuple{}.GetSize() + sizeof(std::uint64_t);
1367 anchorInfo.fLocator.SetType(RNTupleLocator::kTypeFile);
1368
1369 if (auto fileProper = std::get_if<RImplTFile>(&fFile)) {
1370 // Easy case, the ROOT file header and the RNTuple streaming is taken care of by TFile
1371 fileProper->fDirectory->WriteObject(&fNTupleAnchor, fNTupleName.c_str());
1372 WriteStreamerInfoToFile(fileProper->fDirectory->GetFile());
1373 auto key = static_cast<TKey *>(fileProper->fDirectory->GetListOfKeys()->FindObject(fNTupleName.c_str()));
1374 R__ASSERT(key);
1375 anchorInfo.fLocator.SetPosition(key->GetSeekKey() + key->GetKeylen());
1376 anchorInfo.fLocator.SetNBytesOnStorage(key->GetNbytes() - key->GetKeylen());
1377 // NOTE: this must happen after FindObject(), otherwise some TFile implementations, such as TBufferMergerFile,
1378 // may reset the keys list upon write.
1379 fileProper->fDirectory->GetFile()->Write();
1380
1381 if (fIsHidden) {
1382 // Remove the anchor's key from the directory's KeysList to disallow retrieving directly the
1383 // attribute RNTuple from the TFile.
1384 fileProper->fDirectory->GetListOfKeys()->Remove(key);
1385 }
1386 } else if (auto fileRFile = std::get_if<RImplRFile>(&fFile)) {
1387 // Same as the case above but handled via RFile
1388 fileRFile->fFile->Put(fileRFile->fDir + fNTupleName, fNTupleAnchor);
1390 auto key = fileRFile->fFile->GetKeyInfo(fNTupleName);
1391 R__ASSERT(key);
1392 anchorInfo.fLocator.SetPosition(key->GetSeekKey() + key->GetNBytesKey());
1393 anchorInfo.fLocator.SetNBytesOnStorage(key->GetNBytesObj());
1394 fileRFile->fFile->Flush();
1395 } else {
1396 // Writing by C file stream: prepare the container format header and stream the RNTuple anchor object
1397 auto &fileSimple = std::get<RImplSimple>(fFile);
1398
1399 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1400 anchorInfo.fLocator.SetPosition(fileSimple.fControlBlock->fSeekNTuple);
1401
1402 if (fIsBare) {
1403 // Compute the checksum
1404 std::uint64_t checksum = XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData());
1405 memcpy(fileSimple.fHeaderBlock + fileSimple.fControlBlock->fSeekNTuple, &ntupleOnDisk, ntupleOnDisk.GetSize());
1406 memcpy(fileSimple.fHeaderBlock + fileSimple.fControlBlock->fSeekNTuple + ntupleOnDisk.GetSize(), &checksum,
1407 sizeof(checksum));
1408 fileSimple.Flush();
1409
1410 anchorInfo.fLocator.SetNBytesOnStorage(ntupleOnDisk.GetSize());
1411 } else {
1412 auto anchorSize = WriteTFileNTupleKey(compression);
1413 WriteTFileKeysList(anchorSize); // NOTE: this is written uncompressed
1414 WriteTFileStreamerInfo(compression);
1415 WriteTFileFreeList(); // NOTE: this is written uncompressed
1416
1417 // Update header and TFile record
1418 memcpy(fileSimple.fHeaderBlock, &fileSimple.fControlBlock->fHeader,
1419 fileSimple.fControlBlock->fHeader.GetSize());
1420 R__ASSERT(fileSimple.fControlBlock->fSeekFileRecord + fileSimple.fControlBlock->fFileRecord.GetSize() <
1421 RImplSimple::kHeaderBlockSize);
1422 memcpy(fileSimple.fHeaderBlock + fileSimple.fControlBlock->fSeekFileRecord,
1423 &fileSimple.fControlBlock->fFileRecord, fileSimple.fControlBlock->fFileRecord.GetSize());
1424
1425 fileSimple.Flush();
1426
1427 anchorInfo.fLocator.SetNBytesOnStorage(anchorSize);
1428 }
1429 }
1430
1431 return anchorInfo;
1432}
1433
1434std::uint64_t ROOT::Internal::RNTupleFileWriter::WriteBlob(const void *data, size_t nbytes, size_t len)
1435{
1436 auto writeKey = [this](const void *payload, size_t nBytes, size_t length) {
1437 std::uint64_t offset = ReserveBlob(nBytes, length);
1438 WriteIntoReservedBlob(payload, nBytes, offset);
1439 return offset;
1440 };
1441
1442 const std::uint64_t maxKeySize = fNTupleAnchor.fMaxKeySize;
1443 R__ASSERT(maxKeySize > 0);
1444 // We don't need the object length except for seeing compression ratios in TFile::Map()
1445 // Make sure that the on-disk object length fits into the TKey header.
1446 if (static_cast<std::uint64_t>(len) > static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()))
1447 len = nbytes;
1448
1449 if (nbytes <= maxKeySize) {
1450 // Fast path: only write 1 key.
1451 return writeKey(data, nbytes, len);
1452 }
1453
1454 /**
1455 * Writing a key bigger than the max allowed size. In this case we split the payload
1456 * into multiple keys, reserving the end of the first key payload for pointers to the
1457 * next ones. E.g. if a key needs to be split into 3 chunks, the first chunk will have
1458 * the format:
1459 * +--------------------+
1460 * | |
1461 * | Data |
1462 * |--------------------|
1463 * | pointer to chunk 2 |
1464 * | pointer to chunk 3 |
1465 * +--------------------+
1466 */
1467 const size_t nChunks = ComputeNumChunks(nbytes, maxKeySize);
1468 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
1470 // Skip writing the first chunk, it will be written last (in the file) below.
1471
1472 const uint8_t *chunkData = reinterpret_cast<const uint8_t *>(data) + nbytesFirstChunk;
1474
1476 std::uint64_t chunkOffsetIdx = 0;
1477
1478 do {
1479 const size_t bytesNextChunk = std::min<size_t>(remainingBytes, maxKeySize);
1480 const std::uint64_t offset = writeKey(chunkData, bytesNextChunk, bytesNextChunk);
1481
1484
1487
1488 } while (remainingBytes > 0);
1489
1490 // Write the first key, with part of the data and the pointers to (logically) following keys appended.
1491 const std::uint64_t firstOffset = ReserveBlob(maxKeySize, maxKeySize);
1492 WriteIntoReservedBlob(data, nbytesFirstChunk, firstOffset);
1493 const std::uint64_t chunkOffsetsOffset = firstOffset + nbytesFirstChunk;
1494 WriteIntoReservedBlob(chunkOffsetsToWrite.get(), nbytesChunkOffsets, chunkOffsetsOffset);
1495
1496 return firstOffset;
1497}
1498
1499std::uint64_t
1500ROOT::Internal::RNTupleFileWriter::ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen])
1501{
1502 // ReserveBlob cannot be used to reserve a multi-key blob
1503 R__ASSERT(nbytes <= fNTupleAnchor.GetMaxKeySize());
1504
1505 std::uint64_t offset;
1506 if (auto *fileSimple = std::get_if<RImplSimple>(&fFile)) {
1507 if (fIsBare) {
1508 offset = fileSimple->fKeyOffset;
1509 fileSimple->fKeyOffset += nbytes;
1510 } else {
1511 offset = fileSimple->ReserveBlobKey(nbytes, len, keyBuffer);
1512 }
1513 } else if (auto *fileProper = std::get_if<RImplTFile>(&fFile)) {
1514 offset = fileProper->ReserveBlobKey(nbytes, len, keyBuffer);
1515 } else {
1516 auto &fileRFile = std::get<RImplRFile>(fFile);
1517 offset = fileRFile.ReserveBlobKey(nbytes, len, keyBuffer);
1518 }
1519 return offset;
1520}
1521
1522void ROOT::Internal::RNTupleFileWriter::WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
1523{
1524 if (auto *fileSimple = std::get_if<RImplSimple>(&fFile)) {
1525 fileSimple->Write(buffer, nbytes, offset);
1526 } else if (auto *fileProper = std::get_if<RImplTFile>(&fFile)) {
1527 fileProper->Write(buffer, nbytes, offset);
1528 } else {
1529 auto &fileRFile = std::get<RImplRFile>(fFile);
1530 fileRFile.Write(buffer, nbytes, offset);
1531 }
1532}
1533
1535{
1536 auto offset = WriteBlob(data, nbytes, lenHeader);
1537 fNTupleAnchor.fLenHeader = lenHeader;
1538 fNTupleAnchor.fNBytesHeader = nbytes;
1539 fNTupleAnchor.fSeekHeader = offset;
1540 return offset;
1541}
1542
1544{
1545 auto offset = WriteBlob(data, nbytes, lenFooter);
1546 fNTupleAnchor.fLenFooter = lenFooter;
1547 fNTupleAnchor.fNBytesFooter = nbytes;
1548 fNTupleAnchor.fSeekFooter = offset;
1549 return offset;
1550}
1551
1553{
1554 RBareFileHeader bareHeader;
1555 bareHeader.fCompress = defaultCompression;
1556 auto &fileSimple = std::get<RImplSimple>(fFile);
1557 fileSimple.Write(&bareHeader, sizeof(bareHeader), 0);
1558 RTFString ntupleName{fNTupleName};
1559 fileSimple.Write(&ntupleName, ntupleName.GetSize());
1560
1561 // Write zero-initialized ntuple to reserve the space; will be overwritten on commit
1562 RTFNTuple ntupleOnDisk;
1563 fileSimple.fControlBlock->fSeekNTuple = fileSimple.fFilePos;
1564 fileSimple.Write(&ntupleOnDisk, ntupleOnDisk.GetSize());
1565 std::uint64_t checksum = 0;
1566 fileSimple.Write(&checksum, sizeof(checksum));
1567 fileSimple.fKeyOffset = fileSimple.fFilePos;
1568}
1569
1571{
1572 // The streamer info record is a TList of TStreamerInfo object. We cannot use
1573 // RNTupleSerializer::SerializeStreamerInfos because that uses TBufferIO::WriteObject.
1574 // This would prepend the streamed TList with self-decription information.
1575 // The streamer info record is just the streamed TList.
1576
1578 for (auto [_, info] : fStreamerInfoMap) {
1580 }
1581
1582 // We will stream the list with a TBufferFile. When reading the streamer info records back,
1583 // the read buffer includes the key and the streamed list. Therefore, we need to start streaming
1584 // with an offset of the key length. Otherwise, the offset for referencing duplicate objects in the
1585 // buffer will point to the wrong places.
1586
1587 // Figure out key length
1588 RTFString strTList{"TList"};
1589 RTFString strStreamerInfo{"StreamerInfo"};
1590 RTFString strStreamerTitle{"Doubly linked list"};
1591 auto &fileSimple = std::get<RImplSimple>(fFile);
1592 fileSimple.fControlBlock->fHeader.SetSeekInfo(fileSimple.fKeyOffset);
1593 auto keyLen = RTFKey(fileSimple.fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, strTList, strStreamerInfo,
1595 .fKeyLen;
1596
1597 TBufferFile buffer(TBuffer::kWrite, keyLen + 1);
1598 buffer.SetBufferOffset(keyLen);
1599 streamerInfoList.Streamer(buffer);
1600 assert(buffer.Length() > keyLen);
1601 const auto bufPayload = buffer.Buffer() + keyLen;
1602 const auto lenPayload = buffer.Length() - keyLen;
1603
1606
1608 fileSimple.fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, "TList", "StreamerInfo",
1609 "Doubly linked list");
1610 fileSimple.fControlBlock->fHeader.SetNbytesInfo(fileSimple.fFilePos -
1611 fileSimple.fControlBlock->fHeader.GetSeekInfo());
1612}
1613
1615{
1616 RTFString strEmpty;
1617 RTFString strRNTupleClass{"ROOT::RNTuple"};
1618 RTFString strRNTupleName{fNTupleName};
1619 RTFString strFileName{fFileName};
1620
1621 auto &fileSimple = std::get<RImplSimple>(fFile);
1622 RTFKey keyRNTuple(fileSimple.fControlBlock->fSeekNTuple, RTFHeader::kBEGIN, strRNTupleClass, strRNTupleName,
1623 strEmpty, RTFNTuple::GetSizePlusChecksum(), anchorSize);
1624
1625 fileSimple.fControlBlock->fFileRecord.SetSeekKeys(fileSimple.fKeyOffset);
1626 RTFKeyList keyList{1};
1627 RTFKey keyKeyList(fileSimple.fControlBlock->fFileRecord.GetSeekKeys(), RTFHeader::kBEGIN, strEmpty, strFileName,
1628 strEmpty, keyList.GetSize() + keyRNTuple.fKeyLen);
1629 fileSimple.Write(&keyKeyList, keyKeyList.GetHeaderSize(), fileSimple.fControlBlock->fFileRecord.GetSeekKeys());
1630 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1631 fileSimple.Write(&strFileName, strFileName.GetSize());
1632 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1633 fileSimple.Write(&keyList, keyList.GetSize());
1634 fileSimple.Write(&keyRNTuple, keyRNTuple.GetHeaderSize());
1635 // Write class name, object name, and title for this key.
1636 fileSimple.Write(&strRNTupleClass, strRNTupleClass.GetSize());
1637 fileSimple.Write(&strRNTupleName, strRNTupleName.GetSize());
1638 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1639 fileSimple.fControlBlock->fFileRecord.fNBytesKeys =
1640 fileSimple.fFilePos - fileSimple.fControlBlock->fFileRecord.GetSeekKeys();
1641 fileSimple.fKeyOffset = fileSimple.fFilePos;
1642}
1643
1645{
1646 auto &fileSimple = std::get<RImplSimple>(fFile);
1647 fileSimple.fControlBlock->fHeader.SetSeekFree(fileSimple.fKeyOffset);
1648 RTFString strEmpty;
1649 RTFString strFileName{fFileName};
1650 RTFFreeEntry freeEntry;
1651 RTFKey keyFreeList(fileSimple.fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, strEmpty, strFileName,
1652 strEmpty, freeEntry.GetSize());
1653 std::uint64_t firstFree = fileSimple.fControlBlock->fHeader.GetSeekFree() + keyFreeList.GetSize();
1654 freeEntry.Set(firstFree, std::max(2000000000ULL, ((firstFree / 1000000000ULL) + 1) * 1000000000ULL));
1655 fileSimple.WriteKey(&freeEntry, freeEntry.GetSize(), freeEntry.GetSize(),
1656 fileSimple.fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, "", fFileName, "");
1657 fileSimple.fControlBlock->fHeader.SetNbytesFree(fileSimple.fFilePos -
1658 fileSimple.fControlBlock->fHeader.GetSeekFree());
1659 fileSimple.fControlBlock->fHeader.SetEnd(fileSimple.fFilePos);
1660}
1661
1663{
1664 RTFString strRNTupleClass{"ROOT::RNTuple"};
1665 RTFString strRNTupleName{fNTupleName};
1666 RTFString strEmpty;
1667
1668 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1669 RUInt64BE checksum{XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData())};
1670 auto &fileSimple = std::get<RImplSimple>(fFile);
1671 fileSimple.fControlBlock->fSeekNTuple = fileSimple.fKeyOffset;
1672
1673 char keyBuf[RTFNTuple::GetSizePlusChecksum()];
1674
1675 // concatenate the RNTuple anchor with its checksum
1676 memcpy(keyBuf, &ntupleOnDisk, sizeof(RTFNTuple));
1677 memcpy(keyBuf + sizeof(RTFNTuple), &checksum, sizeof(checksum));
1678
1679 const auto sizeAnchor = sizeof(RTFNTuple) + sizeof(checksum);
1680 char zipAnchor[RTFNTuple::GetSizePlusChecksum()];
1682
1683 fileSimple.WriteKey(zipAnchor, szZipAnchor, sizeof(keyBuf), fileSimple.fControlBlock->fSeekNTuple, RTFHeader::kBEGIN,
1684 "ROOT::RNTuple", fNTupleName, "");
1685 return szZipAnchor;
1686}
1687
1689{
1690 RTFString strTFile{"TFile"};
1691 RTFString strFileName{fFileName};
1692 RTFString strEmpty;
1693
1694 auto &fileSimple = std::get<RImplSimple>(fFile);
1695 fileSimple.fControlBlock->fHeader = RTFHeader(defaultCompression);
1696
1697 RTFUUID uuid;
1698
1699 // First record of the file: the TFile object at offset kBEGIN (= 100)
1700 RTFKey keyRoot(RTFHeader::kBEGIN, 0, strTFile, strFileName, strEmpty,
1701 sizeof(RTFDirectory) + strFileName.GetSize() + strEmpty.GetSize() + uuid.GetSize());
1702 std::uint32_t nbytesName = keyRoot.fKeyLen + strFileName.GetSize() + 1;
1703 fileSimple.fControlBlock->fFileRecord.fNBytesName = nbytesName;
1704 fileSimple.fControlBlock->fHeader.SetNbytesName(nbytesName);
1705
1706 fileSimple.Write(&keyRoot, keyRoot.GetHeaderSize(), RTFHeader::kBEGIN);
1707 // Write class name, object name, and title for the TFile key.
1708 fileSimple.Write(&strTFile, strTFile.GetSize());
1709 fileSimple.Write(&strFileName, strFileName.GetSize());
1710 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1711 // Write the name and title of the TNamed preceding the TFile entry.
1712 fileSimple.Write(&strFileName, strFileName.GetSize());
1713 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1714 // Will be overwritten on commit
1715 fileSimple.fControlBlock->fSeekFileRecord = fileSimple.fFilePos;
1716 fileSimple.Write(&fileSimple.fControlBlock->fFileRecord, fileSimple.fControlBlock->fFileRecord.GetSize());
1717 fileSimple.Write(&uuid, uuid.GetSize());
1718
1719 // Padding bytes to allow the TFile record to grow for a big file
1720 RUInt32BE padding{0};
1721 for (int i = 0; i < 3; ++i)
1722 fileSimple.Write(&padding, sizeof(padding));
1723 fileSimple.fKeyOffset = fileSimple.fFilePos;
1724}
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:300
static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
#define ROOT_VERSION_CODE
Definition RVersion.hxx:24
#define ClassDefInlineOverride(name, id)
Definition Rtypes.h:360
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:125
const Int_t kBEGIN
Definition TFile.cxx:203
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
char name[80]
Definition TGX11.cxx:145
Binding & operator=(OUT(*fun)(void))
void ReadBuffer(char *&buffer) override
T1 fFirst
Definition X11Events.mm:86
#define _(A, B)
Definition cfortran.h:108
An interface to read from, or write to, a ROOT file, as well as performing other common operations.
Definition RFile.hxx:253
The RKeyBlob writes an invisible key into a TFile.
bool WasAllocatedInAFreeSlot() const
void Reserve(size_t nbytes, std::uint64_t *seekKey)
Register a new key for a data record of size nbytes.
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
void LoadStreamerInfo()
Attempts to load the streamer info from the file.
RResult< RNTuple > GetNTupleProperAtOffset(std::uint64_t payloadOffset, std::uint64_t compSize, std::uint64_t uncompLen)
Loads an RNTuple anchor from a TFile at the given file offset (unzipping it if necessary).
RResult< RNTuple > GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
ROOT::RResult< void > TryReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Like ReadBuffer but returns a RResult instead of throwing.
std::uint64_t SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName, std::string_view typeName)
Searches for a key with the given name and type in the key index of the directory starting at offsetD...
RResult< RNTuple > GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
RResult< RNTuple > GetNTupleProper(std::string_view ntuplePath)
Used when the file turns out to be a TFile container.
Helper class to compress data blocks in the ROOT compression frame format.
static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to)
Returns the size of the compressed data, written into the provided output buffer.
Helper class to uncompress data blocks in the ROOT compression frame format.
static void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
Write RNTuple data blocks in a TFile or a bare file container.
std::uint64_t ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves a new record as an RBlob key in the file.
void WriteTFileStreamerInfo(int compression)
Write the compressed streamer info record with the description of the RNTuple class.
void WriteTFileKeysList(std::uint64_t anchorSize)
Write the TList with the RNTuple key.
std::uint64_t WriteTFileNTupleKey(int compression)
The only key that will be visible in file->ls() Returns the size on disk of the anchor object.
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
static std::uint64_t ReserveBlobKey(T &caller, TFile &file, std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen])
static std::unique_ptr< RNTupleFileWriter > Append(std::string_view ntupleName, TDirectory &fileOrDirectory, std::uint64_t maxKeySize, bool isHidden)
The directory parameter can also be a TFile object (TFile inherits from TDirectory).
void WriteTFileFreeList()
Last record in the file.
std::unique_ptr< RNTupleFileWriter > CloneAsHidden(std::string_view ntupleName) const
Creates a new RNTupleFileWriter with the same underlying TDirectory as this but writing to a differen...
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the header and TFile object.
RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize, bool isHidden)
void Seek(std::uint64_t offset)
Seek a simple writer to offset.
ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t fStreamerInfoMap
Set of streamer info records that should be written to the file.
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
static std::unique_ptr< RNTupleFileWriter > Recreate(std::string_view ntupleName, std::string_view path, EContainerFormat containerFormat, const ROOT::RNTupleWriteOptions &options)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
void WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
Write into a reserved record; the caller is responsible for making sure that the written byte range i...
RNTupleLink Commit(int compression=RCompressionSetting::EDefaults::kUseGeneralPurpose)
Writes the RNTuple key to the file so that the header and footer keys can be found.
static void PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len, unsigned char buffer[kBlobKeyLen])
Prepares buffer for a new record as an RBlob key at offset.
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
void UpdateStreamerInfos(const ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t &streamerInfos)
Ensures that the streamer info records passed as argument are written to the file.
RNTuple fNTupleAnchor
Header and footer location of the ntuple, written on Commit()
EContainerFormat
For testing purposes, RNTuple data can be written into a bare file container instead of a ROOT file.
A helper class for serializing and deserialization of the RNTuple binary format.
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
The RRawFile provides read-only access to local and remote files.
Definition RRawFile.hxx:43
Base class for all ROOT issued exceptions.
Definition RError.hxx:79
Common user-tunable settings for storing RNTuples.
std::size_t GetWriteBufferSize() const
std::uint64_t GetMaxKeySize() const
std::uint32_t GetCompression() const
Representation of an RNTuple data set in a ROOT file.
Definition RNTuple.hxx:68
std::uint64_t fMaxKeySize
The maximum size for a TKey payload. Payloads bigger than this size will be written as multiple blobs...
Definition RNTuple.hxx:121
static TClass * Class()
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:198
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition TBufferFile.h:47
void TagStreamerInfo(TVirtualStreamerInfo *info) override
Mark the classindex of the current file as using this TStreamerInfo.
void SetBufferDisplacement() override
Definition TBufferIO.h:82
void SetParent(TObject *parent)
Set parent owning this buffer.
Definition TBuffer.cxx:269
@ kWrite
Definition TBuffer.h:73
@ kRead
Definition TBuffer.h:73
void SetBufferOffset(Int_t offset=0)
Definition TBuffer.h:93
Int_t Length() const
Definition TBuffer.h:100
char * Buffer() const
Definition TBuffer.h:96
Describe directory structure in memory.
Definition TDirectory.h:45
A file, usually with extension .root, that stores data and code in the form of serialized objects in ...
Definition TFile.h:130
Bool_t IsBinary() const
Definition TFile.h:347
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition TKey.h:28
Int_t Sizeof() const override
Return the size in bytes of the key header structure.
Definition TKey.cxx:1370
Int_t fVersion
Key version identifier.
Definition TKey.h:41
Int_t fLeft
Number of bytes left in current segment.
Definition TKey.h:50
Short_t fKeylen
Number of bytes for the key itself.
Definition TKey.h:45
Long64_t fSeekKey
Location of object on file.
Definition TKey.h:47
virtual void Create(Int_t nbytes, TFile *f=nullptr)
Create a TKey object of specified size.
Definition TKey.cxx:505
TString fClassName
Object Class name.
Definition TKey.h:49
A doubly linked list.
Definition TList.h:38
Describes a persistent version of a class.
static TClass * Class()
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition TUUID.h:42
static TUUID UUIDv4()
Create a UUID version 4 (variant 1) UUID according to RFC 4122.
Definition TUUID.cxx:157
TFile * GetRFileTFile(RFile &rfile)
Definition RFile.cxx:574
RNTuple CreateAnchor(std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch, std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter, std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize)
Definition RNTuple.cxx:52
std::unique_ptr< T[]> MakeUninitArray(std::size_t size)
Make an array of default-initialized elements.
Helper templated class for swapping bytes; specializations for N={2,4,8} are provided below.
Definition Byteswap.h:124
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
void AllocateBuffers(std::size_t bufferSize)
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
std::uint64_t ReserveBlobKey(std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
If a TFile container is written by a C stream (simple file), on dataset commit, the file header and t...
auto * tt
Definition textangle.C:16