Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleSerialize.cxx
Go to the documentation of this file.
1/// \file RNTupleSerialize.cxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \author Javier Lopez-Gomez <javier.lopez.gomez@cern.ch>
5/// \date 2021-08-02
6/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
7/// is welcome!
8
9/*************************************************************************
10 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
11 * All rights reserved. *
12 * *
13 * For the licensing terms see $ROOTSYS/LICENSE. *
14 * For the list of contributors see $ROOTSYS/README/CREDITS. *
15 *************************************************************************/
16
18#include <ROOT/RColumnModel.hxx>
19#include <ROOT/RError.hxx>
22
23#include <RVersion.h>
24#include <xxhash.h>
25
26#include <cstring> // for memcpy
27#include <deque>
28#include <set>
29#include <unordered_map>
30
31template <typename T>
33
34
35namespace {
37
38std::uint32_t SerializeField(const ROOT::Experimental::RFieldDescriptor &fieldDesc,
39 ROOT::Experimental::DescriptorId_t onDiskParentId, void *buffer)
40{
41
42 auto base = reinterpret_cast<unsigned char *>(buffer);
43 auto pos = base;
44 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
45
46 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
47
48 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetFieldVersion(), *where);
49 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetTypeVersion(), *where);
50 pos += RNTupleSerializer::SerializeUInt32(onDiskParentId, *where);
51 pos += RNTupleSerializer::SerializeFieldStructure(fieldDesc.GetStructure(), *where);
52 if (fieldDesc.GetNRepetitions() > 0) {
53 pos += RNTupleSerializer::SerializeUInt16(RNTupleSerializer::kFlagRepetitiveField, *where);
54 pos += RNTupleSerializer::SerializeUInt64(fieldDesc.GetNRepetitions(), *where);
55 } else {
56 pos += RNTupleSerializer::SerializeUInt16(0, *where);
57 }
58 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldName(), *where);
59 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeName(), *where);
60 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeAlias(), *where);
61 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldDescription(), *where);
62
63 auto size = pos - base;
64 RNTupleSerializer::SerializeFramePostscript(base, size);
65
66 return size;
67}
68
69// clang-format off
70/// Serialize, in order, fields enumerated in `fieldList` to `buffer`. `firstOnDiskId` specifies the on-disk ID for the
71/// first element in the `fieldList` sequence. Before calling this function `RContext::MapSchema()` should have been
72/// called on `context` in order to map in-memory field IDs to their on-disk counterpart.
73/// \return The number of bytes written to the output buffer; if `buffer` is `nullptr` no data is serialized and the
74/// required buffer size is returned
75// clang-format on
76std::uint32_t SerializeFieldList(const ROOT::Experimental::RNTupleDescriptor &desc,
77 std::span<const ROOT::Experimental::DescriptorId_t> fieldList,
78 std::size_t firstOnDiskId,
80{
81 auto base = reinterpret_cast<unsigned char *>(buffer);
82 auto pos = base;
83 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
84
85 auto fieldZeroId = desc.GetFieldZeroId();
86 ROOT::Experimental::DescriptorId_t onDiskFieldId = firstOnDiskId;
87 for (auto fieldId : fieldList) {
88 const auto &f = desc.GetFieldDescriptor(fieldId);
89 auto onDiskParentId =
90 (f.GetParentId() == fieldZeroId) ? onDiskFieldId : context.GetOnDiskFieldId(f.GetParentId());
91 pos += SerializeField(f, onDiskParentId, *where);
92 ++onDiskFieldId;
93 }
94
95 return pos - base;
96}
97
98RResult<std::uint32_t> DeserializeField(const void *buffer, std::uint64_t bufSize,
100{
101 using ENTupleStructure = ROOT::Experimental::ENTupleStructure;
102
103 auto base = reinterpret_cast<const unsigned char *>(buffer);
104 auto bytes = base;
105 std::uint64_t frameSize;
106 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
107 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
108 if (!result)
109 return R__FORWARD_ERROR(result);
110 bytes += result.Unwrap();
111
112 std::uint32_t fieldVersion;
113 std::uint32_t typeVersion;
114 std::uint32_t parentId;
115 // initialize properly for call to SerializeFieldStructure()
116 ENTupleStructure structure{ENTupleStructure::kLeaf};
117 std::uint16_t flags;
118 if (fnFrameSizeLeft() < 3 * sizeof(std::uint32_t) +
119 RNTupleSerializer::SerializeFieldStructure(structure, nullptr) +
120 sizeof(std::uint16_t))
121 {
122 return R__FAIL("field record frame too short");
123 }
124 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldVersion);
125 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeVersion);
126 bytes += RNTupleSerializer::DeserializeUInt32(bytes, parentId);
127 auto res16 = RNTupleSerializer::DeserializeFieldStructure(bytes, structure);
128 if (!res16)
129 return R__FORWARD_ERROR(res16);
130 bytes += res16.Unwrap();
131 bytes += RNTupleSerializer::DeserializeUInt16(bytes, flags);
132 fieldDesc.FieldVersion(fieldVersion).TypeVersion(typeVersion).ParentId(parentId).Structure(structure);
133
134 if (flags & RNTupleSerializer::kFlagRepetitiveField) {
135 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
136 return R__FAIL("field record frame too short");
137 std::uint64_t nRepetitions;
138 bytes += RNTupleSerializer::DeserializeUInt64(bytes, nRepetitions);
139 fieldDesc.NRepetitions(nRepetitions);
140 }
141
142 std::string fieldName;
143 std::string typeName;
144 std::string aliasName;
145 std::string description;
146 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), fieldName).Unwrap();
147 if (!result)
148 return R__FORWARD_ERROR(result);
149 bytes += result.Unwrap();
150 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), typeName).Unwrap();
151 if (!result)
152 return R__FORWARD_ERROR(result);
153 bytes += result.Unwrap();
154 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), aliasName).Unwrap();
155 if (!result)
156 return R__FORWARD_ERROR(result);
157 bytes += result.Unwrap();
158 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), description).Unwrap();
159 if (!result)
160 return R__FORWARD_ERROR(result);
161 bytes += result.Unwrap();
162 fieldDesc.FieldName(fieldName).TypeName(typeName).TypeAlias(aliasName).FieldDescription(description);
163
164 return frameSize;
165}
166
167std::uint32_t SerializeColumnList(const ROOT::Experimental::RNTupleDescriptor &desc,
168 std::span<const ROOT::Experimental::DescriptorId_t> fieldList,
170 void *buffer)
171{
172 using RColumnElementBase = ROOT::Experimental::Internal::RColumnElementBase;
173
174 auto base = reinterpret_cast<unsigned char *>(buffer);
175 auto pos = base;
176 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
177
178 for (auto parentId : fieldList) {
179 for (const auto &c : desc.GetColumnIterable(parentId)) {
180 if (c.IsAliasColumn())
181 continue;
182
183 auto frame = pos;
184 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
185
186 auto type = c.GetModel().GetType();
187 pos += RNTupleSerializer::SerializeColumnType(type, *where);
188 pos += RNTupleSerializer::SerializeUInt16(RColumnElementBase::GetBitsOnStorage(type), *where);
189 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(c.GetFieldId()), *where);
190 std::uint32_t flags = 0;
191 // TODO(jblomer): add support for descending columns in the column model
192 if (c.GetModel().GetIsSorted())
193 flags |= RNTupleSerializer::kFlagSortAscColumn;
194 // TODO(jblomer): fix for unsigned integer types
196 flags |= RNTupleSerializer::kFlagNonNegativeColumn;
197 const std::uint64_t firstElementIdx = c.GetFirstElementIndex();
198 if (firstElementIdx > 0)
199 flags |= RNTupleSerializer::kFlagDeferredColumn;
200 pos += RNTupleSerializer::SerializeUInt32(flags, *where);
201 if (flags & RNTupleSerializer::kFlagDeferredColumn)
202 pos += RNTupleSerializer::SerializeUInt64(firstElementIdx, *where);
203
204 pos += RNTupleSerializer::SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
205 }
206 }
207
208 return pos - base;
209}
210
211RResult<std::uint32_t> DeserializeColumn(const void *buffer, std::uint64_t bufSize,
213{
215
216 auto base = reinterpret_cast<const unsigned char *>(buffer);
217 auto bytes = base;
218 std::uint64_t frameSize;
219 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
220 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
221 if (!result)
222 return R__FORWARD_ERROR(result);
223 bytes += result.Unwrap();
224
225 // Initialize properly for SerializeColumnType
226 EColumnType type{EColumnType::kIndex32};
227 std::uint16_t bitsOnStorage;
228 std::uint32_t fieldId;
229 std::uint32_t flags;
230 std::uint64_t firstElementIdx = 0;
231 if (fnFrameSizeLeft() < RNTupleSerializer::SerializeColumnType(type, nullptr) +
232 sizeof(std::uint16_t) + 2 * sizeof(std::uint32_t))
233 {
234 return R__FAIL("column record frame too short");
235 }
236 auto res16 = RNTupleSerializer::DeserializeColumnType(bytes, type);
237 if (!res16)
238 return R__FORWARD_ERROR(res16);
239 bytes += res16.Unwrap();
240 bytes += RNTupleSerializer::DeserializeUInt16(bytes, bitsOnStorage);
241 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
242 bytes += RNTupleSerializer::DeserializeUInt32(bytes, flags);
243 if (flags & RNTupleSerializer::kFlagDeferredColumn) {
244 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
245 return R__FAIL("column record frame too short");
246 bytes += RNTupleSerializer::DeserializeUInt64(bytes, firstElementIdx);
247 }
248
250 return R__FAIL("column element size mismatch");
251
252 const bool isSorted = (flags & (RNTupleSerializer::kFlagSortAscColumn | RNTupleSerializer::kFlagSortDesColumn));
253 columnDesc.FieldId(fieldId).Model({type, isSorted}).FirstElementIndex(firstElementIdx);
254
255 return frameSize;
256}
257
258std::uint32_t SerializeLocatorPayloadURI(const ROOT::Experimental::RNTupleLocator &locator, unsigned char *buffer)
259{
260 const auto &uri = locator.GetPosition<std::string>();
261 if (uri.length() >= (1 << 16))
262 throw ROOT::Experimental::RException(R__FAIL("locator too large"));
263 if (buffer)
264 memcpy(buffer, uri.data(), uri.length());
265 return uri.length();
266}
267
268void DeserializeLocatorPayloadURI(const unsigned char *buffer, std::uint32_t payloadSize,
270{
271 locator.fBytesOnStorage = 0;
272 auto &uri = locator.fPosition.emplace<std::string>();
273 uri.resize(payloadSize);
274 memcpy(uri.data(), buffer, payloadSize);
275}
276
277std::uint32_t SerializeLocatorPayloadObject64(const ROOT::Experimental::RNTupleLocator &locator, unsigned char *buffer)
278{
280 if (buffer) {
281 RNTupleSerializer::SerializeUInt32(locator.fBytesOnStorage, buffer);
282 RNTupleSerializer::SerializeUInt64(data.fLocation, buffer + sizeof(std::uint32_t));
283 }
284 return sizeof(std::uint32_t) + sizeof(std::uint64_t);
285}
286
287void DeserializeLocatorPayloadObject64(const unsigned char *buffer, ROOT::Experimental::RNTupleLocator &locator)
288{
290 RNTupleSerializer::DeserializeUInt32(buffer, locator.fBytesOnStorage);
291 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint32_t), data.fLocation);
292}
293
294std::uint32_t SerializeAliasColumnList(const ROOT::Experimental::RNTupleDescriptor &desc,
295 std::span<const ROOT::Experimental::DescriptorId_t> fieldList,
297 void *buffer)
298{
299 auto base = reinterpret_cast<unsigned char *>(buffer);
300 auto pos = base;
301 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
302
303 for (auto parentId : fieldList) {
304 for (const auto &c : desc.GetColumnIterable(parentId)) {
305 if (!c.IsAliasColumn())
306 continue;
307
308 auto frame = pos;
309 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
310
311 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskColumnId(c.GetPhysicalId()), *where);
312 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(c.GetFieldId()), *where);
313
314 pos += RNTupleSerializer::SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
315 }
316 }
317
318 return pos - base;
319}
320
321RResult<std::uint32_t> DeserializeAliasColumn(const void *buffer, std::uint64_t bufSize,
322 std::uint32_t &physicalColumnId, std::uint32_t &fieldId)
323{
324 auto base = reinterpret_cast<const unsigned char *>(buffer);
325 auto bytes = base;
326 std::uint64_t frameSize;
327 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
328 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
329 if (!result)
330 return R__FORWARD_ERROR(result);
331 bytes += result.Unwrap();
332
333 if (fnFrameSizeLeft() < 2 * sizeof(std::uint32_t)) {
334 return R__FAIL("alias column record frame too short");
335 }
336
337 bytes += RNTupleSerializer::DeserializeUInt32(bytes, physicalColumnId);
338 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
339
340 return frameSize;
341}
342
343} // anonymous namespace
344
346 std::uint64_t length,
347 std::uint64_t &xxhash3, void *buffer)
348{
349 if (buffer != nullptr) {
350 xxhash3 = XXH3_64bits(data, length);
351 SerializeUInt64(xxhash3, buffer);
352 }
353 return 8;
354}
355
357 std::uint64_t length,
358 std::uint64_t &xxhash3)
359{
360 auto checksumReal = XXH3_64bits(data, length);
361 DeserializeUInt64(data + length, xxhash3);
362 if (xxhash3 != checksumReal)
363 return R__FAIL("XxHash-3 checksum mismatch");
364 return RResult<void>::Success();
365}
366
369{
370 std::uint64_t xxhash3;
371 return R__FORWARD_RESULT(VerifyXxHash3(data, length, xxhash3));
372}
373
374
375std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeInt16(std::int16_t val, void *buffer)
376{
377 if (buffer != nullptr) {
378 auto bytes = reinterpret_cast<unsigned char *>(buffer);
379 bytes[0] = (val & 0x00FF);
380 bytes[1] = (val & 0xFF00) >> 8;
381 }
382 return 2;
383}
384
385std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeInt16(const void *buffer, std::int16_t &val)
386{
387 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
388 val = std::int16_t(bytes[0]) + (std::int16_t(bytes[1]) << 8);
389 return 2;
390}
391
392std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeUInt16(std::uint16_t val, void *buffer)
393{
394 return SerializeInt16(val, buffer);
395}
396
397std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt16(const void *buffer, std::uint16_t &val)
398{
399 return DeserializeInt16(buffer, *reinterpret_cast<std::int16_t *>(&val));
400}
401
402std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeInt32(std::int32_t val, void *buffer)
403{
404 if (buffer != nullptr) {
405 auto bytes = reinterpret_cast<unsigned char *>(buffer);
406 bytes[0] = (val & 0x000000FF);
407 bytes[1] = (val & 0x0000FF00) >> 8;
408 bytes[2] = (val & 0x00FF0000) >> 16;
409 bytes[3] = (val & 0xFF000000) >> 24;
410 }
411 return 4;
412}
413
414std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeInt32(const void *buffer, std::int32_t &val)
415{
416 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
417 val = std::int32_t(bytes[0]) + (std::int32_t(bytes[1]) << 8) +
418 (std::int32_t(bytes[2]) << 16) + (std::int32_t(bytes[3]) << 24);
419 return 4;
420}
421
422std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeUInt32(std::uint32_t val, void *buffer)
423{
424 return SerializeInt32(val, buffer);
425}
426
427std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt32(const void *buffer, std::uint32_t &val)
428{
429 return DeserializeInt32(buffer, *reinterpret_cast<std::int32_t *>(&val));
430}
431
432std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeInt64(std::int64_t val, void *buffer)
433{
434 if (buffer != nullptr) {
435 auto bytes = reinterpret_cast<unsigned char *>(buffer);
436 bytes[0] = (val & 0x00000000000000FF);
437 bytes[1] = (val & 0x000000000000FF00) >> 8;
438 bytes[2] = (val & 0x0000000000FF0000) >> 16;
439 bytes[3] = (val & 0x00000000FF000000) >> 24;
440 bytes[4] = (val & 0x000000FF00000000) >> 32;
441 bytes[5] = (val & 0x0000FF0000000000) >> 40;
442 bytes[6] = (val & 0x00FF000000000000) >> 48;
443 bytes[7] = (val & 0xFF00000000000000) >> 56;
444 }
445 return 8;
446}
447
448std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeInt64(const void *buffer, std::int64_t &val)
449{
450 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
451 val = std::int64_t(bytes[0]) + (std::int64_t(bytes[1]) << 8) +
452 (std::int64_t(bytes[2]) << 16) + (std::int64_t(bytes[3]) << 24) +
453 (std::int64_t(bytes[4]) << 32) + (std::int64_t(bytes[5]) << 40) +
454 (std::int64_t(bytes[6]) << 48) + (std::int64_t(bytes[7]) << 56);
455 return 8;
456}
457
458std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeUInt64(std::uint64_t val, void *buffer)
459{
460 return SerializeInt64(val, buffer);
461}
462
463std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt64(const void *buffer, std::uint64_t &val)
464{
465 return DeserializeInt64(buffer, *reinterpret_cast<std::int64_t *>(&val));
466}
467
468std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeString(const std::string &val, void *buffer)
469{
470 if (buffer) {
471 auto pos = reinterpret_cast<unsigned char *>(buffer);
472 pos += SerializeUInt32(val.length(), pos);
473 memcpy(pos, val.data(), val.length());
474 }
475 return sizeof(std::uint32_t) + val.length();
476}
477
479 std::uint64_t bufSize,
480 std::string &val)
481{
482 if (bufSize < sizeof(std::uint32_t))
483 return R__FAIL("string buffer too short");
484 bufSize -= sizeof(std::uint32_t);
485
486 auto base = reinterpret_cast<const unsigned char *>(buffer);
487 auto bytes = base;
488 std::uint32_t length;
489 bytes += DeserializeUInt32(buffer, length);
490 if (bufSize < length)
491 return R__FAIL("string buffer too short");
492
493 val.resize(length);
494 memcpy(&val[0], bytes, length);
495 return sizeof(std::uint32_t) + length;
496}
497
498
501{
503 switch (type) {
504 case EColumnType::kIndex64: return SerializeUInt16(0x01, buffer);
505 case EColumnType::kIndex32: return SerializeUInt16(0x02, buffer);
506 case EColumnType::kSwitch: return SerializeUInt16(0x03, buffer);
507 case EColumnType::kByte: return SerializeUInt16(0x04, buffer);
508 case EColumnType::kChar: return SerializeUInt16(0x05, buffer);
509 case EColumnType::kBit: return SerializeUInt16(0x06, buffer);
510 case EColumnType::kReal64: return SerializeUInt16(0x07, buffer);
511 case EColumnType::kReal32: return SerializeUInt16(0x08, buffer);
512 case EColumnType::kReal16: return SerializeUInt16(0x09, buffer);
513 case EColumnType::kInt64: return SerializeUInt16(0x16, buffer);
514 case EColumnType::kUInt64: return SerializeUInt16(0x0A, buffer);
515 case EColumnType::kInt32: return SerializeUInt16(0x17, buffer);
516 case EColumnType::kUInt32: return SerializeUInt16(0x0B, buffer);
517 case EColumnType::kInt16: return SerializeUInt16(0x18, buffer);
518 case EColumnType::kUInt16: return SerializeUInt16(0x0C, buffer);
519 case EColumnType::kInt8: return SerializeUInt16(0x19, buffer);
520 case EColumnType::kUInt8: return SerializeUInt16(0x0D, buffer);
521 case EColumnType::kSplitIndex64: return SerializeUInt16(0x0E, buffer);
522 case EColumnType::kSplitIndex32: return SerializeUInt16(0x0F, buffer);
523 case EColumnType::kSplitReal64: return SerializeUInt16(0x10, buffer);
524 case EColumnType::kSplitReal32: return SerializeUInt16(0x11, buffer);
525 case EColumnType::kSplitInt64: return SerializeUInt16(0x1A, buffer);
526 case EColumnType::kSplitUInt64: return SerializeUInt16(0x13, buffer);
527 case EColumnType::kSplitInt32: return SerializeUInt16(0x1B, buffer);
528 case EColumnType::kSplitUInt32: return SerializeUInt16(0x14, buffer);
529 case EColumnType::kSplitInt16: return SerializeUInt16(0x1C, buffer);
530 case EColumnType::kSplitUInt16: return SerializeUInt16(0x15, buffer);
531 default: throw RException(R__FAIL("ROOT bug: unexpected column type"));
532 }
533}
534
535
537 const void *buffer, ROOT::Experimental::EColumnType &type)
538{
540 std::uint16_t onDiskType;
541 auto result = DeserializeUInt16(buffer, onDiskType);
542 switch (onDiskType) {
543 case 0x01: type = EColumnType::kIndex64; break;
544 case 0x02: type = EColumnType::kIndex32; break;
545 case 0x03: type = EColumnType::kSwitch; break;
546 case 0x04: type = EColumnType::kByte; break;
547 case 0x05: type = EColumnType::kChar; break;
548 case 0x06: type = EColumnType::kBit; break;
549 case 0x07: type = EColumnType::kReal64; break;
550 case 0x08: type = EColumnType::kReal32; break;
551 case 0x09: type = EColumnType::kReal16; break;
552 case 0x16: type = EColumnType::kInt64; break;
553 case 0x0A: type = EColumnType::kUInt64; break;
554 case 0x17: type = EColumnType::kInt32; break;
555 case 0x0B: type = EColumnType::kUInt32; break;
556 case 0x18: type = EColumnType::kInt16; break;
557 case 0x0C: type = EColumnType::kUInt16; break;
558 case 0x19: type = EColumnType::kInt8; break;
559 case 0x0D: type = EColumnType::kUInt8; break;
560 case 0x0E: type = EColumnType::kSplitIndex64; break;
561 case 0x0F: type = EColumnType::kSplitIndex32; break;
562 case 0x10: type = EColumnType::kSplitReal64; break;
563 case 0x11: type = EColumnType::kSplitReal32; break;
564 case 0x1A: type = EColumnType::kSplitInt64; break;
565 case 0x13: type = EColumnType::kSplitUInt64; break;
566 case 0x1B: type = EColumnType::kSplitInt32; break;
567 case 0x14: type = EColumnType::kSplitUInt32; break;
568 case 0x1C: type = EColumnType::kSplitInt16; break;
569 case 0x15: type = EColumnType::kSplitUInt16; break;
570 default: return R__FAIL("unexpected on-disk column type");
571 }
572 return result;
573}
574
575
577 ROOT::Experimental::ENTupleStructure structure, void *buffer)
578{
580 switch (structure) {
581 case ENTupleStructure::kLeaf: return SerializeUInt16(0x00, buffer);
582 case ENTupleStructure::kCollection: return SerializeUInt16(0x01, buffer);
583 case ENTupleStructure::kRecord: return SerializeUInt16(0x02, buffer);
584 case ENTupleStructure::kVariant: return SerializeUInt16(0x03, buffer);
585 case ENTupleStructure::kUnsplit: return SerializeUInt16(0x04, buffer);
586 default: throw RException(R__FAIL("ROOT bug: unexpected field structure type"));
587 }
588}
589
590
592 const void *buffer, ROOT::Experimental::ENTupleStructure &structure)
593{
595 std::uint16_t onDiskValue;
596 auto result = DeserializeUInt16(buffer, onDiskValue);
597 switch (onDiskValue) {
598 case 0x00: structure = ENTupleStructure::kLeaf; break;
599 case 0x01: structure = ENTupleStructure::kCollection; break;
600 case 0x02: structure = ENTupleStructure::kRecord; break;
601 case 0x03: structure = ENTupleStructure::kVariant; break;
602 case 0x04: structure = ENTupleStructure::kUnsplit; break;
603 default: return R__FAIL("unexpected on-disk field structure value");
604 }
605 return result;
606}
607
608std::uint32_t
610{
611 auto base = reinterpret_cast<unsigned char *>(buffer);
612 auto pos = base;
613 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
614
615 pos += SerializeUInt64(envelopeType, *where);
616 // The 48bits size information is filled in the postscript
617 return pos - base;
618}
619
621 std::uint64_t size,
622 std::uint64_t &xxhash3)
623{
624 if (size < sizeof(std::uint64_t))
625 throw RException(R__FAIL("envelope size too small"));
626 if (size >= static_cast<uint64_t>(1) << 48)
627 throw RException(R__FAIL("envelope size too big"));
628 if (envelope) {
629 std::uint64_t typeAndSize;
630 DeserializeUInt64(envelope, typeAndSize);
631 typeAndSize |= (size + 8) << 16;
632 SerializeUInt64(typeAndSize, envelope);
633 }
634 return SerializeXxHash3(envelope, size, xxhash3, envelope ? (envelope + size) : nullptr);
635}
636
638 std::uint64_t size)
639{
640 std::uint64_t xxhash3;
641 return SerializeEnvelopePostscript(envelope, size, xxhash3);
642}
643
646 std::uint16_t expectedType, std::uint64_t &xxhash3)
647{
648 const std::uint64_t minEnvelopeSize = sizeof(std::uint64_t) + sizeof(std::uint64_t);
649 if (bufSize < minEnvelopeSize)
650 return R__FAIL("invalid envelope buffer, too short");
651
652 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
653 auto base = bytes;
654
655 std::uint64_t typeAndSize;
656 bytes += DeserializeUInt64(bytes, typeAndSize);
657
658 std::uint16_t envelopeType = typeAndSize & 0xFFFF;
659 if (envelopeType != expectedType) {
660 return R__FAIL("envelope type mismatch: expected " + std::to_string(expectedType) + ", found " +
661 std::to_string(envelopeType));
662 }
663
664 std::uint64_t envelopeSize = typeAndSize >> 16;
665 if (bufSize < envelopeSize)
666 return R__FAIL("envelope buffer size too small");
667 if (envelopeSize < minEnvelopeSize)
668 return R__FAIL("invalid envelope, too short");
669
670 auto result = VerifyXxHash3(base, envelopeSize - 8, xxhash3);
671 if (!result)
672 return R__FORWARD_ERROR(result);
673
674 return sizeof(typeAndSize);
675}
676
678 std::uint64_t bufSize,
679 std::uint16_t expectedType)
680{
681 std::uint64_t xxhash3;
682 return R__FORWARD_RESULT(DeserializeEnvelope(buffer, bufSize, expectedType, xxhash3));
683}
684
685
687{
688 // Marker: multiply the final size with 1
689 return SerializeInt64(1, buffer);
690}
691
692
694 std::uint32_t nitems, void *buffer)
695{
696 auto base = reinterpret_cast<unsigned char *>(buffer);
697 auto pos = base;
698 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
699
700 // Marker: multiply the final size with -1
701 pos += SerializeInt64(-1, *where);
702 pos += SerializeUInt32(nitems, *where);
703 return pos - base;
704}
705
707{
708 auto preambleSize = sizeof(std::int64_t);
709 if (size < preambleSize)
710 throw RException(R__FAIL("frame too short: " + std::to_string(size)));
711 if (frame) {
712 std::int64_t marker;
713 DeserializeInt64(frame, marker);
714 if ((marker < 0) && (size < (sizeof(std::uint32_t) + preambleSize)))
715 throw RException(R__FAIL("frame too short: " + std::to_string(size)));
716 SerializeInt64(marker * static_cast<int64_t>(size), frame);
717 }
718 return 0;
719}
720
723 std::uint64_t &frameSize, std::uint32_t &nitems)
724{
725 std::uint64_t minSize = sizeof(std::int64_t);
726 if (bufSize < minSize)
727 return R__FAIL("frame too short");
728
729 std::int64_t *ssize = reinterpret_cast<std::int64_t *>(&frameSize);
730 DeserializeInt64(buffer, *ssize);
731
732 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
733 bytes += minSize;
734
735 if (*ssize >= 0) {
736 // Record frame
737 nitems = 1;
738 } else {
739 // List frame
740 minSize += sizeof(std::uint32_t);
741 if (bufSize < minSize)
742 return R__FAIL("frame too short");
743 bytes += DeserializeUInt32(bytes, nitems);
744 *ssize = -(*ssize);
745 }
746
747 if (frameSize < minSize)
748 return R__FAIL("corrupt frame size");
749 if (bufSize < frameSize)
750 return R__FAIL("frame too short");
751
752 return bytes - reinterpret_cast<const unsigned char *>(buffer);
753}
754
757 std::uint64_t &frameSize)
758{
759 std::uint32_t nitems;
760 return R__FORWARD_RESULT(DeserializeFrameHeader(buffer, bufSize, frameSize, nitems));
761}
762
763std::uint32_t
765 void *buffer)
766{
767 if (flags.empty())
768 return SerializeUInt64(0, buffer);
769
770 if (buffer) {
771 auto bytes = reinterpret_cast<unsigned char *>(buffer);
772
773 for (unsigned i = 0; i < flags.size(); ++i) {
774 if (flags[i] & 0x8000000000000000)
775 throw RException(R__FAIL("feature flag out of bounds"));
776
777 // The MSb indicates that another Int64 follows; set this bit to 1 for all except the last element
778 if (i == (flags.size() - 1))
779 SerializeUInt64(flags[i], bytes);
780 else
781 bytes += SerializeUInt64(flags[i] | 0x8000000000000000, bytes);
782 }
783 }
784 return (flags.size() * sizeof(std::int64_t));
785}
786
789 std::vector<std::uint64_t> &flags)
790{
791 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
792
793 flags.clear();
794 std::uint64_t f;
795 do {
796 if (bufSize < sizeof(std::uint64_t))
797 return R__FAIL("feature flag buffer too short");
798 bytes += DeserializeUInt64(bytes, f);
799 bufSize -= sizeof(std::uint64_t);
800 flags.emplace_back(f & ~0x8000000000000000);
801 } while (f & 0x8000000000000000);
802
803 return (flags.size() * sizeof(std::uint64_t));
804}
805
807 const RNTupleLocator &locator, void *buffer)
808{
810 throw RException(R__FAIL("locator is not serializable"));
811
812 std::uint32_t size = 0;
813 if (locator.fType == RNTupleLocator::kTypeFile) {
814 if (static_cast<std::int32_t>(locator.fBytesOnStorage) < 0)
815 throw RException(R__FAIL("locator too large"));
816 size += SerializeUInt32(locator.fBytesOnStorage, buffer);
817 size += SerializeUInt64(locator.GetPosition<std::uint64_t>(),
818 buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
819 return size;
820 }
821
822 auto payloadp = buffer ? reinterpret_cast<unsigned char *>(buffer) + sizeof(std::int32_t) : nullptr;
823 switch (locator.fType) {
824 case RNTupleLocator::kTypeURI: size += SerializeLocatorPayloadURI(locator, payloadp); break;
825 case RNTupleLocator::kTypeDAOS: size += SerializeLocatorPayloadObject64(locator, payloadp); break;
826 default: throw RException(R__FAIL("locator has unknown type"));
827 }
828 std::int32_t head = sizeof(std::int32_t) + size;
829 head |= locator.fReserved << 16;
830 head |= static_cast<int>(locator.fType & 0x7F) << 24;
831 head = -head;
833 return size;
834}
835
837 std::uint64_t bufSize,
838 RNTupleLocator &locator)
839{
840 if (bufSize < sizeof(std::int32_t))
841 return R__FAIL("too short locator");
842
843 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
844 std::int32_t head;
845
846 bytes += DeserializeInt32(bytes, head);
847 bufSize -= sizeof(std::int32_t);
848 if (head < 0) {
849 head = -head;
850 const int type = head >> 24;
851 const std::uint32_t payloadSize = (static_cast<std::uint32_t>(head) & 0x0000FFFF) - sizeof(std::int32_t);
852 if (bufSize < payloadSize)
853 return R__FAIL("too short locator");
854 locator.fType = static_cast<RNTupleLocator::ELocatorType>(type);
855 locator.fReserved = static_cast<std::uint32_t>(head >> 16) & 0xFF;
856 switch (type) {
857 case RNTupleLocator::kTypeURI: DeserializeLocatorPayloadURI(bytes, payloadSize, locator); break;
858 case RNTupleLocator::kTypeDAOS: DeserializeLocatorPayloadObject64(bytes, locator); break;
859 default: return R__FAIL("unsupported locator type: " + std::to_string(type));
860 }
861 bytes += payloadSize;
862 } else {
863 if (bufSize < sizeof(std::uint64_t))
864 return R__FAIL("too short locator");
865 auto &offset = locator.fPosition.emplace<std::uint64_t>();
867 bytes += DeserializeUInt64(bytes, offset);
868 locator.fBytesOnStorage = head;
869 }
870
871 return bytes - reinterpret_cast<const unsigned char *>(buffer);
872}
873
875 const REnvelopeLink &envelopeLink, void *buffer)
876{
877 auto size = SerializeUInt64(envelopeLink.fLength, buffer);
878 size += SerializeLocator(envelopeLink.fLocator,
879 buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
880 return size;
881}
882
885 REnvelopeLink &envelopeLink)
886{
887 if (bufSize < sizeof(std::int64_t))
888 return R__FAIL("too short envelope link");
889
890 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
891 bytes += DeserializeUInt64(bytes, envelopeLink.fLength);
892 bufSize -= sizeof(std::uint64_t);
893 auto result = DeserializeLocator(bytes, bufSize, envelopeLink.fLocator);
894 if (!result)
895 return R__FORWARD_ERROR(result);
896 bytes += result.Unwrap();
897 return bytes - reinterpret_cast<const unsigned char *>(buffer);
898}
899
900
902 const RClusterSummary &clusterSummary, void *buffer)
903{
904 auto base = reinterpret_cast<unsigned char *>(buffer);
905 auto pos = base;
906 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
907
908 auto frame = pos;
909 pos += SerializeRecordFramePreamble(*where);
910 pos += SerializeUInt64(clusterSummary.fFirstEntry, *where);
911 if (clusterSummary.fColumnGroupID >= 0) {
912 pos += SerializeInt64(-static_cast<int64_t>(clusterSummary.fNEntries), *where);
913 pos += SerializeUInt32(clusterSummary.fColumnGroupID, *where);
914 } else {
915 pos += SerializeInt64(static_cast<int64_t>(clusterSummary.fNEntries), *where);
916 }
917 auto size = pos - frame;
918 pos += SerializeFramePostscript(frame, size);
919 return size;
920}
921
924 RClusterSummary &clusterSummary)
925{
926 auto base = reinterpret_cast<const unsigned char *>(buffer);
927 auto bytes = base;
928 std::uint64_t frameSize;
929 auto result = DeserializeFrameHeader(bytes, bufSize, frameSize);
930 if (!result)
931 return R__FORWARD_ERROR(result);
932 bytes += result.Unwrap();
933
934 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
935 if (fnFrameSizeLeft() < 2 * sizeof(std::uint64_t))
936 return R__FAIL("too short cluster summary");
937
938 bytes += DeserializeUInt64(bytes, clusterSummary.fFirstEntry);
939 std::int64_t nEntries;
940 bytes += DeserializeInt64(bytes, nEntries);
941
942 if (nEntries < 0) {
943 if (fnFrameSizeLeft() < sizeof(std::uint32_t))
944 return R__FAIL("too short cluster summary");
945 clusterSummary.fNEntries = -nEntries;
946 std::uint32_t columnGroupID;
947 bytes += DeserializeUInt32(bytes, columnGroupID);
948 clusterSummary.fColumnGroupID = columnGroupID;
949 } else {
950 clusterSummary.fNEntries = nEntries;
951 clusterSummary.fColumnGroupID = -1;
952 }
953
954 return frameSize;
955}
956
957
959 const RClusterGroup &clusterGroup, void *buffer)
960{
961 auto base = reinterpret_cast<unsigned char *>(buffer);
962 auto pos = base;
963 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
964
965 auto frame = pos;
966 pos += SerializeRecordFramePreamble(*where);
967 pos += SerializeUInt64(clusterGroup.fMinEntry, *where);
968 pos += SerializeUInt64(clusterGroup.fEntrySpan, *where);
969 pos += SerializeUInt32(clusterGroup.fNClusters, *where);
970 pos += SerializeEnvelopeLink(clusterGroup.fPageListEnvelopeLink, *where);
971 auto size = pos - frame;
972 pos += SerializeFramePostscript(frame, size);
973 return size;
974}
975
978 RClusterGroup &clusterGroup)
979{
980 auto base = reinterpret_cast<const unsigned char *>(buffer);
981 auto bytes = base;
982
983 std::uint64_t frameSize;
984 auto result = DeserializeFrameHeader(bytes, bufSize, frameSize);
985 if (!result)
986 return R__FORWARD_ERROR(result);
987 bytes += result.Unwrap();
988
989 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
990 if (fnFrameSizeLeft() < sizeof(std::uint32_t) + 2 * sizeof(std::uint64_t))
991 return R__FAIL("too short cluster group");
992
993 bytes += DeserializeUInt64(bytes, clusterGroup.fMinEntry);
994 bytes += DeserializeUInt64(bytes, clusterGroup.fEntrySpan);
995 bytes += DeserializeUInt32(bytes, clusterGroup.fNClusters);
996 result = DeserializeEnvelopeLink(bytes, fnFrameSizeLeft(), clusterGroup.fPageListEnvelopeLink);
997 if (!result)
998 return R__FORWARD_ERROR(result);
999
1000 return frameSize;
1001}
1002
1004 bool forHeaderExtension)
1005{
1006 auto fieldZeroId = desc.GetFieldZeroId();
1007 auto depthFirstTraversal = [&](std::span<DescriptorId_t> fieldTrees, auto doForEachField) {
1008 std::deque<DescriptorId_t> idQueue{fieldTrees.begin(), fieldTrees.end()};
1009 while (!idQueue.empty()) {
1010 auto fieldId = idQueue.front();
1011 idQueue.pop_front();
1012 // Field zero has no physical representation nor columns of its own; recurse over its subfields only
1013 if (fieldId != fieldZeroId)
1014 doForEachField(fieldId);
1015 unsigned i = 0;
1016 for (const auto &f : desc.GetFieldIterable(fieldId))
1017 idQueue.insert(idQueue.begin() + i++, f.GetId());
1018 }
1019 };
1020
1021 R__ASSERT(desc.GetNFields() > 0); // we must have at least a zero field
1022 if (!forHeaderExtension)
1023 R__ASSERT(GetHeaderExtensionOffset() == -1U);
1024
1025 std::vector<DescriptorId_t> fieldTrees;
1026 if (!forHeaderExtension) {
1027 fieldTrees.emplace_back(fieldZeroId);
1028 } else if (auto xHeader = desc.GetHeaderExtension()) {
1029 fieldTrees = xHeader->GetTopLevelFields(desc);
1030 }
1031 depthFirstTraversal(fieldTrees, [&](DescriptorId_t fieldId) { MapFieldId(fieldId); });
1032 depthFirstTraversal(fieldTrees, [&](DescriptorId_t fieldId) {
1033 for (const auto &c : desc.GetColumnIterable(fieldId))
1034 if (!c.IsAliasColumn())
1035 MapColumnId(c.GetLogicalId());
1036 });
1037 depthFirstTraversal(fieldTrees, [&](DescriptorId_t fieldId) {
1038 for (const auto &c : desc.GetColumnIterable(fieldId))
1039 if (c.IsAliasColumn())
1040 MapColumnId(c.GetLogicalId());
1041 });
1042
1043 // Anything added after this point is accounted for the header extension
1044 if (!forHeaderExtension)
1045 BeginHeaderExtension();
1046}
1047
1049 const RNTupleDescriptor &desc,
1050 const RContext &context,
1051 bool forHeaderExtension)
1052{
1053 auto base = reinterpret_cast<unsigned char *>(buffer);
1054 auto pos = base;
1055 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
1056
1057 std::size_t nFields = 0, nColumns = 0, nAliasColumns = 0, fieldListOffset = 0;
1058 if (forHeaderExtension) {
1059 // A call to `RNTupleDescriptorBuilder::BeginHeaderExtension()` is not strictly required after serializing the
1060 // header, which may happen, e.g., in unit tests. Ensure an empty schema extension is serialized in this case
1061 if (auto xHeader = desc.GetHeaderExtension()) {
1062 nFields = xHeader->GetNFields();
1063 nColumns = xHeader->GetNPhysicalColumns();
1064 nAliasColumns = xHeader->GetNLogicalColumns() - xHeader->GetNPhysicalColumns();
1065 fieldListOffset = context.GetHeaderExtensionOffset();
1066 }
1067 } else {
1068 nFields = desc.GetNFields() - 1;
1069 nColumns = desc.GetNPhysicalColumns();
1070 nAliasColumns = desc.GetNLogicalColumns() - desc.GetNPhysicalColumns();
1071 }
1072 const auto &onDiskFields = context.GetOnDiskFieldList();
1073 R__ASSERT(onDiskFields.size() >= fieldListOffset);
1074 std::span<const DescriptorId_t> fieldList{onDiskFields.data() + fieldListOffset,
1075 onDiskFields.size() - fieldListOffset};
1076
1077 auto frame = pos;
1078 pos += SerializeListFramePreamble(nFields, *where);
1079 pos += SerializeFieldList(desc, fieldList, /*firstOnDiskId=*/fieldListOffset, context, *where);
1080 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1081
1082 frame = pos;
1083 pos += SerializeListFramePreamble(nColumns, *where);
1084 pos += SerializeColumnList(desc, fieldList, context, *where);
1085 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1086
1087 frame = pos;
1088 pos += SerializeListFramePreamble(nAliasColumns, *where);
1089 pos += SerializeAliasColumnList(desc, fieldList, context, *where);
1090 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1091
1092 // We don't use extra type information yet
1093 frame = pos;
1094 pos += SerializeListFramePreamble(0, *where);
1095 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1096 return static_cast<std::uint32_t>(pos - base);
1097}
1098
1101 RNTupleDescriptorBuilder &descBuilder)
1102{
1103 auto base = reinterpret_cast<const unsigned char *>(buffer);
1104 auto bytes = base;
1105 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1107
1108 std::uint64_t frameSize;
1109 auto frame = bytes;
1110 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1111
1112 std::uint32_t nFields;
1113 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nFields);
1114 if (!result)
1115 return R__FORWARD_ERROR(result);
1116 bytes += result.Unwrap();
1117 // The zero field is always added before `DeserializeSchemaDescription()` is called
1118 const std::uint32_t fieldIdRangeBegin = descBuilder.GetDescriptor().GetNFields() - 1;
1119 for (unsigned i = 0; i < nFields; ++i) {
1120 std::uint32_t fieldId = fieldIdRangeBegin + i;
1121 RFieldDescriptorBuilder fieldBuilder;
1122 result = DeserializeField(bytes, fnFrameSizeLeft(), fieldBuilder);
1123 if (!result)
1124 return R__FORWARD_ERROR(result);
1125 bytes += result.Unwrap();
1126 if (fieldId == fieldBuilder.GetParentId())
1127 fieldBuilder.ParentId(kZeroFieldId);
1128 auto fieldDesc = fieldBuilder.FieldId(fieldId).MakeDescriptor();
1129 if (!fieldDesc)
1130 return R__FORWARD_ERROR(fieldDesc);
1131 auto parentId = fieldDesc.Inspect().GetParentId();
1132 descBuilder.AddField(fieldDesc.Unwrap());
1133 auto resVoid = descBuilder.AddFieldLink(parentId, fieldId);
1134 if (!resVoid)
1135 return R__FORWARD_ERROR(resVoid);
1136 }
1137 bytes = frame + frameSize;
1138
1139 std::uint32_t nColumns;
1140 frame = bytes;
1141 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nColumns);
1142 if (!result)
1143 return R__FORWARD_ERROR(result);
1144 bytes += result.Unwrap();
1145 const std::uint32_t columnIdRangeBegin = descBuilder.GetDescriptor().GetNLogicalColumns();
1146 std::unordered_map<DescriptorId_t, std::uint32_t> maxIndexes;
1147 for (unsigned i = 0; i < nColumns; ++i) {
1148 std::uint32_t columnId = columnIdRangeBegin + i;
1149 RColumnDescriptorBuilder columnBuilder;
1150 result = DeserializeColumn(bytes, fnFrameSizeLeft(), columnBuilder);
1151 if (!result)
1152 return R__FORWARD_ERROR(result);
1153 bytes += result.Unwrap();
1154
1155 std::uint32_t idx = 0;
1156 const auto fieldId = columnBuilder.GetFieldId();
1157 auto maxIdx = maxIndexes.find(fieldId);
1158 if (maxIdx != maxIndexes.end())
1159 idx = maxIdx->second + 1;
1160 maxIndexes[fieldId] = idx;
1161
1162 auto columnDesc = columnBuilder.Index(idx).LogicalColumnId(columnId).PhysicalColumnId(columnId).MakeDescriptor();
1163 if (!columnDesc)
1164 return R__FORWARD_ERROR(columnDesc);
1165 auto resVoid = descBuilder.AddColumn(columnDesc.Unwrap());
1166 if (!resVoid)
1167 return R__FORWARD_ERROR(resVoid);
1168 }
1169 bytes = frame + frameSize;
1170
1171 std::uint32_t nAliasColumns;
1172 frame = bytes;
1173 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nAliasColumns);
1174 if (!result)
1175 return R__FORWARD_ERROR(result);
1176 bytes += result.Unwrap();
1177 const std::uint32_t aliasColumnIdRangeBegin = columnIdRangeBegin + nColumns;
1178 for (unsigned i = 0; i < nAliasColumns; ++i) {
1179 std::uint32_t physicalId;
1180 std::uint32_t fieldId;
1181 result = DeserializeAliasColumn(bytes, fnFrameSizeLeft(), physicalId, fieldId);
1182 if (!result)
1183 return R__FORWARD_ERROR(result);
1184 bytes += result.Unwrap();
1185
1186 RColumnDescriptorBuilder columnBuilder;
1187 columnBuilder.LogicalColumnId(aliasColumnIdRangeBegin + i).PhysicalColumnId(physicalId).FieldId(fieldId);
1188 columnBuilder.Model(descBuilder.GetDescriptor().GetColumnDescriptor(physicalId).GetModel());
1189
1190 std::uint32_t idx = 0;
1191 auto maxIdx = maxIndexes.find(fieldId);
1192 if (maxIdx != maxIndexes.end())
1193 idx = maxIdx->second + 1;
1194 maxIndexes[fieldId] = idx;
1195
1196 auto aliasColumnDesc = columnBuilder.Index(idx).MakeDescriptor();
1197 if (!aliasColumnDesc)
1198 return R__FORWARD_ERROR(aliasColumnDesc);
1199 auto resVoid = descBuilder.AddColumn(aliasColumnDesc.Unwrap());
1200 if (!resVoid)
1201 return R__FORWARD_ERROR(resVoid);
1202 }
1203 bytes = frame + frameSize;
1204
1205 std::uint32_t nTypeInfo;
1206 frame = bytes;
1207 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nTypeInfo);
1208 if (!result)
1209 return R__FORWARD_ERROR(result);
1210 if (nTypeInfo > 0)
1211 R__LOG_WARNING(NTupleLog()) << "Extra type information is still unsupported! ";
1212 bytes = frame + frameSize;
1213
1214 return bytes - base;
1215}
1216
1220{
1221 RContext context;
1222
1223 auto base = reinterpret_cast<unsigned char *>(buffer);
1224 auto pos = base;
1225 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1226
1227 pos += SerializeEnvelopePreamble(kEnvelopeTypeHeader, *where);
1228 // So far we don't make use of feature flags
1229 pos += SerializeFeatureFlags(desc.GetFeatureFlags(), *where);
1230 pos += SerializeString(desc.GetName(), *where);
1231 pos += SerializeString(desc.GetDescription(), *where);
1232 pos += SerializeString(std::string("ROOT v") + ROOT_RELEASE, *where);
1233
1234 context.MapSchema(desc, /*forHeaderExtension=*/false);
1235 pos += SerializeSchemaDescription(*where, desc, context);
1236
1237 std::uint64_t size = pos - base;
1238 std::uint64_t xxhash3 = 0;
1239 size += SerializeEnvelopePostscript(base, size, xxhash3);
1240
1241 context.SetHeaderSize(size);
1242 context.SetHeaderXxHash3(xxhash3);
1243 return context;
1244}
1245
1246std::uint32_t
1248 std::span<DescriptorId_t> physClusterIDs,
1249 const RContext &context)
1250{
1251 auto base = reinterpret_cast<unsigned char *>(buffer);
1252 auto pos = base;
1253 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
1254
1255 pos += SerializeEnvelopePreamble(kEnvelopeTypePageList, *where);
1256
1257 pos += SerializeUInt64(context.GetHeaderXxHash3(), *where);
1258
1259 // Cluster summaries
1260 const auto nClusters = physClusterIDs.size();
1261 auto clusterSummaryFrame = pos;
1262 pos += SerializeListFramePreamble(nClusters, *where);
1263 for (auto clusterId : physClusterIDs) {
1264 const auto &clusterDesc = desc.GetClusterDescriptor(context.GetMemClusterId(clusterId));
1265 RClusterSummary summary{clusterDesc.GetFirstEntryIndex(), clusterDesc.GetNEntries(), -1};
1266 pos += SerializeClusterSummary(summary, *where);
1267 }
1268 pos += SerializeFramePostscript(buffer ? clusterSummaryFrame : nullptr, pos - clusterSummaryFrame);
1269
1270 // Page locations
1271 auto topMostFrame = pos;
1272 pos += SerializeListFramePreamble(nClusters, *where);
1273
1274 for (auto clusterId : physClusterIDs) {
1275 const auto &clusterDesc = desc.GetClusterDescriptor(context.GetMemClusterId(clusterId));
1276 // Get an ordered set of physical column ids
1277 std::set<DescriptorId_t> onDiskColumnIds;
1278 for (auto column : clusterDesc.GetColumnIds())
1279 onDiskColumnIds.insert(context.GetOnDiskColumnId(column));
1280
1281 auto outerFrame = pos;
1282 pos += SerializeListFramePreamble(onDiskColumnIds.size(), *where);
1283 for (auto onDiskId : onDiskColumnIds) {
1284 auto memId = context.GetMemColumnId(onDiskId);
1285 const auto &columnRange = clusterDesc.GetColumnRange(memId);
1286 const auto &pageRange = clusterDesc.GetPageRange(memId);
1287
1288 auto innerFrame = pos;
1289 pos += SerializeListFramePreamble(pageRange.fPageInfos.size(), *where);
1290
1291 for (const auto &pi : pageRange.fPageInfos) {
1292 pos += SerializeUInt32(pi.fNElements, *where);
1293 pos += SerializeLocator(pi.fLocator, *where);
1294 }
1295 pos += SerializeUInt64(columnRange.fFirstElementIndex, *where);
1296 pos += SerializeUInt32(columnRange.fCompressionSettings, *where);
1297
1298 pos += SerializeFramePostscript(buffer ? innerFrame : nullptr, pos - innerFrame);
1299 }
1300 pos += SerializeFramePostscript(buffer ? outerFrame : nullptr, pos - outerFrame);
1301 }
1302
1303 pos += SerializeFramePostscript(buffer ? topMostFrame : nullptr, pos - topMostFrame);
1304 std::uint64_t size = pos - base;
1305 size += SerializeEnvelopePostscript(base, size);
1306 return size;
1307}
1308
1309std::uint32_t
1312 const RContext &context)
1313{
1314 auto base = reinterpret_cast<unsigned char *>(buffer);
1315 auto pos = base;
1316 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
1317
1318 pos += SerializeEnvelopePreamble(kEnvelopeTypeFooter, *where);
1319
1320 // So far we don't make use of footer feature flags
1321 pos += SerializeFeatureFlags(std::vector<std::uint64_t>(), *where);
1322 pos += SerializeUInt64(context.GetHeaderXxHash3(), *where);
1323
1324 // Schema extension, i.e. incremental changes with respect to the header
1325 auto frame = pos;
1326 pos += SerializeRecordFramePreamble(*where);
1327 pos += SerializeSchemaDescription(*where, desc, context, /*forHeaderExtension=*/true);
1328 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1329
1330 // So far no support for shared clusters (no column groups)
1331 frame = pos;
1332 pos += SerializeListFramePreamble(0, *where);
1333 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1334
1335 // Cluster groups
1336 frame = pos;
1337 const auto nClusterGroups = desc.GetNClusterGroups();
1338 pos += SerializeListFramePreamble(nClusterGroups, *where);
1339 for (unsigned int i = 0; i < nClusterGroups; ++i) {
1340 const auto &cgDesc = desc.GetClusterGroupDescriptor(context.GetMemClusterGroupId(i));
1341 RClusterGroup clusterGroup;
1342 clusterGroup.fMinEntry = cgDesc.GetMinEntry();
1343 clusterGroup.fEntrySpan = cgDesc.GetEntrySpan();
1344 clusterGroup.fNClusters = cgDesc.GetNClusters();
1345 clusterGroup.fPageListEnvelopeLink.fLength = cgDesc.GetPageListLength();
1346 clusterGroup.fPageListEnvelopeLink.fLocator = cgDesc.GetPageListLocator();
1347 pos += SerializeClusterGroup(clusterGroup, *where);
1348 }
1349 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1350
1351 // So far no support for meta-data
1352 frame = pos;
1353 pos += SerializeListFramePreamble(0, *where);
1354 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1355
1356 std::uint32_t size = pos - base;
1357 size += SerializeEnvelopePostscript(base, size);
1358 return size;
1359}
1360
1363 RNTupleDescriptorBuilder &descBuilder)
1364{
1365 auto base = reinterpret_cast<const unsigned char *>(buffer);
1366 auto bytes = base;
1367 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1369
1370 std::uint64_t xxhash3{0};
1371 result = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypeHeader, xxhash3);
1372 if (!result)
1373 return R__FORWARD_ERROR(result);
1374 bytes += result.Unwrap();
1375 descBuilder.SetOnDiskHeaderXxHash3(xxhash3);
1376
1377 std::vector<std::uint64_t> featureFlags;
1378 result = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags);
1379 if (!result)
1380 return R__FORWARD_ERROR(result);
1381 bytes += result.Unwrap();
1382 for (std::size_t i = 0; i < featureFlags.size(); ++i) {
1383 if (!featureFlags[i])
1384 continue;
1385 unsigned int bit = 0;
1386 while (!(featureFlags[i] & (static_cast<uint64_t>(1) << bit)))
1387 bit++;
1388 return R__FAIL("unsupported format feature: " + std::to_string(i * 64 + bit));
1389 }
1390
1391 std::string name;
1392 std::string description;
1393 std::string writer;
1394 result = DeserializeString(bytes, fnBufSizeLeft(), name);
1395 if (!result)
1396 return R__FORWARD_ERROR(result);
1397 bytes += result.Unwrap();
1398 result = DeserializeString(bytes, fnBufSizeLeft(), description);
1399 if (!result)
1400 return R__FORWARD_ERROR(result);
1401 bytes += result.Unwrap();
1402 result = DeserializeString(bytes, fnBufSizeLeft(), writer);
1403 if (!result)
1404 return R__FORWARD_ERROR(result);
1405 bytes += result.Unwrap();
1406 descBuilder.SetNTuple(name, description);
1407
1408 // Zero field
1409 descBuilder.AddField(
1410 RFieldDescriptorBuilder().FieldId(kZeroFieldId).Structure(ENTupleStructure::kRecord).MakeDescriptor().Unwrap());
1411 result = DeserializeSchemaDescription(bytes, fnBufSizeLeft(), descBuilder);
1412 if (!result)
1413 return R__FORWARD_ERROR(result);
1414
1415 return RResult<void>::Success();
1416}
1417
1420 RNTupleDescriptorBuilder &descBuilder)
1421{
1422 auto base = reinterpret_cast<const unsigned char *>(buffer);
1423 auto bytes = base;
1424 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1426
1427 result = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypeFooter);
1428 if (!result)
1429 return R__FORWARD_ERROR(result);
1430 bytes += result.Unwrap();
1431
1432 std::vector<std::uint64_t> featureFlags;
1433 result = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags);
1434 if (!result)
1435 return R__FORWARD_ERROR(result);
1436 bytes += result.Unwrap();
1437 for (auto f: featureFlags) {
1438 if (f)
1439 R__LOG_WARNING(NTupleLog()) << "Unsupported feature flag! " << f;
1440 }
1441
1442 std::uint64_t xxhash3{0};
1443 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint64_t)))
1444 return R__FAIL("footer too short");
1445 bytes += DeserializeUInt64(bytes, xxhash3);
1446 if (xxhash3 != descBuilder.GetDescriptor().GetOnDiskHeaderXxHash3())
1447 return R__FAIL("XxHash-3 mismatch between header and footer");
1448
1449 std::uint64_t frameSize;
1450 auto frame = bytes;
1451 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1452
1453 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize);
1454 if (!result)
1455 return R__FORWARD_ERROR(result);
1456 bytes += result.Unwrap();
1457 if (fnFrameSizeLeft() > 0) {
1458 descBuilder.BeginHeaderExtension();
1459 result = DeserializeSchemaDescription(bytes, fnFrameSizeLeft(), descBuilder);
1460 if (!result)
1461 return R__FORWARD_ERROR(result);
1462 }
1463 bytes = frame + frameSize;
1464
1465 std::uint32_t nColumnGroups;
1466 frame = bytes;
1467 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nColumnGroups);
1468 if (!result)
1469 return R__FORWARD_ERROR(result);
1470 if (nColumnGroups > 0)
1471 return R__FAIL("sharded clusters are still unsupported");
1472 bytes = frame + frameSize;
1473
1474 std::uint32_t nClusterGroups;
1475 frame = bytes;
1476 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nClusterGroups);
1477 if (!result)
1478 return R__FORWARD_ERROR(result);
1479 bytes += result.Unwrap();
1480 for (std::uint32_t groupId = 0; groupId < nClusterGroups; ++groupId) {
1481 RClusterGroup clusterGroup;
1482 result = DeserializeClusterGroup(bytes, fnFrameSizeLeft(), clusterGroup);
1483 if (!result)
1484 return R__FORWARD_ERROR(result);
1485 bytes += result.Unwrap();
1486
1488 RClusterGroupDescriptorBuilder clusterGroupBuilder;
1489 clusterGroupBuilder.ClusterGroupId(groupId)
1492 .MinEntry(clusterGroup.fMinEntry)
1493 .EntrySpan(clusterGroup.fEntrySpan)
1494 .NClusters(clusterGroup.fNClusters);
1495 descBuilder.AddClusterGroup(clusterGroupBuilder.MoveDescriptor().Unwrap());
1496 }
1497 bytes = frame + frameSize;
1498
1499 std::uint32_t nMDBlocks;
1500 frame = bytes;
1501 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nMDBlocks);
1502 if (!result)
1503 return R__FORWARD_ERROR(result);
1504 if (nMDBlocks > 0)
1505 R__LOG_WARNING(NTupleLog()) << "meta-data blocks are still unsupported";
1506 bytes = frame + frameSize;
1507
1508 return RResult<void>::Success();
1509}
1510
1513 DescriptorId_t clusterGroupId,
1514 RNTupleDescriptor &desc)
1515{
1516 auto base = reinterpret_cast<const unsigned char *>(buffer);
1517 auto bytes = base;
1518 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1520
1521 result = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypePageList);
1522 if (!result)
1523 return R__FORWARD_ERROR(result);
1524 bytes += result.Unwrap();
1525
1526 std::uint64_t xxhash3{0};
1527 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint64_t)))
1528 return R__FAIL("page list too short");
1529 bytes += DeserializeUInt64(bytes, xxhash3);
1530 if (xxhash3 != desc.GetOnDiskHeaderXxHash3())
1531 return R__FAIL("XxHash-3 mismatch between header and page list");
1532
1533 std::vector<RClusterDescriptorBuilder> clusterBuilders;
1534 DescriptorId_t firstClusterId{0};
1535 for (DescriptorId_t i = 0; i < clusterGroupId; ++i) {
1536 firstClusterId = firstClusterId + desc.GetClusterGroupDescriptor(i).GetNClusters();
1537 }
1538
1539 std::uint64_t clusterSummaryFrameSize;
1540 auto clusterSummaryFrame = bytes;
1541 auto fnClusterSummaryFrameSizeLeft = [&]() { return clusterSummaryFrameSize - (bytes - clusterSummaryFrame); };
1542
1543 std::uint32_t nClusterSummaries;
1544 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), clusterSummaryFrameSize, nClusterSummaries);
1545 if (!result)
1546 return R__FORWARD_ERROR(result);
1547 bytes += result.Unwrap();
1548 for (auto clusterId = firstClusterId; clusterId < firstClusterId + nClusterSummaries; ++clusterId) {
1549 RClusterSummary clusterSummary;
1550 result = DeserializeClusterSummary(bytes, fnClusterSummaryFrameSizeLeft(), clusterSummary);
1551 if (!result)
1552 return R__FORWARD_ERROR(result);
1553 bytes += result.Unwrap();
1554 if (clusterSummary.fColumnGroupID >= 0)
1555 return R__FAIL("sharded clusters are still unsupported");
1556
1558 builder.ClusterId(clusterId).FirstEntryIndex(clusterSummary.fFirstEntry).NEntries(clusterSummary.fNEntries);
1559 clusterBuilders.emplace_back(std::move(builder));
1560 }
1561 bytes = clusterSummaryFrame + clusterSummaryFrameSize;
1562
1563 std::uint64_t topMostFrameSize;
1564 auto topMostFrame = bytes;
1565 auto fnTopMostFrameSizeLeft = [&]() { return topMostFrameSize - (bytes - topMostFrame); };
1566
1567 std::uint32_t nClusters;
1568 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), topMostFrameSize, nClusters);
1569 if (!result)
1570 return R__FORWARD_ERROR(result);
1571 bytes += result.Unwrap();
1572
1573 if (nClusters != nClusterSummaries)
1574 return R__FAIL("mismatch between number of clusters and number of cluster summaries");
1575
1576 std::vector<RClusterDescriptor> clusters;
1577 for (std::uint32_t i = 0; i < nClusters; ++i) {
1578 std::uint64_t outerFrameSize;
1579 auto outerFrame = bytes;
1580 auto fnOuterFrameSizeLeft = [&]() { return outerFrameSize - (bytes - outerFrame); };
1581
1582 std::uint32_t nColumns;
1583 result = DeserializeFrameHeader(bytes, fnTopMostFrameSizeLeft(), outerFrameSize, nColumns);
1584 if (!result)
1585 return R__FORWARD_ERROR(result);
1586 bytes += result.Unwrap();
1587
1588 for (std::uint32_t j = 0; j < nColumns; ++j) {
1589 std::uint64_t innerFrameSize;
1590 auto innerFrame = bytes;
1591 auto fnInnerFrameSizeLeft = [&]() { return innerFrameSize - (bytes - innerFrame); };
1592
1593 std::uint32_t nPages;
1594 result = DeserializeFrameHeader(bytes, fnOuterFrameSizeLeft(), innerFrameSize, nPages);
1595 if (!result)
1596 return R__FORWARD_ERROR(result);
1597 bytes += result.Unwrap();
1598
1600 pageRange.fPhysicalColumnId = j;
1601 for (std::uint32_t k = 0; k < nPages; ++k) {
1602 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
1603 return R__FAIL("inner frame too short");
1604 std::int32_t nElements;
1605 RNTupleLocator locator;
1606 bytes += DeserializeInt32(bytes, nElements);
1607 if (nElements < 0) {
1608 // TODO(jblomer): page with checksum
1609 nElements = -nElements;
1610 }
1611 result = DeserializeLocator(bytes, fnInnerFrameSizeLeft(), locator);
1612 if (!result)
1613 return R__FORWARD_ERROR(result);
1614 pageRange.fPageInfos.push_back({static_cast<std::uint32_t>(nElements), locator});
1615 bytes += result.Unwrap();
1616 }
1617
1618 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t) + sizeof(std::uint64_t)))
1619 return R__FAIL("page list frame too short");
1620 std::uint64_t columnOffset;
1621 bytes += DeserializeUInt64(bytes, columnOffset);
1622 std::uint32_t compressionSettings;
1623 bytes += DeserializeUInt32(bytes, compressionSettings);
1624
1625 clusterBuilders[i].CommitColumnRange(j, columnOffset, compressionSettings, pageRange);
1626 bytes = innerFrame + innerFrameSize;
1627 } // loop over columns
1628
1629 bytes = outerFrame + outerFrameSize;
1630
1631 clusterBuilders[i].AddDeferredColumnRanges(desc);
1632 clusters.emplace_back(clusterBuilders[i].MoveDescriptor().Unwrap());
1633 } // loop over clusters
1634 desc.AddClusterGroupDetails(clusterGroupId, clusters);
1635
1636 return RResult<void>::Success();
1637}
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
Definition RError.hxx:294
#define R__FORWARD_RESULT(res)
Short-hand to return an RResult<T> value from a subroutine to the calling stack frame.
Definition RError.hxx:292
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:290
#define R__LOG_WARNING(...)
Definition RLogger.hxx:363
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
#define ROOT_RELEASE
Definition RVersion.hxx:29
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
#define R__ASSERT(e)
Definition TError.h:118
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t nitems
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:110
The available trivial, native content types of a column.
A helper class for piece-wise construction of an RClusterDescriptor.
RClusterDescriptorBuilder & ClusterId(DescriptorId_t clusterId)
RClusterDescriptorBuilder & NEntries(std::uint64_t nEntries)
RClusterDescriptorBuilder & FirstEntryIndex(std::uint64_t firstEntryIndex)
A helper class for piece-wise construction of an RClusterGroupDescriptor.
RClusterGroupDescriptorBuilder & PageListLocator(const RNTupleLocator &pageListLocator)
RClusterGroupDescriptorBuilder & MinEntry(std::uint64_t minEntry)
RClusterGroupDescriptorBuilder & ClusterGroupId(DescriptorId_t clusterGroupId)
RClusterGroupDescriptorBuilder & EntrySpan(std::uint64_t entrySpan)
RClusterGroupDescriptorBuilder & NClusters(std::uint32_t nClusters)
RClusterGroupDescriptorBuilder & PageListLength(std::uint64_t pageListLength)
A helper class for piece-wise construction of an RColumnDescriptor.
RColumnDescriptorBuilder & PhysicalColumnId(DescriptorId_t physicalColumnId)
RColumnDescriptorBuilder & Model(const RColumnModel &model)
RColumnDescriptorBuilder & FieldId(DescriptorId_t fieldId)
RColumnDescriptorBuilder & Index(std::uint32_t index)
RResult< RColumnDescriptor > MakeDescriptor() const
Attempt to make a column descriptor.
RColumnDescriptorBuilder & LogicalColumnId(DescriptorId_t logicalColumnId)
A column element encapsulates the translation between basic C++ types and their column representation...
A helper class for piece-wise construction of an RFieldDescriptor.
RFieldDescriptorBuilder & TypeVersion(std::uint32_t typeVersion)
RFieldDescriptorBuilder & NRepetitions(std::uint64_t nRepetitions)
RFieldDescriptorBuilder & FieldVersion(std::uint32_t fieldVersion)
RFieldDescriptorBuilder & Structure(const ENTupleStructure &structure)
RFieldDescriptorBuilder & TypeName(const std::string &typeName)
RResult< RFieldDescriptor > MakeDescriptor() const
Attempt to make a field descriptor.
RFieldDescriptorBuilder & FieldName(const std::string &fieldName)
RFieldDescriptorBuilder & ParentId(DescriptorId_t id)
RFieldDescriptorBuilder & TypeAlias(const std::string &typeAlias)
RFieldDescriptorBuilder & FieldId(DescriptorId_t fieldId)
RFieldDescriptorBuilder & FieldDescription(const std::string &fieldDescription)
A helper class for piece-wise construction of an RNTupleDescriptor.
void BeginHeaderExtension()
Mark the beginning of the header extension; any fields and columns added after a call to this functio...
RResult< void > AddFieldLink(DescriptorId_t fieldId, DescriptorId_t linkId)
void AddToOnDiskFooterSize(std::uint64_t size)
The real footer size also include the page list envelopes.
void SetNTuple(const std::string_view name, const std::string_view description)
RResult< void > AddClusterGroup(RClusterGroupDescriptor &&clusterGroup)
RResult< void > AddColumn(DescriptorId_t logicalId, DescriptorId_t physicalId, DescriptorId_t fieldId, const RColumnModel &model, std::uint32_t index, std::uint64_t firstElementIdx=0U)
The serialization context is used for the piecewise serialization of a descriptor.
DescriptorId_t GetOnDiskColumnId(DescriptorId_t memId) const
const std::vector< DescriptorId_t > & GetOnDiskFieldList() const
Return a vector containing the in-memory field ID for each on-disk counterpart, in order,...
DescriptorId_t GetOnDiskFieldId(DescriptorId_t memId) const
DescriptorId_t GetMemColumnId(DescriptorId_t onDiskId) const
std::size_t GetHeaderExtensionOffset() const
Return the offset of the first element in fOnDisk2MemFieldIDs that is part of the schema extension.
DescriptorId_t GetMemClusterGroupId(DescriptorId_t onDiskId) const
DescriptorId_t GetMemClusterId(DescriptorId_t onDiskId) const
void MapSchema(const RNTupleDescriptor &desc, bool forHeaderExtension)
Map in-memory field and column IDs to their on-disk counterparts.
A helper class for serializing and deserialization of the RNTuple binary format.
static std::uint32_t SerializeXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3, void *buffer)
Writes a XxHash-3 64bit checksum of the byte range given by data and length.
static RResult< std::uint32_t > DeserializeClusterSummary(const void *buffer, std::uint64_t bufSize, RClusterSummary &clusterSummary)
static std::uint32_t SerializeListFramePreamble(std::uint32_t nitems, void *buffer)
static RResult< std::uint32_t > DeserializeClusterGroup(const void *buffer, std::uint64_t bufSize, RClusterGroup &clusterGroup)
static std::uint32_t SerializeEnvelopePostscript(unsigned char *envelope, std::uint64_t size)
static RContext SerializeHeader(void *buffer, const RNTupleDescriptor &desc)
static std::uint32_t SerializeFeatureFlags(const std::vector< std::uint64_t > &flags, void *buffer)
static std::uint16_t SerializeColumnType(ROOT::Experimental::EColumnType type, void *buffer)
static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val)
static RResult< void > DeserializeHeader(const void *buffer, std::uint64_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static RResult< void > DeserializeFooter(const void *buffer, std::uint64_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeString(const std::string &val, void *buffer)
static RResult< std::uint32_t > DeserializeFrameHeader(const void *buffer, std::uint64_t bufSize, std::uint64_t &frameSize, std::uint32_t &nitems)
static std::uint32_t SerializePageList(void *buffer, const RNTupleDescriptor &desc, std::span< DescriptorId_t > physClusterIDs, const RContext &context)
static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
static std::uint32_t SerializeEnvelopePreamble(std::uint16_t envelopeType, void *buffer)
static std::uint32_t DeserializeInt16(const void *buffer, std::int16_t &val)
static std::uint32_t SerializeClusterSummary(const RClusterSummary &clusterSummary, void *buffer)
static std::uint32_t SerializeFramePostscript(void *frame, std::uint64_t size)
static std::uint32_t SerializeInt16(std::int16_t val, void *buffer)
static RResult< void > DeserializePageList(const void *buffer, std::uint64_t bufSize, DescriptorId_t clusterGroupId, RNTupleDescriptor &desc)
static std::uint32_t SerializeSchemaDescription(void *buffer, const RNTupleDescriptor &desc, const RContext &context, bool forHeaderExtension=false)
Serialize the schema description in desc into buffer.
static RResult< std::uint32_t > DeserializeSchemaDescription(const void *buffer, std::uint64_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeLocator(const RNTupleLocator &locator, void *buffer)
static std::uint32_t SerializeInt32(std::int32_t val, void *buffer)
static RResult< void > VerifyXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3)
Expects an xxhash3 checksum in the 8 bytes following data + length and verifies it.
static RResult< std::uint16_t > DeserializeColumnType(const void *buffer, ROOT::Experimental::EColumnType &type)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static RResult< std::uint32_t > DeserializeFeatureFlags(const void *buffer, std::uint64_t bufSize, std::vector< std::uint64_t > &flags)
static std::uint32_t DeserializeInt32(const void *buffer, std::int32_t &val)
static std::uint32_t DeserializeInt64(const void *buffer, std::int64_t &val)
static RResult< std::uint16_t > DeserializeFieldStructure(const void *buffer, ROOT::Experimental::ENTupleStructure &structure)
static std::uint32_t SerializeEnvelopeLink(const REnvelopeLink &envelopeLink, void *buffer)
static RResult< std::uint32_t > DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val)
static std::uint32_t SerializeRecordFramePreamble(void *buffer)
static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer)
static std::uint32_t SerializeClusterGroup(const RClusterGroup &clusterGroup, void *buffer)
static RResult< std::uint32_t > DeserializeEnvelopeLink(const void *buffer, std::uint64_t bufSize, REnvelopeLink &envelopeLink)
static std::uint32_t SerializeFooter(void *buffer, const RNTupleDescriptor &desc, const RContext &context)
static std::uint32_t SerializeInt64(std::int64_t val, void *buffer)
static std::uint16_t SerializeFieldStructure(ROOT::Experimental::ENTupleStructure structure, void *buffer)
While we could just interpret the enums as ints, we make the translation explicit in order to avoid a...
static RResult< std::uint32_t > DeserializeEnvelope(const void *buffer, std::uint64_t bufSize, std::uint16_t expectedType)
static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer)
static RResult< std::uint32_t > DeserializeLocator(const void *buffer, std::uint64_t bufSize, RNTupleLocator &locator)
Records the parition of data into pages for a particular column in a particular cluster.
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
Meta-data stored for every field of an ntuple.
The on-storage meta-data of an ntuple.
const RClusterDescriptor & GetClusterDescriptor(DescriptorId_t clusterId) const
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
DescriptorId_t GetFieldZeroId() const
Returns the logical parent of all top-level NTuple data fields.
RResult< void > AddClusterGroupDetails(DescriptorId_t clusterGroupId, std::vector< RClusterDescriptor > &clusterDescs)
Methods to load and drop cluster group details (cluster IDs and page locations)
const RColumnDescriptor & GetColumnDescriptor(DescriptorId_t columnId) const
const RFieldDescriptor & GetFieldDescriptor(DescriptorId_t fieldId) const
const RClusterGroupDescriptor & GetClusterGroupDescriptor(DescriptorId_t clusterGroupId) const
RColumnDescriptorIterable GetColumnIterable() const
const RHeaderExtension * GetHeaderExtension() const
Return header extension information; if the descriptor does not have a header extension,...
std::vector< std::uint64_t > GetFeatureFlags() const
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:194
RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
ENTupleStructure
The fields in the ntuple model tree can carry different structural information about the type system.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
RNTupleLocator payload that is common for object stores using 64bit location information.
Generic information about the physical location of data.
ELocatorType
Values for the Type field in non-disk locators.
std::uint8_t fReserved
Reserved for use by concrete storage backends.
ELocatorType fType
For non-disk locators, the value for the Type field.
std::variant< std::uint64_t, std::string, RNTupleLocatorObject64 > fPosition
Simple on-disk locators consisting of a 64-bit offset use variant type uint64_t; extended locators ha...