Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleSerialize.cxx
Go to the documentation of this file.
1/// \file RNTupleSerialize.cxx
2/// \ingroup NTuple
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \author Javier Lopez-Gomez <javier.lopez.gomez@cern.ch>
5/// \date 2021-08-02
6
7/*************************************************************************
8 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
9 * All rights reserved. *
10 * *
11 * For the licensing terms see $ROOTSYS/LICENSE. *
12 * For the list of contributors see $ROOTSYS/README/CREDITS. *
13 *************************************************************************/
14
16#include <ROOT/RError.hxx>
19#include <ROOT/RNTupleTypes.hxx>
20#include <ROOT/RNTupleUtils.hxx>
21
22#include <RVersion.h>
23#include <TBufferFile.h>
24#include <TClass.h>
25#include <TList.h>
26#include <TStreamerInfo.h>
28#include <xxhash.h>
29
30#include <cassert>
31#include <cmath>
32#include <cstring> // for memcpy
33#include <deque>
34#include <functional>
35#include <limits>
36#include <set>
37#include <unordered_map>
38
45
46namespace {
47using RNTupleSerializer = ROOT::Internal::RNTupleSerializer;
48
52{
53
54 auto base = reinterpret_cast<unsigned char *>(buffer);
55 auto pos = base;
56 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
57
58 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
59
60 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetFieldVersion(), *where);
61 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetTypeVersion(), *where);
62 pos += RNTupleSerializer::SerializeUInt32(onDiskParentId, *where);
63 if (auto res = RNTupleSerializer::SerializeFieldStructure(fieldDesc.GetStructure(), *where)) {
64 pos += res.Unwrap();
65 } else {
66 return R__FORWARD_ERROR(res);
67 }
68
69 std::uint16_t flags = 0;
70 if (fieldDesc.GetNRepetitions() > 0)
71 flags |= RNTupleSerializer::kFlagRepetitiveField;
72 if (fieldDesc.IsProjectedField())
73 flags |= RNTupleSerializer::kFlagProjectedField;
74 if (fieldDesc.GetTypeChecksum().has_value())
75 flags |= RNTupleSerializer::kFlagHasTypeChecksum;
76 pos += RNTupleSerializer::SerializeUInt16(flags, *where);
77
78 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldName(), *where);
79 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeName(), *where);
80 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeAlias(), *where);
81 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldDescription(), *where);
82
83 if (flags & RNTupleSerializer::kFlagRepetitiveField) {
84 pos += RNTupleSerializer::SerializeUInt64(fieldDesc.GetNRepetitions(), *where);
85 }
86 if (flags & RNTupleSerializer::kFlagProjectedField) {
87 pos += RNTupleSerializer::SerializeUInt32(onDiskProjectionSourceId, *where);
88 }
89 if (flags & RNTupleSerializer::kFlagHasTypeChecksum) {
90 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetTypeChecksum().value(), *where);
91 }
92
93 auto size = pos - base;
94 RNTupleSerializer::SerializeFramePostscript(base, size);
95
96 return size;
97}
98
99// clang-format off
100/// Serialize, in order, fields enumerated in `fieldList` to `buffer`. `firstOnDiskId` specifies the on-disk ID for the
101/// first element in the `fieldList` sequence. Before calling this function `RContext::MapSchema()` should have been
102/// called on `context` in order to map in-memory field IDs to their on-disk counterpart.
103/// \return The number of bytes written to the output buffer; if `buffer` is `nullptr` no data is serialized and the
104/// required buffer size is returned
105// clang-format on
107SerializeFieldList(const ROOT::RNTupleDescriptor &desc, std::span<const ROOT::DescriptorId_t> fieldList,
108 std::size_t firstOnDiskId, const ROOT::Internal::RNTupleSerializer::RContext &context, void *buffer)
109{
110 auto base = reinterpret_cast<unsigned char *>(buffer);
111 auto pos = base;
112 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
113
114 auto fieldZeroId = desc.GetFieldZeroId();
116 for (auto fieldId : fieldList) {
117 const auto &f = desc.GetFieldDescriptor(fieldId);
118 auto onDiskParentId =
119 (f.GetParentId() == fieldZeroId) ? onDiskFieldId : context.GetOnDiskFieldId(f.GetParentId());
121 f.IsProjectedField() ? context.GetOnDiskFieldId(f.GetProjectionSourceId()) : ROOT::kInvalidDescriptorId;
123 pos += res.Unwrap();
124 } else {
125 return R__FORWARD_ERROR(res);
126 }
128 }
129
130 return pos - base;
131}
132
135{
136 using ENTupleStructure = ROOT::ENTupleStructure;
137
138 auto base = reinterpret_cast<const unsigned char *>(buffer);
139 auto bytes = base;
140 std::uint64_t frameSize;
141 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
142 if (auto res = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize)) {
143 bytes += res.Unwrap();
144 } else {
145 return R__FORWARD_ERROR(res);
146 }
147
148 std::uint32_t fieldVersion;
149 std::uint32_t typeVersion;
150 std::uint32_t parentId;
151 // initialize properly for call to SerializeFieldStructure()
152 ENTupleStructure structure{ENTupleStructure::kPlain};
153 std::uint16_t flags;
154 std::uint32_t result;
155 if (auto res = RNTupleSerializer::SerializeFieldStructure(structure, nullptr)) {
156 result = res.Unwrap();
157 } else {
158 return R__FORWARD_ERROR(res);
159 }
160 if (fnFrameSizeLeft() < 3 * sizeof(std::uint32_t) + result + sizeof(std::uint16_t)) {
161 return R__FAIL("field record frame too short");
162 }
163 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldVersion);
164 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeVersion);
165 bytes += RNTupleSerializer::DeserializeUInt32(bytes, parentId);
166 if (auto res = RNTupleSerializer::DeserializeFieldStructure(bytes, structure)) {
167 bytes += res.Unwrap();
168 } else {
169 return R__FORWARD_ERROR(res);
170 }
171 bytes += RNTupleSerializer::DeserializeUInt16(bytes, flags);
172 fieldDesc.FieldVersion(fieldVersion).TypeVersion(typeVersion).ParentId(parentId).Structure(structure);
173
174 std::string fieldName;
175 std::string typeName;
176 std::string aliasName;
177 std::string description;
178 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), fieldName)) {
179 bytes += res.Unwrap();
180 } else {
181 return R__FORWARD_ERROR(res);
182 }
183 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), typeName)) {
184 bytes += res.Unwrap();
185 } else {
186 return R__FORWARD_ERROR(res);
187 }
188 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), aliasName)) {
189 bytes += res.Unwrap();
190 } else {
191 return R__FORWARD_ERROR(res);
192 }
193 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), description)) {
194 bytes += res.Unwrap();
195 } else {
196 return R__FORWARD_ERROR(res);
197 }
198 fieldDesc.FieldName(fieldName).TypeName(typeName).TypeAlias(aliasName).FieldDescription(description);
199
200 if (flags & RNTupleSerializer::kFlagRepetitiveField) {
201 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
202 return R__FAIL("field record frame too short");
203 std::uint64_t nRepetitions;
204 bytes += RNTupleSerializer::DeserializeUInt64(bytes, nRepetitions);
205 fieldDesc.NRepetitions(nRepetitions);
206 }
207
208 if (flags & RNTupleSerializer::kFlagProjectedField) {
209 if (fnFrameSizeLeft() < sizeof(std::uint32_t))
210 return R__FAIL("field record frame too short");
211 std::uint32_t projectionSourceId;
212 bytes += RNTupleSerializer::DeserializeUInt32(bytes, projectionSourceId);
213 fieldDesc.ProjectionSourceId(projectionSourceId);
214 }
215
216 if (flags & RNTupleSerializer::kFlagHasTypeChecksum) {
217 if (fnFrameSizeLeft() < sizeof(std::uint32_t))
218 return R__FAIL("field record frame too short");
219 std::uint32_t typeChecksum;
220 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeChecksum);
221 fieldDesc.TypeChecksum(typeChecksum);
222 }
223
224 return frameSize;
225}
226
229 void *buffer)
230{
231 R__ASSERT(!columnDesc.IsAliasColumn());
232
233 auto base = reinterpret_cast<unsigned char *>(buffer);
234 auto pos = base;
235 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
236
237 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
238
239 if (auto res = RNTupleSerializer::SerializeColumnType(columnDesc.GetType(), *where)) {
240 pos += res.Unwrap();
241 } else {
242 return R__FORWARD_ERROR(res);
243 }
244 pos += RNTupleSerializer::SerializeUInt16(columnDesc.GetBitsOnStorage(), *where);
245 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(columnDesc.GetFieldId()), *where);
246 std::uint16_t flags = 0;
247 if (columnDesc.IsDeferredColumn())
248 flags |= RNTupleSerializer::kFlagDeferredColumn;
249 if (columnDesc.GetValueRange().has_value())
250 flags |= RNTupleSerializer::kFlagHasValueRange;
251 std::int64_t firstElementIdx = columnDesc.GetFirstElementIndex();
252 if (columnDesc.IsSuppressedDeferredColumn())
254 pos += RNTupleSerializer::SerializeUInt16(flags, *where);
255 pos += RNTupleSerializer::SerializeUInt16(columnDesc.GetRepresentationIndex(), *where);
256 if (flags & RNTupleSerializer::kFlagDeferredColumn)
257 pos += RNTupleSerializer::SerializeInt64(firstElementIdx, *where);
258 if (flags & RNTupleSerializer::kFlagHasValueRange) {
259 auto [min, max] = *columnDesc.GetValueRange();
260 std::uint64_t intMin, intMax;
261 static_assert(sizeof(min) == sizeof(intMin) && sizeof(max) == sizeof(intMax));
262 memcpy(&intMin, &min, sizeof(min));
263 memcpy(&intMax, &max, sizeof(max));
264 pos += RNTupleSerializer::SerializeUInt64(intMin, *where);
265 pos += RNTupleSerializer::SerializeUInt64(intMax, *where);
266 }
267
268 if (auto res = RNTupleSerializer::SerializeFramePostscript(buffer ? base : nullptr, pos - base)) {
269 pos += res.Unwrap();
270 } else {
271 return R__FORWARD_ERROR(res);
272 }
273
274 return pos - base;
275}
276
278 std::span<const ROOT::DescriptorId_t> fieldList,
280 void *buffer, bool forHeaderExtension)
281{
282 auto base = reinterpret_cast<unsigned char *>(buffer);
283 auto pos = base;
284 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
285
286 const auto *xHeader = !forHeaderExtension ? desc.GetHeaderExtension() : nullptr;
287
288 for (auto parentId : fieldList) {
289 // If we're serializing the non-extended header and we already have a header extension (which may happen if
290 // we load an RNTuple for incremental merging), we need to skip all the extended fields, as they need to be
291 // written in the header extension, not in the regular header.
292 if (xHeader && xHeader->ContainsField(parentId))
293 continue;
294
295 for (const auto &c : desc.GetColumnIterable(parentId)) {
296 if (c.IsAliasColumn() || (xHeader && xHeader->ContainsExtendedColumnRepresentation(c.GetLogicalId())))
297 continue;
298
299 if (auto res = SerializePhysicalColumn(c, context, *where)) {
300 pos += res.Unwrap();
301 } else {
302 return R__FORWARD_ERROR(res);
303 }
304 }
305 }
306
307 return pos - base;
308}
309
312{
314
315 auto base = reinterpret_cast<const unsigned char *>(buffer);
316 auto bytes = base;
317 std::uint64_t frameSize;
318 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
319 if (auto res = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize)) {
320 bytes += res.Unwrap();
321 } else {
322 return R__FORWARD_ERROR(res);
323 }
324
325 // Initialize properly for SerializeColumnType
326 ENTupleColumnType type{ENTupleColumnType::kIndex32};
327 std::uint16_t bitsOnStorage;
328 std::uint32_t fieldId;
329 std::uint16_t flags;
330 std::uint16_t representationIndex;
331 std::int64_t firstElementIdx = 0;
332 if (fnFrameSizeLeft() < RNTupleSerializer::SerializeColumnType(type, nullptr).Unwrap() + sizeof(std::uint16_t) +
333 2 * sizeof(std::uint32_t)) {
334 return R__FAIL("column record frame too short");
335 }
336 if (auto res = RNTupleSerializer::DeserializeColumnType(bytes, type)) {
337 bytes += res.Unwrap();
338 } else {
339 return R__FORWARD_ERROR(res);
340 }
341 bytes += RNTupleSerializer::DeserializeUInt16(bytes, bitsOnStorage);
342 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
343 bytes += RNTupleSerializer::DeserializeUInt16(bytes, flags);
344 bytes += RNTupleSerializer::DeserializeUInt16(bytes, representationIndex);
345 if (flags & RNTupleSerializer::kFlagDeferredColumn) {
346 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
347 return R__FAIL("column record frame too short");
348 bytes += RNTupleSerializer::DeserializeInt64(bytes, firstElementIdx);
349 }
350 if (flags & RNTupleSerializer::kFlagHasValueRange) {
351 if (fnFrameSizeLeft() < 2 * sizeof(std::uint64_t))
352 return R__FAIL("field record frame too short");
353 std::uint64_t minInt, maxInt;
354 bytes += RNTupleSerializer::DeserializeUInt64(bytes, minInt);
355 bytes += RNTupleSerializer::DeserializeUInt64(bytes, maxInt);
356 double min, max;
357 memcpy(&min, &minInt, sizeof(min));
358 memcpy(&max, &maxInt, sizeof(max));
359 columnDesc.ValueRange(min, max);
360 }
361
362 columnDesc.FieldId(fieldId).BitsOnStorage(bitsOnStorage).Type(type).RepresentationIndex(representationIndex);
363 columnDesc.FirstElementIndex(std::abs(firstElementIdx));
364 if (firstElementIdx < 0)
365 columnDesc.SetSuppressedDeferred();
366
367 return frameSize;
368}
369
371{
372 auto base = reinterpret_cast<unsigned char *>(buffer);
373 auto pos = base;
374 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
375
376 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
377
378 if (auto res = RNTupleSerializer::SerializeExtraTypeInfoId(desc.GetContentId(), *where)) {
379 pos += res.Unwrap();
380 } else {
381 return R__FORWARD_ERROR(res);
382 }
383 pos += RNTupleSerializer::SerializeUInt32(desc.GetTypeVersion(), *where);
384 pos += RNTupleSerializer::SerializeString(desc.GetTypeName(), *where);
385 pos += RNTupleSerializer::SerializeString(desc.GetContent(), *where);
386
387 auto size = pos - base;
388 RNTupleSerializer::SerializeFramePostscript(base, size);
389
390 return size;
391}
392
394{
395 auto base = reinterpret_cast<unsigned char *>(buffer);
396 auto pos = base;
397 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
398
399 for (const auto &extraTypeInfoDesc : ntplDesc.GetExtraTypeInfoIterable()) {
401 pos += res.Unwrap();
402 } else {
403 return R__FORWARD_ERROR(res);
404 }
405 }
406
407 return pos - base;
408}
409
410ROOT::RResult<std::uint32_t> DeserializeExtraTypeInfo(const void *buffer, std::uint64_t bufSize,
412{
414
415 auto base = reinterpret_cast<const unsigned char *>(buffer);
416 auto bytes = base;
417 std::uint64_t frameSize;
418 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
419 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
420 if (!result)
421 return R__FORWARD_ERROR(result);
422 bytes += result.Unwrap();
423
424 EExtraTypeInfoIds contentId{EExtraTypeInfoIds::kInvalid};
425 std::uint32_t typeVersion;
426 if (fnFrameSizeLeft() < 2 * sizeof(std::uint32_t)) {
427 return R__FAIL("extra type info record frame too short");
428 }
429 result = RNTupleSerializer::DeserializeExtraTypeInfoId(bytes, contentId);
430 if (!result)
431 return R__FORWARD_ERROR(result);
432 bytes += result.Unwrap();
433 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeVersion);
434
435 std::string typeName;
436 std::string content;
437 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), typeName).Unwrap();
438 if (!result)
439 return R__FORWARD_ERROR(result);
440 bytes += result.Unwrap();
441 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), content).Unwrap();
442 if (!result)
443 return R__FORWARD_ERROR(result);
444 bytes += result.Unwrap();
445
447
448 return frameSize;
449}
450
451std::uint32_t SerializeLocatorPayloadLarge(const ROOT::RNTupleLocator &locator, unsigned char *buffer)
452{
453 if (buffer) {
454 RNTupleSerializer::SerializeUInt64(locator.GetNBytesOnStorage(), buffer);
455 RNTupleSerializer::SerializeUInt64(locator.GetPosition<std::uint64_t>(), buffer + sizeof(std::uint64_t));
456 }
457 return sizeof(std::uint64_t) + sizeof(std::uint64_t);
458}
459
460void DeserializeLocatorPayloadLarge(const unsigned char *buffer, ROOT::RNTupleLocator &locator)
461{
462 std::uint64_t nBytesOnStorage;
463 std::uint64_t position;
464 RNTupleSerializer::DeserializeUInt64(buffer, nBytesOnStorage);
465 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint64_t), position);
466 locator.SetNBytesOnStorage(nBytesOnStorage);
467 locator.SetPosition(position);
468}
469
470std::uint32_t SerializeLocatorPayloadObject64(const ROOT::RNTupleLocator &locator, unsigned char *buffer)
471{
472 const auto &data = locator.GetPosition<ROOT::RNTupleLocatorObject64>();
473 const uint32_t sizeofNBytesOnStorage = (locator.GetNBytesOnStorage() > std::numeric_limits<std::uint32_t>::max())
474 ? sizeof(std::uint64_t)
475 : sizeof(std::uint32_t);
476 if (buffer) {
477 if (sizeofNBytesOnStorage == sizeof(std::uint32_t)) {
478 RNTupleSerializer::SerializeUInt32(locator.GetNBytesOnStorage(), buffer);
479 } else {
480 RNTupleSerializer::SerializeUInt64(locator.GetNBytesOnStorage(), buffer);
481 }
482 RNTupleSerializer::SerializeUInt64(data.GetLocation(), buffer + sizeofNBytesOnStorage);
483 }
484 return sizeofNBytesOnStorage + sizeof(std::uint64_t);
485}
486
487ROOT::RResult<void> DeserializeLocatorPayloadObject64(const unsigned char *buffer, std::uint32_t sizeofLocatorPayload,
489{
490 std::uint64_t location;
491 if (sizeofLocatorPayload == 12) {
492 std::uint32_t nBytesOnStorage;
493 RNTupleSerializer::DeserializeUInt32(buffer, nBytesOnStorage);
494 locator.SetNBytesOnStorage(nBytesOnStorage);
495 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint32_t), location);
496 } else if (sizeofLocatorPayload == 16) {
497 std::uint64_t nBytesOnStorage;
498 RNTupleSerializer::DeserializeUInt64(buffer, nBytesOnStorage);
499 locator.SetNBytesOnStorage(nBytesOnStorage);
500 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint64_t), location);
501 } else {
502 return R__FAIL("invalid DAOS locator payload size: " + std::to_string(sizeofLocatorPayload));
503 }
504 locator.SetPosition(ROOT::RNTupleLocatorObject64{location});
506}
507
509 const ROOT::Internal::RNTupleSerializer::RContext &context, void *buffer)
510{
511 R__ASSERT(columnDesc.IsAliasColumn());
512
513 auto base = reinterpret_cast<unsigned char *>(buffer);
514 auto pos = base;
515 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
516
517 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
518
519 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskColumnId(columnDesc.GetPhysicalId()), *where);
520 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(columnDesc.GetFieldId()), *where);
521
522 pos += RNTupleSerializer::SerializeFramePostscript(buffer ? base : nullptr, pos - base).Unwrap();
523
524 return pos - base;
525}
526
528 std::span<const ROOT::DescriptorId_t> fieldList,
529 const ROOT::Internal::RNTupleSerializer::RContext &context, void *buffer,
531{
532 auto base = reinterpret_cast<unsigned char *>(buffer);
533 auto pos = base;
534 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
535
536 const auto *xHeader = !forHeaderExtension ? desc.GetHeaderExtension() : nullptr;
537
538 for (auto parentId : fieldList) {
539 if (xHeader && xHeader->ContainsField(parentId))
540 continue;
541
542 for (const auto &c : desc.GetColumnIterable(parentId)) {
543 if (!c.IsAliasColumn() || (xHeader && xHeader->ContainsExtendedColumnRepresentation(c.GetLogicalId())))
544 continue;
545
546 pos += SerializeAliasColumn(c, context, *where);
547 }
548 }
549
550 return pos - base;
551}
552
553ROOT::RResult<std::uint32_t> DeserializeAliasColumn(const void *buffer, std::uint64_t bufSize,
554 std::uint32_t &physicalColumnId, std::uint32_t &fieldId)
555{
556 auto base = reinterpret_cast<const unsigned char *>(buffer);
557 auto bytes = base;
558 std::uint64_t frameSize;
559 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
560 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
561 if (!result)
562 return R__FORWARD_ERROR(result);
563 bytes += result.Unwrap();
564
565 if (fnFrameSizeLeft() < 2 * sizeof(std::uint32_t)) {
566 return R__FAIL("alias column record frame too short");
567 }
568
569 bytes += RNTupleSerializer::DeserializeUInt32(bytes, physicalColumnId);
570 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
571
572 return frameSize;
573}
574
575} // anonymous namespace
576
577std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeXxHash3(const unsigned char *data, std::uint64_t length,
578 std::uint64_t &xxhash3, void *buffer)
579{
580 if (buffer != nullptr) {
582 SerializeUInt64(xxhash3, buffer);
583 }
584 return 8;
585}
586
588 std::uint64_t &xxhash3)
589{
591 DeserializeUInt64(data + length, xxhash3);
592 if (xxhash3 != checksumReal)
593 return R__FAIL("XxHash-3 checksum mismatch");
594 return RResult<void>::Success();
595}
596
598{
599 std::uint64_t xxhash3;
600 return R__FORWARD_RESULT(VerifyXxHash3(data, length, xxhash3));
601}
602
603std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeInt16(std::int16_t val, void *buffer)
604{
605 if (buffer != nullptr) {
606 auto bytes = reinterpret_cast<unsigned char *>(buffer);
607 bytes[0] = (val & 0x00FF);
608 bytes[1] = (val & 0xFF00) >> 8;
609 }
610 return 2;
611}
612
613std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeInt16(const void *buffer, std::int16_t &val)
614{
615 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
616 val = std::int16_t(bytes[0]) + (std::int16_t(bytes[1]) << 8);
617 return 2;
618}
619
620std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeUInt16(std::uint16_t val, void *buffer)
621{
622 return SerializeInt16(val, buffer);
623}
624
625std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeUInt16(const void *buffer, std::uint16_t &val)
626{
627 return DeserializeInt16(buffer, *reinterpret_cast<std::int16_t *>(&val));
628}
629
630std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeInt32(std::int32_t val, void *buffer)
631{
632 if (buffer != nullptr) {
633 auto bytes = reinterpret_cast<unsigned char *>(buffer);
634 bytes[0] = (val & 0x000000FF);
635 bytes[1] = (val & 0x0000FF00) >> 8;
636 bytes[2] = (val & 0x00FF0000) >> 16;
637 bytes[3] = (val & 0xFF000000) >> 24;
638 }
639 return 4;
640}
641
642std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeInt32(const void *buffer, std::int32_t &val)
643{
644 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
645 val = std::int32_t(bytes[0]) + (std::int32_t(bytes[1]) << 8) + (std::int32_t(bytes[2]) << 16) +
646 (std::int32_t(bytes[3]) << 24);
647 return 4;
648}
649
650std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeUInt32(std::uint32_t val, void *buffer)
651{
652 return SerializeInt32(val, buffer);
653}
654
655std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeUInt32(const void *buffer, std::uint32_t &val)
656{
657 return DeserializeInt32(buffer, *reinterpret_cast<std::int32_t *>(&val));
658}
659
660std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeInt64(std::int64_t val, void *buffer)
661{
662 if (buffer != nullptr) {
663 auto bytes = reinterpret_cast<unsigned char *>(buffer);
664 bytes[0] = (val & 0x00000000000000FF);
665 bytes[1] = (val & 0x000000000000FF00) >> 8;
666 bytes[2] = (val & 0x0000000000FF0000) >> 16;
667 bytes[3] = (val & 0x00000000FF000000) >> 24;
668 bytes[4] = (val & 0x000000FF00000000) >> 32;
669 bytes[5] = (val & 0x0000FF0000000000) >> 40;
670 bytes[6] = (val & 0x00FF000000000000) >> 48;
671 bytes[7] = (val & 0xFF00000000000000) >> 56;
672 }
673 return 8;
674}
675
676std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeInt64(const void *buffer, std::int64_t &val)
677{
678 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
679 val = std::int64_t(bytes[0]) + (std::int64_t(bytes[1]) << 8) + (std::int64_t(bytes[2]) << 16) +
680 (std::int64_t(bytes[3]) << 24) + (std::int64_t(bytes[4]) << 32) + (std::int64_t(bytes[5]) << 40) +
681 (std::int64_t(bytes[6]) << 48) + (std::int64_t(bytes[7]) << 56);
682 return 8;
683}
684
685std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeUInt64(std::uint64_t val, void *buffer)
686{
687 return SerializeInt64(val, buffer);
688}
689
690std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeUInt64(const void *buffer, std::uint64_t &val)
691{
692 return DeserializeInt64(buffer, *reinterpret_cast<std::int64_t *>(&val));
693}
694
695std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeString(const std::string &val, void *buffer)
696{
697 if (buffer) {
698 auto pos = reinterpret_cast<unsigned char *>(buffer);
699 pos += SerializeUInt32(val.length(), pos);
700 memcpy(pos, val.data(), val.length());
701 }
702 return sizeof(std::uint32_t) + val.length();
703}
704
706ROOT::Internal::RNTupleSerializer::DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val)
707{
708 if (bufSize < sizeof(std::uint32_t))
709 return R__FAIL("string buffer too short");
710 bufSize -= sizeof(std::uint32_t);
711
712 auto base = reinterpret_cast<const unsigned char *>(buffer);
713 auto bytes = base;
714 std::uint32_t length;
715 bytes += DeserializeUInt32(buffer, length);
716 if (bufSize < length)
717 return R__FAIL("string buffer too short");
718
719 val.resize(length);
720 memcpy(&val[0], bytes, length);
721 return sizeof(std::uint32_t) + length;
722}
723
726{
727 switch (type) {
728 case ENTupleColumnType::kBit: return SerializeUInt16(0x00, buffer);
729 case ENTupleColumnType::kByte: return SerializeUInt16(0x01, buffer);
730 case ENTupleColumnType::kChar: return SerializeUInt16(0x02, buffer);
731 case ENTupleColumnType::kInt8: return SerializeUInt16(0x03, buffer);
732 case ENTupleColumnType::kUInt8: return SerializeUInt16(0x04, buffer);
733 case ENTupleColumnType::kInt16: return SerializeUInt16(0x05, buffer);
734 case ENTupleColumnType::kUInt16: return SerializeUInt16(0x06, buffer);
735 case ENTupleColumnType::kInt32: return SerializeUInt16(0x07, buffer);
736 case ENTupleColumnType::kUInt32: return SerializeUInt16(0x08, buffer);
737 case ENTupleColumnType::kInt64: return SerializeUInt16(0x09, buffer);
738 case ENTupleColumnType::kUInt64: return SerializeUInt16(0x0A, buffer);
739 case ENTupleColumnType::kReal16: return SerializeUInt16(0x0B, buffer);
740 case ENTupleColumnType::kReal32: return SerializeUInt16(0x0C, buffer);
741 case ENTupleColumnType::kReal64: return SerializeUInt16(0x0D, buffer);
742 case ENTupleColumnType::kIndex32: return SerializeUInt16(0x0E, buffer);
743 case ENTupleColumnType::kIndex64: return SerializeUInt16(0x0F, buffer);
744 case ENTupleColumnType::kSwitch: return SerializeUInt16(0x10, buffer);
745 case ENTupleColumnType::kSplitInt16: return SerializeUInt16(0x11, buffer);
746 case ENTupleColumnType::kSplitUInt16: return SerializeUInt16(0x12, buffer);
747 case ENTupleColumnType::kSplitInt32: return SerializeUInt16(0x13, buffer);
748 case ENTupleColumnType::kSplitUInt32: return SerializeUInt16(0x14, buffer);
749 case ENTupleColumnType::kSplitInt64: return SerializeUInt16(0x15, buffer);
750 case ENTupleColumnType::kSplitUInt64: return SerializeUInt16(0x16, buffer);
751 case ENTupleColumnType::kSplitReal32: return SerializeUInt16(0x18, buffer);
752 case ENTupleColumnType::kSplitReal64: return SerializeUInt16(0x19, buffer);
753 case ENTupleColumnType::kSplitIndex32: return SerializeUInt16(0x1A, buffer);
754 case ENTupleColumnType::kSplitIndex64: return SerializeUInt16(0x1B, buffer);
755 case ENTupleColumnType::kReal32Trunc: return SerializeUInt16(0x1C, buffer);
756 case ENTupleColumnType::kReal32Quant: return SerializeUInt16(0x1D, buffer);
757 default:
759 return SerializeUInt16(0x99, buffer);
760 return R__FAIL("unexpected column type");
761 }
762}
763
766{
767 std::uint16_t onDiskType;
768 auto result = DeserializeUInt16(buffer, onDiskType);
769
770 switch (onDiskType) {
771 case 0x00: type = ENTupleColumnType::kBit; break;
772 case 0x01: type = ENTupleColumnType::kByte; break;
773 case 0x02: type = ENTupleColumnType::kChar; break;
774 case 0x03: type = ENTupleColumnType::kInt8; break;
775 case 0x04: type = ENTupleColumnType::kUInt8; break;
776 case 0x05: type = ENTupleColumnType::kInt16; break;
777 case 0x06: type = ENTupleColumnType::kUInt16; break;
778 case 0x07: type = ENTupleColumnType::kInt32; break;
779 case 0x08: type = ENTupleColumnType::kUInt32; break;
780 case 0x09: type = ENTupleColumnType::kInt64; break;
781 case 0x0A: type = ENTupleColumnType::kUInt64; break;
782 case 0x0B: type = ENTupleColumnType::kReal16; break;
783 case 0x0C: type = ENTupleColumnType::kReal32; break;
784 case 0x0D: type = ENTupleColumnType::kReal64; break;
785 case 0x0E: type = ENTupleColumnType::kIndex32; break;
786 case 0x0F: type = ENTupleColumnType::kIndex64; break;
787 case 0x10: type = ENTupleColumnType::kSwitch; break;
788 case 0x11: type = ENTupleColumnType::kSplitInt16; break;
789 case 0x12: type = ENTupleColumnType::kSplitUInt16; break;
790 case 0x13: type = ENTupleColumnType::kSplitInt32; break;
791 case 0x14: type = ENTupleColumnType::kSplitUInt32; break;
792 case 0x15: type = ENTupleColumnType::kSplitInt64; break;
793 case 0x16: type = ENTupleColumnType::kSplitUInt64; break;
794 case 0x18: type = ENTupleColumnType::kSplitReal32; break;
795 case 0x19: type = ENTupleColumnType::kSplitReal64; break;
796 case 0x1A: type = ENTupleColumnType::kSplitIndex32; break;
797 case 0x1B: type = ENTupleColumnType::kSplitIndex64; break;
798 case 0x1C: type = ENTupleColumnType::kReal32Trunc; break;
799 case 0x1D: type = ENTupleColumnType::kReal32Quant; break;
800 // case 0x99 => kTestFutureColumnType missing on purpose
801 default:
802 // may be a column type introduced by a future version
804 break;
805 }
806 return result;
807}
808
811{
813 switch (structure) {
814 case ENTupleStructure::kPlain: return SerializeUInt16(0x00, buffer);
815 case ENTupleStructure::kCollection: return SerializeUInt16(0x01, buffer);
816 case ENTupleStructure::kRecord: return SerializeUInt16(0x02, buffer);
817 case ENTupleStructure::kVariant: return SerializeUInt16(0x03, buffer);
818 case ENTupleStructure::kStreamer: return SerializeUInt16(0x04, buffer);
819 default:
821 return SerializeUInt16(0x99, buffer);
822 return R__FAIL("unexpected field structure type");
823 }
824}
825
828{
830 std::uint16_t onDiskValue;
831 auto result = DeserializeUInt16(buffer, onDiskValue);
832 switch (onDiskValue) {
833 case 0x00: structure = ENTupleStructure::kPlain; break;
834 case 0x01: structure = ENTupleStructure::kCollection; break;
835 case 0x02: structure = ENTupleStructure::kRecord; break;
836 case 0x03: structure = ENTupleStructure::kVariant; break;
837 case 0x04: structure = ENTupleStructure::kStreamer; break;
838 // case 0x99 => kTestFutureFieldStructure intentionally missing
839 default: structure = ENTupleStructure::kUnknown;
840 }
841 return result;
842}
843
846{
847 switch (id) {
848 case ROOT::EExtraTypeInfoIds::kStreamerInfo: return SerializeUInt32(0x00, buffer);
849 default: return R__FAIL("unexpected extra type info id");
850 }
851}
852
855{
856 std::uint32_t onDiskValue;
857 auto result = DeserializeUInt32(buffer, onDiskValue);
858 switch (onDiskValue) {
859 case 0x00: id = ROOT::EExtraTypeInfoIds::kStreamerInfo; break;
860 default:
862 R__LOG_DEBUG(0, ROOT::Internal::NTupleLog()) << "Unknown extra type info id: " << onDiskValue;
863 }
864 return result;
865}
866
868{
869 auto base = reinterpret_cast<unsigned char *>(buffer);
870 auto pos = base;
871 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
872
873 pos += SerializeUInt64(envelopeType, *where);
874 // The 48bits size information is filled in the postscript
875 return pos - base;
876}
877
879 std::uint64_t size,
880 std::uint64_t &xxhash3)
881{
882 if (size < sizeof(std::uint64_t))
883 return R__FAIL("envelope size too small");
884 if (size >= static_cast<uint64_t>(1) << 48)
885 return R__FAIL("envelope size too big");
886 if (envelope) {
887 std::uint64_t typeAndSize;
888 DeserializeUInt64(envelope, typeAndSize);
889 typeAndSize |= (size + 8) << 16;
890 SerializeUInt64(typeAndSize, envelope);
891 }
892 return SerializeXxHash3(envelope, size, xxhash3, envelope ? (envelope + size) : nullptr);
893}
894
897{
898 std::uint64_t xxhash3;
899 return R__FORWARD_RESULT(SerializeEnvelopePostscript(envelope, size, xxhash3));
900}
901
904 std::uint16_t expectedType, std::uint64_t &xxhash3)
905{
906 const std::uint64_t minEnvelopeSize = sizeof(std::uint64_t) + sizeof(std::uint64_t);
908 return R__FAIL("invalid envelope buffer, too short");
909
910 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
911 auto base = bytes;
912
913 std::uint64_t typeAndSize;
914 bytes += DeserializeUInt64(bytes, typeAndSize);
915
916 std::uint16_t envelopeType = typeAndSize & 0xFFFF;
917 if (envelopeType != expectedType) {
918 return R__FAIL("envelope type mismatch: expected " + std::to_string(expectedType) + ", found " +
919 std::to_string(envelopeType));
920 }
921
922 std::uint64_t envelopeSize = typeAndSize >> 16;
923 if (bufSize < envelopeSize)
924 return R__FAIL("envelope buffer size too small");
926 return R__FAIL("invalid envelope, too short");
927
928 auto result = VerifyXxHash3(base, envelopeSize - 8, xxhash3);
929 if (!result)
930 return R__FORWARD_ERROR(result);
931
932 return sizeof(typeAndSize);
933}
934
936 std::uint64_t bufSize,
937 std::uint16_t expectedType)
938{
939 std::uint64_t xxhash3;
940 return R__FORWARD_RESULT(DeserializeEnvelope(buffer, bufSize, expectedType, xxhash3));
941}
942
944{
945 // Marker: multiply the final size with 1
946 return SerializeInt64(1, buffer);
947}
948
950{
951 auto base = reinterpret_cast<unsigned char *>(buffer);
952 auto pos = base;
953 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
954
955 // Marker: multiply the final size with -1
956 pos += SerializeInt64(-1, *where);
957 pos += SerializeUInt32(nitems, *where);
958 return pos - base;
959}
960
963{
964 auto preambleSize = sizeof(std::int64_t);
965 if (size < preambleSize)
966 return R__FAIL("frame too short: " + std::to_string(size));
967 if (frame) {
968 std::int64_t marker;
969 DeserializeInt64(frame, marker);
970 if ((marker < 0) && (size < (sizeof(std::uint32_t) + preambleSize)))
971 return R__FAIL("frame too short: " + std::to_string(size));
972 SerializeInt64(marker * static_cast<int64_t>(size), frame);
973 }
974 return 0;
975}
976
979 std::uint64_t &frameSize, std::uint32_t &nitems)
980{
981 std::uint64_t minSize = sizeof(std::int64_t);
982 if (bufSize < minSize)
983 return R__FAIL("frame too short");
984
985 std::int64_t *ssize = reinterpret_cast<std::int64_t *>(&frameSize);
986 DeserializeInt64(buffer, *ssize);
987
988 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
989 bytes += minSize;
990
991 if (*ssize >= 0) {
992 // Record frame
993 nitems = 1;
994 } else {
995 // List frame
996 minSize += sizeof(std::uint32_t);
997 if (bufSize < minSize)
998 return R__FAIL("frame too short");
999 bytes += DeserializeUInt32(bytes, nitems);
1000 *ssize = -(*ssize);
1001 }
1002
1003 if (frameSize < minSize)
1004 return R__FAIL("corrupt frame size");
1005 if (bufSize < frameSize)
1006 return R__FAIL("frame too short");
1007
1008 return bytes - reinterpret_cast<const unsigned char *>(buffer);
1009}
1010
1012 std::uint64_t bufSize,
1013 std::uint64_t &frameSize)
1014{
1015 std::uint32_t nitems;
1016 return R__FORWARD_RESULT(DeserializeFrameHeader(buffer, bufSize, frameSize, nitems));
1017}
1018
1020ROOT::Internal::RNTupleSerializer::SerializeFeatureFlags(const std::vector<std::uint64_t> &flags, void *buffer)
1021{
1022 if (flags.empty())
1023 return SerializeUInt64(0, buffer);
1024
1025 if (buffer) {
1026 auto bytes = reinterpret_cast<unsigned char *>(buffer);
1027
1028 for (unsigned i = 0; i < flags.size(); ++i) {
1029 if (flags[i] & 0x8000000000000000)
1030 return R__FAIL("feature flag out of bounds");
1031
1032 // The MSb indicates that another Int64 follows; set this bit to 1 for all except the last element
1033 if (i == (flags.size() - 1))
1034 SerializeUInt64(flags[i], bytes);
1035 else
1036 bytes += SerializeUInt64(flags[i] | 0x8000000000000000, bytes);
1037 }
1038 }
1039 return (flags.size() * sizeof(std::int64_t));
1040}
1041
1044 std::vector<std::uint64_t> &flags)
1045{
1046 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
1047
1048 flags.clear();
1049 std::uint64_t f;
1050 do {
1051 if (bufSize < sizeof(std::uint64_t))
1052 return R__FAIL("feature flag buffer too short");
1053 bytes += DeserializeUInt64(bytes, f);
1054 bufSize -= sizeof(std::uint64_t);
1055 flags.emplace_back(f & ~0x8000000000000000);
1056 } while (f & 0x8000000000000000);
1057
1058 return (flags.size() * sizeof(std::uint64_t));
1059}
1060
1063{
1065 return R__FAIL("locator is not serializable");
1066
1067 std::uint32_t size = 0;
1068 if ((locator.GetType() == RNTupleLocator::kTypeFile) &&
1069 (locator.GetNBytesOnStorage() <= std::numeric_limits<std::int32_t>::max())) {
1070 size += SerializeUInt32(locator.GetNBytesOnStorage(), buffer);
1071 size += SerializeUInt64(locator.GetPosition<std::uint64_t>(),
1072 buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
1073 return size;
1074 }
1075
1076 std::uint8_t locatorType = 0;
1077 auto payloadp = buffer ? reinterpret_cast<unsigned char *>(buffer) + sizeof(std::int32_t) : nullptr;
1078 switch (locator.GetType()) {
1081 locatorType = 0x01;
1082 break;
1085 locatorType = 0x02;
1086 break;
1087 default:
1088 if (locator.GetType() == ROOT::Internal::kTestLocatorType) {
1089 // For the testing locator, use the same payload as Object64. We're not gonna really read it back anyway.
1091 locatorType = 0x7e;
1092 } else {
1093 return R__FAIL("locator has unknown type");
1094 }
1095 }
1096 std::int32_t head = sizeof(std::int32_t) + size;
1097 head |= locator.GetReserved() << 16;
1098 head |= static_cast<int>(locatorType & 0x7F) << 24;
1099 head = -head;
1100 size += RNTupleSerializer::SerializeInt32(head, buffer);
1101 return size;
1102}
1103
1105 std::uint64_t bufSize,
1107{
1108 if (bufSize < sizeof(std::int32_t))
1109 return R__FAIL("too short locator");
1110
1111 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
1112 std::int32_t head;
1113
1114 bytes += DeserializeInt32(bytes, head);
1115 bufSize -= sizeof(std::int32_t);
1116 if (head < 0) {
1117 head = -head;
1118 const int type = head >> 24;
1119 const std::uint32_t payloadSize = (static_cast<std::uint32_t>(head) & 0x0000FFFF) - sizeof(std::int32_t);
1120 if (bufSize < payloadSize)
1121 return R__FAIL("too short locator");
1122
1123 locator.SetReserved(static_cast<std::uint32_t>(head >> 16) & 0xFF);
1124 switch (type) {
1125 case 0x01:
1128 break;
1129 case 0x02:
1132 break;
1133 default: locator.SetType(RNTupleLocator::kTypeUnknown);
1134 }
1135 bytes += payloadSize;
1136 } else {
1137 if (bufSize < sizeof(std::uint64_t))
1138 return R__FAIL("too short locator");
1139 std::uint64_t offset;
1140 bytes += DeserializeUInt64(bytes, offset);
1142 locator.SetNBytesOnStorage(head);
1143 locator.SetPosition(offset);
1144 }
1145
1146 return bytes - reinterpret_cast<const unsigned char *>(buffer);
1147}
1148
1151{
1152 auto size = SerializeUInt64(envelopeLink.fLength, buffer);
1153 auto res =
1154 SerializeLocator(envelopeLink.fLocator, buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
1155 if (res)
1156 size += res.Unwrap();
1157 else
1158 return R__FORWARD_ERROR(res);
1159 return size;
1160}
1161
1163 std::uint64_t bufSize,
1165{
1166 if (bufSize < sizeof(std::int64_t))
1167 return R__FAIL("too short envelope link");
1168
1169 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
1170 bytes += DeserializeUInt64(bytes, envelopeLink.fLength);
1171 bufSize -= sizeof(std::uint64_t);
1172 if (auto res = DeserializeLocator(bytes, bufSize, envelopeLink.fLocator)) {
1173 bytes += res.Unwrap();
1174 } else {
1175 return R__FORWARD_ERROR(res);
1176 }
1177 return bytes - reinterpret_cast<const unsigned char *>(buffer);
1178}
1179
1182{
1183 if (clusterSummary.fNEntries >= (static_cast<std::uint64_t>(1) << 56)) {
1184 return R__FAIL("number of entries in cluster exceeds maximum of 2^56");
1185 }
1186
1187 auto base = reinterpret_cast<unsigned char *>(buffer);
1188 auto pos = base;
1189 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1190
1191 auto frame = pos;
1192 pos += SerializeRecordFramePreamble(*where);
1193 pos += SerializeUInt64(clusterSummary.fFirstEntry, *where);
1194 const std::uint64_t nEntriesAndFlags =
1195 (static_cast<std::uint64_t>(clusterSummary.fFlags) << 56) | clusterSummary.fNEntries;
1196 pos += SerializeUInt64(nEntriesAndFlags, *where);
1197
1198 auto size = pos - frame;
1199 if (auto res = SerializeFramePostscript(frame, size)) {
1200 pos += res.Unwrap();
1201 } else {
1202 return R__FORWARD_ERROR(res);
1203 }
1204 return size;
1205}
1206
1210{
1211 auto base = reinterpret_cast<const unsigned char *>(buffer);
1212 auto bytes = base;
1213 std::uint64_t frameSize;
1214 if (auto res = DeserializeFrameHeader(bytes, bufSize, frameSize)) {
1215 bytes += res.Unwrap();
1216 } else {
1217 return R__FORWARD_ERROR(res);
1218 }
1219
1220 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
1221 if (fnFrameSizeLeft() < 2 * sizeof(std::uint64_t))
1222 return R__FAIL("too short cluster summary");
1223
1224 bytes += DeserializeUInt64(bytes, clusterSummary.fFirstEntry);
1225 std::uint64_t nEntriesAndFlags;
1226 bytes += DeserializeUInt64(bytes, nEntriesAndFlags);
1227
1228 const std::uint64_t nEntries = (nEntriesAndFlags << 8) >> 8;
1229 const std::uint8_t flags = nEntriesAndFlags >> 56;
1230
1231 if (flags & 0x01) {
1232 return R__FAIL("sharded cluster flag set in cluster summary; sharded clusters are currently unsupported.");
1233 }
1234
1235 clusterSummary.fNEntries = nEntries;
1236 clusterSummary.fFlags = flags;
1237
1238 return frameSize;
1239}
1240
1243{
1244 auto base = reinterpret_cast<unsigned char *>(buffer);
1245 auto pos = base;
1246 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1247
1248 auto frame = pos;
1249 pos += SerializeRecordFramePreamble(*where);
1250 pos += SerializeUInt64(clusterGroup.fMinEntry, *where);
1251 pos += SerializeUInt64(clusterGroup.fEntrySpan, *where);
1252 pos += SerializeUInt32(clusterGroup.fNClusters, *where);
1253 if (auto res = SerializeEnvelopeLink(clusterGroup.fPageListEnvelopeLink, *where)) {
1254 pos += res.Unwrap();
1255 } else {
1256 return R__FORWARD_ERROR(res);
1257 }
1258 auto size = pos - frame;
1259 if (auto res = SerializeFramePostscript(frame, size)) {
1260 return size;
1261 } else {
1262 return R__FORWARD_ERROR(res);
1263 }
1264}
1265
1267 std::uint64_t bufSize,
1269{
1270 auto base = reinterpret_cast<const unsigned char *>(buffer);
1271 auto bytes = base;
1272
1273 std::uint64_t frameSize;
1274 if (auto res = DeserializeFrameHeader(bytes, bufSize, frameSize)) {
1275 bytes += res.Unwrap();
1276 } else {
1277 return R__FORWARD_ERROR(res);
1278 }
1279
1280 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
1281 if (fnFrameSizeLeft() < sizeof(std::uint32_t) + 2 * sizeof(std::uint64_t))
1282 return R__FAIL("too short cluster group");
1283
1284 bytes += DeserializeUInt64(bytes, clusterGroup.fMinEntry);
1285 bytes += DeserializeUInt64(bytes, clusterGroup.fEntrySpan);
1286 bytes += DeserializeUInt32(bytes, clusterGroup.fNClusters);
1287 if (auto res = DeserializeEnvelopeLink(bytes, fnFrameSizeLeft(), clusterGroup.fPageListEnvelopeLink)) {
1288 bytes += res.Unwrap();
1289 } else {
1290 return R__FORWARD_ERROR(res);
1291 }
1292
1293 return frameSize;
1294}
1295
1297 bool forHeaderExtension)
1298{
1299 auto fieldZeroId = desc.GetFieldZeroId();
1300 auto depthFirstTraversal = [&](std::span<ROOT::DescriptorId_t> fieldTrees, auto doForEachField) {
1301 std::deque<ROOT::DescriptorId_t> idQueue{fieldTrees.begin(), fieldTrees.end()};
1302 while (!idQueue.empty()) {
1303 auto fieldId = idQueue.front();
1304 idQueue.pop_front();
1305 // Field zero has no physical representation nor columns of its own; recurse over its subfields only
1306 if (fieldId != fieldZeroId)
1308 unsigned i = 0;
1309 for (const auto &f : desc.GetFieldIterable(fieldId))
1310 idQueue.insert(idQueue.begin() + i++, f.GetId());
1311 }
1312 };
1313
1314 R__ASSERT(desc.GetNFields() > 0); // we must have at least a zero field
1315
1316 std::vector<ROOT::DescriptorId_t> fieldTrees;
1317 if (!forHeaderExtension) {
1318 fieldTrees.emplace_back(fieldZeroId);
1319 } else if (auto xHeader = desc.GetHeaderExtension()) {
1320 fieldTrees = xHeader->GetTopLevelFields(desc);
1321 }
1324 for (const auto &c : desc.GetColumnIterable(fieldId)) {
1325 if (!c.IsAliasColumn()) {
1326 MapPhysicalColumnId(c.GetPhysicalId());
1327 }
1328 }
1329 });
1330
1331 if (forHeaderExtension) {
1332 // Create physical IDs for column representations that extend fields of the regular header.
1333 // First the physical columns then the alias columns.
1334 for (auto memId : desc.GetHeaderExtension()->GetExtendedColumnRepresentations()) {
1335 const auto &columnDesc = desc.GetColumnDescriptor(memId);
1336 if (!columnDesc.IsAliasColumn()) {
1337 MapPhysicalColumnId(columnDesc.GetPhysicalId());
1338 }
1339 }
1340 }
1341}
1342
1345 const RContext &context, bool forHeaderExtension)
1346{
1347 auto base = reinterpret_cast<unsigned char *>(buffer);
1348 auto pos = base;
1349 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1350
1351 std::size_t nFields = 0, nColumns = 0, nAliasColumns = 0, fieldListOffset = 0;
1352 // Columns in the extension header that are attached to a field of the regular header
1353 std::vector<std::reference_wrapper<const ROOT::RColumnDescriptor>> extraColumns;
1354 if (forHeaderExtension) {
1355 // A call to `RNTupleDescriptorBuilder::BeginHeaderExtension()` is not strictly required after serializing the
1356 // header, which may happen, e.g., in unit tests. Ensure an empty schema extension is serialized in this case
1357 if (auto xHeader = desc.GetHeaderExtension()) {
1358 nFields = xHeader->GetNFields();
1359 nColumns = xHeader->GetNPhysicalColumns();
1360 nAliasColumns = xHeader->GetNLogicalColumns() - xHeader->GetNPhysicalColumns();
1361 fieldListOffset = desc.GetNFields() - nFields - 1;
1362
1363 extraColumns.reserve(xHeader->GetExtendedColumnRepresentations().size());
1364 for (auto columnId : xHeader->GetExtendedColumnRepresentations()) {
1365 extraColumns.emplace_back(desc.GetColumnDescriptor(columnId));
1366 }
1367 }
1368 } else {
1369 if (auto xHeader = desc.GetHeaderExtension()) {
1370 nFields = desc.GetNFields() - xHeader->GetNFields() - 1;
1371 nColumns = desc.GetNPhysicalColumns() - xHeader->GetNPhysicalColumns();
1373 (xHeader->GetNLogicalColumns() - xHeader->GetNPhysicalColumns());
1374 } else {
1375 nFields = desc.GetNFields() - 1;
1378 }
1379 }
1380 const auto nExtraTypeInfos = desc.GetNExtraTypeInfos();
1381 const auto &onDiskFields = context.GetOnDiskFieldList();
1383 std::span<const ROOT::DescriptorId_t> fieldList{onDiskFields.data() + fieldListOffset, nFields};
1384
1385 auto frame = pos;
1386 pos += SerializeListFramePreamble(nFields, *where);
1387 if (auto res = SerializeFieldList(desc, fieldList, /*firstOnDiskId=*/fieldListOffset, context, *where)) {
1388 pos += res.Unwrap();
1389 } else {
1390 return R__FORWARD_ERROR(res);
1391 }
1392 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1393 pos += res.Unwrap();
1394 } else {
1395 return R__FORWARD_ERROR(res);
1396 }
1397
1398 frame = pos;
1399 pos += SerializeListFramePreamble(nColumns, *where);
1400 if (auto res = SerializeColumnsOfFields(desc, fieldList, context, *where, forHeaderExtension)) {
1401 pos += res.Unwrap();
1402 } else {
1403 return R__FORWARD_ERROR(res);
1404 }
1405 for (const auto &c : extraColumns) {
1406 if (!c.get().IsAliasColumn()) {
1407 if (auto res = SerializePhysicalColumn(c.get(), context, *where)) {
1408 pos += res.Unwrap();
1409 } else {
1410 return R__FORWARD_ERROR(res);
1411 }
1412 }
1413 }
1414 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1415 pos += res.Unwrap();
1416 } else {
1417 return R__FORWARD_ERROR(res);
1418 }
1419
1420 frame = pos;
1421 pos += SerializeListFramePreamble(nAliasColumns, *where);
1423 for (const auto &c : extraColumns) {
1424 if (c.get().IsAliasColumn()) {
1425 pos += SerializeAliasColumn(c.get(), context, *where);
1426 }
1427 }
1428 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1429 pos += res.Unwrap();
1430 } else {
1431 return R__FORWARD_ERROR(res);
1432 }
1433
1434 frame = pos;
1435 // We only serialize the extra type info list in the header extension.
1436 if (forHeaderExtension) {
1437 pos += SerializeListFramePreamble(nExtraTypeInfos, *where);
1438 if (auto res = SerializeExtraTypeInfoList(desc, *where)) {
1439 pos += res.Unwrap();
1440 } else {
1441 return R__FORWARD_ERROR(res);
1442 }
1443 } else {
1444 pos += SerializeListFramePreamble(0, *where);
1445 }
1446 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1447 pos += res.Unwrap();
1448 } else {
1449 return R__FORWARD_ERROR(res);
1450 }
1451
1452 return static_cast<std::uint32_t>(pos - base);
1453}
1454
1458{
1459 auto base = reinterpret_cast<const unsigned char *>(buffer);
1460 auto bytes = base;
1461 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1462
1463 std::uint64_t frameSize;
1464 auto frame = bytes;
1465 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1466
1467 std::uint32_t nFields;
1468 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nFields)) {
1469 bytes += res.Unwrap();
1470 } else {
1471 return R__FORWARD_ERROR(res);
1472 }
1473 // The zero field is always added before `DeserializeSchemaDescription()` is called
1474 const std::uint32_t fieldIdRangeBegin = descBuilder.GetDescriptor().GetNFields() - 1;
1475 for (unsigned i = 0; i < nFields; ++i) {
1476 std::uint32_t fieldId = fieldIdRangeBegin + i;
1478 if (auto res = DeserializeField(bytes, fnFrameSizeLeft(), fieldBuilder)) {
1479 bytes += res.Unwrap();
1480 } else {
1481 return R__FORWARD_ERROR(res);
1482 }
1483 if (fieldId == fieldBuilder.GetParentId())
1484 fieldBuilder.ParentId(kZeroFieldId);
1485 auto fieldDesc = fieldBuilder.FieldId(fieldId).MakeDescriptor();
1486 if (!fieldDesc)
1488 const auto parentId = fieldDesc.Inspect().GetParentId();
1489 const auto projectionSourceId = fieldDesc.Inspect().GetProjectionSourceId();
1490 descBuilder.AddField(fieldDesc.Unwrap());
1491 auto resVoid = descBuilder.AddFieldLink(parentId, fieldId);
1492 if (!resVoid)
1493 return R__FORWARD_ERROR(resVoid);
1495 resVoid = descBuilder.AddFieldProjection(projectionSourceId, fieldId);
1496 if (!resVoid)
1497 return R__FORWARD_ERROR(resVoid);
1498 }
1499 }
1500 bytes = frame + frameSize;
1501
1502 // As columns are added in order of representation index and column index, determine the column index
1503 // for the currently deserialized column from the columns already added.
1505 std::uint16_t representationIndex) -> std::uint32_t {
1506 const auto &existingColumns = descBuilder.GetDescriptor().GetFieldDescriptor(fieldId).GetLogicalColumnIds();
1507 if (existingColumns.empty())
1508 return 0;
1509 const auto &lastColumnDesc = descBuilder.GetDescriptor().GetColumnDescriptor(existingColumns.back());
1510 return (representationIndex == lastColumnDesc.GetRepresentationIndex()) ? (lastColumnDesc.GetIndex() + 1) : 0;
1511 };
1512
1513 std::uint32_t nColumns;
1514 frame = bytes;
1515 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nColumns)) {
1516 bytes += res.Unwrap();
1517 } else {
1518 return R__FORWARD_ERROR(res);
1519 }
1520
1521 if (descBuilder.GetDescriptor().GetNLogicalColumns() > descBuilder.GetDescriptor().GetNPhysicalColumns())
1522 descBuilder.ShiftAliasColumns(nColumns);
1523
1524 const std::uint32_t columnIdRangeBegin = descBuilder.GetDescriptor().GetNPhysicalColumns();
1525 for (unsigned i = 0; i < nColumns; ++i) {
1526 std::uint32_t columnId = columnIdRangeBegin + i;
1529 bytes += res.Unwrap();
1530 } else {
1531 return R__FORWARD_ERROR(res);
1532 }
1533
1534 columnBuilder.Index(fnNextColumnIndex(columnBuilder.GetFieldId(), columnBuilder.GetRepresentationIndex()));
1535 columnBuilder.LogicalColumnId(columnId);
1536 columnBuilder.PhysicalColumnId(columnId);
1537 auto columnDesc = columnBuilder.MakeDescriptor();
1538 if (!columnDesc)
1540 auto resVoid = descBuilder.AddColumn(columnDesc.Unwrap());
1541 if (!resVoid)
1542 return R__FORWARD_ERROR(resVoid);
1543 }
1544 bytes = frame + frameSize;
1545
1546 std::uint32_t nAliasColumns;
1547 frame = bytes;
1548 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nAliasColumns)) {
1549 bytes += res.Unwrap();
1550 } else {
1551 return R__FORWARD_ERROR(res);
1552 }
1553 const std::uint32_t aliasColumnIdRangeBegin = descBuilder.GetDescriptor().GetNLogicalColumns();
1554 for (unsigned i = 0; i < nAliasColumns; ++i) {
1555 std::uint32_t physicalId;
1556 std::uint32_t fieldId;
1558 bytes += res.Unwrap();
1559 } else {
1560 return R__FORWARD_ERROR(res);
1561 }
1562
1564 columnBuilder.LogicalColumnId(aliasColumnIdRangeBegin + i).PhysicalColumnId(physicalId).FieldId(fieldId);
1565 const auto &physicalColumnDesc = descBuilder.GetDescriptor().GetColumnDescriptor(physicalId);
1566 columnBuilder.BitsOnStorage(physicalColumnDesc.GetBitsOnStorage());
1567 columnBuilder.ValueRange(physicalColumnDesc.GetValueRange());
1568 columnBuilder.Type(physicalColumnDesc.GetType());
1569 columnBuilder.RepresentationIndex(physicalColumnDesc.GetRepresentationIndex());
1570 columnBuilder.Index(fnNextColumnIndex(columnBuilder.GetFieldId(), columnBuilder.GetRepresentationIndex()));
1571
1572 auto aliasColumnDesc = columnBuilder.MakeDescriptor();
1573 if (!aliasColumnDesc)
1575 auto resVoid = descBuilder.AddColumn(aliasColumnDesc.Unwrap());
1576 if (!resVoid)
1577 return R__FORWARD_ERROR(resVoid);
1578 }
1579 bytes = frame + frameSize;
1580
1581 std::uint32_t nExtraTypeInfos;
1582 frame = bytes;
1583 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nExtraTypeInfos)) {
1584 bytes += res.Unwrap();
1585 } else {
1586 return R__FORWARD_ERROR(res);
1587 }
1588 for (unsigned i = 0; i < nExtraTypeInfos; ++i) {
1591 bytes += res.Unwrap();
1592 } else {
1593 return R__FORWARD_ERROR(res);
1594 }
1595
1596 auto extraTypeInfoDesc = extraTypeInfoBuilder.MoveDescriptor();
1597 // We ignore unknown extra type information
1599 descBuilder.AddExtraTypeInfo(extraTypeInfoDesc.Unwrap());
1600 }
1601 bytes = frame + frameSize;
1602
1603 return bytes - base;
1604}
1605
1608{
1609 RContext context;
1610
1611 auto base = reinterpret_cast<unsigned char *>(buffer);
1612 auto pos = base;
1613 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1614
1615 pos += SerializeEnvelopePreamble(kEnvelopeTypeHeader, *where);
1616 // So far we don't make use of feature flags
1617 if (auto res = SerializeFeatureFlags(desc.GetFeatureFlags(), *where)) {
1618 pos += res.Unwrap();
1619 } else {
1620 return R__FORWARD_ERROR(res);
1621 }
1622 pos += SerializeString(desc.GetName(), *where);
1623 pos += SerializeString(desc.GetDescription(), *where);
1624 pos += SerializeString(std::string("ROOT v") + ROOT_RELEASE, *where);
1625
1626 context.MapSchema(desc, /*forHeaderExtension=*/false);
1627
1628 if (auto res = SerializeSchemaDescription(*where, desc, context)) {
1629 pos += res.Unwrap();
1630 } else {
1631 return R__FORWARD_ERROR(res);
1632 }
1633
1634 std::uint64_t size = pos - base;
1635 std::uint64_t xxhash3 = 0;
1636 if (auto res = SerializeEnvelopePostscript(base, size, xxhash3)) {
1637 size += res.Unwrap();
1638 } else {
1639 return R__FORWARD_ERROR(res);
1640 }
1641
1642 context.SetHeaderSize(size);
1643 context.SetHeaderXxHash3(xxhash3);
1644 return context;
1645}
1646
1649 std::span<ROOT::DescriptorId_t> physClusterIDs,
1650 const RContext &context)
1651{
1652 auto base = reinterpret_cast<unsigned char *>(buffer);
1653 auto pos = base;
1654 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1655
1656 pos += SerializeEnvelopePreamble(kEnvelopeTypePageList, *where);
1657
1658 pos += SerializeUInt64(context.GetHeaderXxHash3(), *where);
1659
1660 // Cluster summaries
1661 const auto nClusters = physClusterIDs.size();
1662 auto clusterSummaryFrame = pos;
1663 pos += SerializeListFramePreamble(nClusters, *where);
1664 for (auto clusterId : physClusterIDs) {
1665 const auto &clusterDesc = desc.GetClusterDescriptor(context.GetMemClusterId(clusterId));
1666 RClusterSummary summary{clusterDesc.GetFirstEntryIndex(), clusterDesc.GetNEntries(), 0};
1667 if (auto res = SerializeClusterSummary(summary, *where)) {
1668 pos += res.Unwrap();
1669 } else {
1670 return R__FORWARD_ERROR(res);
1671 }
1672 }
1673 if (auto res = SerializeFramePostscript(buffer ? clusterSummaryFrame : nullptr, pos - clusterSummaryFrame)) {
1674 pos += res.Unwrap();
1675 } else {
1676 return R__FORWARD_ERROR(res);
1677 }
1678
1679 // Page locations
1680 auto topMostFrame = pos;
1681 pos += SerializeListFramePreamble(nClusters, *where);
1682
1683 for (auto clusterId : physClusterIDs) {
1684 const auto &clusterDesc = desc.GetClusterDescriptor(context.GetMemClusterId(clusterId));
1685 // Get an ordered set of physical column ids
1686 std::set<ROOT::DescriptorId_t> onDiskColumnIds;
1687 for (const auto &columnRange : clusterDesc.GetColumnRangeIterable())
1688 onDiskColumnIds.insert(context.GetOnDiskColumnId(columnRange.GetPhysicalColumnId()));
1689
1690 auto outerFrame = pos;
1691 pos += SerializeListFramePreamble(onDiskColumnIds.size(), *where);
1692 for (auto onDiskId : onDiskColumnIds) {
1693 auto memId = context.GetMemColumnId(onDiskId);
1694 const auto &columnRange = clusterDesc.GetColumnRange(memId);
1695
1696 auto innerFrame = pos;
1697 if (columnRange.IsSuppressed()) {
1698 // Empty page range
1699 pos += SerializeListFramePreamble(0, *where);
1700 pos += SerializeInt64(kSuppressedColumnMarker, *where);
1701 } else {
1702 const auto &pageRange = clusterDesc.GetPageRange(memId);
1703 pos += SerializeListFramePreamble(pageRange.GetPageInfos().size(), *where);
1704
1705 for (const auto &pi : pageRange.GetPageInfos()) {
1706 std::int32_t nElements =
1707 pi.HasChecksum() ? -static_cast<std::int32_t>(pi.GetNElements()) : pi.GetNElements();
1708 pos += SerializeUInt32(nElements, *where);
1709 if (auto res = SerializeLocator(pi.GetLocator(), *where)) {
1710 pos += res.Unwrap();
1711 } else {
1712 return R__FORWARD_ERROR(res);
1713 }
1714 }
1715 pos += SerializeInt64(columnRange.GetFirstElementIndex(), *where);
1716 pos += SerializeUInt32(columnRange.GetCompressionSettings().value(), *where);
1717 }
1718
1719 if (auto res = SerializeFramePostscript(buffer ? innerFrame : nullptr, pos - innerFrame)) {
1720 pos += res.Unwrap();
1721 } else {
1722 return R__FORWARD_ERROR(res);
1723 }
1724 }
1725 if (auto res = SerializeFramePostscript(buffer ? outerFrame : nullptr, pos - outerFrame)) {
1726 pos += res.Unwrap();
1727 } else {
1728 return R__FORWARD_ERROR(res);
1729 }
1730 }
1731
1732 if (auto res = SerializeFramePostscript(buffer ? topMostFrame : nullptr, pos - topMostFrame)) {
1733 pos += res.Unwrap();
1734 } else {
1735 return R__FORWARD_ERROR(res);
1736 }
1737 std::uint64_t size = pos - base;
1738 if (auto res = SerializeEnvelopePostscript(base, size)) {
1739 size += res.Unwrap();
1740 } else {
1741 return R__FORWARD_ERROR(res);
1742 }
1743 return size;
1744}
1745
1747 const ROOT::RNTupleDescriptor &desc,
1748 const RContext &context)
1749{
1750 auto base = reinterpret_cast<unsigned char *>(buffer);
1751 auto pos = base;
1752 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1753
1754 pos += SerializeEnvelopePreamble(kEnvelopeTypeFooter, *where);
1755
1756 // So far we don't make use of footer feature flags
1757 if (auto res = SerializeFeatureFlags(std::vector<std::uint64_t>(), *where)) {
1758 pos += res.Unwrap();
1759 } else {
1760 return R__FORWARD_ERROR(res);
1761 }
1762 pos += SerializeUInt64(context.GetHeaderXxHash3(), *where);
1763
1764 // Schema extension, i.e. incremental changes with respect to the header
1765 auto frame = pos;
1766 pos += SerializeRecordFramePreamble(*where);
1767 if (auto res = SerializeSchemaDescription(*where, desc, context, /*forHeaderExtension=*/true)) {
1768 pos += res.Unwrap();
1769 } else {
1770 return R__FORWARD_ERROR(res);
1771 }
1772 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1773 pos += res.Unwrap();
1774 } else {
1775 return R__FORWARD_ERROR(res);
1776 }
1777
1778 // Cluster groups
1779 frame = pos;
1780 const auto nClusterGroups = desc.GetNClusterGroups();
1781 pos += SerializeListFramePreamble(nClusterGroups, *where);
1782 for (unsigned int i = 0; i < nClusterGroups; ++i) {
1783 const auto &cgDesc = desc.GetClusterGroupDescriptor(context.GetMemClusterGroupId(i));
1785 clusterGroup.fMinEntry = cgDesc.GetMinEntry();
1786 clusterGroup.fEntrySpan = cgDesc.GetEntrySpan();
1787 clusterGroup.fNClusters = cgDesc.GetNClusters();
1788 clusterGroup.fPageListEnvelopeLink.fLength = cgDesc.GetPageListLength();
1789 clusterGroup.fPageListEnvelopeLink.fLocator = cgDesc.GetPageListLocator();
1790 if (auto res = SerializeClusterGroup(clusterGroup, *where)) {
1791 pos += res.Unwrap();
1792 } else {
1793 return R__FORWARD_ERROR(res);
1794 }
1795 }
1796 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1797 pos += res.Unwrap();
1798 } else {
1799 return R__FORWARD_ERROR(res);
1800 }
1801
1802 std::uint32_t size = pos - base;
1803 if (auto res = SerializeEnvelopePostscript(base, size)) {
1804 size += res.Unwrap();
1805 } else {
1806 return R__FORWARD_ERROR(res);
1807 }
1808 return size;
1809}
1810
1813{
1814 auto base = reinterpret_cast<const unsigned char *>(buffer);
1815 auto bytes = base;
1816 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1817
1818 std::uint64_t xxhash3{0};
1819 if (auto res = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypeHeader, xxhash3)) {
1820 bytes += res.Unwrap();
1821 } else {
1822 return R__FORWARD_ERROR(res);
1823 }
1824 descBuilder.SetOnDiskHeaderXxHash3(xxhash3);
1825
1826 std::vector<std::uint64_t> featureFlags;
1827 if (auto res = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags)) {
1828 bytes += res.Unwrap();
1829 } else {
1830 return R__FORWARD_ERROR(res);
1831 }
1832 for (std::size_t i = 0; i < featureFlags.size(); ++i) {
1833 if (!featureFlags[i])
1834 continue;
1835 unsigned int bit = 0;
1836 while (!(featureFlags[i] & (static_cast<uint64_t>(1) << bit)))
1837 bit++;
1838 return R__FAIL("unsupported format feature: " + std::to_string(i * 64 + bit));
1839 }
1840
1841 std::string name;
1842 std::string description;
1843 std::string writer;
1844 if (auto res = DeserializeString(bytes, fnBufSizeLeft(), name)) {
1845 bytes += res.Unwrap();
1846 } else {
1847 return R__FORWARD_ERROR(res);
1848 }
1849 if (auto res = DeserializeString(bytes, fnBufSizeLeft(), description)) {
1850 bytes += res.Unwrap();
1851 } else {
1852 return R__FORWARD_ERROR(res);
1853 }
1854 if (auto res = DeserializeString(bytes, fnBufSizeLeft(), writer)) {
1855 bytes += res.Unwrap();
1856 } else {
1857 return R__FORWARD_ERROR(res);
1858 }
1859 descBuilder.SetNTuple(name, description);
1860
1861 // Zero field
1863 .FieldId(kZeroFieldId)
1865 .MakeDescriptor()
1866 .Unwrap());
1867 if (auto res = DeserializeSchemaDescription(bytes, fnBufSizeLeft(), descBuilder)) {
1868 return RResult<void>::Success();
1869 } else {
1870 return R__FORWARD_ERROR(res);
1871 }
1872}
1873
1876{
1877 auto base = reinterpret_cast<const unsigned char *>(buffer);
1878 auto bytes = base;
1879 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1880 if (auto res = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypeFooter)) {
1881 bytes += res.Unwrap();
1882 } else {
1883 return R__FORWARD_ERROR(res);
1884 }
1885
1886 std::vector<std::uint64_t> featureFlags;
1887 if (auto res = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags)) {
1888 bytes += res.Unwrap();
1889 } else {
1890 return R__FORWARD_ERROR(res);
1891 }
1892 for (auto f : featureFlags) {
1893 if (f)
1894 R__LOG_WARNING(ROOT::Internal::NTupleLog()) << "Unsupported feature flag! " << f;
1895 }
1896
1897 std::uint64_t xxhash3{0};
1898 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint64_t)))
1899 return R__FAIL("footer too short");
1900 bytes += DeserializeUInt64(bytes, xxhash3);
1901 if (xxhash3 != descBuilder.GetDescriptor().GetOnDiskHeaderXxHash3())
1902 return R__FAIL("XxHash-3 mismatch between header and footer");
1903
1904 std::uint64_t frameSize;
1905 auto frame = bytes;
1906 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1907
1908 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize)) {
1909 bytes += res.Unwrap();
1910 } else {
1911 return R__FORWARD_ERROR(res);
1912 }
1913 if (fnFrameSizeLeft() > 0) {
1914 descBuilder.BeginHeaderExtension();
1915 if (auto res = DeserializeSchemaDescription(bytes, fnFrameSizeLeft(), descBuilder); !res) {
1916 return R__FORWARD_ERROR(res);
1917 }
1918 }
1919 bytes = frame + frameSize;
1920
1921 std::uint32_t nClusterGroups;
1922 frame = bytes;
1923 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nClusterGroups)) {
1924 bytes += res.Unwrap();
1925 } else {
1926 return R__FORWARD_ERROR(res);
1927 }
1928 for (std::uint32_t groupId = 0; groupId < nClusterGroups; ++groupId) {
1930 if (auto res = DeserializeClusterGroup(bytes, fnFrameSizeLeft(), clusterGroup)) {
1931 bytes += res.Unwrap();
1932 } else {
1933 return R__FORWARD_ERROR(res);
1934 }
1935
1936 descBuilder.AddToOnDiskFooterSize(clusterGroup.fPageListEnvelopeLink.fLocator.GetNBytesOnStorage());
1938 clusterGroupBuilder.ClusterGroupId(groupId)
1939 .PageListLocator(clusterGroup.fPageListEnvelopeLink.fLocator)
1940 .PageListLength(clusterGroup.fPageListEnvelopeLink.fLength)
1941 .MinEntry(clusterGroup.fMinEntry)
1942 .EntrySpan(clusterGroup.fEntrySpan)
1943 .NClusters(clusterGroup.fNClusters);
1944 descBuilder.AddClusterGroup(clusterGroupBuilder.MoveDescriptor().Unwrap());
1945 }
1946 bytes = frame + frameSize;
1947
1948 return RResult<void>::Success();
1949}
1950
1954 const ROOT::RNTupleDescriptor &desc)
1955{
1956 auto base = reinterpret_cast<const unsigned char *>(buffer);
1957 auto bytes = base;
1958 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1959
1960 if (auto res = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypePageList)) {
1961 bytes += res.Unwrap();
1962 } else {
1963 return R__FORWARD_ERROR(res);
1964 }
1965
1966 std::uint64_t xxhash3{0};
1967 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint64_t)))
1968 return R__FAIL("page list too short");
1969 bytes += DeserializeUInt64(bytes, xxhash3);
1970 if (xxhash3 != desc.GetOnDiskHeaderXxHash3())
1971 return R__FAIL("XxHash-3 mismatch between header and page list");
1972
1973 std::vector<RClusterDescriptorBuilder> clusterBuilders;
1975 for (ROOT::DescriptorId_t i = 0; i < clusterGroupId; ++i) {
1976 firstClusterId = firstClusterId + desc.GetClusterGroupDescriptor(i).GetNClusters();
1977 }
1978
1979 std::uint64_t clusterSummaryFrameSize;
1982
1983 std::uint32_t nClusterSummaries;
1984 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), clusterSummaryFrameSize, nClusterSummaries)) {
1985 bytes += res.Unwrap();
1986 } else {
1987 return R__FORWARD_ERROR(res);
1988 }
1991 if (auto res = DeserializeClusterSummary(bytes, fnClusterSummaryFrameSizeLeft(), clusterSummary)) {
1992 bytes += res.Unwrap();
1993 } else {
1994 return R__FORWARD_ERROR(res);
1995 }
1996
1999 clusterBuilders.emplace_back(std::move(builder));
2000 }
2002
2003 std::uint64_t topMostFrameSize;
2004 auto topMostFrame = bytes;
2005 auto fnTopMostFrameSizeLeft = [&]() { return topMostFrameSize - (bytes - topMostFrame); };
2006
2007 std::uint32_t nClusters;
2008 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), topMostFrameSize, nClusters)) {
2009 bytes += res.Unwrap();
2010 } else {
2011 return R__FORWARD_ERROR(res);
2012 }
2013
2015 return R__FAIL("mismatch between number of clusters and number of cluster summaries");
2016
2017 for (std::uint32_t i = 0; i < nClusters; ++i) {
2018 std::uint64_t outerFrameSize;
2019 auto outerFrame = bytes;
2020 auto fnOuterFrameSizeLeft = [&]() { return outerFrameSize - (bytes - outerFrame); };
2021
2022 std::uint32_t nColumns;
2023 if (auto res = DeserializeFrameHeader(bytes, fnTopMostFrameSizeLeft(), outerFrameSize, nColumns)) {
2024 bytes += res.Unwrap();
2025 } else {
2026 return R__FORWARD_ERROR(res);
2027 }
2028
2029 for (std::uint32_t j = 0; j < nColumns; ++j) {
2030 std::uint64_t innerFrameSize;
2031 auto innerFrame = bytes;
2032 auto fnInnerFrameSizeLeft = [&]() { return innerFrameSize - (bytes - innerFrame); };
2033
2034 std::uint32_t nPages;
2035 if (auto res = DeserializeFrameHeader(bytes, fnOuterFrameSizeLeft(), innerFrameSize, nPages)) {
2036 bytes += res.Unwrap();
2037 } else {
2038 return R__FORWARD_ERROR(res);
2039 }
2040
2042 pageRange.SetPhysicalColumnId(j);
2043 for (std::uint32_t k = 0; k < nPages; ++k) {
2044 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
2045 return R__FAIL("inner frame too short");
2046 std::int32_t nElements;
2047 bool hasChecksum = false;
2049 bytes += DeserializeInt32(bytes, nElements);
2050 if (nElements < 0) {
2052 hasChecksum = true;
2053 }
2054 if (auto res = DeserializeLocator(bytes, fnInnerFrameSizeLeft(), locator)) {
2055 bytes += res.Unwrap();
2056 } else {
2057 return R__FORWARD_ERROR(res);
2058 }
2059 pageRange.GetPageInfos().push_back({static_cast<std::uint32_t>(nElements), locator, hasChecksum});
2060 }
2061
2062 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::int64_t)))
2063 return R__FAIL("page list frame too short");
2064 std::int64_t columnOffset;
2065 bytes += DeserializeInt64(bytes, columnOffset);
2066 if (columnOffset < 0) {
2067 if (nPages > 0)
2068 return R__FAIL("unexpected non-empty page list");
2069 clusterBuilders[i].MarkSuppressedColumnRange(j);
2070 } else {
2071 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
2072 return R__FAIL("page list frame too short");
2073 std::uint32_t compressionSettings;
2074 bytes += DeserializeUInt32(bytes, compressionSettings);
2076 }
2077
2079 } // loop over columns
2080
2082 } // loop over clusters
2083
2084 return clusterBuilders;
2085}
2086
2091{
2093 if (!clusterBuildersRes)
2095
2096 auto clusterBuilders = clusterBuildersRes.Unwrap();
2097
2098 std::vector<ROOT::RClusterDescriptor> clusters;
2099 clusters.reserve(clusterBuilders.size());
2100
2101 // Conditionally fixup the clusters depending on the attach purpose
2102 switch (mode) {
2103 case EDescriptorDeserializeMode::kForReading:
2104 for (auto &builder : clusterBuilders) {
2105 if (auto res = builder.CommitSuppressedColumnRanges(desc); !res)
2106 return R__FORWARD_RESULT(res);
2107 builder.AddExtendedColumnRanges(desc);
2108 clusters.emplace_back(builder.MoveDescriptor().Unwrap());
2109 }
2110 break;
2111 case EDescriptorDeserializeMode::kForWriting:
2112 for (auto &builder : clusterBuilders) {
2113 if (auto res = builder.CommitSuppressedColumnRanges(desc); !res)
2114 return R__FORWARD_RESULT(res);
2115 clusters.emplace_back(builder.MoveDescriptor().Unwrap());
2116 }
2117 break;
2118 case EDescriptorDeserializeMode::kRaw:
2119 for (auto &builder : clusterBuilders)
2120 clusters.emplace_back(builder.MoveDescriptor().Unwrap());
2121 break;
2122 }
2123
2125
2126 return RResult<void>::Success();
2127}
2128
2130{
2132 for (auto si : infos) {
2133 assert(si.first == si.second->GetNumber());
2134 streamerInfos.Add(si.second);
2135 }
2137 buffer.WriteObject(&streamerInfos);
2138 assert(buffer.Length() > 0);
2139 return std::string{buffer.Buffer(), static_cast<UInt_t>(buffer.Length())};
2140}
2141
2144{
2146
2147 TBufferFile buffer(TBuffer::kRead, extraTypeInfoContent.length(), const_cast<char *>(extraTypeInfoContent.data()),
2148 false /* adopt */);
2149 auto infoList = reinterpret_cast<TList *>(buffer.ReadObject(TList::Class()));
2150
2151 TObjLink *lnk = infoList->FirstLink();
2152 while (lnk) {
2153 auto info = reinterpret_cast<TStreamerInfo *>(lnk->GetObject());
2154 info->BuildCheck();
2155 infoMap[info->GetNumber()] = info->GetClass()->GetStreamerInfo(info->GetClassVersion());
2156 assert(info->GetNumber() == infoMap[info->GetNumber()]->GetNumber());
2157 lnk = lnk->Next();
2158 }
2159
2160 delete infoList;
2161
2162 return infoMap;
2163}
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
Definition RError.hxx:304
#define R__FORWARD_RESULT(res)
Short-hand to return an RResult<T> value from a subroutine to the calling stack frame.
Definition RError.hxx:302
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:300
#define R__LOG_WARNING(...)
Definition RLogger.hxx:358
#define R__LOG_DEBUG(DEBUGLEVEL,...)
Definition RLogger.hxx:360
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
#define ROOT_RELEASE
Definition RVersion.hxx:44
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:125
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t nitems
Option_t Option_t TPoint TPoint const char mode
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:110
The available trivial, native content types of a column.
A helper class for piece-wise construction of an RClusterDescriptor.
RClusterDescriptorBuilder & NEntries(std::uint64_t nEntries)
RClusterDescriptorBuilder & ClusterId(ROOT::DescriptorId_t clusterId)
RClusterDescriptorBuilder & FirstEntryIndex(std::uint64_t firstEntryIndex)
A helper class for piece-wise construction of an RClusterGroupDescriptor.
A helper class for piece-wise construction of an RColumnDescriptor.
A helper class for piece-wise construction of an RExtraTypeInfoDescriptor.
RExtraTypeInfoDescriptorBuilder & ContentId(EExtraTypeInfoIds contentId)
RExtraTypeInfoDescriptorBuilder & TypeName(const std::string &typeName)
RExtraTypeInfoDescriptorBuilder & Content(const std::string &content)
RExtraTypeInfoDescriptorBuilder & TypeVersion(std::uint32_t typeVersion)
A helper class for piece-wise construction of an RFieldDescriptor.
A helper class for piece-wise construction of an RNTupleDescriptor.
The serialization context is used for the piecewise serialization of a descriptor.
ROOT::DescriptorId_t GetOnDiskFieldId(ROOT::DescriptorId_t memId) const
ROOT::DescriptorId_t GetMemColumnId(ROOT::DescriptorId_t onDiskId) const
ROOT::DescriptorId_t GetMemClusterGroupId(ROOT::DescriptorId_t onDiskId) const
ROOT::DescriptorId_t GetOnDiskColumnId(ROOT::DescriptorId_t memId) const
void MapSchema(const RNTupleDescriptor &desc, bool forHeaderExtension)
Map in-memory field and column IDs to their on-disk counterparts.
ROOT::DescriptorId_t GetMemClusterId(ROOT::DescriptorId_t onDiskId) const
const std::vector< ROOT::DescriptorId_t > & GetOnDiskFieldList() const
Return a vector containing the in-memory field ID for each on-disk counterpart, in order,...
A helper class for serializing and deserialization of the RNTuple binary format.
static std::uint32_t SerializeXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3, void *buffer)
Writes a XxHash-3 64bit checksum of the byte range given by data and length.
static RResult< std::vector< ROOT::Internal::RClusterDescriptorBuilder > > DeserializePageListRaw(const void *buffer, std::uint64_t bufSize, ROOT::DescriptorId_t clusterGroupId, const RNTupleDescriptor &desc)
static RResult< std::uint32_t > SerializeSchemaDescription(void *buffer, const RNTupleDescriptor &desc, const RContext &context, bool forHeaderExtension=false)
Serialize the schema description in desc into buffer.
static RResult< std::uint32_t > DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val)
static RResult< std::uint32_t > SerializeEnvelopeLink(const REnvelopeLink &envelopeLink, void *buffer)
static std::uint32_t SerializeInt32(std::int32_t val, void *buffer)
static RResult< std::uint32_t > DeserializeEnvelopeLink(const void *buffer, std::uint64_t bufSize, REnvelopeLink &envelopeLink)
static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer)
static RResult< std::uint32_t > SerializeFieldStructure(ROOT::ENTupleStructure structure, void *buffer)
While we could just interpret the enums as ints, we make the translation explicit in order to avoid a...
static RResult< std::uint32_t > SerializeEnvelopePostscript(unsigned char *envelope, std::uint64_t size)
static RResult< std::uint32_t > SerializeFeatureFlags(const std::vector< std::uint64_t > &flags, void *buffer)
static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val)
static RResult< std::uint32_t > DeserializeFrameHeader(const void *buffer, std::uint64_t bufSize, std::uint64_t &frameSize, std::uint32_t &nitems)
static RResult< std::uint32_t > DeserializeEnvelope(const void *buffer, std::uint64_t bufSize, std::uint16_t expectedType)
static RResult< std::uint32_t > SerializeColumnType(ROOT::ENTupleColumnType type, void *buffer)
static std::uint32_t SerializeListFramePreamble(std::uint32_t nitems, void *buffer)
static std::uint32_t SerializeInt16(std::int16_t val, void *buffer)
static RResult< std::uint32_t > SerializeFramePostscript(void *frame, std::uint64_t size)
static RResult< std::uint32_t > DeserializeClusterGroup(const void *buffer, std::uint64_t bufSize, RClusterGroup &clusterGroup)
static RResult< std::uint32_t > DeserializeLocator(const void *buffer, std::uint64_t bufSize, RNTupleLocator &locator)
static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer)
static RResult< void > DeserializePageList(const void *buffer, std::uint64_t bufSize, ROOT::DescriptorId_t clusterGroupId, RNTupleDescriptor &desc, EDescriptorDeserializeMode mode)
static RResult< void > DeserializeFooter(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t DeserializeInt64(const void *buffer, std::int64_t &val)
static std::uint32_t SerializeEnvelopePreamble(std::uint16_t envelopeType, void *buffer)
static std::uint32_t DeserializeInt32(const void *buffer, std::int32_t &val)
static std::uint32_t SerializeString(const std::string &val, void *buffer)
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
static RResult< std::uint32_t > SerializeExtraTypeInfoId(ROOT::EExtraTypeInfoIds id, void *buffer)
static RResult< StreamerInfoMap_t > DeserializeStreamerInfos(const std::string &extraTypeInfoContent)
static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val)
static RResult< void > VerifyXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3)
Expects an xxhash3 checksum in the 8 bytes following data + length and verifies it.
static RResult< std::uint32_t > SerializeClusterSummary(const RClusterSummary &clusterSummary, void *buffer)
static RResult< std::uint32_t > DeserializeColumnType(const void *buffer, ROOT::ENTupleColumnType &type)
static std::uint32_t DeserializeInt16(const void *buffer, std::int16_t &val)
static RResult< std::uint32_t > DeserializeExtraTypeInfoId(const void *buffer, ROOT::EExtraTypeInfoIds &id)
static RResult< std::uint32_t > DeserializeClusterSummary(const void *buffer, std::uint64_t bufSize, RClusterSummary &clusterSummary)
static RResult< std::uint32_t > SerializeClusterGroup(const RClusterGroup &clusterGroup, void *buffer)
static std::uint32_t SerializeRecordFramePreamble(void *buffer)
static RResult< std::uint32_t > SerializePageList(void *buffer, const RNTupleDescriptor &desc, std::span< ROOT::DescriptorId_t > physClusterIDs, const RContext &context)
static RResult< std::uint32_t > DeserializeFieldStructure(const void *buffer, ROOT::ENTupleStructure &structure)
static std::uint32_t SerializeInt64(std::int64_t val, void *buffer)
static RResult< void > DeserializeHeader(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder)
static RResult< std::uint32_t > SerializeFooter(void *buffer, const RNTupleDescriptor &desc, const RContext &context)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static RResult< std::uint32_t > SerializeLocator(const RNTupleLocator &locator, void *buffer)
static RResult< std::uint32_t > DeserializeSchemaDescription(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
static RResult< std::uint32_t > DeserializeFeatureFlags(const void *buffer, std::uint64_t bufSize, std::vector< std::uint64_t > &flags)
static RResult< RContext > SerializeHeader(void *buffer, const RNTupleDescriptor &desc)
static std::string SerializeStreamerInfos(const StreamerInfoMap_t &infos)
Records the partition of data into pages for a particular column in a particular cluster.
Metadata stored for every column of an RNTuple.
Field specific extra type information from the header / extenstion header.
const std::string & GetContent() const
const std::string & GetTypeName() const
EExtraTypeInfoIds GetContentId() const
Metadata stored for every field of an RNTuple.
The on-storage metadata of an RNTuple.
const RClusterGroupDescriptor & GetClusterGroupDescriptor(ROOT::DescriptorId_t clusterGroupId) const
const RColumnDescriptor & GetColumnDescriptor(ROOT::DescriptorId_t columnId) const
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
const RFieldDescriptor & GetFieldDescriptor(ROOT::DescriptorId_t fieldId) const
std::size_t GetNExtraTypeInfos() const
RColumnDescriptorIterable GetColumnIterable() const
const std::string & GetName() const
std::vector< std::uint64_t > GetFeatureFlags() const
ROOT::DescriptorId_t GetFieldZeroId() const
Returns the logical parent of all top-level RNTuple data fields.
std::size_t GetNPhysicalColumns() const
const RHeaderExtension * GetHeaderExtension() const
Return header extension information; if the descriptor does not have a header extension,...
const RClusterDescriptor & GetClusterDescriptor(ROOT::DescriptorId_t clusterId) const
std::uint64_t GetOnDiskHeaderXxHash3() const
std::size_t GetNFields() const
RResult< void > AddClusterGroupDetails(ROOT::DescriptorId_t clusterGroupId, std::vector< RClusterDescriptor > &clusterDescs)
Methods to load and drop cluster group details (cluster IDs and page locations)
std::size_t GetNLogicalColumns() const
std::size_t GetNClusterGroups() const
const std::string & GetDescription() const
RNTupleLocator payload that is common for object stores using 64bit location information.
Generic information about the physical location of data.
const_iterator begin() const
const_iterator end() const
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:198
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition TBufferFile.h:47
TObject * ReadObject(const TClass *cl) override
Read object from I/O buffer.
void WriteObject(const TObject *obj, Bool_t cacheReuse=kTRUE) override
Write object to I/O buffer.
@ kWrite
Definition TBuffer.h:73
@ kRead
Definition TBuffer.h:73
Int_t Length() const
Definition TBuffer.h:100
char * Buffer() const
Definition TBuffer.h:96
A doubly linked list.
Definition TList.h:38
static TClass * Class()
Describes a persistent version of a class.
constexpr ROOT::ENTupleStructure kTestFutureFieldStructure
ROOT::RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
constexpr ENTupleColumnType kTestFutureColumnType
constexpr RNTupleLocator::ELocatorType kTestLocatorType
EExtraTypeInfoIds
Used in RExtraTypeInfoDescriptor.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
constexpr DescriptorId_t kInvalidDescriptorId
ENTupleStructure
The fields in the RNTuple data model tree can carry different structural information about the type s...
ENTupleColumnType