Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleSerialize.cxx
Go to the documentation of this file.
1/// \file RNTupleSerialize.cxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \author Javier Lopez-Gomez <javier.lopez.gomez@cern.ch>
5/// \date 2021-08-02
6/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
7/// is welcome!
8
9/*************************************************************************
10 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
11 * All rights reserved. *
12 * *
13 * For the licensing terms see $ROOTSYS/LICENSE. *
14 * For the list of contributors see $ROOTSYS/README/CREDITS. *
15 *************************************************************************/
16
18#include <ROOT/RColumnModel.hxx>
19#include <ROOT/RError.hxx>
22
23#include <RVersion.h>
24#include <RZip.h> // for R__crc32
25
26#include <cstring> // for memcpy
27#include <deque>
28#include <set>
29#include <unordered_map>
30
31template <typename T>
33
34
35namespace {
37
38std::uint32_t SerializeFieldV1(const ROOT::Experimental::RFieldDescriptor &fieldDesc,
39 ROOT::Experimental::DescriptorId_t onDiskParentId, void *buffer)
40{
41
42 auto base = reinterpret_cast<unsigned char *>(buffer);
43 auto pos = base;
44 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
45
46 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
47
48 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetFieldVersion(), *where);
49 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetTypeVersion(), *where);
50 pos += RNTupleSerializer::SerializeUInt32(onDiskParentId, *where);
51 pos += RNTupleSerializer::SerializeFieldStructure(fieldDesc.GetStructure(), *where);
52 if (fieldDesc.GetNRepetitions() > 0) {
53 pos += RNTupleSerializer::SerializeUInt16(RNTupleSerializer::kFlagRepetitiveField, *where);
54 pos += RNTupleSerializer::SerializeUInt64(fieldDesc.GetNRepetitions(), *where);
55 } else {
56 pos += RNTupleSerializer::SerializeUInt16(0, *where);
57 }
58 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldName(), *where);
59 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeName(), *where);
60 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeAlias(), *where);
61 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldDescription(), *where);
62
63 auto size = pos - base;
64 RNTupleSerializer::SerializeFramePostscript(base, size);
65
66 return size;
67}
68
69// clang-format off
70/// Serialize, in order, fields enumerated in `fieldList` to `buffer`. `firstOnDiskId` specifies the on-disk ID for the
71/// first element in the `fieldList` sequence. Before calling this function `RContext::MapSchema()` should have been
72/// called on `context` in order to map in-memory field IDs to their on-disk counterpart.
73/// \return The number of bytes written to the output buffer; if `buffer` is `nullptr` no data is serialized and the
74/// required buffer size is returned
75// clang-format on
76std::uint32_t SerializeFieldList(const ROOT::Experimental::RNTupleDescriptor &desc,
77 std::span<const ROOT::Experimental::DescriptorId_t> fieldList,
78 std::size_t firstOnDiskId,
80{
81 auto base = reinterpret_cast<unsigned char *>(buffer);
82 auto pos = base;
83 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
84
85 auto fieldZeroId = desc.GetFieldZeroId();
86 ROOT::Experimental::DescriptorId_t onDiskFieldId = firstOnDiskId;
87 for (auto fieldId : fieldList) {
88 const auto &f = desc.GetFieldDescriptor(fieldId);
89 auto onDiskParentId =
90 (f.GetParentId() == fieldZeroId) ? onDiskFieldId : context.GetOnDiskFieldId(f.GetParentId());
91 pos += SerializeFieldV1(f, onDiskParentId, *where);
92 ++onDiskFieldId;
93 }
94
95 return pos - base;
96}
97
98RResult<std::uint32_t> DeserializeFieldV1(
99 const void *buffer,
100 std::uint32_t bufSize,
102{
103 using ENTupleStructure = ROOT::Experimental::ENTupleStructure;
104
105 auto base = reinterpret_cast<const unsigned char *>(buffer);
106 auto bytes = base;
107 std::uint32_t frameSize;
108 auto fnFrameSizeLeft = [&]() { return frameSize - static_cast<std::uint32_t>(bytes - base); };
109 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
110 if (!result)
111 return R__FORWARD_ERROR(result);
112 bytes += result.Unwrap();
113
114 std::uint32_t fieldVersion;
115 std::uint32_t typeVersion;
116 std::uint32_t parentId;
117 // initialize properly for call to SerializeFieldStructure()
118 ENTupleStructure structure{ENTupleStructure::kLeaf};
119 std::uint16_t flags;
120 if (fnFrameSizeLeft() < 3 * sizeof(std::uint32_t) +
121 RNTupleSerializer::SerializeFieldStructure(structure, nullptr) +
122 sizeof(std::uint16_t))
123 {
124 return R__FAIL("field record frame too short");
125 }
126 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldVersion);
127 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeVersion);
128 bytes += RNTupleSerializer::DeserializeUInt32(bytes, parentId);
129 auto res16 = RNTupleSerializer::DeserializeFieldStructure(bytes, structure);
130 if (!res16)
131 return R__FORWARD_ERROR(res16);
132 bytes += res16.Unwrap();
133 bytes += RNTupleSerializer::DeserializeUInt16(bytes, flags);
134 fieldDesc.FieldVersion(fieldVersion).TypeVersion(typeVersion).ParentId(parentId).Structure(structure);
135
136 if (flags & RNTupleSerializer::kFlagRepetitiveField) {
137 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
138 return R__FAIL("field record frame too short");
139 std::uint64_t nRepetitions;
140 bytes += RNTupleSerializer::DeserializeUInt64(bytes, nRepetitions);
141 fieldDesc.NRepetitions(nRepetitions);
142 }
143
144 std::string fieldName;
145 std::string typeName;
146 std::string aliasName;
147 std::string description;
148 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), fieldName).Unwrap();
149 if (!result)
150 return R__FORWARD_ERROR(result);
151 bytes += result.Unwrap();
152 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), typeName).Unwrap();
153 if (!result)
154 return R__FORWARD_ERROR(result);
155 bytes += result.Unwrap();
156 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), aliasName).Unwrap();
157 if (!result)
158 return R__FORWARD_ERROR(result);
159 bytes += result.Unwrap();
160 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), description).Unwrap();
161 if (!result)
162 return R__FORWARD_ERROR(result);
163 bytes += result.Unwrap();
164 fieldDesc.FieldName(fieldName).TypeName(typeName).TypeAlias(aliasName).FieldDescription(description);
165
166 return frameSize;
167}
168
169std::uint32_t SerializeColumnListV1(const ROOT::Experimental::RNTupleDescriptor &desc,
170 std::span<const ROOT::Experimental::DescriptorId_t> fieldList,
172 void *buffer)
173{
174 using RColumnElementBase = ROOT::Experimental::Detail::RColumnElementBase;
175
176 auto base = reinterpret_cast<unsigned char *>(buffer);
177 auto pos = base;
178 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
179
180 for (auto parentId : fieldList) {
181 for (const auto &c : desc.GetColumnIterable(parentId)) {
182 if (c.IsAliasColumn())
183 continue;
184
185 auto frame = pos;
186 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
187
188 auto type = c.GetModel().GetType();
189 pos += RNTupleSerializer::SerializeColumnType(type, *where);
190 pos += RNTupleSerializer::SerializeUInt16(RColumnElementBase::GetBitsOnStorage(type), *where);
191 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(c.GetFieldId()), *where);
192 std::uint32_t flags = 0;
193 // TODO(jblomer): add support for descending columns in the column model
194 if (c.GetModel().GetIsSorted())
195 flags |= RNTupleSerializer::kFlagSortAscColumn;
196 // TODO(jblomer): fix for unsigned integer types
198 flags |= RNTupleSerializer::kFlagNonNegativeColumn;
199 const std::uint64_t firstElementIdx = c.GetFirstElementIndex();
200 if (firstElementIdx > 0)
201 flags |= RNTupleSerializer::kFlagDeferredColumn;
202 pos += RNTupleSerializer::SerializeUInt32(flags, *where);
203 if (flags & RNTupleSerializer::kFlagDeferredColumn)
204 pos += RNTupleSerializer::SerializeUInt64(firstElementIdx, *where);
205
206 pos += RNTupleSerializer::SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
207 }
208 }
209
210 return pos - base;
211}
212
213RResult<std::uint32_t> DeserializeColumnV1(
214 const void *buffer,
215 std::uint32_t bufSize,
217{
219
220 auto base = reinterpret_cast<const unsigned char *>(buffer);
221 auto bytes = base;
222 std::uint32_t frameSize;
223 auto fnFrameSizeLeft = [&]() { return frameSize - static_cast<std::uint32_t>(bytes - base); };
224 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
225 if (!result)
226 return R__FORWARD_ERROR(result);
227 bytes += result.Unwrap();
228
229 // Initialize properly for SerializeColumnType
230 EColumnType type{EColumnType::kIndex32};
231 std::uint16_t bitsOnStorage;
232 std::uint32_t fieldId;
233 std::uint32_t flags;
234 std::uint64_t firstElementIdx = 0;
235 if (fnFrameSizeLeft() < RNTupleSerializer::SerializeColumnType(type, nullptr) +
236 sizeof(std::uint16_t) + 2 * sizeof(std::uint32_t))
237 {
238 return R__FAIL("column record frame too short");
239 }
240 auto res16 = RNTupleSerializer::DeserializeColumnType(bytes, type);
241 if (!res16)
242 return R__FORWARD_ERROR(res16);
243 bytes += res16.Unwrap();
244 bytes += RNTupleSerializer::DeserializeUInt16(bytes, bitsOnStorage);
245 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
246 bytes += RNTupleSerializer::DeserializeUInt32(bytes, flags);
247 if (flags & RNTupleSerializer::kFlagDeferredColumn) {
248 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
249 return R__FAIL("column record frame too short");
250 bytes += RNTupleSerializer::DeserializeUInt64(bytes, firstElementIdx);
251 }
252
254 return R__FAIL("column element size mismatch");
255
256 const bool isSorted = (flags & (RNTupleSerializer::kFlagSortAscColumn | RNTupleSerializer::kFlagSortDesColumn));
257 columnDesc.FieldId(fieldId).Model({type, isSorted}).FirstElementIndex(firstElementIdx);
258
259 return frameSize;
260}
261
262std::uint32_t SerializeLocatorPayloadURI(const ROOT::Experimental::RNTupleLocator &locator, unsigned char *buffer)
263{
264 const auto &uri = locator.GetPosition<std::string>();
265 if (uri.length() >= (1 << 16))
266 throw ROOT::Experimental::RException(R__FAIL("locator too large"));
267 if (buffer)
268 memcpy(buffer, uri.data(), uri.length());
269 return uri.length();
270}
271
272void DeserializeLocatorPayloadURI(const unsigned char *buffer, std::uint32_t payloadSize,
274{
275 locator.fBytesOnStorage = 0;
276 auto &uri = locator.fPosition.emplace<std::string>();
277 uri.resize(payloadSize);
278 memcpy(uri.data(), buffer, payloadSize);
279}
280
281std::uint32_t SerializeLocatorPayloadObject64(const ROOT::Experimental::RNTupleLocator &locator, unsigned char *buffer)
282{
284 if (buffer) {
285 RNTupleSerializer::SerializeUInt32(locator.fBytesOnStorage, buffer);
286 RNTupleSerializer::SerializeUInt64(data.fLocation, buffer + sizeof(std::uint32_t));
287 }
288 return sizeof(std::uint32_t) + sizeof(std::uint64_t);
289}
290
291void DeserializeLocatorPayloadObject64(const unsigned char *buffer, ROOT::Experimental::RNTupleLocator &locator)
292{
294 RNTupleSerializer::DeserializeUInt32(buffer, locator.fBytesOnStorage);
295 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint32_t), data.fLocation);
296}
297
298std::uint32_t SerializeAliasColumnList(const ROOT::Experimental::RNTupleDescriptor &desc,
299 std::span<const ROOT::Experimental::DescriptorId_t> fieldList,
301 void *buffer)
302{
303 auto base = reinterpret_cast<unsigned char *>(buffer);
304 auto pos = base;
305 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
306
307 for (auto parentId : fieldList) {
308 for (const auto &c : desc.GetColumnIterable(parentId)) {
309 if (!c.IsAliasColumn())
310 continue;
311
312 auto frame = pos;
313 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
314
315 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskColumnId(c.GetPhysicalId()), *where);
316 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(c.GetFieldId()), *where);
317
318 pos += RNTupleSerializer::SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
319 }
320 }
321
322 return pos - base;
323}
324
325RResult<std::uint32_t> DeserializeAliasColumn(const void *buffer, std::uint32_t bufSize,
326 std::uint32_t &physicalColumnId, std::uint32_t &fieldId)
327{
328 auto base = reinterpret_cast<const unsigned char *>(buffer);
329 auto bytes = base;
330 std::uint32_t frameSize;
331 auto fnFrameSizeLeft = [&]() { return frameSize - static_cast<std::uint32_t>(bytes - base); };
332 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
333 if (!result)
334 return R__FORWARD_ERROR(result);
335 bytes += result.Unwrap();
336
337 if (fnFrameSizeLeft() < 2 * sizeof(std::uint32_t)) {
338 return R__FAIL("alias column record frame too short");
339 }
340
341 bytes += RNTupleSerializer::DeserializeUInt32(bytes, physicalColumnId);
342 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
343
344 return frameSize;
345}
346
347} // anonymous namespace
348
349
351 const unsigned char *data, std::uint32_t length, std::uint32_t &crc32, void *buffer)
352{
353 if (buffer != nullptr) {
354 crc32 = R__crc32(0, nullptr, 0);
355 crc32 = R__crc32(crc32, data, length);
356 SerializeUInt32(crc32, buffer);
357 }
358 return 4;
359}
360
362 const unsigned char *data, std::uint32_t length, std::uint32_t &crc32)
363{
364 auto checksumReal = R__crc32(0, nullptr, 0);
365 checksumReal = R__crc32(checksumReal, data, length);
366 DeserializeUInt32(data + length, crc32);
367 if (crc32 != checksumReal)
368 return R__FAIL("CRC32 checksum mismatch");
369 return RResult<void>::Success();
370}
371
372
374 const unsigned char *data, std::uint32_t length)
375{
376 std::uint32_t crc32;
377 return R__FORWARD_RESULT(VerifyCRC32(data, length, crc32));
378}
379
380
381std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeInt16(std::int16_t val, void *buffer)
382{
383 if (buffer != nullptr) {
384 auto bytes = reinterpret_cast<unsigned char *>(buffer);
385 bytes[0] = (val & 0x00FF);
386 bytes[1] = (val & 0xFF00) >> 8;
387 }
388 return 2;
389}
390
391std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeInt16(const void *buffer, std::int16_t &val)
392{
393 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
394 val = std::int16_t(bytes[0]) + (std::int16_t(bytes[1]) << 8);
395 return 2;
396}
397
398std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeUInt16(std::uint16_t val, void *buffer)
399{
400 return SerializeInt16(val, buffer);
401}
402
403std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt16(const void *buffer, std::uint16_t &val)
404{
405 return DeserializeInt16(buffer, *reinterpret_cast<std::int16_t *>(&val));
406}
407
408std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeInt32(std::int32_t val, void *buffer)
409{
410 if (buffer != nullptr) {
411 auto bytes = reinterpret_cast<unsigned char *>(buffer);
412 bytes[0] = (val & 0x000000FF);
413 bytes[1] = (val & 0x0000FF00) >> 8;
414 bytes[2] = (val & 0x00FF0000) >> 16;
415 bytes[3] = (val & 0xFF000000) >> 24;
416 }
417 return 4;
418}
419
420std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeInt32(const void *buffer, std::int32_t &val)
421{
422 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
423 val = std::int32_t(bytes[0]) + (std::int32_t(bytes[1]) << 8) +
424 (std::int32_t(bytes[2]) << 16) + (std::int32_t(bytes[3]) << 24);
425 return 4;
426}
427
428std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeUInt32(std::uint32_t val, void *buffer)
429{
430 return SerializeInt32(val, buffer);
431}
432
433std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt32(const void *buffer, std::uint32_t &val)
434{
435 return DeserializeInt32(buffer, *reinterpret_cast<std::int32_t *>(&val));
436}
437
438std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeInt64(std::int64_t val, void *buffer)
439{
440 if (buffer != nullptr) {
441 auto bytes = reinterpret_cast<unsigned char *>(buffer);
442 bytes[0] = (val & 0x00000000000000FF);
443 bytes[1] = (val & 0x000000000000FF00) >> 8;
444 bytes[2] = (val & 0x0000000000FF0000) >> 16;
445 bytes[3] = (val & 0x00000000FF000000) >> 24;
446 bytes[4] = (val & 0x000000FF00000000) >> 32;
447 bytes[5] = (val & 0x0000FF0000000000) >> 40;
448 bytes[6] = (val & 0x00FF000000000000) >> 48;
449 bytes[7] = (val & 0xFF00000000000000) >> 56;
450 }
451 return 8;
452}
453
454std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeInt64(const void *buffer, std::int64_t &val)
455{
456 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
457 val = std::int64_t(bytes[0]) + (std::int64_t(bytes[1]) << 8) +
458 (std::int64_t(bytes[2]) << 16) + (std::int64_t(bytes[3]) << 24) +
459 (std::int64_t(bytes[4]) << 32) + (std::int64_t(bytes[5]) << 40) +
460 (std::int64_t(bytes[6]) << 48) + (std::int64_t(bytes[7]) << 56);
461 return 8;
462}
463
464std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeUInt64(std::uint64_t val, void *buffer)
465{
466 return SerializeInt64(val, buffer);
467}
468
469std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::DeserializeUInt64(const void *buffer, std::uint64_t &val)
470{
471 return DeserializeInt64(buffer, *reinterpret_cast<std::int64_t *>(&val));
472}
473
474std::uint32_t ROOT::Experimental::Internal::RNTupleSerializer::SerializeString(const std::string &val, void *buffer)
475{
476 if (buffer) {
477 auto pos = reinterpret_cast<unsigned char *>(buffer);
478 pos += SerializeUInt32(val.length(), pos);
479 memcpy(pos, val.data(), val.length());
480 }
481 return sizeof(std::uint32_t) + val.length();
482}
483
485 const void *buffer, std::uint32_t bufSize, std::string &val)
486{
487 if (bufSize < sizeof(std::uint32_t))
488 return R__FAIL("string buffer too short");
489 bufSize -= sizeof(std::uint32_t);
490
491 auto base = reinterpret_cast<const unsigned char *>(buffer);
492 auto bytes = base;
493 std::uint32_t length;
494 bytes += DeserializeUInt32(buffer, length);
495 if (bufSize < length)
496 return R__FAIL("string buffer too short");
497
498 val.resize(length);
499 memcpy(&val[0], bytes, length);
500 return sizeof(std::uint32_t) + length;
501}
502
503
506{
508 switch (type) {
509 case EColumnType::kIndex64: return SerializeUInt16(0x01, buffer);
510 case EColumnType::kIndex32: return SerializeUInt16(0x02, buffer);
511 case EColumnType::kSwitch: return SerializeUInt16(0x03, buffer);
512 case EColumnType::kByte: return SerializeUInt16(0x04, buffer);
513 case EColumnType::kChar: return SerializeUInt16(0x05, buffer);
514 case EColumnType::kBit: return SerializeUInt16(0x06, buffer);
515 case EColumnType::kReal64: return SerializeUInt16(0x07, buffer);
516 case EColumnType::kReal32: return SerializeUInt16(0x08, buffer);
517 case EColumnType::kReal16: return SerializeUInt16(0x09, buffer);
518 case EColumnType::kInt64: return SerializeUInt16(0x16, buffer);
519 case EColumnType::kUInt64: return SerializeUInt16(0x0A, buffer);
520 case EColumnType::kInt32: return SerializeUInt16(0x17, buffer);
521 case EColumnType::kUInt32: return SerializeUInt16(0x0B, buffer);
522 case EColumnType::kInt16: return SerializeUInt16(0x18, buffer);
523 case EColumnType::kUInt16: return SerializeUInt16(0x0C, buffer);
524 case EColumnType::kInt8: return SerializeUInt16(0x19, buffer);
525 case EColumnType::kUInt8: return SerializeUInt16(0x0D, buffer);
526 case EColumnType::kSplitIndex64: return SerializeUInt16(0x0E, buffer);
527 case EColumnType::kSplitIndex32: return SerializeUInt16(0x0F, buffer);
528 case EColumnType::kSplitReal64: return SerializeUInt16(0x10, buffer);
529 case EColumnType::kSplitReal32: return SerializeUInt16(0x11, buffer);
530 case EColumnType::kSplitInt64: return SerializeUInt16(0x1A, buffer);
531 case EColumnType::kSplitUInt64: return SerializeUInt16(0x13, buffer);
532 case EColumnType::kSplitInt32: return SerializeUInt16(0x1B, buffer);
533 case EColumnType::kSplitUInt32: return SerializeUInt16(0x14, buffer);
534 case EColumnType::kSplitInt16: return SerializeUInt16(0x1C, buffer);
535 case EColumnType::kSplitUInt16: return SerializeUInt16(0x15, buffer);
536 default: throw RException(R__FAIL("ROOT bug: unexpected column type"));
537 }
538}
539
540
542 const void *buffer, ROOT::Experimental::EColumnType &type)
543{
545 std::uint16_t onDiskType;
546 auto result = DeserializeUInt16(buffer, onDiskType);
547 switch (onDiskType) {
548 case 0x01: type = EColumnType::kIndex64; break;
549 case 0x02: type = EColumnType::kIndex32; break;
550 case 0x03: type = EColumnType::kSwitch; break;
551 case 0x04: type = EColumnType::kByte; break;
552 case 0x05: type = EColumnType::kChar; break;
553 case 0x06: type = EColumnType::kBit; break;
554 case 0x07: type = EColumnType::kReal64; break;
555 case 0x08: type = EColumnType::kReal32; break;
556 case 0x09: type = EColumnType::kReal16; break;
557 case 0x16: type = EColumnType::kInt64; break;
558 case 0x0A: type = EColumnType::kUInt64; break;
559 case 0x17: type = EColumnType::kInt32; break;
560 case 0x0B: type = EColumnType::kUInt32; break;
561 case 0x18: type = EColumnType::kInt16; break;
562 case 0x0C: type = EColumnType::kUInt16; break;
563 case 0x19: type = EColumnType::kInt8; break;
564 case 0x0D: type = EColumnType::kUInt8; break;
565 case 0x0E: type = EColumnType::kSplitIndex64; break;
566 case 0x0F: type = EColumnType::kSplitIndex32; break;
567 case 0x10: type = EColumnType::kSplitReal64; break;
568 case 0x11: type = EColumnType::kSplitReal32; break;
569 case 0x1A: type = EColumnType::kSplitInt64; break;
570 case 0x13: type = EColumnType::kSplitUInt64; break;
571 case 0x1B: type = EColumnType::kSplitInt32; break;
572 case 0x14: type = EColumnType::kSplitUInt32; break;
573 case 0x1C: type = EColumnType::kSplitInt16; break;
574 case 0x15: type = EColumnType::kSplitUInt16; break;
575 default: return R__FAIL("unexpected on-disk column type");
576 }
577 return result;
578}
579
580
582 ROOT::Experimental::ENTupleStructure structure, void *buffer)
583{
585 switch (structure) {
587 return SerializeUInt16(0x00, buffer);
589 return SerializeUInt16(0x01, buffer);
591 return SerializeUInt16(0x02, buffer);
593 return SerializeUInt16(0x03, buffer);
595 return SerializeUInt16(0x04, buffer);
596 default:
597 throw RException(R__FAIL("ROOT bug: unexpected field structure type"));
598 }
599}
600
601
603 const void *buffer, ROOT::Experimental::ENTupleStructure &structure)
604{
606 std::uint16_t onDiskValue;
607 auto result = DeserializeUInt16(buffer, onDiskValue);
608 switch (onDiskValue) {
609 case 0x00:
610 structure = ENTupleStructure::kLeaf;
611 break;
612 case 0x01:
614 break;
615 case 0x02:
616 structure = ENTupleStructure::kRecord;
617 break;
618 case 0x03:
619 structure = ENTupleStructure::kVariant;
620 break;
621 case 0x04:
623 break;
624 default:
625 return R__FAIL("unexpected on-disk field structure value");
626 }
627 return result;
628}
629
630
631/// Currently all enevelopes have the same version number (1). At a later point, different envelope types
632/// may have different version numbers
634{
635 auto base = reinterpret_cast<unsigned char *>(buffer);
636 auto pos = base;
637 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
638
639 pos += SerializeUInt16(kEnvelopeCurrentVersion, *where);
640 pos += SerializeUInt16(kEnvelopeMinVersion, *where);
641 return pos - base;
642}
643
644
646 const unsigned char *envelope, std::uint32_t size, std::uint32_t &crc32, void *buffer)
647{
648 return SerializeCRC32(envelope, size, crc32, buffer);
649}
650
652 const unsigned char *envelope, std::uint32_t size, void *buffer)
653{
654 std::uint32_t crc32;
655 return SerializeEnvelopePostscript(envelope, size, crc32, buffer);
656}
657
658/// Currently all enevelopes have the same version number (1). At a later point, different envelope types
659/// may have different version numbers
661 const void *buffer, std::uint32_t bufSize, std::uint32_t &crc32)
662{
663 if (bufSize < (2 * sizeof(std::uint16_t) + sizeof(std::uint32_t)))
664 return R__FAIL("invalid envelope, too short");
665
666 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
667 auto base = bytes;
668
669 std::uint16_t protocolVersionAtWrite;
670 std::uint16_t protocolVersionMinRequired;
671 bytes += DeserializeUInt16(bytes, protocolVersionAtWrite);
672 // RNTuple compatible back to version 1 (but not to version 0)
673 if (protocolVersionAtWrite < 1)
674 return R__FAIL("The RNTuple format is too old (version 0)");
675
676 bytes += DeserializeUInt16(bytes, protocolVersionMinRequired);
677 if (protocolVersionMinRequired > kEnvelopeCurrentVersion) {
678 return R__FAIL(std::string("The RNTuple format is too new (version ") +
679 std::to_string(protocolVersionMinRequired) + ")");
680 }
681
682 // We defer the CRC32 check to the end to faciliate testing of forward/backward incompatibilities
683 auto result = VerifyCRC32(base, bufSize - 4, crc32);
684 if (!result)
685 return R__FORWARD_ERROR(result);
686
687 return sizeof(protocolVersionAtWrite) + sizeof(protocolVersionMinRequired);
688}
689
690
692 const void *buffer, std::uint32_t bufSize)
693{
694 std::uint32_t crc32;
695 return R__FORWARD_RESULT(DeserializeEnvelope(buffer, bufSize, crc32));
696}
697
698
700{
701 // Marker: multiply the final size with 1
702 return SerializeInt32(1, buffer);
703}
704
705
707 std::uint32_t nitems, void *buffer)
708{
709 if (nitems >= (1 << 28))
710 throw RException(R__FAIL("list frame too large: " + std::to_string(nitems)));
711
712 auto base = reinterpret_cast<unsigned char *>(buffer);
713 auto pos = base;
714 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
715
716 // Marker: multiply the final size with -1
717 pos += RNTupleSerializer::SerializeInt32(-1, *where);
718 pos += SerializeUInt32(nitems, *where);
719 return pos - base;
720}
721
723 void *frame, std::int32_t size)
724{
725 if (size < 0)
726 throw RException(R__FAIL("frame too large: " + std::to_string(size)));
727 if (size < static_cast<std::int32_t>(sizeof(std::int32_t)))
728 throw RException(R__FAIL("frame too short: " + std::to_string(size)));
729 if (frame) {
730 std::int32_t marker;
731 DeserializeInt32(frame, marker);
732 if ((marker < 0) && (size < static_cast<std::int32_t>(2 * sizeof(std::int32_t))))
733 throw RException(R__FAIL("frame too short: " + std::to_string(size)));
734
735 SerializeInt32(marker * size, frame);
736 }
737 return 0;
738}
739
741 const void *buffer, std::uint32_t bufSize, std::uint32_t &frameSize, std::uint32_t &nitems)
742{
743 if (bufSize < sizeof(std::int32_t))
744 return R__FAIL("frame too short");
745
746 std::int32_t *ssize = reinterpret_cast<std::int32_t *>(&frameSize);
747 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
748 bytes += DeserializeInt32(bytes, *ssize);
749 if (*ssize >= 0) {
750 // Record frame
751 nitems = 1;
752 if (frameSize < sizeof(std::int32_t))
753 return R__FAIL("corrupt record frame size");
754 } else {
755 // List frame
756 if (bufSize < 2 * sizeof(std::int32_t))
757 return R__FAIL("frame too short");
758 bytes += DeserializeUInt32(bytes, nitems);
759 nitems &= (2 << 28) - 1;
760 *ssize = -(*ssize);
761 if (frameSize < 2 * sizeof(std::int32_t))
762 return R__FAIL("corrupt list frame size");
763 }
764
765 if (bufSize < frameSize)
766 return R__FAIL("frame too short");
767
768 return bytes - reinterpret_cast<const unsigned char *>(buffer);
769}
770
772 const void *buffer, std::uint32_t bufSize, std::uint32_t &frameSize)
773{
774 std::uint32_t nitems;
775 return R__FORWARD_RESULT(DeserializeFrameHeader(buffer, bufSize, frameSize, nitems));
776}
777
779 const std::vector<std::int64_t> &flags, void *buffer)
780{
781 if (flags.empty())
782 return SerializeInt64(0, buffer);
783
784 if (buffer) {
785 auto bytes = reinterpret_cast<unsigned char *>(buffer);
786
787 for (unsigned i = 0; i < flags.size(); ++i) {
788 if (flags[i] < 0)
789 throw RException(R__FAIL("feature flag out of bounds"));
790
791 // The MSb indicates that another Int64 follows; set this bit to 1 for all except the last element
792 if (i == (flags.size() - 1))
793 SerializeInt64(flags[i], bytes);
794 else
795 bytes += SerializeInt64(flags[i] | 0x8000000000000000, bytes);
796 }
797 }
798 return (flags.size() * sizeof(std::int64_t));
799}
800
802 const void *buffer, std::uint32_t bufSize, std::vector<std::int64_t> &flags)
803{
804 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
805
806 flags.clear();
807 std::int64_t f;
808 do {
809 if (bufSize < sizeof(std::int64_t))
810 return R__FAIL("feature flag buffer too short");
811 bytes += DeserializeInt64(bytes, f);
812 bufSize -= sizeof(std::int64_t);
813 flags.emplace_back(f & ~0x8000000000000000);
814 } while (f < 0);
815
816 return (flags.size() * sizeof(std::int64_t));
817}
818
820 const RNTupleLocator &locator, void *buffer)
821{
823 throw RException(R__FAIL("locator is not serializable"));
824
825 std::uint32_t size = 0;
826 if (locator.fType == RNTupleLocator::kTypeFile) {
827 if (static_cast<std::int32_t>(locator.fBytesOnStorage) < 0)
828 throw RException(R__FAIL("locator too large"));
829 size += SerializeUInt32(locator.fBytesOnStorage, buffer);
830 size += SerializeUInt64(locator.GetPosition<std::uint64_t>(),
831 buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
832 return size;
833 }
834
835 auto payloadp = buffer ? reinterpret_cast<unsigned char *>(buffer) + sizeof(std::int32_t) : nullptr;
836 switch (locator.fType) {
837 case RNTupleLocator::kTypeURI: size += SerializeLocatorPayloadURI(locator, payloadp); break;
838 case RNTupleLocator::kTypeDAOS: size += SerializeLocatorPayloadObject64(locator, payloadp); break;
839 default: throw RException(R__FAIL("locator has unknown type"));
840 }
841 std::int32_t head = sizeof(std::int32_t) + size;
842 head |= locator.fReserved << 16;
843 head |= static_cast<int>(locator.fType & 0x7F) << 24;
844 head = -head;
846 return size;
847}
848
850 const void *buffer, std::uint32_t bufSize, RNTupleLocator &locator)
851{
852 if (bufSize < sizeof(std::int32_t))
853 return R__FAIL("too short locator");
854
855 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
856 std::int32_t head;
857
858 bytes += DeserializeInt32(bytes, head);
859 bufSize -= sizeof(std::int32_t);
860 if (head < 0) {
861 head = -head;
862 const int type = head >> 24;
863 const std::uint32_t payloadSize = (static_cast<std::uint32_t>(head) & 0x0000FFFF) - sizeof(std::int32_t);
864 if (bufSize < payloadSize)
865 return R__FAIL("too short locator");
866 locator.fType = static_cast<RNTupleLocator::ELocatorType>(type);
867 locator.fReserved = static_cast<std::uint32_t>(head >> 16) & 0xFF;
868 switch (type) {
869 case RNTupleLocator::kTypeURI: DeserializeLocatorPayloadURI(bytes, payloadSize, locator); break;
870 case RNTupleLocator::kTypeDAOS: DeserializeLocatorPayloadObject64(bytes, locator); break;
871 default: return R__FAIL("unsupported locator type: " + std::to_string(type));
872 }
873 bytes += payloadSize;
874 } else {
875 if (bufSize < sizeof(std::uint64_t))
876 return R__FAIL("too short locator");
877 auto &offset = locator.fPosition.emplace<std::uint64_t>();
879 bytes += DeserializeUInt64(bytes, offset);
880 locator.fBytesOnStorage = head;
881 }
882
883 return bytes - reinterpret_cast<const unsigned char *>(buffer);
884}
885
887 const REnvelopeLink &envelopeLink, void *buffer)
888{
889 auto size = SerializeUInt32(envelopeLink.fUnzippedSize, buffer);
890 size += SerializeLocator(envelopeLink.fLocator,
891 buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
892 return size;
893}
894
896 const void *buffer, std::uint32_t bufSize, REnvelopeLink &envelopeLink)
897{
898 if (bufSize < sizeof(std::int32_t))
899 return R__FAIL("too short envelope link");
900
901 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
902 bytes += DeserializeUInt32(bytes, envelopeLink.fUnzippedSize);
903 bufSize -= sizeof(std::uint32_t);
904 auto result = DeserializeLocator(bytes, bufSize, envelopeLink.fLocator);
905 if (!result)
906 return R__FORWARD_ERROR(result);
907 bytes += result.Unwrap();
908 return bytes - reinterpret_cast<const unsigned char *>(buffer);
909}
910
911
913 const RClusterSummary &clusterSummary, void *buffer)
914{
915 auto base = reinterpret_cast<unsigned char *>(buffer);
916 auto pos = base;
917 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
918
919 auto frame = pos;
920 pos += SerializeRecordFramePreamble(*where);
921 pos += SerializeUInt64(clusterSummary.fFirstEntry, *where);
922 if (clusterSummary.fColumnGroupID >= 0) {
923 pos += SerializeInt64(-static_cast<int64_t>(clusterSummary.fNEntries), *where);
924 pos += SerializeUInt32(clusterSummary.fColumnGroupID, *where);
925 } else {
926 pos += SerializeInt64(static_cast<int64_t>(clusterSummary.fNEntries), *where);
927 }
928 auto size = pos - frame;
929 pos += SerializeFramePostscript(frame, size);
930 return size;
931}
932
933
935 const void *buffer, std::uint32_t bufSize, RClusterSummary &clusterSummary)
936{
937 auto base = reinterpret_cast<const unsigned char *>(buffer);
938 auto bytes = base;
939 std::uint32_t frameSize;
940 auto result = DeserializeFrameHeader(bytes, bufSize, frameSize);
941 if (!result)
942 return R__FORWARD_ERROR(result);
943 bytes += result.Unwrap();
944
945 auto fnBufSizeLeft = [&]() { return frameSize - static_cast<std::uint32_t>(bytes - base); };
946 if (fnBufSizeLeft() < 2 * sizeof(std::uint64_t))
947 return R__FAIL("too short cluster summary");
948
949 bytes += DeserializeUInt64(bytes, clusterSummary.fFirstEntry);
950 std::int64_t nEntries;
951 bytes += DeserializeInt64(bytes, nEntries);
952
953 if (nEntries < 0) {
954 if (fnBufSizeLeft() < sizeof(std::uint32_t))
955 return R__FAIL("too short cluster summary");
956 clusterSummary.fNEntries = -nEntries;
957 std::uint32_t columnGroupID;
958 bytes += DeserializeUInt32(bytes, columnGroupID);
959 clusterSummary.fColumnGroupID = columnGroupID;
960 } else {
961 clusterSummary.fNEntries = nEntries;
962 clusterSummary.fColumnGroupID = -1;
963 }
964
965 return frameSize;
966}
967
968
970 const RClusterGroup &clusterGroup, void *buffer)
971{
972 auto base = reinterpret_cast<unsigned char *>(buffer);
973 auto pos = base;
974 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
975
976 auto frame = pos;
977 pos += SerializeRecordFramePreamble(*where);
978 pos += SerializeUInt32(clusterGroup.fNClusters, *where);
979 pos += SerializeEnvelopeLink(clusterGroup.fPageListEnvelopeLink, *where);
980 auto size = pos - frame;
981 pos += SerializeFramePostscript(frame, size);
982 return size;
983}
984
985
987 const void *buffer, std::uint32_t bufSize, RClusterGroup &clusterGroup)
988{
989 auto base = reinterpret_cast<const unsigned char *>(buffer);
990 auto bytes = base;
991
992 std::uint32_t frameSize;
993 auto result = DeserializeFrameHeader(bytes, bufSize, frameSize);
994 if (!result)
995 return R__FORWARD_ERROR(result);
996 bytes += result.Unwrap();
997
998 auto fnFrameSizeLeft = [&]() { return frameSize - static_cast<std::uint32_t>(bytes - base); };
999 if (fnFrameSizeLeft() < sizeof(std::uint32_t))
1000 return R__FAIL("too short cluster group");
1001
1002 bytes += DeserializeUInt32(bytes, clusterGroup.fNClusters);
1003 result = DeserializeEnvelopeLink(bytes, fnFrameSizeLeft(), clusterGroup.fPageListEnvelopeLink);
1004 if (!result)
1005 return R__FORWARD_ERROR(result);
1006
1007 return frameSize;
1008}
1009
1011 bool forHeaderExtension)
1012{
1013 auto fieldZeroId = desc.GetFieldZeroId();
1014 auto depthFirstTraversal = [&](std::span<DescriptorId_t> fieldTrees, auto doForEachField) {
1015 std::deque<DescriptorId_t> idQueue{fieldTrees.begin(), fieldTrees.end()};
1016 while (!idQueue.empty()) {
1017 auto fieldId = idQueue.front();
1018 idQueue.pop_front();
1019 // Field zero has no physical representation nor columns of its own; recurse over its subfields only
1020 if (fieldId != fieldZeroId)
1021 doForEachField(fieldId);
1022 unsigned i = 0;
1023 for (const auto &f : desc.GetFieldIterable(fieldId))
1024 idQueue.insert(idQueue.begin() + i++, f.GetId());
1025 }
1026 };
1027
1028 R__ASSERT(desc.GetNFields() > 0); // we must have at least a zero field
1029 if (!forHeaderExtension)
1030 R__ASSERT(GetHeaderExtensionOffset() == -1U);
1031
1032 std::vector<DescriptorId_t> fieldTrees;
1033 if (!forHeaderExtension) {
1034 fieldTrees.emplace_back(fieldZeroId);
1035 } else if (auto xHeader = desc.GetHeaderExtension()) {
1036 fieldTrees = xHeader->GetTopLevelFields(desc);
1037 }
1038 depthFirstTraversal(fieldTrees, [&](DescriptorId_t fieldId) { MapFieldId(fieldId); });
1039 depthFirstTraversal(fieldTrees, [&](DescriptorId_t fieldId) {
1040 for (const auto &c : desc.GetColumnIterable(fieldId))
1041 if (!c.IsAliasColumn())
1042 MapColumnId(c.GetLogicalId());
1043 });
1044 depthFirstTraversal(fieldTrees, [&](DescriptorId_t fieldId) {
1045 for (const auto &c : desc.GetColumnIterable(fieldId))
1046 if (c.IsAliasColumn())
1047 MapColumnId(c.GetLogicalId());
1048 });
1049
1050 // Anything added after this point is accounted for the header extension
1051 if (!forHeaderExtension)
1052 BeginHeaderExtension();
1053}
1054
1056 const RNTupleDescriptor &desc,
1057 const RContext &context,
1058 bool forHeaderExtension)
1059{
1060 auto base = reinterpret_cast<unsigned char *>(buffer);
1061 auto pos = base;
1062 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
1063
1064 std::size_t nFields = 0, nColumns = 0, nAliasColumns = 0, fieldListOffset = 0;
1065 if (forHeaderExtension) {
1066 // A call to `RNTupleDescriptorBuilder::BeginHeaderExtension()` is not strictly required after serializing the
1067 // header, which may happen, e.g., in unit tests. Ensure an empty schema extension is serialized in this case
1068 if (auto xHeader = desc.GetHeaderExtension()) {
1069 nFields = xHeader->GetNFields();
1070 nColumns = xHeader->GetNPhysicalColumns();
1071 nAliasColumns = xHeader->GetNLogicalColumns() - xHeader->GetNPhysicalColumns();
1072 fieldListOffset = context.GetHeaderExtensionOffset();
1073 }
1074 } else {
1075 nFields = desc.GetNFields() - 1;
1076 nColumns = desc.GetNPhysicalColumns();
1077 nAliasColumns = desc.GetNLogicalColumns() - desc.GetNPhysicalColumns();
1078 }
1079 const auto &onDiskFields = context.GetOnDiskFieldList();
1080 R__ASSERT(onDiskFields.size() >= fieldListOffset);
1081 std::span<const DescriptorId_t> fieldList{onDiskFields.data() + fieldListOffset,
1082 onDiskFields.size() - fieldListOffset};
1083
1084 auto frame = pos;
1085 pos += SerializeListFramePreamble(nFields, *where);
1086 pos += SerializeFieldList(desc, fieldList, /*firstOnDiskId=*/fieldListOffset, context, *where);
1087 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1088
1089 frame = pos;
1090 pos += SerializeListFramePreamble(nColumns, *where);
1091 pos += SerializeColumnListV1(desc, fieldList, context, *where);
1092 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1093
1094 frame = pos;
1095 pos += SerializeListFramePreamble(nAliasColumns, *where);
1096 pos += SerializeAliasColumnList(desc, fieldList, context, *where);
1097 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1098
1099 // We don't use extra type information yet
1100 frame = pos;
1101 pos += SerializeListFramePreamble(0, *where);
1102 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1103 return static_cast<std::uint32_t>(pos - base);
1104}
1105
1108 RNTupleDescriptorBuilder &descBuilder)
1109{
1110 auto base = reinterpret_cast<const unsigned char *>(buffer);
1111 auto bytes = base;
1112 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1114
1115 std::uint32_t frameSize;
1116 auto frame = bytes;
1117 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1118
1119 std::uint32_t nFields;
1120 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nFields);
1121 if (!result)
1122 return R__FORWARD_ERROR(result);
1123 bytes += result.Unwrap();
1124 // The zero field is always added before `DeserializeSchemaDescription()` is called
1125 const std::uint32_t fieldIdRangeBegin = descBuilder.GetDescriptor().GetNFields() - 1;
1126 for (unsigned i = 0; i < nFields; ++i) {
1127 std::uint32_t fieldId = fieldIdRangeBegin + i;
1128 RFieldDescriptorBuilder fieldBuilder;
1129 result = DeserializeFieldV1(bytes, fnFrameSizeLeft(), fieldBuilder);
1130 if (!result)
1131 return R__FORWARD_ERROR(result);
1132 bytes += result.Unwrap();
1133 if (fieldId == fieldBuilder.GetParentId())
1134 fieldBuilder.ParentId(kZeroFieldId);
1135 auto fieldDesc = fieldBuilder.FieldId(fieldId).MakeDescriptor();
1136 if (!fieldDesc)
1137 return R__FORWARD_ERROR(fieldDesc);
1138 auto parentId = fieldDesc.Inspect().GetParentId();
1139 descBuilder.AddField(fieldDesc.Unwrap());
1140 auto resVoid = descBuilder.AddFieldLink(parentId, fieldId);
1141 if (!resVoid)
1142 return R__FORWARD_ERROR(resVoid);
1143 }
1144 bytes = frame + frameSize;
1145
1146 std::uint32_t nColumns;
1147 frame = bytes;
1148 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nColumns);
1149 if (!result)
1150 return R__FORWARD_ERROR(result);
1151 bytes += result.Unwrap();
1152 const std::uint32_t columnIdRangeBegin = descBuilder.GetDescriptor().GetNLogicalColumns();
1153 std::unordered_map<DescriptorId_t, std::uint32_t> maxIndexes;
1154 for (unsigned i = 0; i < nColumns; ++i) {
1155 std::uint32_t columnId = columnIdRangeBegin + i;
1156 RColumnDescriptorBuilder columnBuilder;
1157 result = DeserializeColumnV1(bytes, fnFrameSizeLeft(), columnBuilder);
1158 if (!result)
1159 return R__FORWARD_ERROR(result);
1160 bytes += result.Unwrap();
1161
1162 std::uint32_t idx = 0;
1163 const auto fieldId = columnBuilder.GetFieldId();
1164 auto maxIdx = maxIndexes.find(fieldId);
1165 if (maxIdx != maxIndexes.end())
1166 idx = maxIdx->second + 1;
1167 maxIndexes[fieldId] = idx;
1168
1169 auto columnDesc = columnBuilder.Index(idx).LogicalColumnId(columnId).PhysicalColumnId(columnId).MakeDescriptor();
1170 if (!columnDesc)
1171 return R__FORWARD_ERROR(columnDesc);
1172 auto resVoid = descBuilder.AddColumn(columnDesc.Unwrap());
1173 if (!resVoid)
1174 return R__FORWARD_ERROR(resVoid);
1175 }
1176 bytes = frame + frameSize;
1177
1178 std::uint32_t nAliasColumns;
1179 frame = bytes;
1180 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nAliasColumns);
1181 if (!result)
1182 return R__FORWARD_ERROR(result);
1183 bytes += result.Unwrap();
1184 const std::uint32_t aliasColumnIdRangeBegin = columnIdRangeBegin + nColumns;
1185 for (unsigned i = 0; i < nAliasColumns; ++i) {
1186 std::uint32_t physicalId;
1187 std::uint32_t fieldId;
1188 result = DeserializeAliasColumn(bytes, fnFrameSizeLeft(), physicalId, fieldId);
1189 if (!result)
1190 return R__FORWARD_ERROR(result);
1191 bytes += result.Unwrap();
1192
1193 RColumnDescriptorBuilder columnBuilder;
1194 columnBuilder.LogicalColumnId(aliasColumnIdRangeBegin + i).PhysicalColumnId(physicalId).FieldId(fieldId);
1195 columnBuilder.Model(descBuilder.GetDescriptor().GetColumnDescriptor(physicalId).GetModel());
1196
1197 std::uint32_t idx = 0;
1198 auto maxIdx = maxIndexes.find(fieldId);
1199 if (maxIdx != maxIndexes.end())
1200 idx = maxIdx->second + 1;
1201 maxIndexes[fieldId] = idx;
1202
1203 auto aliasColumnDesc = columnBuilder.Index(idx).MakeDescriptor();
1204 if (!aliasColumnDesc)
1205 return R__FORWARD_ERROR(aliasColumnDesc);
1206 auto resVoid = descBuilder.AddColumn(aliasColumnDesc.Unwrap());
1207 if (!resVoid)
1208 return R__FORWARD_ERROR(resVoid);
1209 }
1210 bytes = frame + frameSize;
1211
1212 std::uint32_t nTypeInfo;
1213 frame = bytes;
1214 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nTypeInfo);
1215 if (!result)
1216 return R__FORWARD_ERROR(result);
1217 if (nTypeInfo > 0)
1218 R__LOG_WARNING(NTupleLog()) << "Extra type information is still unsupported! ";
1219 bytes = frame + frameSize;
1220
1221 return bytes - base;
1222}
1223
1227{
1228 RContext context;
1229
1230 auto base = reinterpret_cast<unsigned char *>(buffer);
1231 auto pos = base;
1232 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1233
1234 pos += SerializeEnvelopePreamble(*where);
1235 // So far we don't make use of feature flags
1236 pos += SerializeFeatureFlags(std::vector<std::int64_t>(), *where);
1237 pos += SerializeUInt32(kReleaseCandidateTag, *where);
1238 pos += SerializeString(desc.GetName(), *where);
1239 pos += SerializeString(desc.GetDescription(), *where);
1240 pos += SerializeString(std::string("ROOT v") + ROOT_RELEASE, *where);
1241
1242 context.MapSchema(desc, /*forHeaderExtension=*/false);
1243 pos += SerializeSchemaDescription(*where, desc, context);
1244
1245 std::uint32_t size = pos - base;
1246 std::uint32_t crc32 = 0;
1247 size += SerializeEnvelopePostscript(base, size, crc32, *where);
1248
1249 context.SetHeaderSize(size);
1250 context.SetHeaderCRC32(crc32);
1251 return context;
1252}
1253
1255 void *buffer, const RNTupleDescriptor &desc, std::span<DescriptorId_t> physClusterIDs, const RContext &context)
1256{
1257 auto base = reinterpret_cast<unsigned char *>(buffer);
1258 auto pos = base;
1259 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
1260
1261 pos += SerializeEnvelopePreamble(*where);
1262 auto topMostFrame = pos;
1263 pos += SerializeListFramePreamble(physClusterIDs.size(), *where);
1264
1265 for (auto clusterId : physClusterIDs) {
1266 const auto &clusterDesc = desc.GetClusterDescriptor(context.GetMemClusterId(clusterId));
1267 // Get an ordered set of physical column ids
1268 std::set<DescriptorId_t> onDiskColumnIds;
1269 for (auto column : clusterDesc.GetColumnIds())
1270 onDiskColumnIds.insert(context.GetOnDiskColumnId(column));
1271
1272 auto outerFrame = pos;
1273 pos += SerializeListFramePreamble(onDiskColumnIds.size(), *where);
1274 for (auto onDiskId : onDiskColumnIds) {
1275 auto memId = context.GetMemColumnId(onDiskId);
1276 const auto &columnRange = clusterDesc.GetColumnRange(memId);
1277 const auto &pageRange = clusterDesc.GetPageRange(memId);
1278
1279 auto innerFrame = pos;
1280 pos += SerializeListFramePreamble(pageRange.fPageInfos.size(), *where);
1281
1282 for (const auto &pi : pageRange.fPageInfos) {
1283 pos += SerializeUInt32(pi.fNElements, *where);
1284 pos += SerializeLocator(pi.fLocator, *where);
1285 }
1286 pos += SerializeUInt64(columnRange.fFirstElementIndex, *where);
1287 pos += SerializeUInt32(columnRange.fCompressionSettings, *where);
1288
1289 pos += SerializeFramePostscript(buffer ? innerFrame : nullptr, pos - innerFrame);
1290 }
1291 pos += SerializeFramePostscript(buffer ? outerFrame : nullptr, pos - outerFrame);
1292 }
1293
1294 pos += SerializeFramePostscript(buffer ? topMostFrame : nullptr, pos - topMostFrame);
1295 std::uint32_t size = pos - base;
1296 size += SerializeEnvelopePostscript(base, size, *where);
1297 return size;
1298}
1299
1300std::uint32_t
1303 const RContext &context)
1304{
1305 auto base = reinterpret_cast<unsigned char *>(buffer);
1306 auto pos = base;
1307 void** where = (buffer == nullptr) ? &buffer : reinterpret_cast<void**>(&pos);
1308
1309 pos += SerializeEnvelopePreamble(*where);
1310
1311 // So far we don't make use of feature flags
1312 pos += SerializeFeatureFlags(std::vector<std::int64_t>(), *where);
1313 pos += SerializeUInt32(context.GetHeaderCRC32(), *where);
1314
1315 // Schema extension, i.e. incremental changes with respect to the header
1316 auto frame = pos;
1317 pos += SerializeRecordFramePreamble(*where);
1318 pos += SerializeSchemaDescription(*where, desc, context, /*forHeaderExtension=*/true);
1319 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1320
1321 // So far no support for shared clusters (no column groups)
1322 frame = pos;
1323 pos += SerializeListFramePreamble(0, *where);
1324 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1325
1326 // Cluster summaries
1327 const auto nClusterGroups = desc.GetNClusterGroups();
1328 unsigned int nClusters = 0;
1329 for (const auto &cgDesc : desc.GetClusterGroupIterable())
1330 nClusters += cgDesc.GetNClusters();
1331 frame = pos;
1332 pos += SerializeListFramePreamble(nClusters, *where);
1333 for (unsigned int i = 0; i < nClusterGroups; ++i) {
1334 const auto &cgDesc = desc.GetClusterGroupDescriptor(context.GetMemClusterGroupId(i));
1335 const auto nClustersInGroup = cgDesc.GetNClusters();
1336 const auto &clusterIds = cgDesc.GetClusterIds();
1337 for (unsigned int j = 0; j < nClustersInGroup; ++j) {
1338 const auto &clusterDesc = desc.GetClusterDescriptor(clusterIds[j]);
1339 RClusterSummary summary{clusterDesc.GetFirstEntryIndex(), clusterDesc.GetNEntries(), -1};
1340 pos += SerializeClusterSummary(summary, *where);
1341 }
1342 }
1343 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1344
1345 // Cluster groups
1346 frame = pos;
1347 pos += SerializeListFramePreamble(nClusterGroups, *where);
1348 for (unsigned int i = 0; i < nClusterGroups; ++i) {
1349 const auto &cgDesc = desc.GetClusterGroupDescriptor(context.GetMemClusterGroupId(i));
1350 RClusterGroup clusterGroup;
1351 clusterGroup.fNClusters = cgDesc.GetNClusters();
1352 clusterGroup.fPageListEnvelopeLink.fUnzippedSize = cgDesc.GetPageListLength();
1353 clusterGroup.fPageListEnvelopeLink.fLocator = cgDesc.GetPageListLocator();
1354 pos += SerializeClusterGroup(clusterGroup, *where);
1355 }
1356 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1357
1358 // So far no support for meta-data
1359 frame = pos;
1360 pos += SerializeListFramePreamble(0, *where);
1361 pos += SerializeFramePostscript(buffer ? frame : nullptr, pos - frame);
1362
1363 std::uint32_t size = pos - base;
1364 size += SerializeEnvelopePostscript(base, size, *where);
1365 return size;
1366}
1367
1369 const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
1370{
1371 auto base = reinterpret_cast<const unsigned char *>(buffer);
1372 auto bytes = base;
1373 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1375
1376 std::uint32_t crc32{0};
1377 result = DeserializeEnvelope(bytes, fnBufSizeLeft(), crc32);
1378 if (!result)
1379 return R__FORWARD_ERROR(result);
1380 bytes += result.Unwrap();
1381 descBuilder.SetHeaderCRC32(crc32);
1382
1383 std::vector<std::int64_t> featureFlags;
1384 result = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags);
1385 if (!result)
1386 return R__FORWARD_ERROR(result);
1387 bytes += result.Unwrap();
1388 for (auto f: featureFlags) {
1389 if (f)
1390 R__LOG_WARNING(NTupleLog()) << "Unsupported feature flag! " << f;
1391 }
1392
1393 std::uint32_t rcTag;
1394 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
1395 return R__FAIL("header too short");
1396 bytes += DeserializeUInt32(bytes, rcTag);
1397 if (rcTag > 0) {
1398 R__LOG_WARNING(NTupleLog()) << "Pre-release format version: RC " << rcTag;
1399 }
1400
1401 std::string name;
1402 std::string description;
1403 std::string writer;
1404 result = DeserializeString(bytes, fnBufSizeLeft(), name);
1405 if (!result)
1406 return R__FORWARD_ERROR(result);
1407 bytes += result.Unwrap();
1408 result = DeserializeString(bytes, fnBufSizeLeft(), description);
1409 if (!result)
1410 return R__FORWARD_ERROR(result);
1411 bytes += result.Unwrap();
1412 result = DeserializeString(bytes, fnBufSizeLeft(), writer);
1413 if (!result)
1414 return R__FORWARD_ERROR(result);
1415 bytes += result.Unwrap();
1416 descBuilder.SetNTuple(name, description);
1417
1418 // Zero field
1419 descBuilder.AddField(
1420 RFieldDescriptorBuilder().FieldId(kZeroFieldId).Structure(ENTupleStructure::kRecord).MakeDescriptor().Unwrap());
1421 result = DeserializeSchemaDescription(bytes, fnBufSizeLeft(), descBuilder);
1422 if (!result)
1423 return R__FORWARD_ERROR(result);
1424
1425 return RResult<void>::Success();
1426}
1427
1428
1430 const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
1431{
1432 auto base = reinterpret_cast<const unsigned char *>(buffer);
1433 auto bytes = base;
1434 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1436
1437 result = DeserializeEnvelope(bytes, fnBufSizeLeft());
1438 if (!result)
1439 return R__FORWARD_ERROR(result);
1440 bytes += result.Unwrap();
1441
1442 std::vector<std::int64_t> featureFlags;
1443 result = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags);
1444 if (!result)
1445 return R__FORWARD_ERROR(result);
1446 bytes += result.Unwrap();
1447 for (auto f: featureFlags) {
1448 if (f)
1449 R__LOG_WARNING(NTupleLog()) << "Unsupported feature flag! " << f;
1450 }
1451
1452 std::uint32_t crc32{0};
1453 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
1454 return R__FAIL("footer too short");
1455 bytes += DeserializeUInt32(bytes, crc32);
1456 if (crc32 != descBuilder.GetHeaderCRC32())
1457 return R__FAIL("CRC32 mismatch between header and footer");
1458
1459 std::uint32_t frameSize;
1460 auto frame = bytes;
1461 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1462
1463 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize);
1464 if (!result)
1465 return R__FORWARD_ERROR(result);
1466 bytes += result.Unwrap();
1467 if (fnFrameSizeLeft() > 0) {
1468 descBuilder.BeginHeaderExtension();
1469 result = DeserializeSchemaDescription(bytes, fnFrameSizeLeft(), descBuilder);
1470 if (!result)
1471 return R__FORWARD_ERROR(result);
1472 }
1473 bytes = frame + frameSize;
1474
1475 std::uint32_t nColumnGroups;
1476 frame = bytes;
1477 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nColumnGroups);
1478 if (!result)
1479 return R__FORWARD_ERROR(result);
1480 if (nColumnGroups > 0)
1481 return R__FAIL("sharded clusters are still unsupported");
1482 bytes = frame + frameSize;
1483
1484 std::uint32_t nClusterSummaries;
1485 frame = bytes;
1486 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nClusterSummaries);
1487 if (!result)
1488 return R__FORWARD_ERROR(result);
1489 bytes += result.Unwrap();
1490 for (std::uint32_t clusterId = 0; clusterId < nClusterSummaries; ++clusterId) {
1491 RClusterSummary clusterSummary;
1492 result = DeserializeClusterSummary(bytes, fnFrameSizeLeft(), clusterSummary);
1493 if (!result)
1494 return R__FORWARD_ERROR(result);
1495 bytes += result.Unwrap();
1496 if (clusterSummary.fColumnGroupID >= 0)
1497 return R__FAIL("sharded clusters are still unsupported");
1498 descBuilder.AddClusterSummary(clusterId, clusterSummary.fFirstEntry, clusterSummary.fNEntries);
1499 }
1500 bytes = frame + frameSize;
1501
1502 std::uint32_t nClusterGroups;
1503 frame = bytes;
1504 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nClusterGroups);
1505 if (!result)
1506 return R__FORWARD_ERROR(result);
1507 bytes += result.Unwrap();
1508 std::uint64_t clusterId = 0;
1509 for (std::uint32_t groupId = 0; groupId < nClusterGroups; ++groupId) {
1510 RClusterGroup clusterGroup;
1511 result = DeserializeClusterGroup(bytes, fnFrameSizeLeft(), clusterGroup);
1512 if (!result)
1513 return R__FORWARD_ERROR(result);
1514 bytes += result.Unwrap();
1515
1517 RClusterGroupDescriptorBuilder clusterGroupBuilder;
1518 clusterGroupBuilder.ClusterGroupId(groupId)
1521 for (std::uint64_t i = 0; i < clusterGroup.fNClusters; ++i)
1522 clusterGroupBuilder.AddCluster(clusterId + i);
1523 clusterId += clusterGroup.fNClusters;
1524 descBuilder.AddClusterGroup(std::move(clusterGroupBuilder));
1525 }
1526 bytes = frame + frameSize;
1527
1528 std::uint32_t nMDBlocks;
1529 frame = bytes;
1530 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nMDBlocks);
1531 if (!result)
1532 return R__FORWARD_ERROR(result);
1533 if (nMDBlocks > 0)
1534 R__LOG_WARNING(NTupleLog()) << "meta-data blocks are still unsupported";
1535 bytes = frame + frameSize;
1536
1537 return RResult<void>::Success();
1538}
1539
1540
1542 const void *buffer, std::uint32_t bufSize, std::vector<RClusterDescriptorBuilder> &clusters)
1543{
1544 auto base = reinterpret_cast<const unsigned char *>(buffer);
1545 auto bytes = base;
1546 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1548
1549 result = DeserializeEnvelope(bytes, fnBufSizeLeft());
1550 if (!result)
1551 return R__FORWARD_ERROR(result);
1552 bytes += result.Unwrap();
1553
1554 std::uint32_t topMostFrameSize;
1555 auto topMostFrame = bytes;
1556 auto fnTopMostFrameSizeLeft = [&]() { return topMostFrameSize - (bytes - topMostFrame); };
1557
1558 std::uint32_t nClusters;
1559 result = DeserializeFrameHeader(bytes, fnBufSizeLeft(), topMostFrameSize, nClusters);
1560 if (!result)
1561 return R__FORWARD_ERROR(result);
1562 bytes += result.Unwrap();
1563
1564 if (nClusters != clusters.size())
1565 return R__FAIL("mismatch of page list and cluster summaries");
1566
1567 for (std::uint32_t i = 0; i < nClusters; ++i) {
1568 std::uint32_t outerFrameSize;
1569 auto outerFrame = bytes;
1570 auto fnOuterFrameSizeLeft = [&]() { return outerFrameSize - (bytes - outerFrame); };
1571
1572 std::uint32_t nColumns;
1573 result = DeserializeFrameHeader(bytes, fnTopMostFrameSizeLeft(), outerFrameSize, nColumns);
1574 if (!result)
1575 return R__FORWARD_ERROR(result);
1576 bytes += result.Unwrap();
1577
1578 for (std::uint32_t j = 0; j < nColumns; ++j) {
1579 std::uint32_t innerFrameSize;
1580 auto innerFrame = bytes;
1581 auto fnInnerFrameSizeLeft = [&]() { return innerFrameSize - (bytes - innerFrame); };
1582
1583 std::uint32_t nPages;
1584 result = DeserializeFrameHeader(bytes, fnOuterFrameSizeLeft(), innerFrameSize, nPages);
1585 if (!result)
1586 return R__FORWARD_ERROR(result);
1587 bytes += result.Unwrap();
1588
1590 pageRange.fPhysicalColumnId = j;
1591 for (std::uint32_t k = 0; k < nPages; ++k) {
1592 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
1593 return R__FAIL("inner frame too short");
1594 std::uint32_t nElements;
1595 RNTupleLocator locator;
1596 bytes += DeserializeUInt32(bytes, nElements);
1597 result = DeserializeLocator(bytes, fnInnerFrameSizeLeft(), locator);
1598 if (!result)
1599 return R__FORWARD_ERROR(result);
1600 pageRange.fPageInfos.push_back({nElements, locator});
1601 bytes += result.Unwrap();
1602 }
1603
1604 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t) + sizeof(std::uint64_t)))
1605 return R__FAIL("page list frame too short");
1606 std::uint64_t columnOffset;
1607 bytes += DeserializeUInt64(bytes, columnOffset);
1608 std::uint32_t compressionSettings;
1609 bytes += DeserializeUInt32(bytes, compressionSettings);
1610
1611 clusters[i].CommitColumnRange(j, columnOffset, compressionSettings, pageRange);
1612 bytes = innerFrame + innerFrameSize;
1613 }
1614
1615 bytes = outerFrame + outerFrameSize;
1616 }
1617
1618 return RResult<void>::Success();
1619}
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
Definition RError.hxx:307
#define R__FORWARD_RESULT(res)
Short-hand to return an RResult<T> value from a subroutine to the calling stack frame.
Definition RError.hxx:305
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:303
#define R__LOG_WARNING(...)
Definition RLogger.hxx:363
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
#define ROOT_RELEASE
Definition RVersion.hxx:44
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
#define R__ASSERT(e)
Definition TError.h:118
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t nitems
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:110
A column element encapsulates the translation between basic C++ types and their column representation...
The available trivial, native content types of a column.
The serialization context is used for the piecewise serialization of a descriptor.
DescriptorId_t GetOnDiskColumnId(DescriptorId_t memId) const
const std::vector< DescriptorId_t > & GetOnDiskFieldList() const
Return a vector containing the in-memory field ID for each on-disk counterpart, in order,...
DescriptorId_t GetOnDiskFieldId(DescriptorId_t memId) const
DescriptorId_t GetMemColumnId(DescriptorId_t onDiskId) const
std::size_t GetHeaderExtensionOffset() const
Return the offset of the first element in fOnDisk2MemFieldIDs that is part of the schema extension.
DescriptorId_t GetMemClusterGroupId(DescriptorId_t onDiskId) const
DescriptorId_t GetMemClusterId(DescriptorId_t onDiskId) const
void MapSchema(const RNTupleDescriptor &desc, bool forHeaderExtension)
Map in-memory field and column IDs to their on-disk counterparts.
A helper class for serializing and deserialization of the RNTuple binary format.
static RResult< std::uint32_t > DeserializeString(const void *buffer, std::uint32_t bufSize, std::string &val)
static std::uint32_t SerializeFeatureFlags(const std::vector< std::int64_t > &flags, void *buffer)
static std::uint32_t SerializePageListV1(void *buffer, const RNTupleDescriptor &desc, std::span< DescriptorId_t > physClusterIDs, const RContext &context)
static std::uint32_t SerializeListFramePreamble(std::uint32_t nitems, void *buffer)
static RResult< std::uint32_t > DeserializeLocator(const void *buffer, std::uint32_t bufSize, RNTupleLocator &locator)
static std::uint32_t SerializeCRC32(const unsigned char *data, std::uint32_t length, std::uint32_t &crc32, void *buffer)
Writes a CRC32 checksum of the byte range given by data and length.
static std::uint16_t SerializeColumnType(ROOT::Experimental::EColumnType type, void *buffer)
static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val)
static RResult< void > DeserializePageListV1(const void *buffer, std::uint32_t bufSize, std::vector< RClusterDescriptorBuilder > &clusters)
static std::uint32_t SerializeString(const std::string &val, void *buffer)
static RResult< std::uint32_t > DeserializeEnvelope(const void *buffer, std::uint32_t bufSize)
static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
static std::uint32_t DeserializeInt16(const void *buffer, std::int16_t &val)
static std::uint32_t SerializeClusterSummary(const RClusterSummary &clusterSummary, void *buffer)
static RContext SerializeHeaderV1(void *buffer, const RNTupleDescriptor &desc)
static RResult< void > DeserializeFooterV1(const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeInt16(std::int16_t val, void *buffer)
static RResult< std::uint32_t > DeserializeSchemaDescription(const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeSchemaDescription(void *buffer, const RNTupleDescriptor &desc, const RContext &context, bool forHeaderExtension=false)
Serialize the schema description in desc into buffer.
static std::uint32_t SerializeLocator(const RNTupleLocator &locator, void *buffer)
static std::uint32_t SerializeInt32(std::int32_t val, void *buffer)
static std::uint32_t SerializeEnvelopePreamble(void *buffer)
Currently all enevelopes have the same version number (1).
static RResult< std::uint16_t > DeserializeColumnType(const void *buffer, ROOT::Experimental::EColumnType &type)
static RResult< std::uint32_t > DeserializeClusterGroup(const void *buffer, std::uint32_t bufSize, RClusterGroup &clusterGroup)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static std::uint32_t DeserializeInt32(const void *buffer, std::int32_t &val)
static std::uint32_t DeserializeInt64(const void *buffer, std::int64_t &val)
static RResult< std::uint32_t > DeserializeEnvelopeLink(const void *buffer, std::uint32_t bufSize, REnvelopeLink &envelopeLink)
static std::uint32_t SerializeEnvelopePostscript(const unsigned char *envelope, std::uint32_t size, void *buffer)
static RResult< std::uint16_t > DeserializeFieldStructure(const void *buffer, ROOT::Experimental::ENTupleStructure &structure)
static std::uint32_t SerializeEnvelopeLink(const REnvelopeLink &envelopeLink, void *buffer)
static std::uint32_t SerializeRecordFramePreamble(void *buffer)
static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer)
static RResult< std::uint32_t > DeserializeFrameHeader(const void *buffer, std::uint32_t bufSize, std::uint32_t &frameSize, std::uint32_t &nitems)
static RResult< std::uint32_t > DeserializeFeatureFlags(const void *buffer, std::uint32_t bufSize, std::vector< std::int64_t > &flags)
static std::uint32_t SerializeClusterGroup(const RClusterGroup &clusterGroup, void *buffer)
static std::uint32_t SerializeFramePostscript(void *frame, std::int32_t size)
static std::uint32_t SerializeFooterV1(void *buffer, const RNTupleDescriptor &desc, const RContext &context)
static std::uint32_t SerializeInt64(std::int64_t val, void *buffer)
static RResult< void > VerifyCRC32(const unsigned char *data, std::uint32_t length, std::uint32_t &crc32)
Expects a CRC32 checksum in the 4 bytes following data + length and verifies it.
static std::uint16_t SerializeFieldStructure(ROOT::Experimental::ENTupleStructure structure, void *buffer)
While we could just interpret the enums as ints, we make the translation explicit in order to avoid a...
static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer)
static RResult< void > DeserializeHeaderV1(const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static RResult< std::uint32_t > DeserializeClusterSummary(const void *buffer, std::uint32_t bufSize, RClusterSummary &clusterSummary)
A helper class for piece-wise construction of an RClusterGroupDescriptor.
RClusterGroupDescriptorBuilder & ClusterGroupId(DescriptorId_t clusterGroupId)
RClusterGroupDescriptorBuilder & PageListLength(std::uint32_t pageListLength)
RClusterGroupDescriptorBuilder & PageListLocator(const RNTupleLocator &pageListLocator)
A helper class for piece-wise construction of an RColumnDescriptor.
RColumnDescriptorBuilder & Model(const RColumnModel &model)
RResult< RColumnDescriptor > MakeDescriptor() const
Attempt to make a column descriptor.
RColumnDescriptorBuilder & FieldId(DescriptorId_t fieldId)
RColumnDescriptorBuilder & Index(std::uint32_t index)
RColumnDescriptorBuilder & LogicalColumnId(DescriptorId_t logicalColumnId)
RColumnDescriptorBuilder & PhysicalColumnId(DescriptorId_t physicalColumnId)
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
A helper class for piece-wise construction of an RFieldDescriptor.
RFieldDescriptorBuilder & FieldName(const std::string &fieldName)
RFieldDescriptorBuilder & NRepetitions(std::uint64_t nRepetitions)
RFieldDescriptorBuilder & Structure(const ENTupleStructure &structure)
RResult< RFieldDescriptor > MakeDescriptor() const
Attempt to make a field descriptor.
RFieldDescriptorBuilder & TypeName(const std::string &typeName)
RFieldDescriptorBuilder & TypeVersion(std::uint32_t typeVersion)
RFieldDescriptorBuilder & ParentId(DescriptorId_t id)
RFieldDescriptorBuilder & FieldDescription(const std::string &fieldDescription)
RFieldDescriptorBuilder & FieldVersion(std::uint32_t fieldVersion)
RFieldDescriptorBuilder & TypeAlias(const std::string &typeAlias)
RFieldDescriptorBuilder & FieldId(DescriptorId_t fieldId)
Meta-data stored for every field of an ntuple.
A helper class for piece-wise construction of an RNTupleDescriptor.
void AddToOnDiskFooterSize(std::uint64_t size)
The real footer size also include the page list envelopes.
RResult< void > AddClusterSummary(DescriptorId_t clusterId, std::uint64_t firstEntry, std::uint64_t nEntries)
void AddColumn(DescriptorId_t logicalId, DescriptorId_t physicalId, DescriptorId_t fieldId, const RColumnModel &model, std::uint32_t index, std::uint64_t firstElementIdx=0U)
RResult< void > AddFieldLink(DescriptorId_t fieldId, DescriptorId_t linkId)
void SetNTuple(const std::string_view name, const std::string_view description)
const RNTupleDescriptor & GetDescriptor() const
void AddClusterGroup(RClusterGroupDescriptorBuilder &&clusterGroup)
void BeginHeaderExtension()
Mark the beginning of the header extension; any fields and columns added after a call to this functio...
void AddField(const RFieldDescriptor &fieldDesc)
The on-storage meta-data of an ntuple.
RClusterGroupDescriptorIterable GetClusterGroupIterable() const
DescriptorId_t GetFieldZeroId() const
Returns the logical parent of all top-level NTuple data fields.
const RClusterDescriptor & GetClusterDescriptor(DescriptorId_t clusterId) const
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
const RColumnDescriptor & GetColumnDescriptor(DescriptorId_t columnId) const
const RFieldDescriptor & GetFieldDescriptor(DescriptorId_t fieldId) const
const RClusterGroupDescriptor & GetClusterGroupDescriptor(DescriptorId_t clusterGroupId) const
RColumnDescriptorIterable GetColumnIterable() const
const RHeaderExtension * GetHeaderExtension() const
Return header extension information; if the descriptor does not have a header extension,...
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:207
RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
ENTupleStructure
The fields in the ntuple model tree can carry different structural information about the type system.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
Records the parition of data into pages for a particular column in a particular cluster.
RNTupleLocator payload that is common for object stores using 64bit location information.
Generic information about the physical location of data.
ELocatorType
Values for the Type field in non-disk locators.
std::uint8_t fReserved
Reserved for use by concrete storage backends.
ELocatorType fType
For non-disk locators, the value for the Type field.
std::variant< std::uint64_t, std::string, RNTupleLocatorObject64 > fPosition
Simple on-disk locators consisting of a 64-bit offset use variant type uint64_t; extended locators ha...