Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleDescriptor.cxx
Go to the documentation of this file.
1/// \file RNTupleDescriptor.cxx
2/// \author Jakob Blomer <jblomer@cern.ch>
3/// \author Javier Lopez-Gomez <javier.lopez.gomez@cern.ch>
4/// \date 2018-10-04
5
6/*************************************************************************
7 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
8 * All rights reserved. *
9 * *
10 * For the licensing terms see $ROOTSYS/LICENSE. *
11 * For the list of contributors see $ROOTSYS/README/CREDITS. *
12 *************************************************************************/
13
14#include <ROOT/RError.hxx>
15#include <ROOT/RFieldBase.hxx>
16#include <ROOT/RNTuple.hxx>
18#include <ROOT/RNTupleModel.hxx>
19#include <ROOT/RNTupleTypes.hxx>
20#include <ROOT/RNTupleUtils.hxx>
21#include <ROOT/RPage.hxx>
22#include <string_view>
23
24#include <RZip.h>
25#include <TError.h>
26
27#include <algorithm>
28#include <cstdint>
29#include <deque>
30#include <functional>
31#include <iostream>
32#include <set>
33#include <utility>
34
36{
37 static std::string gEmpty;
38 return gEmpty;
39}
40
42{
43 return fFieldId == other.fFieldId && fFieldVersion == other.fFieldVersion && fTypeVersion == other.fTypeVersion &&
44 GetFieldName() == other.GetFieldName() && GetFieldDescription() == other.GetFieldDescription() &&
45 GetTypeName() == other.GetTypeName() && GetTypeAlias() == other.GetTypeAlias() &&
46 fNRepetitions == other.fNRepetitions && fStructure == other.fStructure && fParentId == other.fParentId &&
47 fProjectionSourceId == other.fProjectionSourceId && fLinkIds == other.fLinkIds &&
48 fLogicalColumnIds == other.fLogicalColumnIds && fTypeChecksum == other.fTypeChecksum &&
49 fIsSoACollection == other.fIsSoACollection;
50}
51
53{
54 fFieldId = source.fFieldId;
55 fFieldVersion = source.fFieldVersion;
56 fTypeVersion = source.fTypeVersion;
57 fNRepetitions = source.fNRepetitions;
58 fStructure = source.fStructure;
59 fParentId = source.fParentId;
60 fProjectionSourceId = source.fProjectionSourceId;
61 fLinkIds = source.fLinkIds;
62 fColumnCardinality = source.fColumnCardinality;
63 fLogicalColumnIds = source.fLogicalColumnIds;
64 fTypeChecksum = source.fTypeChecksum;
65 fIsSoACollection = source.fIsSoACollection;
66
67 fFieldName = stringPool.Intern(source.GetFieldName());
68 fFieldDescription = stringPool.Intern(source.GetFieldDescription());
69 fTypeName = stringPool.Intern(source.GetTypeName());
70 fTypeAlias = stringPool.Intern(source.GetTypeAlias());
71}
72
74{
75 RFieldDescriptor clone;
76 clone.fStringPool = std::make_unique<Internal::RStringPool>();
77 clone.InitFrom(*this, *clone.fStringPool);
78 return clone;
79}
80
81std::unique_ptr<ROOT::RFieldBase>
83{
84 if (GetStructure() == ROOT::ENTupleStructure::kStreamer) {
85 const auto &typeName = GetTypeAlias().empty() ? GetTypeName() : GetTypeAlias();
86 auto streamerField = std::make_unique<ROOT::RStreamerField>(GetFieldName(), typeName);
87 if ((streamerField->GetTraits() & RFieldBase::kTraitEmulatedField) && !options.GetEmulateUnknownTypes()) {
88 throw RException(
89 R__FAIL("streamer field " + GetFieldName() + " has an emulated class but emulation is turned off"));
90 }
91 streamerField->SetOnDiskId(fFieldId);
92 return streamerField;
93 }
94
95 // The structure may be unknown if the descriptor comes from a deserialized field with an unknown structural role.
96 // For forward compatibility, we allow this case and return an InvalidField.
97 if (GetStructure() == ROOT::ENTupleStructure::kUnknown) {
98 if (options.GetReturnInvalidOnError()) {
99 auto invalidField = std::make_unique<ROOT::RInvalidField>(GetFieldName(), GetTypeName(), "",
101 invalidField->SetOnDiskId(fFieldId);
102 return invalidField;
103 } else {
104 throw RException(R__FAIL("unexpected on-disk field structure value for field \"" + GetFieldName() + "\""));
105 }
106 }
107
108 // Untyped records and collections
109 if (GetTypeName().empty()) {
110 switch (GetStructure()) {
112 std::vector<std::unique_ptr<ROOT::RFieldBase>> memberFields;
113 memberFields.reserve(fLinkIds.size());
114 for (auto id : fLinkIds) {
115 const auto &memberDesc = ntplDesc.GetFieldDescriptor(id);
116 auto field = memberDesc.CreateField(ntplDesc, options);
117 if (field->GetTraits() & ROOT::RFieldBase::kTraitInvalidField)
118 return field;
119 memberFields.emplace_back(std::move(field));
120 }
121 auto recordField = std::make_unique<ROOT::RRecordField>(GetFieldName(), std::move(memberFields));
122 recordField->SetOnDiskId(fFieldId);
123 return recordField;
124 }
126 if (fLinkIds.size() != 1) {
127 throw RException(R__FAIL("unsupported untyped collection for field \"" + GetFieldName() + "\""));
128 }
129 auto itemField = ntplDesc.GetFieldDescriptor(fLinkIds[0]).CreateField(ntplDesc, options);
130 if (itemField->GetTraits() & ROOT::RFieldBase::kTraitInvalidField)
131 return itemField;
132 auto collectionField = ROOT::RVectorField::CreateUntyped(GetFieldName(), std::move(itemField));
133 collectionField->SetOnDiskId(fFieldId);
134 return collectionField;
135 }
136 default: throw RException(R__FAIL("unsupported untyped field structure for field \"" + GetFieldName() + "\""));
137 }
138 }
139
140 try {
141 const auto &fieldName = GetFieldName();
142 const auto &typeName = GetTypeAlias().empty() ? GetTypeName() : GetTypeAlias();
143 // NOTE: Unwrap() here may throw an exception, hence the try block.
144 // If options.fReturnInvalidOnError is false we just rethrow it, otherwise we return an InvalidField wrapping the
145 // error.
146 auto field = ROOT::Internal::CallFieldBaseCreate(fieldName, typeName, options, &ntplDesc, fFieldId).Unwrap();
147 field->SetOnDiskId(fFieldId);
148
149 for (auto &subfield : *field) {
150 const auto subfieldId = ntplDesc.FindFieldId(subfield.GetFieldName(), subfield.GetParent()->GetOnDiskId());
151 subfield.SetOnDiskId(subfieldId);
152 if (subfield.GetTraits() & ROOT::RFieldBase::kTraitInvalidField) {
153 auto &invalidField = static_cast<ROOT::RInvalidField &>(subfield);
154 // A subfield being invalid "infects" its entire ancestry.
155 return invalidField.Clone(fieldName);
156 }
157 }
158
159 return field;
160 } catch (const RException &ex) {
161 if (options.GetReturnInvalidOnError())
162 return std::make_unique<ROOT::RInvalidField>(GetFieldName(), GetTypeName(), ex.what(),
164 else
165 throw ex;
166 }
167}
168
169////////////////////////////////////////////////////////////////////////////////
170
172{
173 return fLogicalColumnId == other.fLogicalColumnId && fPhysicalColumnId == other.fPhysicalColumnId &&
174 fBitsOnStorage == other.fBitsOnStorage && fType == other.fType && fFieldId == other.fFieldId &&
175 fIndex == other.fIndex && fRepresentationIndex == other.fRepresentationIndex &&
176 fValueRange == other.fValueRange;
177}
178
180{
181 RColumnDescriptor clone;
182 clone.fLogicalColumnId = fLogicalColumnId;
183 clone.fPhysicalColumnId = fPhysicalColumnId;
184 clone.fBitsOnStorage = fBitsOnStorage;
185 clone.fType = fType;
186 clone.fFieldId = fFieldId;
187 clone.fIndex = fIndex;
188 clone.fFirstElementIndex = fFirstElementIndex;
189 clone.fRepresentationIndex = fRepresentationIndex;
190 if (fValueRange)
191 clone.fValueRange = std::make_unique<RValueRange>(*fValueRange);
192 return clone;
193}
194
195////////////////////////////////////////////////////////////////////////////////
196
199{
200 if (!fCumulativeNElements) {
201 // Small range, just iterate through fPageInfos
202 NTupleSize_t pageNumber = 0;
203 NTupleSize_t firstInPage = 0;
204 for (const auto &pi : fPageInfos) {
205 if (firstInPage + pi.GetNElements() > idxInCluster) {
206 return RPageInfoExtended{pi, firstInPage, pageNumber};
207 }
208 pageNumber++;
209 firstInPage += pi.GetNElements();
210 }
211 R__ASSERT(false);
212 }
213
214 const auto N = fCumulativeNElements->size();
215 R__ASSERT(N > 0);
216 R__ASSERT(N == fPageInfos.size());
217
218 std::size_t left = 0;
219 std::size_t right = N - 1;
220 std::size_t midpoint = N;
221 while (left <= right) {
222 midpoint = (left + right) / 2;
223 if ((*fCumulativeNElements)[midpoint] <= idxInCluster) {
224 left = midpoint + 1;
225 continue;
226 }
227
228 if ((midpoint == 0) || ((*fCumulativeNElements)[midpoint - 1] <= idxInCluster))
229 break;
230
231 right = midpoint - 1;
232 }
233 R__ASSERT(midpoint < N);
234
235 auto pageInfo = fPageInfos[midpoint];
236 decltype(idxInCluster) firstInPage = (midpoint == 0) ? 0 : (*fCumulativeNElements)[midpoint - 1];
237 R__ASSERT(firstInPage <= idxInCluster);
238 R__ASSERT((firstInPage + pageInfo.GetNElements()) > idxInCluster);
239 return RPageInfoExtended{pageInfo, firstInPage, midpoint};
240}
241
242std::size_t
245 std::size_t pageSize)
246{
247 R__ASSERT(fPhysicalColumnId == columnRange.GetPhysicalColumnId());
248 R__ASSERT(!columnRange.IsSuppressed());
249
250 const auto nElements =
251 std::accumulate(fPageInfos.begin(), fPageInfos.end(), 0U,
252 [](std::size_t n, const auto &pageInfo) { return n + pageInfo.GetNElements(); });
253 const auto nElementsRequired = static_cast<std::uint64_t>(columnRange.GetNElements());
254
255 if (nElementsRequired == nElements)
256 return 0U;
257 R__ASSERT((nElementsRequired > nElements) && "invalid attempt to shrink RPageRange");
258
259 std::vector<RPageInfo> pageInfos;
260 // Synthesize new `RPageInfo`s as needed
261 const std::uint64_t nElementsPerPage = pageSize / element.GetSize();
262 R__ASSERT(nElementsPerPage > 0);
263 for (auto nRemainingElements = nElementsRequired - nElements; nRemainingElements > 0;) {
264 RPageInfo pageInfo;
265 pageInfo.SetNElements(std::min(nElementsPerPage, nRemainingElements));
266 RNTupleLocator locator;
268 locator.SetNBytesOnStorage(element.GetPackedSize(pageInfo.GetNElements()));
269 pageInfo.SetLocator(locator);
270 pageInfos.emplace_back(pageInfo);
271 nRemainingElements -= pageInfo.GetNElements();
272 }
273
274 pageInfos.insert(pageInfos.end(), std::make_move_iterator(fPageInfos.begin()),
275 std::make_move_iterator(fPageInfos.end()));
276 std::swap(fPageInfos, pageInfos);
277 return nElementsRequired - nElements;
278}
279
281{
282 return fClusterId == other.fClusterId && fFirstEntryIndex == other.fFirstEntryIndex &&
283 fNEntries == other.fNEntries && fColumnRanges == other.fColumnRanges && fPageRanges == other.fPageRanges;
284}
285
287{
288 std::uint64_t nbytes = 0;
289 for (const auto &pr : fPageRanges) {
290 for (const auto &pi : pr.second.GetPageInfos()) {
291 nbytes += pi.GetLocator().GetNBytesOnStorage();
292 }
293 }
294 return nbytes;
295}
296
298{
299 RClusterDescriptor clone;
300 clone.fClusterId = fClusterId;
301 clone.fFirstEntryIndex = fFirstEntryIndex;
302 clone.fNEntries = fNEntries;
303 clone.fColumnRanges = fColumnRanges;
304 for (const auto &d : fPageRanges)
305 clone.fPageRanges.emplace(d.first, d.second.Clone());
306 return clone;
307}
308
309////////////////////////////////////////////////////////////////////////////////
310
312{
313 return fContentId == other.fContentId && fTypeName == other.fTypeName && fTypeVersion == other.fTypeVersion;
314}
315
317{
319 clone.fContentId = fContentId;
320 clone.fTypeVersion = fTypeVersion;
321 clone.fTypeName = fTypeName;
322 clone.fContent = fContent;
323 return clone;
324}
325
326////////////////////////////////////////////////////////////////////////////////
327
328ROOT::RNTupleDescriptor::RNTupleDescriptor() : fStringPool(std::make_shared<Internal::RStringPool>())
329{
330 // We need to make sure that for cloning fields, the empty string that is not explicitly entered for
331 // default constructed field descriptors is available.
332 fStringPool->Intern("");
333}
334
336{
337 // clang-format off
338 return fName == other.fName &&
339 fDescription == other.fDescription &&
340 fNEntries == other.fNEntries &&
341 fGeneration == other.fGeneration &&
342 fFieldZeroId == other.fFieldZeroId &&
343 fFieldDescriptors == other.fFieldDescriptors &&
344 fColumnDescriptors == other.fColumnDescriptors &&
345 fClusterGroupDescriptors == other.fClusterGroupDescriptors &&
346 fClusterDescriptors == other.fClusterDescriptors;
347 // clang-format on
348}
349
350ROOT::NTupleSize_t ROOT::RNTupleDescriptor::GetNElements(ROOT::DescriptorId_t physicalColumnId) const
351{
353 for (const auto &cd : fClusterDescriptors) {
354 if (!cd.second.ContainsColumn(physicalColumnId))
355 continue;
356 auto columnRange = cd.second.GetColumnRange(physicalColumnId);
357 result = std::max(result, columnRange.GetFirstElementIndex() + columnRange.GetNElements());
358 }
359 return result;
360}
361
362////////////////////////////////////////////////////////////////////////////////
363/// Return the cluster boundaries for each cluster in this RNTuple.
364std::vector<ROOT::Internal::RNTupleClusterBoundaries>
366{
367 std::vector<Internal::RNTupleClusterBoundaries> boundaries;
368 boundaries.reserve(desc.GetNClusters());
369 R__ASSERT(desc.GetNClusters() == desc.GetNActiveClusters());
370 for (const auto &clusterDesc : desc.GetActiveClusterIterable()) {
371 R__ASSERT(clusterDesc.GetNEntries() > 0);
372 boundaries.emplace_back(ROOT::Internal::RNTupleClusterBoundaries{
373 clusterDesc.GetFirstEntryIndex(), clusterDesc.GetFirstEntryIndex() + clusterDesc.GetNEntries()});
374 }
375 return boundaries;
376}
377
379ROOT::RNTupleDescriptor::FindFieldId(std::string_view fieldName, ROOT::DescriptorId_t parentId) const
380{
381 std::string leafName(fieldName);
382 auto posDot = leafName.find_last_of('.');
383 if (posDot != std::string::npos) {
384 auto parentName = leafName.substr(0, posDot);
385 leafName = leafName.substr(posDot + 1);
386 parentId = FindFieldId(parentName, parentId);
387 }
388 auto itrFieldDesc = fFieldDescriptors.find(parentId);
389 if (itrFieldDesc == fFieldDescriptors.end())
391 for (const auto linkId : itrFieldDesc->second.GetLinkIds()) {
392 if (fFieldDescriptors.at(linkId).GetFieldName() == leafName)
393 return linkId;
394 }
396}
397
399{
400 if (fieldId == ROOT::kInvalidDescriptorId)
401 return "";
402
403 const auto &fieldDescriptor = fFieldDescriptors.at(fieldId);
404 auto prefix = GetQualifiedFieldName(fieldDescriptor.GetParentId());
405 if (prefix.empty())
406 return fieldDescriptor.GetFieldName();
407 return prefix + "." + fieldDescriptor.GetFieldName();
408}
409
411{
412 R__ASSERT(fVersionEpoch == 1);
413 return fVersionMajor == 0 && fVersionMinor == 0 && fVersionPatch < 1;
414}
415
417{
418 std::string typeName = fieldDesc.GetTypeName();
419
420 if (FieldTypeNamesMayNeedFixup()) {
421 typeName = ROOT::Internal::GetRenormalizedTypeName(typeName);
422 }
423
424 return typeName;
425}
426
428{
429 return FindFieldId(fieldName, GetFieldZeroId());
430}
431
433 std::uint32_t columnIndex,
434 std::uint16_t representationIndex) const
435{
436 auto itr = fFieldDescriptors.find(fieldId);
437 if (itr == fFieldDescriptors.cend())
439 if (columnIndex >= itr->second.GetColumnCardinality())
441 const auto idx = representationIndex * itr->second.GetColumnCardinality() + columnIndex;
442 if (itr->second.GetLogicalColumnIds().size() <= idx)
444 return itr->second.GetLogicalColumnIds()[idx];
445}
446
448 std::uint32_t columnIndex,
449 std::uint16_t representationIndex) const
450{
451 auto logicalId = FindLogicalColumnId(fieldId, columnIndex, representationIndex);
452 if (logicalId == ROOT::kInvalidDescriptorId)
454 return GetColumnDescriptor(logicalId).GetPhysicalId();
455}
456
459{
460 if (GetNClusterGroups() == 0)
462
463 // Binary search in the cluster group list, followed by a binary search in the clusters of that cluster group
464
465 std::size_t cgLeft = 0;
466 std::size_t cgRight = GetNClusterGroups() - 1;
467 while (cgLeft <= cgRight) {
468 const std::size_t cgMidpoint = (cgLeft + cgRight) / 2;
469 const auto &clusterIds = GetClusterGroupDescriptor(fSortedClusterGroupIds[cgMidpoint]).GetClusterIds();
470 R__ASSERT(!clusterIds.empty());
471
472 const auto &clusterDesc = GetClusterDescriptor(clusterIds.front());
473 // this may happen if the RNTuple has an empty schema
474 if (!clusterDesc.ContainsColumn(physicalColumnId))
476
477 const auto firstElementInGroup = clusterDesc.GetColumnRange(physicalColumnId).GetFirstElementIndex();
478 if (firstElementInGroup > index) {
479 // Look into the lower half of cluster groups
480 R__ASSERT(cgMidpoint > 0);
481 cgRight = cgMidpoint - 1;
482 continue;
483 }
484
485 const auto &lastColumnRange = GetClusterDescriptor(clusterIds.back()).GetColumnRange(physicalColumnId);
486 if ((lastColumnRange.GetFirstElementIndex() + lastColumnRange.GetNElements()) <= index) {
487 // Look into the upper half of cluster groups
488 cgLeft = cgMidpoint + 1;
489 continue;
490 }
491
492 // Binary search in the current cluster group; since we already checked the element range boundaries,
493 // the element must be in that cluster group.
494 std::size_t clusterLeft = 0;
495 std::size_t clusterRight = clusterIds.size() - 1;
496 while (clusterLeft <= clusterRight) {
497 const std::size_t clusterMidpoint = (clusterLeft + clusterRight) / 2;
498 const auto clusterId = clusterIds[clusterMidpoint];
499 const auto &columnRange = GetClusterDescriptor(clusterId).GetColumnRange(physicalColumnId);
500
501 if (columnRange.Contains(index))
502 return clusterId;
503
504 if (columnRange.GetFirstElementIndex() > index) {
505 R__ASSERT(clusterMidpoint > 0);
506 clusterRight = clusterMidpoint - 1;
507 continue;
508 }
509
510 if (columnRange.GetFirstElementIndex() + columnRange.GetNElements() <= index) {
511 clusterLeft = clusterMidpoint + 1;
512 continue;
513 }
514 }
515 R__ASSERT(false);
516 }
518}
519
521{
522 if (GetNClusterGroups() == 0)
524
525 // Binary search in the cluster group list, followed by a binary search in the clusters of that cluster group
526
527 std::size_t cgLeft = 0;
528 std::size_t cgRight = GetNClusterGroups() - 1;
529 while (cgLeft <= cgRight) {
530 const std::size_t cgMidpoint = (cgLeft + cgRight) / 2;
531 const auto &cgDesc = GetClusterGroupDescriptor(fSortedClusterGroupIds[cgMidpoint]);
532
533 if (cgDesc.GetMinEntry() > entryIdx) {
534 R__ASSERT(cgMidpoint > 0);
535 cgRight = cgMidpoint - 1;
536 continue;
537 }
538
539 if (cgDesc.GetMinEntry() + cgDesc.GetEntrySpan() <= entryIdx) {
540 cgLeft = cgMidpoint + 1;
541 continue;
542 }
543
544 // Binary search in the current cluster group; since we already checked the element range boundaries,
545 // the element must be in that cluster group.
546 const auto &clusterIds = cgDesc.GetClusterIds();
547 R__ASSERT(!clusterIds.empty());
548 std::size_t clusterLeft = 0;
549 std::size_t clusterRight = clusterIds.size() - 1;
550 while (clusterLeft <= clusterRight) {
551 const std::size_t clusterMidpoint = (clusterLeft + clusterRight) / 2;
552 const auto &clusterDesc = GetClusterDescriptor(clusterIds[clusterMidpoint]);
553
554 if (clusterDesc.GetFirstEntryIndex() > entryIdx) {
555 R__ASSERT(clusterMidpoint > 0);
556 clusterRight = clusterMidpoint - 1;
557 continue;
558 }
559
560 if (clusterDesc.GetFirstEntryIndex() + clusterDesc.GetNEntries() <= entryIdx) {
561 clusterLeft = clusterMidpoint + 1;
562 continue;
563 }
564
565 return clusterIds[clusterMidpoint];
566 }
567 R__ASSERT(false);
568 }
570}
571
572ROOT::DescriptorId_t ROOT::RNTupleDescriptor::FindNextClusterId(ROOT::DescriptorId_t clusterId) const
573{
574 // TODO(jblomer): we may want to shortcut the common case and check if clusterId + 1 contains
575 // firstEntryInNextCluster. This shortcut would currently always trigger. We do not want, however, to depend
576 // on the linearity of the descriptor IDs, so we should only enable the shortcut if we can ensure that the
577 // binary search code path remains tested.
578 const auto &clusterDesc = GetClusterDescriptor(clusterId);
579 const auto firstEntryInNextCluster = clusterDesc.GetFirstEntryIndex() + clusterDesc.GetNEntries();
580 return FindClusterId(firstEntryInNextCluster);
581}
582
583ROOT::DescriptorId_t ROOT::RNTupleDescriptor::FindPrevClusterId(ROOT::DescriptorId_t clusterId) const
584{
585 // TODO(jblomer): we may want to shortcut the common case and check if clusterId - 1 contains
586 // firstEntryInNextCluster. This shortcut would currently always trigger. We do not want, however, to depend
587 // on the linearity of the descriptor IDs, so we should only enable the shortcut if we can ensure that the
588 // binary search code path remains tested.
589 const auto &clusterDesc = GetClusterDescriptor(clusterId);
590 if (clusterDesc.GetFirstEntryIndex() == 0)
592 return FindClusterId(clusterDesc.GetFirstEntryIndex() - 1);
593}
594
595std::vector<ROOT::DescriptorId_t>
597{
598 std::vector<ROOT::DescriptorId_t> fields;
599 for (const auto fieldId : fFieldIdsOrder) {
600 if (fFieldIdsLookup.count(desc.GetFieldDescriptor(fieldId).GetParentId()) == 0)
601 fields.emplace_back(fieldId);
602 }
603 return fields;
604}
605
607 const RFieldDescriptor &field)
608 : fNTuple(ntuple), fColumns(field.GetLogicalColumnIds())
609{
610}
611
613 : fNTuple(ntuple)
614{
615 std::deque<ROOT::DescriptorId_t> fieldIdQueue{ntuple.GetFieldZeroId()};
616
617 while (!fieldIdQueue.empty()) {
618 auto currFieldId = fieldIdQueue.front();
619 fieldIdQueue.pop_front();
620
621 const auto &columns = ntuple.GetFieldDescriptor(currFieldId).GetLogicalColumnIds();
622 fColumns.insert(fColumns.end(), columns.begin(), columns.end());
623
624 for (const auto &field : ntuple.GetFieldIterable(currFieldId)) {
625 auto fieldId = field.GetId();
626 fieldIdQueue.push_back(fieldId);
627 }
628 }
629}
630
631std::vector<std::uint64_t> ROOT::RNTupleDescriptor::GetFeatureFlags() const
632{
633 std::vector<std::uint64_t> result;
634 unsigned int base = 0;
635 std::uint64_t flags = 0;
636 for (auto f : fFeatureFlags) {
637 if ((f > 0) && ((f % 64) == 0))
638 throw RException(R__FAIL("invalid feature flag: " + std::to_string(f)));
639 while (f > base + 64) {
640 result.emplace_back(flags);
641 flags = 0;
642 base += 64;
643 }
644 // Note that in the following iterations of the outer loop over fFeatureFlags, we can never have the situation
645 // where base is larger than the feature flag, because they are stored ordered in the std::set.
646 assert(f >= base);
647 f -= base;
648 flags |= std::uint64_t(1) << f;
649 }
650 result.emplace_back(flags);
651 return result;
652}
653
655 std::vector<RClusterDescriptor> &clusterDescs)
656{
657 auto iter = fClusterGroupDescriptors.find(clusterGroupId);
658 if (iter == fClusterGroupDescriptors.end())
659 return R__FAIL("invalid attempt to add details of unknown cluster group");
660 if (iter->second.HasClusterDetails())
661 return R__FAIL("invalid attempt to re-populate cluster group details");
662 if (iter->second.GetNClusters() != clusterDescs.size())
663 return R__FAIL("mismatch of number of clusters");
664
665 std::vector<ROOT::DescriptorId_t> clusterIds;
666 for (unsigned i = 0; i < clusterDescs.size(); ++i) {
667 clusterIds.emplace_back(clusterDescs[i].GetId());
668 auto [_, success] = fClusterDescriptors.emplace(clusterIds.back(), std::move(clusterDescs[i]));
669 if (!success) {
670 return R__FAIL("invalid attempt to re-populate existing cluster");
671 }
672 }
673 std::sort(clusterIds.begin(), clusterIds.end(), [this](ROOT::DescriptorId_t a, ROOT::DescriptorId_t b) {
674 return fClusterDescriptors[a].GetFirstEntryIndex() < fClusterDescriptors[b].GetFirstEntryIndex();
675 });
676 auto cgBuilder = Internal::RClusterGroupDescriptorBuilder::FromSummary(iter->second);
677 cgBuilder.AddSortedClusters(clusterIds);
678 iter->second = cgBuilder.MoveDescriptor().Unwrap();
679 return RResult<void>::Success();
680}
681
683{
684 auto iter = fClusterGroupDescriptors.find(clusterGroupId);
685 if (iter == fClusterGroupDescriptors.end())
686 return R__FAIL("invalid attempt to drop cluster details of unknown cluster group");
687 if (!iter->second.HasClusterDetails())
688 return R__FAIL("invalid attempt to drop details of cluster group summary");
689
690 for (auto clusterId : iter->second.GetClusterIds())
691 fClusterDescriptors.erase(clusterId);
692 iter->second = iter->second.CloneSummary();
693 return RResult<void>::Success();
694}
695
696std::unique_ptr<ROOT::RNTupleModel> ROOT::RNTupleDescriptor::CreateModel(const RCreateModelOptions &options) const
697{
698 // Collect all top-level fields that have invalid columns (recursively): by default if we find any we throw an
699 // exception; if we are in ForwardCompatible mode, we proceed but skip of all those top-level fields.
700 std::unordered_set<ROOT::DescriptorId_t> invalidFields;
701 for (const auto &colDesc : GetColumnIterable()) {
702 if (colDesc.GetType() == ROOT::ENTupleColumnType::kUnknown) {
703 auto fieldId = colDesc.GetFieldId();
704 while (1) {
705 const auto &field = GetFieldDescriptor(fieldId);
706 if (field.GetParentId() == GetFieldZeroId())
707 break;
708 fieldId = field.GetParentId();
709 }
710 invalidFields.insert(fieldId);
711
712 // No need to look for all invalid fields if we're gonna error out anyway
713 if (!options.GetForwardCompatible())
714 break;
715 }
716 }
717
718 if (!options.GetForwardCompatible() && !invalidFields.empty())
720 "cannot create Model: descriptor contains unknown column types. Use 'SetForwardCompatible(true)' on the "
721 "RCreateModelOptions to create a partial model containing only the fields made up by known columns."));
722
723 auto fieldZero = std::make_unique<ROOT::RFieldZero>();
724 fieldZero->SetOnDiskId(GetFieldZeroId());
725 auto model = options.GetCreateBare() ? RNTupleModel::CreateBare(std::move(fieldZero))
726 : RNTupleModel::Create(std::move(fieldZero));
727 ROOT::RCreateFieldOptions createFieldOpts;
728 createFieldOpts.SetReturnInvalidOnError(options.GetForwardCompatible());
729 createFieldOpts.SetEmulateUnknownTypes(options.GetEmulateUnknownTypes());
730 for (const auto &topDesc : GetTopLevelFields()) {
731 if (invalidFields.count(topDesc.GetId()) > 0) {
732 // Field contains invalid columns: skip it
733 continue;
734 }
735
736 auto field = topDesc.CreateField(*this, createFieldOpts);
737
738 // If we got an InvalidField here, figure out if it's a hard error or if the field must simply be skipped.
739 // The only case where it's not a hard error is if the field has an unknown structure, as that case is
740 // covered by the ForwardCompatible flag (note that if the flag is off we would not get here
741 // in the first place, so we don't need to check for that flag again).
742 if (field->GetTraits() & ROOT::RFieldBase::kTraitInvalidField) {
743 const auto &invalid = static_cast<const RInvalidField &>(*field);
744 const auto cat = invalid.GetCategory();
745 bool mustThrow = cat != RInvalidField::ECategory::kUnknownStructure;
746 if (mustThrow)
747 throw RException(R__FAIL(invalid.GetError()));
748
749 // Not a hard error: skip the field and go on.
750 continue;
751 }
752
753 if (options.GetReconstructProjections() && topDesc.IsProjectedField()) {
754 model->AddProjectedField(std::move(field), [this](const std::string &targetName) -> std::string {
755 return GetQualifiedFieldName(GetFieldDescriptor(FindFieldId(targetName)).GetProjectionSourceId());
756 });
757 } else {
758 model->AddField(std::move(field));
759 }
760 }
761 model->Freeze();
762 return model;
763}
764
766{
767 RNTupleDescriptor clone;
768 clone.fName = fName;
773 // OnDiskHeaderSize, OnDiskHeaderXxHash3 not copied because they may come from a merged header + extension header
774 // and therefore not represent the actual sources's header.
775 // OnDiskFooterSize not copied because it contains information beyond the schema, for example the clustering.
776
778 shareStringPool = false;
779
780 if (shareStringPool)
781 clone.fStringPool = fStringPool;
782
784 // In case we are copying the schema from a pre-1.0.0.1 RNTuple we need to patch all field type names
785 // to use the proper normalization. In this case, the clone will get a new string pool because we may need
786 // to insert.
787 for (const auto &d : fFieldDescriptors) {
788 Internal::RFieldDescriptorBuilder fieldDescBuilder(d.second.Clone(), *clone.fStringPool);
789 fieldDescBuilder.TypeName(ROOT::Internal::GetRenormalizedTypeName(d.second.GetTypeName()));
790 clone.fFieldDescriptors.emplace(d.first, fieldDescBuilder.MoveDescriptor().Unwrap());
791 }
792 } else {
793 for (const auto &d : fFieldDescriptors) {
794 clone.fFieldDescriptors.emplace(
796 }
797 }
798
799 for (const auto &d : fColumnDescriptors)
800 clone.fColumnDescriptors.emplace(d.first, d.second.Clone());
801
802 for (const auto &d : fExtraTypeInfoDescriptors)
803 clone.fExtraTypeInfoDescriptors.emplace_back(d.Clone());
805 clone.fHeaderExtension = std::make_unique<RHeaderExtension>(*fHeaderExtension);
806
807 return clone;
808}
809
811{
812 RNTupleDescriptor clone = CloneSchema(true /* shareStringPool */);
813 clone.fStringPool->Freeze();
814
819
823 clone.fNEntries = fNEntries;
824 clone.fNClusters = fNClusters;
825 clone.fGeneration = fGeneration;
826 for (const auto &d : fClusterGroupDescriptors)
827 clone.fClusterGroupDescriptors.emplace(d.first, d.second.Clone());
829 for (const auto &d : fClusterDescriptors)
830 clone.fClusterDescriptors.emplace(d.first, d.second.Clone());
831 for (const auto &d : fAttributeSets)
832 clone.fAttributeSets.emplace_back(d.Clone());
833 return clone;
834}
835
836////////////////////////////////////////////////////////////////////////////////
837
839{
840 return fClusterGroupId == other.fClusterGroupId && fClusterIds == other.fClusterIds &&
841 fMinEntry == other.fMinEntry && fEntrySpan == other.fEntrySpan && fNClusters == other.fNClusters;
842}
843
845{
847 clone.fClusterGroupId = fClusterGroupId;
848 clone.fPageListLocator = fPageListLocator;
849 clone.fPageListLength = fPageListLength;
850 clone.fMinEntry = fMinEntry;
851 clone.fEntrySpan = fEntrySpan;
852 clone.fNClusters = fNClusters;
853 return clone;
854}
855
857{
858 RClusterGroupDescriptor clone = CloneSummary();
859 clone.fClusterIds = fClusterIds;
860 return clone;
861}
862
863////////////////////////////////////////////////////////////////////////////////
864
867 std::uint64_t firstElementIndex,
868 std::uint32_t compressionSettings,
869 const RClusterDescriptor::RPageRange &pageRange)
870{
871 if (physicalId != pageRange.fPhysicalColumnId)
872 return R__FAIL("column ID mismatch");
873 if (fCluster.fColumnRanges.count(physicalId) > 0)
874 return R__FAIL("column ID conflict");
875 RClusterDescriptor::RColumnRange columnRange{physicalId, firstElementIndex, 0, compressionSettings};
876 for (const auto &pi : pageRange.fPageInfos) {
877 columnRange.IncrementNElements(pi.GetNElements());
878 }
879 fCluster.fPageRanges[physicalId] = pageRange.Clone();
880 fCluster.fColumnRanges[physicalId] = columnRange;
881 return RResult<void>::Success();
882}
883
886{
887 if (fCluster.fColumnRanges.count(physicalId) > 0)
888 return R__FAIL("column ID conflict");
889
891 columnRange.SetPhysicalColumnId(physicalId);
892 columnRange.SetIsSuppressed(true);
893 fCluster.fColumnRanges[physicalId] = columnRange;
894 return RResult<void>::Success();
895}
896
899{
900 for (auto &[_, columnRange] : fCluster.fColumnRanges) {
901 if (!columnRange.IsSuppressed())
902 continue;
903 R__ASSERT(columnRange.GetFirstElementIndex() == ROOT::kInvalidNTupleIndex);
904
905 const auto &columnDesc = desc.GetColumnDescriptor(columnRange.GetPhysicalColumnId());
906 const auto &fieldDesc = desc.GetFieldDescriptor(columnDesc.GetFieldId());
907 // We expect only few columns and column representations per field, so we do a linear search
908 for (const auto otherColumnLogicalId : fieldDesc.GetLogicalColumnIds()) {
909 const auto &otherColumnDesc = desc.GetColumnDescriptor(otherColumnLogicalId);
910 if (otherColumnDesc.GetRepresentationIndex() == columnDesc.GetRepresentationIndex())
911 continue;
912 if (otherColumnDesc.GetIndex() != columnDesc.GetIndex())
913 continue;
914
915 // Found corresponding column of a different column representation
916 const auto &otherColumnRange = fCluster.GetColumnRange(otherColumnDesc.GetPhysicalId());
917 if (otherColumnRange.IsSuppressed())
918 continue;
919
920 columnRange.SetFirstElementIndex(otherColumnRange.GetFirstElementIndex());
921 columnRange.SetNElements(otherColumnRange.GetNElements());
922 break;
923 }
924
925 if (columnRange.GetFirstElementIndex() == ROOT::kInvalidNTupleIndex) {
926 return R__FAIL(std::string("cannot find non-suppressed column for column ID ") +
927 std::to_string(columnRange.GetPhysicalColumnId()) +
928 ", cluster ID: " + std::to_string(fCluster.GetId()));
929 }
930 }
931 return RResult<void>::Success();
932}
933
936{
937 /// Carries out a depth-first traversal of a field subtree rooted at `rootFieldId`. For each field, `visitField` is
938 /// called passing the field ID and the number of overall repetitions, taking into account the repetitions of each
939 /// parent field in the hierarchy.
940 auto fnTraverseSubtree = [&](ROOT::DescriptorId_t rootFieldId, std::uint64_t nRepetitionsAtThisLevel,
941 const auto &visitField, const auto &enterSubtree) -> void {
942 visitField(rootFieldId, nRepetitionsAtThisLevel);
943 for (const auto &f : desc.GetFieldIterable(rootFieldId)) {
944 const std::uint64_t nRepetitions = std::max(f.GetNRepetitions(), std::uint64_t{1U}) * nRepetitionsAtThisLevel;
945 enterSubtree(f.GetId(), nRepetitions, visitField, enterSubtree);
946 }
947 };
948
949 // Extended columns can only be part of the header extension
950 if (!desc.GetHeaderExtension())
951 return *this;
952
953 // Ensure that all columns in the header extension have their associated `R(Column|Page)Range`
954 // Extended columns can be attached both to fields of the regular header and to fields of the extension header
955 for (const auto &topLevelField : desc.GetTopLevelFields()) {
956 fnTraverseSubtree(
957 topLevelField.GetId(), std::max(topLevelField.GetNRepetitions(), std::uint64_t{1U}),
958 [&](ROOT::DescriptorId_t fieldId, std::uint64_t nRepetitions) {
959 for (const auto &c : desc.GetColumnIterable(fieldId)) {
960 const ROOT::DescriptorId_t physicalId = c.GetPhysicalId();
961 auto &columnRange = fCluster.fColumnRanges[physicalId];
962
963 // Initialize a RColumnRange for `physicalId` if it was not there. Columns that were created during model
964 // extension won't have on-disk metadata for the clusters that were already committed before the model
965 // was extended. Therefore, these need to be synthetically initialized upon reading.
966 if (columnRange.GetPhysicalColumnId() == ROOT::kInvalidDescriptorId) {
967 columnRange.SetPhysicalColumnId(physicalId);
968 columnRange.SetFirstElementIndex(0);
969 columnRange.SetNElements(0);
970 columnRange.SetIsSuppressed(c.IsSuppressedDeferredColumn());
971 }
972 // Fixup the RColumnRange and RPageRange in deferred columns. We know what the first element index and
973 // number of elements should have been if the column was not deferred; fix those and let
974 // `ExtendToFitColumnRange()` synthesize RPageInfos accordingly.
975 if (c.IsDeferredColumn()) {
976 if (c.GetRepresentationIndex() == 0) {
977 // Note that a deferred column (i.e, whose first element index is > 0) for the 0th representation
978 // index already met the criteria of `ROOT::RFieldBase::EntryToColumnElementIndex()`, i.e. it is a
979 // principal column reachable from the field zero excluding subfields of collection and variant
980 // fields.
981 columnRange.SetFirstElementIndex(fCluster.GetFirstEntryIndex() * nRepetitions);
982 columnRange.SetNElements(fCluster.GetNEntries() * nRepetitions);
983 } else {
984 // Deferred representations which are not the first cannot count on the number of elements being
985 // equal to Entries * nRepetitions because they might have been added in a later cluster. But they
986 // can rely on the first representation having the correct FirstElement/NElements (by definition
987 // the first representation cannot be an "extended" one), therefore they can just copy the value
988 // from it.
989 const auto &field = desc.GetFieldDescriptor(fieldId);
990 const auto firstReprColumnId = field.GetLogicalColumnIds()[c.GetIndex()];
991 const auto &firstReprColumnRange = fCluster.fColumnRanges[firstReprColumnId];
992 columnRange.SetFirstElementIndex(firstReprColumnRange.GetFirstElementIndex());
993 columnRange.SetNElements(firstReprColumnRange.GetNElements());
994 }
995 if (!columnRange.IsSuppressed()) {
996 auto &pageRange = fCluster.fPageRanges[physicalId];
997 pageRange.fPhysicalColumnId = physicalId;
998 const auto element = ROOT::Internal::RColumnElementBase::Generate(c);
999 pageRange.ExtendToFitColumnRange(columnRange, *element, ROOT::Internal::RPage::kPageZeroSize);
1000 }
1001 } else if (!columnRange.IsSuppressed()) {
1002 fCluster.fPageRanges[physicalId].fPhysicalColumnId = physicalId;
1003 }
1004 }
1005 },
1006 fnTraverseSubtree);
1007 }
1008 return *this;
1009}
1010
1012{
1013 if (fCluster.fClusterId == ROOT::kInvalidDescriptorId)
1014 return R__FAIL("unset cluster ID");
1015 if (fCluster.fNEntries == 0)
1016 return R__FAIL("empty cluster");
1017 for (auto &pr : fCluster.fPageRanges) {
1018 if (fCluster.fColumnRanges.count(pr.first) == 0) {
1019 return R__FAIL("missing column range");
1020 }
1021 pr.second.fCumulativeNElements.reset();
1022 const auto nPages = pr.second.fPageInfos.size();
1024 pr.second.fCumulativeNElements = std::make_unique<std::vector<NTupleSize_t>>();
1025 pr.second.fCumulativeNElements->reserve(nPages);
1027 for (const auto &pi : pr.second.fPageInfos) {
1028 sum += pi.GetNElements();
1029 pr.second.fCumulativeNElements->emplace_back(sum);
1030 }
1031 }
1032 }
1034 std::swap(result, fCluster);
1035 return result;
1036}
1037
1038////////////////////////////////////////////////////////////////////////////////
1039
1042{
1044 builder.ClusterGroupId(clusterGroupDesc.GetId())
1045 .PageListLocator(clusterGroupDesc.GetPageListLocator())
1046 .PageListLength(clusterGroupDesc.GetPageListLength())
1047 .MinEntry(clusterGroupDesc.GetMinEntry())
1048 .EntrySpan(clusterGroupDesc.GetEntrySpan())
1049 .NClusters(clusterGroupDesc.GetNClusters());
1050 return builder;
1051}
1052
1054{
1055 if (fClusterGroup.fClusterGroupId == ROOT::kInvalidDescriptorId)
1056 return R__FAIL("unset cluster group ID");
1058 std::swap(result, fClusterGroup);
1059 return result;
1060}
1061
1062////////////////////////////////////////////////////////////////////////////////
1063
1065{
1066 if (fExtraTypeInfo.fContentId == EExtraTypeInfoIds::kInvalid)
1067 throw RException(R__FAIL("invalid extra type info content id"));
1069 std::swap(result, fExtraTypeInfo);
1070 return result;
1071}
1072
1073////////////////////////////////////////////////////////////////////////////////
1074
1076{
1077 if (fDescriptor.fFieldDescriptors.count(fieldId) == 0)
1078 return R__FAIL("field with id '" + std::to_string(fieldId) + "' doesn't exist");
1079 return RResult<void>::Success();
1080}
1081
1083{
1084 if (fDescriptor.fVersionEpoch != RNTuple::kVersionEpoch) {
1085 return R__FAIL("unset or unsupported RNTuple epoch version");
1086 }
1087
1088 // Reuse field name validity check
1089 auto validName = ROOT::Internal::EnsureValidNameForRNTuple(fDescriptor.GetName(), "Field");
1090 if (!validName) {
1091 return R__FORWARD_ERROR(validName);
1092 }
1093
1094 for (const auto &[fieldId, fieldDesc] : fDescriptor.fFieldDescriptors) {
1095 // parent not properly set?
1096 if (fieldId != fDescriptor.GetFieldZeroId() && fieldDesc.GetParentId() == ROOT::kInvalidDescriptorId) {
1097 return R__FAIL("field with id '" + std::to_string(fieldId) + "' has an invalid parent id");
1098 }
1099
1100 // Same number of columns in every column representation?
1101 const auto columnCardinality = fieldDesc.GetColumnCardinality();
1102 if (columnCardinality == 0)
1103 continue;
1104
1105 // In AddColumn, we already checked that all but the last representation are complete.
1106 // Check that the last column representation is complete, i.e. has all columns.
1107 const auto &logicalColumnIds = fieldDesc.GetLogicalColumnIds();
1108 const auto nColumns = logicalColumnIds.size();
1109 // If we have only a single column representation, the following condition is true by construction
1110 if ((nColumns + 1) == columnCardinality)
1111 continue;
1112
1113 const auto &lastColumn = fDescriptor.GetColumnDescriptor(logicalColumnIds.back());
1114 if (lastColumn.GetIndex() + 1 != columnCardinality)
1115 return R__FAIL("field with id '" + std::to_string(fieldId) + "' has incomplete column representations");
1116 }
1117
1118 return RResult<void>::Success();
1119}
1120
1122{
1123 EnsureValidDescriptor().ThrowOnError();
1124 fDescriptor.fSortedClusterGroupIds.reserve(fDescriptor.fClusterGroupDescriptors.size());
1125 for (const auto &[id, _] : fDescriptor.fClusterGroupDescriptors)
1126 fDescriptor.fSortedClusterGroupIds.emplace_back(id);
1127 std::sort(fDescriptor.fSortedClusterGroupIds.begin(), fDescriptor.fSortedClusterGroupIds.end(),
1129 return fDescriptor.fClusterGroupDescriptors[a].GetMinEntry() <
1130 fDescriptor.fClusterGroupDescriptors[b].GetMinEntry();
1131 });
1132 fDescriptor.fStringPool->Freeze();
1134 std::swap(result, fDescriptor);
1135 return result;
1136}
1137
1138void ROOT::Internal::RNTupleDescriptorBuilder::SetVersion(std::uint16_t versionEpoch, std::uint16_t versionMajor,
1139 std::uint16_t versionMinor, std::uint16_t versionPatch)
1140{
1141 if (versionEpoch != RNTuple::kVersionEpoch) {
1142 throw RException(R__FAIL("unsupported RNTuple epoch version: " + std::to_string(versionEpoch)));
1143 }
1144 fDescriptor.fVersionEpoch = versionEpoch;
1145 fDescriptor.fVersionMajor = versionMajor;
1146 fDescriptor.fVersionMinor = versionMinor;
1147 fDescriptor.fVersionPatch = versionPatch;
1148}
1149
1151{
1152 fDescriptor.fVersionEpoch = RNTuple::kVersionEpoch;
1153 fDescriptor.fVersionMajor = RNTuple::kVersionMajor;
1154 fDescriptor.fVersionMinor = RNTuple::kVersionMinor;
1155 fDescriptor.fVersionPatch = RNTuple::kVersionPatch;
1156}
1157
1158void ROOT::Internal::RNTupleDescriptorBuilder::SetNTuple(std::string_view name, std::string_view description)
1159{
1160 fDescriptor.fName = std::string(name);
1161 fDescriptor.fDescription = std::string(description);
1162}
1163
1165{
1166 if (flag > 0 && flag % 64 == 0)
1167 throw RException(R__FAIL("invalid feature flag: " + std::to_string(flag)));
1168 fDescriptor.fFeatureFlags.insert(flag);
1169}
1170
1173{
1174 if (fDesc.fName.empty())
1175 return R__FAIL("attribute set name cannot be empty");
1176 if (fDesc.fAnchorLength == 0)
1177 return R__FAIL("invalid anchor length");
1178 if (fDesc.fAnchorLocator.GetType() == RNTupleLocator::kTypeUnknown)
1179 return R__FAIL("invalid locator type");
1180
1181 return std::move(fDesc);
1182}
1183
1185{
1186 if (fColumn.GetLogicalId() == ROOT::kInvalidDescriptorId)
1187 return R__FAIL("invalid logical column id");
1188 if (fColumn.GetPhysicalId() == ROOT::kInvalidDescriptorId)
1189 return R__FAIL("invalid physical column id");
1190 if (fColumn.GetFieldId() == ROOT::kInvalidDescriptorId)
1191 return R__FAIL("invalid field id, dangling column");
1192
1193 // NOTE: if the column type is unknown we don't want to fail, as we might be reading an RNTuple
1194 // created with a future version of ROOT. In this case we just skip the valid bit range check,
1195 // as we have no idea what the valid range is.
1196 // In general, reading the metadata of an unknown column is fine, it becomes an error only when
1197 // we try to read the actual data contained in it.
1198 if (fColumn.GetType() != ENTupleColumnType::kUnknown) {
1199 const auto [minBits, maxBits] = ROOT::Internal::RColumnElementBase::GetValidBitRange(fColumn.GetType());
1200 if (fColumn.GetBitsOnStorage() < minBits || fColumn.GetBitsOnStorage() > maxBits)
1201 return R__FAIL("invalid column bit width");
1202 }
1203
1205 std::swap(result, fColumn);
1206 return result;
1207}
1208
1211{
1212 RFieldDescriptor clone;
1213 clone.InitFrom(source, stringPool);
1214 return clone;
1215}
1216
1218{
1219 if (fField.GetId() == ROOT::kInvalidDescriptorId) {
1220 return R__FAIL("invalid field id");
1221 }
1222 if (fField.GetStructure() == ROOT::ENTupleStructure::kInvalid) {
1223 return R__FAIL("invalid field structure");
1224 }
1225 if (fField.IsSoACollection() && (fField.GetStructure() != ROOT::ENTupleStructure::kCollection)) {
1226 return R__FAIL("invalid SoA flag on non-collection field");
1227 }
1228 // FieldZero is usually named "" and would be a false positive here
1229 if (fField.GetParentId() != ROOT::kInvalidDescriptorId) {
1230 auto validName = ROOT::Internal::EnsureValidNameForRNTuple(fField.GetFieldName(), "Field");
1231 if (!validName) {
1232 return R__FORWARD_ERROR(validName);
1233 }
1234 if (fField.GetFieldName().empty()) {
1235 return R__FAIL("name cannot be empty string \"\"");
1236 }
1237 }
1238
1240 std::swap(result, fField);
1241 return result;
1242}
1243
1245{
1246 RFieldDescriptorBuilder fieldDesc(GetStringPool());
1247 fieldDesc.FieldId(fieldId)
1249 .TypeVersion(field.GetTypeVersion())
1250 .FieldName(field.GetFieldName())
1252 .TypeName(field.GetTypeName())
1253 .TypeAlias(field.GetTypeAlias())
1254 .Structure(field.GetStructure())
1255 .NRepetitions(field.GetNRepetitions());
1257 fieldDesc.TypeChecksum(field.GetTypeChecksum());
1260 fieldDesc.IsSoACollection(true);
1261 }
1262 AddField(fieldDesc.MoveDescriptor().Unwrap());
1263}
1264
1266{
1267 const auto id = fieldDesc.GetId();
1268 if (fDescriptor.fHeaderExtension)
1269 fDescriptor.fHeaderExtension->MarkExtendedField(fieldDesc);
1270 if (fieldDesc.GetFieldName().empty() && fieldDesc.GetParentId() == ROOT::kInvalidDescriptorId) {
1271 fDescriptor.fFieldZeroId = id;
1272 }
1273
1274 fDescriptor.fFieldDescriptors.emplace(id, std::move(fieldDesc));
1275}
1276
1279{
1280 auto fieldExists = RResult<void>::Success();
1281 if (!(fieldExists = EnsureFieldExists(fieldId)))
1282 return R__FORWARD_ERROR(fieldExists);
1283 if (!(fieldExists = EnsureFieldExists(linkId)))
1284 return R__FAIL("child field with id '" + std::to_string(linkId) + "' doesn't exist in NTuple");
1285
1286 if (linkId == fDescriptor.GetFieldZeroId()) {
1287 return R__FAIL("cannot make FieldZero a child field");
1288 }
1289 // fail if field already has another valid parent
1290 auto parentId = fDescriptor.fFieldDescriptors.at(linkId).GetParentId();
1291 if ((parentId != ROOT::kInvalidDescriptorId) && (parentId != fieldId)) {
1292 return R__FAIL("field '" + std::to_string(linkId) + "' already has a parent ('" + std::to_string(parentId) + ")");
1293 }
1294 if (fieldId == linkId) {
1295 return R__FAIL("cannot make field '" + std::to_string(fieldId) + "' a child of itself");
1296 }
1297 fDescriptor.fFieldDescriptors.at(linkId).fParentId = fieldId;
1298 fDescriptor.fFieldDescriptors.at(fieldId).fLinkIds.push_back(linkId);
1299 return RResult<void>::Success();
1300}
1301
1303 ROOT::DescriptorId_t targetId)
1304{
1305 auto fieldExists = RResult<void>::Success();
1306 if (!(fieldExists = EnsureFieldExists(sourceId)))
1307 return R__FORWARD_ERROR(fieldExists);
1308 if (!(fieldExists = EnsureFieldExists(targetId)))
1309 return R__FAIL("projected field with id '" + std::to_string(targetId) + "' doesn't exist in NTuple");
1310
1311 if (targetId == fDescriptor.GetFieldZeroId()) {
1312 return R__FAIL("cannot make FieldZero a projected field");
1313 }
1314 if (sourceId == targetId) {
1315 return R__FAIL("cannot make field '" + std::to_string(targetId) + "' a projection of itself");
1316 }
1317 if (fDescriptor.fFieldDescriptors.at(sourceId).IsProjectedField()) {
1318 return R__FAIL("cannot make field '" + std::to_string(targetId) + "' a projection of an already projected field");
1319 }
1320 // fail if target field already has another valid projection source
1321 auto &targetDesc = fDescriptor.fFieldDescriptors.at(targetId);
1322 if (targetDesc.IsProjectedField() && targetDesc.GetProjectionSourceId() != sourceId) {
1323 return R__FAIL("field '" + std::to_string(targetId) + "' has already a projection source ('" +
1324 std::to_string(targetDesc.GetProjectionSourceId()) + ")");
1325 }
1326 fDescriptor.fFieldDescriptors.at(targetId).fProjectionSourceId = sourceId;
1327 return RResult<void>::Success();
1328}
1329
1331{
1332 const auto fieldId = columnDesc.GetFieldId();
1333 const auto columnIndex = columnDesc.GetIndex();
1334 const auto representationIndex = columnDesc.GetRepresentationIndex();
1335
1336 auto fieldExists = EnsureFieldExists(fieldId);
1337 if (!fieldExists) {
1338 return R__FORWARD_ERROR(fieldExists);
1339 }
1340 auto &fieldDesc = fDescriptor.fFieldDescriptors.find(fieldId)->second;
1341
1342 if (columnDesc.IsAliasColumn()) {
1343 if (columnDesc.GetType() != fDescriptor.GetColumnDescriptor(columnDesc.GetPhysicalId()).GetType())
1344 return R__FAIL("alias column type mismatch");
1345 }
1346 if (fDescriptor.FindLogicalColumnId(fieldId, columnIndex, representationIndex) != ROOT::kInvalidDescriptorId) {
1347 return R__FAIL("column index clash");
1348 }
1349 if (columnIndex > 0) {
1350 if (fDescriptor.FindLogicalColumnId(fieldId, columnIndex - 1, representationIndex) == ROOT::kInvalidDescriptorId)
1351 return R__FAIL("out of bounds column index");
1352 }
1353 if (representationIndex > 0) {
1354 if (fDescriptor.FindLogicalColumnId(fieldId, 0, representationIndex - 1) == ROOT::kInvalidDescriptorId) {
1355 return R__FAIL("out of bounds representation index");
1356 }
1357 if (columnIndex == 0) {
1358 assert(fieldDesc.fColumnCardinality > 0);
1359 if (fDescriptor.FindLogicalColumnId(fieldId, fieldDesc.fColumnCardinality - 1, representationIndex - 1) ==
1361 return R__FAIL("incomplete column representations");
1362 }
1363 } else {
1364 if (columnIndex >= fieldDesc.fColumnCardinality)
1365 return R__FAIL("irregular column representations");
1366 }
1367 } else {
1368 // This will set the column cardinality to the number of columns of the first representation
1369 fieldDesc.fColumnCardinality = columnIndex + 1;
1370 }
1371
1372 const auto logicalId = columnDesc.GetLogicalId();
1373 fieldDesc.fLogicalColumnIds.emplace_back(logicalId);
1374
1375 if (!columnDesc.IsAliasColumn())
1376 fDescriptor.fNPhysicalColumns++;
1377 if (fDescriptor.fHeaderExtension)
1378 fDescriptor.fHeaderExtension->MarkExtendedColumn(columnDesc);
1379 fDescriptor.fColumnDescriptors.emplace(logicalId, std::move(columnDesc));
1380
1381 return RResult<void>::Success();
1382}
1383
1385{
1386 const auto id = clusterGroup.GetId();
1387 if (fDescriptor.fClusterGroupDescriptors.count(id) > 0)
1388 return R__FAIL("cluster group id clash");
1389 fDescriptor.fNEntries = std::max(fDescriptor.fNEntries, clusterGroup.GetMinEntry() + clusterGroup.GetEntrySpan());
1390 fDescriptor.fNClusters += clusterGroup.GetNClusters();
1391 fDescriptor.fClusterGroupDescriptors.emplace(id, std::move(clusterGroup));
1392 return RResult<void>::Success();
1393}
1394
1396{
1397 // We expect the resulting descriptor to be ammended, so use a dedicated string pool
1398 fDescriptor = descriptor.CloneSchema(false /* shareStringPool */);
1399}
1400
1402{
1403 if (!fDescriptor.fHeaderExtension)
1404 fDescriptor.fHeaderExtension = std::make_unique<RNTupleDescriptor::RHeaderExtension>();
1405}
1406
1408{
1409 if (fDescriptor.GetNLogicalColumns() == 0)
1410 return;
1411 R__ASSERT(fDescriptor.GetNPhysicalColumns() > 0);
1412
1413 for (ROOT::DescriptorId_t id = fDescriptor.GetNLogicalColumns() - 1; id >= fDescriptor.GetNPhysicalColumns(); --id) {
1414 auto c = fDescriptor.fColumnDescriptors[id].Clone();
1415 R__ASSERT(c.IsAliasColumn());
1416 R__ASSERT(id == c.GetLogicalId());
1417 fDescriptor.fColumnDescriptors.erase(id);
1418 for (auto &link : fDescriptor.fFieldDescriptors[c.fFieldId].fLogicalColumnIds) {
1419 if (link == c.fLogicalColumnId) {
1420 link += offset;
1421 break;
1422 }
1423 }
1424 c.fLogicalColumnId += offset;
1425 R__ASSERT(fDescriptor.fColumnDescriptors.count(c.fLogicalColumnId) == 0);
1426 fDescriptor.fColumnDescriptors.emplace(c.fLogicalColumnId, std::move(c));
1427 }
1428
1429 // Patch up column ids in the header extension
1430 if (auto &xHeader = fDescriptor.fHeaderExtension) {
1431 for (auto &columnId : xHeader->fExtendedColumnRepresentations) {
1432 if (columnId >= fDescriptor.GetNPhysicalColumns())
1433 columnId += offset;
1434 }
1435 }
1436}
1437
1439{
1440 auto clusterId = clusterDesc.GetId();
1441 if (fDescriptor.fClusterDescriptors.count(clusterId) > 0)
1442 return R__FAIL("cluster id clash");
1443 fDescriptor.fClusterDescriptors.emplace(clusterId, std::move(clusterDesc));
1444 return RResult<void>::Success();
1445}
1446
1449{
1450 // Make sure we have no duplicates
1451 if (std::find(fDescriptor.fExtraTypeInfoDescriptors.begin(), fDescriptor.fExtraTypeInfoDescriptors.end(),
1452 extraTypeInfoDesc) != fDescriptor.fExtraTypeInfoDescriptors.end()) {
1453 return R__FAIL("extra type info duplicates");
1454 }
1455 fDescriptor.fExtraTypeInfoDescriptors.emplace_back(std::move(extraTypeInfoDesc));
1456 return RResult<void>::Success();
1457}
1458
1460{
1461 auto it = std::find(fDescriptor.fExtraTypeInfoDescriptors.begin(), fDescriptor.fExtraTypeInfoDescriptors.end(),
1462 extraTypeInfoDesc);
1463 if (it != fDescriptor.fExtraTypeInfoDescriptors.end())
1464 *it = std::move(extraTypeInfoDesc);
1465 else
1466 fDescriptor.fExtraTypeInfoDescriptors.emplace_back(std::move(extraTypeInfoDesc));
1467}
1468
1471{
1472 auto &attrSets = fDescriptor.fAttributeSets;
1473 if (std::find_if(attrSets.begin(), attrSets.end(), [&name = attrSetDesc.GetName()](const auto &desc) {
1474 return desc.GetName() == name;
1475 }) != attrSets.end()) {
1476 return R__FAIL("attribute sets with duplicate names");
1477 }
1478 attrSets.push_back(std::move(attrSetDesc));
1479 return RResult<void>::Success();
1480}
1481
1486
1489{
1490 return RFieldDescriptorIterable(*this, fieldDesc);
1491}
1492
1494 const RFieldDescriptor &fieldDesc,
1495 const std::function<bool(ROOT::DescriptorId_t, ROOT::DescriptorId_t)> &comparator) const
1496{
1497 return RFieldDescriptorIterable(*this, fieldDesc, comparator);
1498}
1499
1505
1512
1517
1523
1528
1531{
1532 return RColumnDescriptorIterable(*this, fieldDesc);
1533}
1534
1540
1545
1546ROOT::RNTupleDescriptor::RClusterDescriptorIterable ROOT::RNTupleDescriptor::GetClusterIterable() const
1547{
1548 return GetActiveClusterIterable();
1549}
1550
1555
1560
1565
1567{
1568 return fAnchorLength == other.fAnchorLength && fSchemaVersionMajor == other.fSchemaVersionMajor &&
1569 fSchemaVersionMinor == other.fSchemaVersionMinor && fAnchorLocator == other.fAnchorLocator &&
1570 fName == other.fName;
1571};
1572
1574{
1576 desc.fAnchorLength = fAnchorLength;
1577 desc.fSchemaVersionMajor = fSchemaVersionMajor;
1578 desc.fSchemaVersionMinor = fSchemaVersionMinor;
1579 desc.fAnchorLocator = fAnchorLocator;
1580 desc.fName = fName;
1581 return desc;
1582}
1583
1585{
1587 return false;
1588 if (fieldDesc.GetTypeName().rfind("std::", 0) == 0)
1589 return false;
1590
1591 auto subFieldId = desc.FindFieldId("_0", fieldDesc.GetId());
1592 if (subFieldId == kInvalidDescriptorId)
1593 return false;
1594
1595 static const std::string gIntTypeNames[] = {"bool", "char", "std::int8_t", "std::uint8_t",
1596 "std::int16_t", "std::uint16_t", "std::int32_t", "std::uint32_t",
1597 "std::int64_t", "std::uint64_t"};
1598 return std::find(std::begin(gIntTypeNames), std::end(gIntTypeNames),
1599 desc.GetFieldDescriptor(subFieldId).GetTypeName()) != std::end(gIntTypeNames);
1600}
1601
1603{
1605 return false;
1606 return (fieldDesc.GetTypeName().rfind("std::atomic<", 0) == 0);
1607}
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
Definition RError.hxx:326
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:322
#define d(i)
Definition RSha256.hxx:102
#define b(i)
Definition RSha256.hxx:100
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
#define a(i)
Definition RSha256.hxx:99
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:139
#define N
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize id
char name[80]
Definition TGX11.cxx:142
#define _(A, B)
Definition cfortran.h:112
A helper class for piece-wise construction of an RClusterDescriptor.
RResult< void > MarkSuppressedColumnRange(ROOT::DescriptorId_t physicalId)
Books the given column ID as being suppressed in this cluster.
RResult< void > CommitColumnRange(ROOT::DescriptorId_t physicalId, std::uint64_t firstElementIndex, std::uint32_t compressionSettings, const RClusterDescriptor::RPageRange &pageRange)
RClusterDescriptorBuilder & AddExtendedColumnRanges(const RNTupleDescriptor &desc)
Add column and page ranges for columns created during late model extension missing in this cluster.
RResult< void > CommitSuppressedColumnRanges(const RNTupleDescriptor &desc)
Sets the first element index and number of elements for all the suppressed column ranges.
RResult< RClusterDescriptor > MoveDescriptor()
Move out the full cluster descriptor including page locations.
A helper class for piece-wise construction of an RClusterGroupDescriptor.
RClusterGroupDescriptorBuilder & EntrySpan(std::uint64_t entrySpan)
RClusterGroupDescriptorBuilder & PageListLocator(const RNTupleLocator &pageListLocator)
static RClusterGroupDescriptorBuilder FromSummary(const RClusterGroupDescriptor &clusterGroupDesc)
RClusterGroupDescriptorBuilder & PageListLength(std::uint64_t pageListLength)
RClusterGroupDescriptorBuilder & MinEntry(std::uint64_t minEntry)
RResult< RClusterGroupDescriptor > MoveDescriptor()
RClusterGroupDescriptorBuilder & ClusterGroupId(ROOT::DescriptorId_t clusterGroupId)
RClusterGroupDescriptorBuilder & NClusters(std::uint32_t nClusters)
RResult< RColumnDescriptor > MoveDescriptor()
Attempt to make a column descriptor.
A column element encapsulates the translation between basic C++ types and their column representation...
static std::pair< std::uint16_t, std::uint16_t > GetValidBitRange(ROOT::ENTupleColumnType type)
Most types have a fixed on-disk bit width.
std::size_t GetPackedSize(std::size_t nElements=1U) const
RResult< RExtraTypeInfoDescriptor > MoveDescriptor()
A helper class for piece-wise construction of an RFieldDescriptor.
RResult< RFieldDescriptor > MoveDescriptor()
Attempt to make a field descriptor.
RFieldDescriptorBuilder & NRepetitions(std::uint64_t nRepetitions)
RFieldDescriptorBuilder & Structure(const ROOT::ENTupleStructure &structure)
RFieldDescriptorBuilder & TypeAlias(const std::string &typeAlias)
static RFieldDescriptor CloneDescriptor(const RFieldDescriptor &source, RStringPool &stringPool)
RFieldDescriptorBuilder & TypeVersion(std::uint32_t typeVersion)
RFieldDescriptorBuilder & IsSoACollection(bool val)
RFieldDescriptorBuilder & TypeChecksum(const std::optional< std::uint32_t > typeChecksum)
RFieldDescriptorBuilder & FieldDescription(const std::string &fieldDescription)
RFieldDescriptorBuilder & FieldVersion(std::uint32_t fieldVersion)
RFieldDescriptorBuilder & FieldName(const std::string &fieldName)
RFieldDescriptorBuilder & FieldId(ROOT::DescriptorId_t fieldId)
RFieldDescriptorBuilder & TypeName(const std::string &typeName)
RResult< ROOT::RNTupleAttrSetDescriptor > MoveDescriptor()
Attempt to make an AttributeSet descriptor.
void SetNTuple(std::string_view name, std::string_view description)
void SetSchemaFromExisting(const RNTupleDescriptor &descriptor)
Copies the "schema" part of descriptor into the builder's descriptor.
RResult< void > AddColumn(RColumnDescriptor &&columnDesc)
void AddField(RFieldDescriptor fieldDesc)
RResult< void > AddFieldProjection(ROOT::DescriptorId_t sourceId, ROOT::DescriptorId_t targetId)
void ReplaceExtraTypeInfo(RExtraTypeInfoDescriptor &&extraTypeInfoDesc)
RResult< void > AddExtraTypeInfo(RExtraTypeInfoDescriptor &&extraTypeInfoDesc)
void ShiftAliasColumns(std::uint32_t offset)
Shift column IDs of alias columns by offset
RResult< void > AddAttributeSet(RNTupleAttrSetDescriptor &&attrSetDesc)
void SetVersion(std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch)
void BeginHeaderExtension()
Mark the beginning of the header extension; any fields and columns added after a call to this functio...
RResult< void > AddCluster(RClusterDescriptor &&clusterDesc)
RResult< void > EnsureValidDescriptor() const
Checks whether invariants hold:
RResult< void > AddFieldLink(ROOT::DescriptorId_t fieldId, ROOT::DescriptorId_t linkId)
RResult< void > AddClusterGroup(RClusterGroupDescriptor &&clusterGroup)
RResult< void > EnsureFieldExists(ROOT::DescriptorId_t fieldId) const
void SetFeature(unsigned int flag)
Sets the flag-th bit of the feature flag to 1.
Used for string interning of repeated type names, field names, etc.
const std::string * Intern(std::string_view str)
Searches for str in the pool.
The window of element indexes of a particular column in a particular cluster.
void SetPhysicalColumnId(ROOT::DescriptorId_t id)
ROOT::DescriptorId_t GetPhysicalColumnId() const
void IncrementNElements(ROOT::NTupleSize_t by)
Records the partition of data into pages for a particular column in a particular cluster.
static constexpr std::size_t kLargeRangeThreshold
Create the fCumulativeNElements only when its needed, i.e. when there are many pages to search throug...
RPageInfoExtended Find(ROOT::NTupleSize_t idxInCluster) const
Find the page in the RPageRange that contains the given element. The element must exist.
std::size_t ExtendToFitColumnRange(const RColumnRange &columnRange, const ROOT::Internal::RColumnElementBase &element, std::size_t pageSize)
Extend this RPageRange to fit the given RColumnRange.
Metadata for RNTuple clusters.
ROOT::NTupleSize_t fFirstEntryIndex
Clusters can be swapped by adjusting the entry offsets of the cluster and all ranges.
std::unordered_map< ROOT::DescriptorId_t, RColumnRange > fColumnRanges
ROOT::DescriptorId_t fClusterId
RClusterDescriptor Clone() const
bool operator==(const RClusterDescriptor &other) const
RColumnRangeIterable GetColumnRangeIterable() const
Returns an iterator over pairs { columnId, columnRange }. The iteration order is unspecified.
std::unordered_map< ROOT::DescriptorId_t, RPageRange > fPageRanges
std::uint64_t GetNBytesOnStorage() const
Clusters are bundled in cluster groups.
RNTupleLocator fPageListLocator
The page list that corresponds to the cluster group.
RClusterGroupDescriptor Clone() const
std::vector< ROOT::DescriptorId_t > fClusterIds
The cluster IDs can be empty if the corresponding page list is not loaded.
ROOT::DescriptorId_t GetId() const
std::uint64_t fMinEntry
The minimum first entry number of the clusters in the cluster group.
std::uint32_t fNClusters
Number of clusters is always known even if the cluster IDs are not (yet) populated.
std::uint64_t fPageListLength
Uncompressed size of the page list.
std::uint64_t GetPageListLength() const
RNTupleLocator GetPageListLocator() const
std::uint64_t fEntrySpan
Number of entries that are (partially for sharded clusters) covered by this cluster group.
bool operator==(const RClusterGroupDescriptor &other) const
RClusterGroupDescriptor CloneSummary() const
Creates a clone without the cluster IDs.
Metadata stored for every column of an RNTuple.
ROOT::DescriptorId_t fPhysicalColumnId
Usually identical to the logical column ID, except for alias columns where it references the shadowed...
bool operator==(const RColumnDescriptor &other) const
ROOT::DescriptorId_t fLogicalColumnId
The actual column identifier, which is the link to the corresponding field.
std::unique_ptr< RValueRange > fValueRange
Optional value range (used e.g. by quantized real fields)
ROOT::DescriptorId_t fFieldId
Every column belongs to one and only one field.
std::int64_t fFirstElementIndex
The absolute value specifies the index for the first stored element for this column.
std::uint32_t fIndex
A field can be serialized into several columns, which are numbered from zero to $n$.
std::uint16_t fBitsOnStorage
The size in bits of elements of this column.
std::uint16_t fRepresentationIndex
A field may use multiple column representations, which are numbered from zero to $m$.
ROOT::ENTupleColumnType fType
The on-disk column type.
RColumnDescriptor Clone() const
Get a copy of the descriptor.
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
Field specific extra type information from the header / extenstion header.
bool operator==(const RExtraTypeInfoDescriptor &other) const
RExtraTypeInfoDescriptor Clone() const
EExtraTypeInfoIds fContentId
Specifies the meaning of the extra information.
std::string fTypeName
The type name the extra information refers to; empty for RNTuple-wide extra information.
std::string fContent
The content format depends on the content ID and may be binary.
std::uint32_t fTypeVersion
Type version the extra type information is bound to.
A field translates read and write calls from/to underlying columns to/from tree values.
ROOT::ENTupleStructure GetStructure() const
@ kTraitEmulatedField
This field is a user defined type that was missing dictionaries and was reconstructed from the on-dis...
@ kTraitSoACollection
The field represents a collection in SoA layout.
@ kTraitInvalidField
This field is an instance of RInvalidField and can be safely static_cast to it.
@ kTraitTypeChecksum
The TClass checksum is set and valid.
const std::string & GetFieldName() const
const std::string & GetTypeAlias() const
const std::string & GetDescription() const
Get the field's description.
virtual std::uint32_t GetFieldVersion() const
Indicates an evolution of the mapping scheme from C++ type to columns.
virtual std::uint32_t GetTypeChecksum() const
Return the current TClass reported checksum of this class. Only valid if kTraitTypeChecksum is set.
std::uint32_t GetTraits() const
std::size_t GetNRepetitions() const
const std::string & GetTypeName() const
std::unique_ptr< RFieldBase > Clone(std::string_view newName) const
Copies the field and its subfields using a possibly new name and a new, unconnected set of columns.
virtual std::uint32_t GetTypeVersion() const
Indicates an evolution of the C++ type itself.
Metadata stored for every field of an RNTuple.
const std::string & GetTypeAlias() const
std::unique_ptr< ROOT::RFieldBase > CreateField(const RNTupleDescriptor &ntplDesc, const ROOT::RCreateFieldOptions &options={}) const
In general, we create a field simply from the C++ type name.
std::uint32_t fFieldVersion
The version of the C++-type-to-column translation mechanics.
ROOT::DescriptorId_t fFieldId
RFieldDescriptor Clone() const
Get a copy of the descriptor.
ROOT::DescriptorId_t GetId() const
std::uint64_t fNRepetitions
The number of elements per entry for fixed-size arrays.
const std::vector< ROOT::DescriptorId_t > & GetLogicalColumnIds() const
std::uint32_t fColumnCardinality
The number of columns in the column representations of the field.
ROOT::DescriptorId_t fProjectionSourceId
For projected fields, the source field ID.
static const std::string & GetEmptyString()
ROOT::ENTupleStructure GetStructure() const
bool operator==(const RFieldDescriptor &other) const
ROOT::DescriptorId_t fParentId
Establishes sub field relationships, such as classes and collections.
ROOT::DescriptorId_t GetParentId() const
ROOT::ENTupleStructure fStructure
The structural information carried by this field in the data model tree.
std::vector< ROOT::DescriptorId_t > fLinkIds
The pointers in the other direction from parent to children.
const std::string & GetFieldDescription() const
void InitFrom(const RFieldDescriptor &source, Internal::RStringPool &stringPool)
bool fIsSoACollection
Indicates if this is a collection that should be represented in memory by a SoA layout.
std::uint32_t fTypeVersion
The version of the C++ type itself.
const std::string & GetFieldName() const
std::unique_ptr< Internal::RStringPool > fStringPool
Optional string storage for a free-standing field descriptor.
std::vector< ROOT::DescriptorId_t > fLogicalColumnIds
The ordered list of columns attached to this field: first by representation index then by column inde...
const std::string & GetTypeName() const
std::optional< std::uint32_t > fTypeChecksum
For custom classes, we store the ROOT TClass reported checksum to facilitate the use of I/O rules tha...
Used in RFieldBase::Check() to record field creation failures.
Definition RField.hxx:96
@ kGeneric
Generic unrecoverable error.
@ kUnknownStructure
The field could not be created because its descriptor had an unknown structural role.
ECategory GetCategory() const
Definition RField.hxx:128
Metadata stored for every Attribute Set linked to an RNTuple.
RNTupleAttrSetDescriptor Clone() const
bool operator==(const RNTupleAttrSetDescriptor &other) const
std::uint32_t fAnchorLength
uncompressed size of the linked anchor
Loop over all the clusters of an RNTuple, ordered by first entry number.
Used to loop over all the cluster groups of an RNTuple in order of entry ranges.
Used to loop over a field's associated columns.
std::vector< ROOT::DescriptorId_t > fColumns
The descriptor ids of the columns ordered by field, representation, and column index.
RColumnDescriptorIterable(const RNTupleDescriptor &ntuple, const RFieldDescriptor &fieldDesc)
Used to loop over all the extra type info record of an RNTuple (in unspecified order)
Used to loop over a field's child fields.
std::vector< ROOT::DescriptorId_t > GetTopMostFields(const RNTupleDescriptor &desc) const
Return a vector containing the IDs of the top-level fields defined in the extension header,...
The on-storage metadata of an RNTuple.
std::vector< RNTupleAttrSetDescriptor > fAttributeSets
List of AttributeSets linked to this RNTuple.
const RColumnDescriptor & GetColumnDescriptor(ROOT::DescriptorId_t columnId) const
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
std::set< unsigned int > fFeatureFlags
std::unordered_map< ROOT::DescriptorId_t, RClusterGroupDescriptor > fClusterGroupDescriptors
const RFieldDescriptor & GetFieldDescriptor(ROOT::DescriptorId_t fieldId) const
RNTupleAttrSetDescriptorIterable GetAttrSetIterable() const
std::uint64_t fNPhysicalColumns
Updated by the descriptor builder when columns are added.
ROOT::DescriptorId_t fFieldZeroId
Set by the descriptor builder.
std::uint64_t fNEntries
Updated by the descriptor builder when the cluster groups are added.
RClusterDescriptorIterable GetActiveClusterIterable() const
RClusterGroupDescriptorIterable GetClusterGroupIterable() const
std::size_t GetNActiveClusters() const
RColumnDescriptorIterable GetColumnIterable() const
bool operator==(const RNTupleDescriptor &other) const
std::uint64_t fOnDiskFooterSize
Like fOnDiskHeaderSize, contains both cluster summaries and page locations.
std::uint16_t fVersionMinor
Set by the descriptor builder when deserialized.
ROOT::DescriptorId_t FindClusterId(ROOT::NTupleSize_t entryIdx) const
ROOT::NTupleSize_t R__DEPRECATED(6, 46, "This function is ill-conceived in the descriptor " "as not all cluster descriptors may be present. This interface is no longer publicly exposed.") GetNElements(ROOT ROOT::DescriptorId_t GetFieldZeroId() const
Returns the logical parent of all top-level RNTuple data fields.
std::vector< std::uint64_t > GetFeatureFlags() const
std::unique_ptr< ROOT::RNTupleModel > CreateModel(const RCreateModelOptions &options=RCreateModelOptions()) const
Re-create the C++ model from the stored metadata.
std::string GetTypeNameForComparison(const RFieldDescriptor &fieldDesc) const
Adjust the type name of the passed RFieldDescriptor for comparison with another renormalized type nam...
std::unordered_map< ROOT::DescriptorId_t, RClusterDescriptor > fClusterDescriptors
Potentially a subset of all the available clusters.
std::size_t GetNClusters() const
ROOT::DescriptorId_t FindPhysicalColumnId(ROOT::DescriptorId_t fieldId, std::uint32_t columnIndex, std::uint16_t representationIndex) const
RExtraTypeInfoDescriptorIterable GetExtraTypeInfoIterable() const
std::shared_ptr< Internal::RStringPool > fStringPool
Storage for all the field strings. Shared among descriptor clones.
const RHeaderExtension * GetHeaderExtension() const
Return header extension information; if the descriptor does not have a header extension,...
std::uint64_t fNClusters
Updated by the descriptor builder when the cluster groups are added.
RNTupleDescriptor CloneSchema(bool shareStringPool) const
Creates a descriptor containing only the schema information about this RNTuple, i....
std::uint64_t fOnDiskHeaderXxHash3
Set by the descriptor builder when deserialized.
ROOT::DescriptorId_t FindFieldId(std::string_view fieldName, ROOT::DescriptorId_t parentId) const
std::string fName
The RNTuple name needs to be unique in a given storage location (file)
std::uint64_t fOnDiskHeaderSize
Set by the descriptor builder when deserialized.
RResult< void > DropClusterGroupDetails(ROOT::DescriptorId_t clusterGroupId)
std::uint16_t fVersionMajor
Set by the descriptor builder when deserialized.
std::vector< ROOT::DescriptorId_t > fSortedClusterGroupIds
References cluster groups sorted by entry range and thus allows for binary search.
std::unordered_map< ROOT::DescriptorId_t, RColumnDescriptor > fColumnDescriptors
ROOT::DescriptorId_t FindLogicalColumnId(ROOT::DescriptorId_t fieldId, std::uint32_t columnIndex, std::uint16_t representationIndex) const
std::unordered_map< ROOT::DescriptorId_t, RFieldDescriptor > fFieldDescriptors
RResult< void > AddClusterGroupDetails(ROOT::DescriptorId_t clusterGroupId, std::vector< RClusterDescriptor > &clusterDescs)
Methods to load and drop cluster group details (cluster IDs and page locations)
std::uint16_t fVersionPatch
Set by the descriptor builder when deserialized.
std::string fDescription
Free text from the user.
RFieldDescriptorIterable GetTopLevelFields() const
std::uint16_t fVersionEpoch
Set by the descriptor builder when deserialized.
std::vector< RExtraTypeInfoDescriptor > fExtraTypeInfoDescriptors
RNTupleDescriptor Clone() const
ROOT::DescriptorId_t R__DEPRECATED(6, 46, "This function is ill-defined in the descriptor " "as not all cluster descriptors may be present. This interface is no longer exposed.") FindClusterId(ROOT ROOT::DescriptorId_t R__DEPRECATED(6, 46, "This function is ill-defined in the descriptor " "as not all cluster descriptors may be present. This interface is no longer exposed.") FindNextClusterId(ROOT ROOT::DescriptorId_t R__DEPRECATED(6, 46, "This function is ill-defined in the descriptor " "as not all cluster descriptors may be present. This interface is no longer exposed.") FindPrevClusterId(ROOT std::string GetQualifiedFieldName(ROOT::DescriptorId_t fieldId) const
Walks up the parents of the field ID and returns a field name of the form a.b.c.d In case of invalid ...
bool FieldTypeNamesMayNeedFixup() const
ROOT v6.34, with spec versions before 1.0.0.1, did not properly renormalize the type name.
std::uint64_t fGeneration
The generation of the descriptor.
std::unique_ptr< RHeaderExtension > fHeaderExtension
Generic information about the physical location of data.
void SetType(ELocatorType type)
void SetNBytesOnStorage(std::uint64_t nBytesOnStorage)
static std::unique_ptr< RNTupleModel > Create()
static std::unique_ptr< RNTupleModel > CreateBare()
Creates a "bare model", i.e. an RNTupleModel with no default entry.
static constexpr std::uint16_t kVersionPatch
Definition RNTuple.hxx:82
static constexpr std::uint16_t kVersionMajor
Definition RNTuple.hxx:80
static constexpr std::uint16_t kVersionEpoch
Definition RNTuple.hxx:79
static constexpr std::uint16_t kVersionMinor
Definition RNTuple.hxx:81
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:222
static std::unique_ptr< RVectorField > CreateUntyped(std::string_view fieldName, std::unique_ptr< RFieldBase > itemField)
const Int_t n
Definition legend1.C:16
Double_t ex[n]
Definition legend1.C:17
RResult< void > EnsureValidNameForRNTuple(std::string_view name, std::string_view where)
Check whether a given string is a valid name according to the RNTuple specification.
ROOT::RResult< std::unique_ptr< ROOT::RFieldBase > > CallFieldBaseCreate(const std::string &fieldName, const std::string &typeName, const ROOT::RCreateFieldOptions &options, const ROOT::RNTupleDescriptor *desc, ROOT::DescriptorId_t fieldId)
bool IsCustomEnumFieldDesc(const RNTupleDescriptor &desc, const RFieldDescriptor &fieldDesc)
Tells if the field describes a user-defined enum type.
std::vector< ROOT::Internal::RNTupleClusterBoundaries > GetClusterBoundaries(const RNTupleDescriptor &desc)
Return the cluster boundaries for each cluster in this RNTuple.
std::string GetRenormalizedTypeName(const std::string &metaNormalizedName)
Given a type name normalized by ROOT meta, renormalize it for RNTuple. E.g., insert std::prefix.
bool IsStdAtomicFieldDesc(const RFieldDescriptor &fieldDesc)
Tells if the field describes a std::atomic<T> type.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
constexpr NTupleSize_t kInvalidNTupleIndex
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
constexpr DescriptorId_t kInvalidDescriptorId
Additional information about a page in an in-memory RPageRange.
Information about a single page in the context of a cluster's page range.
void SetLocator(const RNTupleLocator &locator)
static uint64_t sum(uint64_t i)
Definition Factory.cxx:2335