24#ifndef R__LITTLE_ENDIAN
27#define R__LITTLE_ENDIAN 1
29#define R__LITTLE_ENDIAN 0
41constexpr std::size_t
MinBufSize(std::size_t count, std::size_t nDstBits)
43 return (count * nDstBits + 7) / 8;
51void PackBits(
void *dst,
const void *src, std::size_t count, std::size_t sizeofSrc, std::size_t nDstBits);
57void UnpackBits(
void *dst,
const void *src, std::size_t count, std::size_t sizeofDst, std::size_t nSrcBits);
89template <std::
size_t N>
90inline void CopyBswap(
void *destination,
const void *source, std::size_t count)
94 for (std::size_t i = 0; i < count; ++i) {
99template <std::
size_t N>
100void InPlaceBswap(
void *array, std::size_t count)
103 for (std::size_t i = 0; i < count; ++i) {
109#if R__LITTLE_ENDIAN == 0
111inline void ByteSwapIfNecessary(T &value)
113 constexpr auto N =
sizeof(T);
115 void *valuePtr = &value;
117 *
reinterpret_cast<bswap_value_type *
>(valuePtr) = swapped;
120inline void ByteSwapIfNecessary<char>(
char &)
124inline void ByteSwapIfNecessary<signed char>(
signed char &)
128inline void ByteSwapIfNecessary<unsigned char>(
unsigned char &)
132#define ByteSwapIfNecessary(x) ((void)0)
136template <
typename DestT,
typename SourceT>
137inline void EnsureValidRange(SourceT val [[maybe_unused]])
141 if constexpr (!std::is_integral_v<DestT> || !std::is_integral_v<SourceT>)
144 if constexpr (
static_cast<double>(std::numeric_limits<SourceT>::min()) <
145 static_cast<double>(std::numeric_limits<DestT>::min())) {
146 if constexpr (!std::is_signed_v<DestT>) {
148 throw RException(
R__FAIL(std::string(
"value out of range: ") + std::to_string(val) +
" for type " +
149 typeid(DestT).
name()));
151 }
else if (val < std::numeric_limits<DestT>::min()) {
153 R__FAIL(std::string(
"value out of range: ") + std::to_string(val) +
" for type " +
typeid(DestT).
name()));
157 if constexpr (
static_cast<double>(std::numeric_limits<SourceT>::max()) >
158 static_cast<double>(std::numeric_limits<DestT>::max())) {
159 if (val > std::numeric_limits<DestT>::max()) {
161 R__FAIL(std::string(
"value out of range: ") + std::to_string(val) +
" for type " +
typeid(DestT).
name()));
167template <
typename DestT,
typename SourceT>
168inline void EnsureValidConversion(DestT dst [[maybe_unused]], SourceT src [[maybe_unused]])
170 if constexpr (std::is_same_v<DestT, float> && std::is_same_v<SourceT, double>) {
171 if (std::fpclassify(src) != std::fpclassify(dst)) {
173 " on disk to " + std::to_string(dst) +
" in memory"));
182template <
typename DestT,
typename SourceT>
183inline void CastPack(
void *destination,
const void *source, std::size_t count)
185 static_assert(std::is_convertible_v<SourceT, DestT>);
186 auto dst =
reinterpret_cast<DestT *
>(destination);
187 auto src =
reinterpret_cast<const SourceT *
>(source);
188 for (std::size_t i = 0; i < count; ++i) {
190 ByteSwapIfNecessary(dst[i]);
198template <
typename DestT,
typename SourceT>
199inline void CastUnpack(
void *destination,
const void *source, std::size_t count)
201 auto dst =
reinterpret_cast<DestT *
>(destination);
202 auto src =
reinterpret_cast<const SourceT *
>(source);
203 for (std::size_t i = 0; i < count; ++i) {
204 SourceT val = src[i];
205 ByteSwapIfNecessary(val);
206 EnsureValidRange<DestT, SourceT>(val);
208 EnsureValidConversion<DestT, SourceT>(dst[i], val);
215template <
typename DestT,
typename SourceT>
216inline void CastSplitPack(
void *destination,
const void *source, std::size_t count)
218 constexpr std::size_t
N =
sizeof(DestT);
219 auto splitArray =
reinterpret_cast<char *
>(destination);
220 auto src =
reinterpret_cast<const SourceT *
>(source);
221 for (std::size_t i = 0; i < count; ++i) {
223 ByteSwapIfNecessary(val);
224 for (std::size_t
b = 0;
b <
N; ++
b) {
225 splitArray[
b * count + i] =
reinterpret_cast<const char *
>(&val)[
b];
233template <
typename DestT,
typename SourceT>
234inline void CastSplitUnpack(
void *destination,
const void *source, std::size_t count)
236 constexpr std::size_t
N =
sizeof(SourceT);
237 auto dst =
reinterpret_cast<DestT *
>(destination);
238 auto splitArray =
reinterpret_cast<const char *
>(source);
239 for (std::size_t i = 0; i < count; ++i) {
241 for (std::size_t
b = 0;
b <
N; ++
b) {
242 reinterpret_cast<char *
>(&val)[
b] = splitArray[
b * count + i];
244 ByteSwapIfNecessary(val);
245 EnsureValidRange<DestT, SourceT>(val);
247 EnsureValidConversion<DestT, SourceT>(dst[i], val);
254template <
typename DestT,
typename SourceT>
255inline void CastDeltaSplitPack(
void *destination,
const void *source, std::size_t count)
257 constexpr std::size_t
N =
sizeof(DestT);
258 auto src =
reinterpret_cast<const SourceT *
>(source);
259 auto splitArray =
reinterpret_cast<char *
>(destination);
260 for (std::size_t i = 0; i < count; ++i) {
261 DestT val = (i == 0) ? src[0] : src[i] - src[i - 1];
262 ByteSwapIfNecessary(val);
263 for (std::size_t
b = 0;
b <
N; ++
b) {
264 splitArray[
b * count + i] =
reinterpret_cast<char *
>(&val)[
b];
272template <
typename DestT,
typename SourceT>
273inline void CastDeltaSplitUnpack(
void *destination,
const void *source, std::size_t count)
275 constexpr std::size_t
N =
sizeof(SourceT);
276 auto splitArray =
reinterpret_cast<const char *
>(source);
277 auto dst =
reinterpret_cast<DestT *
>(destination);
278 for (std::size_t i = 0; i < count; ++i) {
280 for (std::size_t
b = 0;
b <
N; ++
b) {
281 reinterpret_cast<char *
>(&val)[
b] = splitArray[
b * count + i];
283 ByteSwapIfNecessary(val);
284 val = (i == 0) ? val : val + dst[i - 1];
285 EnsureValidRange<DestT, SourceT>(val);
293template <
typename DestT,
typename SourceT>
294inline void CastZigzagSplitPack(
void *destination,
const void *source, std::size_t count)
296 using UDestT = std::make_unsigned_t<DestT>;
297 constexpr std::size_t kNBitsDestT =
sizeof(DestT) * 8;
298 constexpr std::size_t
N =
sizeof(DestT);
299 auto src =
reinterpret_cast<const SourceT *
>(source);
300 auto splitArray =
reinterpret_cast<char *
>(destination);
301 for (std::size_t i = 0; i < count; ++i) {
302 UDestT val = (
static_cast<DestT
>(src[i]) << 1) ^ (
static_cast<DestT
>(src[i]) >> (kNBitsDestT - 1));
303 ByteSwapIfNecessary(val);
304 for (std::size_t
b = 0;
b <
N; ++
b) {
305 splitArray[
b * count + i] =
reinterpret_cast<char *
>(&val)[
b];
313template <
typename DestT,
typename SourceT>
314inline void CastZigzagSplitUnpack(
void *destination,
const void *source, std::size_t count)
316 using USourceT = std::make_unsigned_t<SourceT>;
317 constexpr std::size_t
N =
sizeof(SourceT);
318 auto splitArray =
reinterpret_cast<const char *
>(source);
319 auto dst =
reinterpret_cast<DestT *
>(destination);
320 for (std::size_t i = 0; i < count; ++i) {
322 for (std::size_t
b = 0;
b <
N; ++
b) {
323 reinterpret_cast<char *
>(&val)[
b] = splitArray[
b * count + i];
325 ByteSwapIfNecessary(val);
326 SourceT sval =
static_cast<SourceT
>((val >> 1) ^ -(
static_cast<SourceT
>(val) & 1));
327 EnsureValidRange<DestT, SourceT>(sval);
341template <
typename CppT, ENTupleColumnType>
344template <
typename CppT>
345std::unique_ptr<RColumnElementBase> GenerateColumnElementInternal(
ENTupleColumnType onDiskType)
347 switch (onDiskType) {
348 case ENTupleColumnType::kIndex64:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kIndex64>>();
349 case ENTupleColumnType::kIndex32:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kIndex32>>();
350 case ENTupleColumnType::kSwitch:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSwitch>>();
351 case ENTupleColumnType::kByte:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kByte>>();
352 case ENTupleColumnType::kChar:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kChar>>();
353 case ENTupleColumnType::kBit:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kBit>>();
354 case ENTupleColumnType::kReal64:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kReal64>>();
355 case ENTupleColumnType::kReal32:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kReal32>>();
356 case ENTupleColumnType::kReal16:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kReal16>>();
357 case ENTupleColumnType::kInt64:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kInt64>>();
358 case ENTupleColumnType::kUInt64:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kUInt64>>();
359 case ENTupleColumnType::kInt32:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kInt32>>();
360 case ENTupleColumnType::kUInt32:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kUInt32>>();
361 case ENTupleColumnType::kInt16:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kInt16>>();
362 case ENTupleColumnType::kUInt16:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kUInt16>>();
363 case ENTupleColumnType::kInt8:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kInt8>>();
364 case ENTupleColumnType::kUInt8:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kUInt8>>();
365 case ENTupleColumnType::kSplitIndex64:
366 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitIndex64>>();
367 case ENTupleColumnType::kSplitIndex32:
368 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitIndex32>>();
369 case ENTupleColumnType::kSplitReal64:
370 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitReal64>>();
371 case ENTupleColumnType::kSplitReal32:
372 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitReal32>>();
373 case ENTupleColumnType::kSplitInt64:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitInt64>>();
374 case ENTupleColumnType::kSplitUInt64:
375 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitUInt64>>();
376 case ENTupleColumnType::kSplitInt32:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitInt32>>();
377 case ENTupleColumnType::kSplitUInt32:
378 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitUInt32>>();
379 case ENTupleColumnType::kSplitInt16:
return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitInt16>>();
380 case ENTupleColumnType::kSplitUInt16:
381 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kSplitUInt16>>();
382 case ENTupleColumnType::kReal32Trunc:
383 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kReal32Trunc>>();
384 case ENTupleColumnType::kReal32Quant:
385 return std::make_unique<RColumnElement<CppT, ENTupleColumnType::kReal32Quant>>();
387 if (onDiskType == kTestFutureColumnType)
388 return std::make_unique<RColumnElement<CppT, kTestFutureColumnType>>();
399template <
typename CppT>
402 explicit RColumnElementLE(std::size_t
size, std::size_t bitsOnStorage) : RColumnElementBase(
size, bitsOnStorage) {}
407 void Pack(
void *dst,
const void *src, std::size_t count)
const final
409#if R__LITTLE_ENDIAN == 1
412 CopyBswap<sizeof(CppT)>(dst, src, count);
415 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
417#if R__LITTLE_ENDIAN == 1
420 CopyBswap<sizeof(CppT)>(dst, src, count);
429template <
typename CppT,
typename NarrowT>
432 explicit RColumnElementCastLE(std::size_t
size, std::size_t bitsOnStorage) : RColumnElementBase(
size, bitsOnStorage)
437 static constexpr bool kIsMappable =
false;
439 void Pack(
void *dst,
const void *src, std::size_t count)
const final { CastPack<NarrowT, CppT>(dst, src, count); }
440 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
442 CastUnpack<CppT, NarrowT>(dst, src, count);
451template <
typename CppT,
typename NarrowT>
454 explicit RColumnElementSplitLE(std::size_t
size, std::size_t bitsOnStorage) : RColumnElementBase(
size, bitsOnStorage)
459 static constexpr bool kIsMappable =
false;
461 void Pack(
void *dst,
const void *src, std::size_t count)
const final
463 CastSplitPack<NarrowT, CppT>(dst, src, count);
465 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
467 CastSplitUnpack<CppT, NarrowT>(dst, src, count);
476template <
typename CppT,
typename NarrowT>
479 explicit RColumnElementDeltaSplitLE(std::size_t
size, std::size_t bitsOnStorage)
480 : RColumnElementBase(
size, bitsOnStorage)
485 static constexpr bool kIsMappable =
false;
487 void Pack(
void *dst,
const void *src, std::size_t count)
const final
489 CastDeltaSplitPack<NarrowT, CppT>(dst, src, count);
491 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
493 CastDeltaSplitUnpack<CppT, NarrowT>(dst, src, count);
498template <
typename CppIntT>
501 explicit RColumnElementBoolAsUnsplitInt(std::size_t
size, std::size_t bitsOnStorage)
502 : RColumnElementBase(
size, bitsOnStorage)
507 static constexpr bool kIsMappable =
false;
510 void Pack(
void *,
const void *, std::size_t)
const final {
R__ASSERT(
false); }
512 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
514 auto *boolArray =
reinterpret_cast<bool *
>(dst);
515 auto *intArray =
reinterpret_cast<const CppIntT *
>(src);
516 for (std::size_t i = 0; i < count; ++i) {
517 boolArray[i] = intArray[i] != 0;
523template <
typename CppIntT>
526 explicit RColumnElementBoolAsSplitInt(std::size_t
size, std::size_t bitsOnStorage)
527 : RColumnElementBase(
size, bitsOnStorage)
532 static constexpr bool kIsMappable =
false;
535 void Pack(
void *,
const void *, std::size_t)
const final {
R__ASSERT(
false); }
537 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
539 constexpr std::size_t
N =
sizeof(CppIntT);
540 auto *boolArray =
reinterpret_cast<bool *
>(dst);
541 auto *splitArray =
reinterpret_cast<const char *
>(src);
542 for (std::size_t i = 0; i < count; ++i) {
543 boolArray[i] =
false;
544 for (std::size_t
b = 0;
b <
N; ++
b) {
545 if (splitArray[
b * count + i]) {
555template <
typename CppIntT>
558 explicit RColumnElementIntAsBool(std::size_t
size, std::size_t bitsOnStorage)
559 : RColumnElementBase(
size, bitsOnStorage)
564 static constexpr bool kIsMappable =
false;
567 void Pack(
void *,
const void *, std::size_t)
const final {
R__ASSERT(
false); }
569 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
571 auto *intArray =
reinterpret_cast<CppIntT *
>(dst);
572 const char *charArray =
reinterpret_cast<const char *
>(src);
573 std::bitset<8> bitSet;
574 for (std::size_t i = 0; i < count; i += 8) {
575 bitSet = charArray[i / 8];
576 for (std::size_t j = i; j < std::min(count, i + 8); ++j) {
577 intArray[j] = bitSet[j % 8];
588template <
typename CppT,
typename NarrowT>
591 explicit RColumnElementZigzagSplitLE(std::size_t
size, std::size_t bitsOnStorage)
592 : RColumnElementBase(
size, bitsOnStorage)
597 static constexpr bool kIsMappable =
false;
599 void Pack(
void *dst,
const void *src, std::size_t count)
const final
601 CastZigzagSplitPack<NarrowT, CppT>(dst, src, count);
603 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
605 CastZigzagSplitUnpack<CppT, NarrowT>(dst, src, count);
617template <
typename CppT, ENTupleColumnType ColumnT = ENTupleColumnType::kUnknown>
620 RColumnElement() : RColumnElementBase(sizeof(CppT))
622 throw ROOT::RException(
R__FAIL(std::string(
"internal error: no column mapping for this C++ type: ") +
623 typeid(CppT).
name() +
" --> " + GetColumnTypeName(ColumnT)));
626 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(CppT), ENTupleColumnType::kUnknown}; }
632 static constexpr std::size_t
kSize =
sizeof(bool);
633 RColumnElement() : RColumnElementBase(
kSize) {}
634 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(bool), ENTupleColumnType::kUnknown}; }
638class RColumnElement<std::byte, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
640 static constexpr std::size_t
kSize =
sizeof(std::byte);
641 RColumnElement() : RColumnElementBase(
kSize) {}
642 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::byte), ENTupleColumnType::kUnknown}; }
648 static constexpr std::size_t
kSize =
sizeof(char);
649 RColumnElement() : RColumnElementBase(
kSize) {}
650 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(char), ENTupleColumnType::kUnknown}; }
654class RColumnElement<std::int8_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
656 static constexpr std::size_t
kSize =
sizeof(std::int8_t);
657 RColumnElement() : RColumnElementBase(
kSize) {}
658 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::int8_t), ENTupleColumnType::kUnknown}; }
662class RColumnElement<std::uint8_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
664 static constexpr std::size_t
kSize =
sizeof(std::uint8_t);
665 RColumnElement() : RColumnElementBase(
kSize) {}
666 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::uint8_t), ENTupleColumnType::kUnknown}; }
670class RColumnElement<std::int16_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
672 static constexpr std::size_t
kSize =
sizeof(std::int16_t);
673 RColumnElement() : RColumnElementBase(
kSize) {}
674 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::int16_t), ENTupleColumnType::kUnknown}; }
678class RColumnElement<std::uint16_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
680 static constexpr std::size_t
kSize =
sizeof(std::uint16_t);
681 RColumnElement() : RColumnElementBase(
kSize) {}
682 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::uint16_t), ENTupleColumnType::kUnknown}; }
686class RColumnElement<std::int32_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
688 static constexpr std::size_t
kSize =
sizeof(std::int32_t);
689 RColumnElement() : RColumnElementBase(
kSize) {}
690 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::int32_t), ENTupleColumnType::kUnknown}; }
694class RColumnElement<std::uint32_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
696 static constexpr std::size_t
kSize =
sizeof(std::uint32_t);
697 RColumnElement() : RColumnElementBase(
kSize) {}
698 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::uint32_t), ENTupleColumnType::kUnknown}; }
702class RColumnElement<std::int64_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
704 static constexpr std::size_t
kSize =
sizeof(std::int64_t);
705 RColumnElement() : RColumnElementBase(
kSize) {}
706 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::int64_t), ENTupleColumnType::kUnknown}; }
710class RColumnElement<std::uint64_t, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
712 static constexpr std::size_t
kSize =
sizeof(std::uint64_t);
713 RColumnElement() : RColumnElementBase(
kSize) {}
714 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(std::uint64_t), ENTupleColumnType::kUnknown}; }
720 static constexpr std::size_t
kSize =
sizeof(float);
721 RColumnElement() : RColumnElementBase(
kSize) {}
722 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(float), ENTupleColumnType::kUnknown}; }
728 static constexpr std::size_t
kSize =
sizeof(double);
729 RColumnElement() : RColumnElementBase(
kSize) {}
730 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(double), ENTupleColumnType::kUnknown}; }
734class RColumnElement<
ROOT::Internal::RColumnIndex, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
736 static constexpr std::size_t
kSize =
sizeof(ROOT::Internal::RColumnIndex);
737 RColumnElement() : RColumnElementBase(
kSize) {}
738 RIdentifier GetIdentifier() const final
740 return RIdentifier{
typeid(ROOT::Internal::RColumnIndex), ENTupleColumnType::kUnknown};
745class RColumnElement<
ROOT::Internal::RColumnSwitch, ENTupleColumnType::kUnknown> :
public RColumnElementBase {
747 static constexpr std::size_t
kSize =
sizeof(ROOT::Internal::RColumnSwitch);
748 RColumnElement() : RColumnElementBase(
kSize) {}
749 RIdentifier GetIdentifier() const final
751 return RIdentifier{
typeid(ROOT::Internal::RColumnSwitch), ENTupleColumnType::kUnknown};
761class RColumnElement<
ROOT::Internal::RColumnSwitch, ENTupleColumnType::kSwitch> :
public RColumnElementBase {
763 struct RSwitchElement {
764 std::uint64_t fIndex;
769 static constexpr bool kIsMappable =
false;
770 static constexpr std::size_t
kSize =
sizeof(ROOT::Internal::RColumnSwitch);
771 static constexpr std::size_t kBitsOnStorage = 96;
772 RColumnElement() : RColumnElementBase(
kSize, kBitsOnStorage) {}
773 bool IsMappable() const final {
return kIsMappable; }
775 void Pack(
void *dst,
const void *src, std::size_t count)
const final
777 auto srcArray =
reinterpret_cast<const ROOT::Internal::RColumnSwitch *
>(src);
778 auto dstArray =
reinterpret_cast<unsigned char *
>(dst);
779 for (std::size_t i = 0; i < count; ++i) {
780 RSwitchElement element{srcArray[i].GetIndex(), srcArray[i].GetTag()};
781#if R__LITTLE_ENDIAN == 0
785 memcpy(dstArray + i * 12, &element, 12);
789 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
791 auto srcArray =
reinterpret_cast<const unsigned char *
>(src);
792 auto dstArray =
reinterpret_cast<ROOT::Internal::RColumnSwitch *
>(dst);
793 for (std::size_t i = 0; i < count; ++i) {
794 RSwitchElement element;
795 memcpy(&element, srcArray + i * 12, 12);
796#if R__LITTLE_ENDIAN == 0
800 dstArray[i] = ROOT::Internal::RColumnSwitch(element.fIndex, element.fTag);
804 RIdentifier GetIdentifier() const final
806 return RIdentifier{
typeid(ROOT::Internal::RColumnSwitch), ENTupleColumnType::kSwitch};
813 static constexpr bool kIsMappable =
false;
814 static constexpr std::size_t
kSize =
sizeof(bool);
815 static constexpr std::size_t kBitsOnStorage = 1;
816 RColumnElement() : RColumnElementBase(
kSize, kBitsOnStorage) {}
817 bool IsMappable() const final {
return kIsMappable; }
819 void Pack(
void *dst,
const void *src, std::size_t count)
const final;
820 void Unpack(
void *dst,
const void *src, std::size_t count)
const final;
822 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(bool), ENTupleColumnType::kBit}; }
828 static constexpr bool kIsMappable =
false;
829 static constexpr std::size_t
kSize =
sizeof(float);
830 static constexpr std::size_t kBitsOnStorage = 16;
831 RColumnElement() : RColumnElementBase(
kSize, kBitsOnStorage) {}
832 bool IsMappable() const final {
return kIsMappable; }
834 void Pack(
void *dst,
const void *src, std::size_t count)
const final
836 const float *floatArray =
reinterpret_cast<const float *
>(src);
837 std::uint16_t *uint16Array =
reinterpret_cast<std::uint16_t *
>(dst);
839 for (std::size_t i = 0; i < count; ++i) {
841 ByteSwapIfNecessary(uint16Array[i]);
845 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
847 float *floatArray =
reinterpret_cast<float *
>(dst);
848 const std::uint16_t *uint16Array =
reinterpret_cast<const std::uint16_t *
>(src);
850 for (std::size_t i = 0; i < count; ++i) {
851 std::uint16_t val = uint16Array[i];
852 ByteSwapIfNecessary(val);
857 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(float), ENTupleColumnType::kReal16}; }
863 static constexpr bool kIsMappable =
false;
864 static constexpr std::size_t
kSize =
sizeof(double);
865 static constexpr std::size_t kBitsOnStorage = 16;
866 RColumnElement() : RColumnElementBase(
kSize, kBitsOnStorage) {}
867 bool IsMappable() const final {
return kIsMappable; }
869 void Pack(
void *dst,
const void *src, std::size_t count)
const final
871 const double *doubleArray =
reinterpret_cast<const double *
>(src);
872 std::uint16_t *uint16Array =
reinterpret_cast<std::uint16_t *
>(dst);
874 for (std::size_t i = 0; i < count; ++i) {
876 ByteSwapIfNecessary(uint16Array[i]);
880 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
882 double *doubleArray =
reinterpret_cast<double *
>(dst);
883 const std::uint16_t *uint16Array =
reinterpret_cast<const std::uint16_t *
>(src);
885 for (std::size_t i = 0; i < count; ++i) {
886 std::uint16_t val = uint16Array[i];
887 ByteSwapIfNecessary(val);
892 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(double), ENTupleColumnType::kReal16}; }
898 static_assert(std::is_floating_point_v<T>);
899 static constexpr bool kIsMappable =
false;
900 static constexpr std::size_t kSize =
sizeof(
T);
905 RColumnElementTrunc() : RColumnElementBase(kSize, 0) {}
907 void SetBitsOnStorage(std::size_t bitsOnStorage)
final
909 const auto &[minBits, maxBits] = GetValidBitRange(ENTupleColumnType::kReal32Trunc);
910 R__ASSERT(bitsOnStorage >= minBits && bitsOnStorage <= maxBits);
911 fBitsOnStorage = bitsOnStorage;
914 bool IsMappable() const final {
return kIsMappable; }
916 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(
T), ENTupleColumnType::kReal32Trunc}; }
922 void Pack(
void *dst,
const void *src, std::size_t count)
const final
924 using namespace ROOT::Internal::BitPacking;
928#if R__LITTLE_ENDIAN == 0
932 CopyBswap<sizeof(float)>(bswapped.get(), src, count);
933 const auto *srcLe = bswapped.get();
935 const auto *srcLe =
reinterpret_cast<const float *
>(src);
937 PackBits(dst, srcLe, count,
sizeof(
float), fBitsOnStorage);
940 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
942 using namespace ROOT::Internal::BitPacking;
946 UnpackBits(dst, src, count,
sizeof(
float), fBitsOnStorage);
947#if R__LITTLE_ENDIAN == 0
948 InPlaceBswap<sizeof(float)>(dst, count);
956 void Pack(
void *dst,
const void *src, std::size_t count)
const final
958 using namespace ROOT::Internal::BitPacking;
965 const double *srcDouble =
reinterpret_cast<const double *
>(src);
966 for (std::size_t i = 0; i < count; ++i)
967 srcFloat[i] =
static_cast<float>(srcDouble[i]);
969#if R__LITTLE_ENDIAN == 0
973 CopyBswap<sizeof(float)>(bswapped.get(), srcFloat.get(), count);
974 const float *srcLe = bswapped.get();
976 const float *srcLe =
reinterpret_cast<const float *
>(srcFloat.get());
978 PackBits(dst, srcLe, count,
sizeof(
float), fBitsOnStorage);
981 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
983 using namespace ROOT::Internal::BitPacking;
989 UnpackBits(dstFloat.get(), src, count,
sizeof(
float), fBitsOnStorage);
990#if R__LITTLE_ENDIAN == 0
991 InPlaceBswap<sizeof(float)>(dstFloat.get(), count);
994 double *dstDouble =
reinterpret_cast<double *
>(dst);
995 for (std::size_t i = 0; i < count; ++i)
996 dstDouble[i] =
static_cast<double>(dstFloat[i]);
1002using Quantized_t = std::uint32_t;
1010template <
typename T>
1011int QuantizeReals(Quantized_t *dst,
const T *src, std::size_t count,
double min,
double max, std::size_t nQuantBits)
1013 static_assert(std::is_floating_point_v<T>);
1014 static_assert(
sizeof(T) <=
sizeof(
double));
1015 assert(1 <= nQuantBits && nQuantBits <= 8 *
sizeof(Quantized_t));
1019 assert(min ==
static_cast<double>(
static_cast<T
>(min)));
1020 assert(max ==
static_cast<double>(
static_cast<T
>(max)));
1022 const std::size_t quantMax = (1ull << nQuantBits) - 1;
1023 const double scale = quantMax / (max - min);
1024 const std::size_t unusedBits =
sizeof(Quantized_t) * 8 - nQuantBits;
1026 int nOutOfRange = 0;
1028 for (std::size_t i = 0; i < count; ++i) {
1029 const T elem = src[i];
1031 bool outOfRange = !(min <= elem && elem <= max);
1032 nOutOfRange += outOfRange;
1034 const double e = 0.5 + (elem - min) * scale;
1035 Quantized_t
q =
static_cast<Quantized_t
>(
e);
1036 ByteSwapIfNecessary(
q);
1043 dst[i] =
q << unusedBits;
1051template <
typename T>
1052int UnquantizeReals(T *dst,
const Quantized_t *src, std::size_t count,
double min,
double max, std::size_t nQuantBits)
1054 static_assert(std::is_floating_point_v<T>);
1055 static_assert(
sizeof(T) <=
sizeof(
double));
1056 assert(1 <= nQuantBits && nQuantBits <= 8 *
sizeof(Quantized_t));
1058 const std::size_t quantMax = (1ull << nQuantBits) - 1;
1059 const double scale = (max - min) / quantMax;
1060 const std::size_t unusedBits =
sizeof(Quantized_t) * 8 - nQuantBits;
1061 const double eps = std::numeric_limits<double>::epsilon();
1062 const double emax = max + std::max(1.0, std::abs(max)) * eps;
1064 int nOutOfRange = 0;
1066 for (std::size_t i = 0; i < count; ++i) {
1067 Quantized_t elem = src[i];
1070 elem >>= unusedBits;
1071 ByteSwapIfNecessary(elem);
1073 const double fq =
static_cast<double>(elem);
1074 double e = fq * scale + min;
1081 nOutOfRange += !(min <=
e &&
e <= emax);
1085 e = std::min(
e, max);
1087 dst[i] =
static_cast<T
>(
e);
1094template <
typename T>
1096 static_assert(std::is_floating_point_v<T>);
1099 static constexpr bool kIsMappable =
false;
1100 static constexpr std::size_t
kSize =
sizeof(T);
1104 void SetBitsOnStorage(std::size_t bitsOnStorage)
final
1106 const auto [minBits, maxBits] = GetValidBitRange(ENTupleColumnType::kReal32Quant);
1107 R__ASSERT(bitsOnStorage >= minBits && bitsOnStorage <= maxBits);
1108 fBitsOnStorage = bitsOnStorage;
1111 void SetValueRange(
double min,
double max)
final
1113 R__ASSERT(min >= std::numeric_limits<T>::lowest());
1114 R__ASSERT(max <= std::numeric_limits<T>::max());
1116 R__ASSERT(std::isnormal(min) || min == 0.0);
1117 R__ASSERT(std::isnormal(max) || max == 0.0);
1118 fValueRange = {min,
max};
1121 bool IsMappable() const final {
return kIsMappable; }
1123 void Pack(
void *dst,
const void *src, std::size_t count)
const final
1125 using namespace ROOT::Internal;
1129 assert(fValueRange);
1130 const auto [min,
max] = *fValueRange;
1131 const int nOutOfRange =
1132 Quantize::QuantizeReals(quantized.get(),
reinterpret_cast<const T *
>(src), count, min, max, fBitsOnStorage);
1134 throw ROOT::RException(
R__FAIL(std::to_string(nOutOfRange) +
1135 " values were found of of range for quantization while packing (range is [" +
1136 std::to_string(min) +
", " + std::to_string(max) +
"])"));
1138 BitPacking::PackBits(dst, quantized.get(), count,
sizeof(Quantize::Quantized_t), fBitsOnStorage);
1141 void Unpack(
void *dst,
const void *src, std::size_t count)
const final
1143 using namespace ROOT::Internal;
1147 assert(fValueRange);
1148 const auto [min,
max] = *fValueRange;
1149 BitPacking::UnpackBits(quantized.get(), src, count,
sizeof(Quantize::Quantized_t), fBitsOnStorage);
1150 [[maybe_unused]]
const int nOutOfRange =
1151 Quantize::UnquantizeReals(
reinterpret_cast<T *
>(dst), quantized.get(), count, min, max, fBitsOnStorage);
1155 assert(nOutOfRange == 0);
1158 RIdentifier GetIdentifier() const final {
return RIdentifier{
typeid(
T), ENTupleColumnType::kReal32Quant}; }
1167#define __RCOLUMNELEMENT_SPEC_BODY(CppT, ColumnT, BaseT, BitsOnStorage) \
1168 static constexpr std::size_t kSize = sizeof(CppT); \
1169 static constexpr std::size_t kBitsOnStorage = BitsOnStorage; \
1170 RColumnElement() : BaseT(kSize, kBitsOnStorage) {} \
1171 bool IsMappable() const final \
1173 return kIsMappable; \
1175 RIdentifier GetIdentifier() const final \
1177 return RIdentifier{typeid(CppT), ColumnT}; \
1185#define DECLARE_RCOLUMNELEMENT_SPEC(CppT, ColumnT, BitsOnStorage, BaseT, ...) \
1187 class RColumnElement<CppT, ColumnT> : public BaseT __VA_ARGS__ { \
1189 __RCOLUMNELEMENT_SPEC_BODY(CppT, ColumnT, BaseT, BitsOnStorage) \
1191#define DECLARE_RCOLUMNELEMENT_SPEC_SIMPLE(CppT, ColumnT, BitsOnStorage) \
1193 class RColumnElement<CppT, ColumnT> : public RColumnElementBase { \
1195 static constexpr bool kIsMappable = true; \
1196 __RCOLUMNELEMENT_SPEC_BODY(CppT, ColumnT, RColumnElementBase, BitsOnStorage) \
1227 <
char, std::int16_t>);
1230 <
char, std::int32_t>);
1233 <
char, std::int64_t>);
1240 <std::int8_t, std::uint8_t>);
1242 <std::int8_t, std::int16_t>);
1244 <std::int8_t, std::uint16_t>);
1246 <std::int8_t, std::int32_t>);
1248 <std::int8_t, std::uint32_t>);
1250 <std::int8_t, std::int64_t>);
1252 <std::int8_t, std::uint64_t>);
1254 <std::int8_t, std::int16_t>);
1256 <std::int8_t, std::uint16_t>);
1258 <std::int8_t, std::int32_t>);
1260 <std::int8_t, std::uint32_t>);
1262 <std::int8_t, std::int64_t>);
1264 <std::int8_t, std::uint64_t>);
1270 <std::uint8_t, std::int8_t>);
1272 <std::uint8_t, std::int16_t>);
1274 <std::uint8_t, std::uint16_t>);
1276 <std::uint8_t, std::int32_t>);
1278 <std::uint8_t, std::uint32_t>);
1280 <std::uint8_t, std::int64_t>);
1282 <std::uint8_t, std::uint64_t>);
1284 <std::uint8_t, std::int16_t>);
1286 <std::uint8_t, std::uint16_t>);
1288 <std::uint8_t, std::int32_t>);
1290 <std::uint8_t, std::uint32_t>);
1292 <std::uint8_t, std::int64_t>);
1294 <std::uint8_t, std::uint64_t>);
1299 <std::int16_t, std::int16_t>);
1302 <std::int16_t, std::int8_t>);
1304 <std::int16_t, std::uint8_t>);
1306 <std::int16_t, std::uint16_t>);
1308 <std::int16_t, std::int32_t>);
1310 <std::int16_t, std::uint32_t>);
1312 <std::int16_t, std::int64_t>);
1314 <std::int16_t, std::uint64_t>);
1316 <std::int16_t, std::uint16_t>);
1318 <std::int16_t, std::int32_t>);
1320 <std::int16_t, std::uint32_t>);
1322 <std::int16_t, std::int64_t>);
1324 <std::int16_t, std::uint64_t>);
1329 <std::uint16_t, std::uint16_t>);
1332 <std::uint16_t, std::int8_t>);
1334 <std::uint16_t, std::uint8_t>);
1336 <std::uint16_t, std::int16_t>);
1338 <std::uint16_t, std::int32_t>);
1340 <std::uint16_t, std::uint32_t>);
1342 <std::uint16_t, std::int64_t>);
1344 <std::uint16_t, std::uint64_t>);
1346 <std::uint16_t, std::int16_t>);
1348 <std::uint16_t, std::int32_t>);
1350 <std::uint16_t, std::uint32_t>);
1352 <std::uint16_t, std::int64_t>);
1354 <std::uint16_t, std::uint64_t>);
1359 <std::int32_t, std::int32_t>);
1362 <std::int32_t, std::int8_t>);
1364 <std::int32_t, std::uint8_t>);
1366 <std::int32_t, std::int16_t>);
1368 <std::int32_t, std::uint16_t>);
1370 <std::int32_t, std::uint32_t>);
1372 <std::int32_t, std::int64_t>);
1374 <std::int32_t, std::uint64_t>);
1376 <std::int32_t, std::int16_t>);
1378 <std::int32_t, std::uint16_t>);
1380 <std::int32_t, std::uint32_t>);
1382 <std::int32_t, std::int64_t>);
1384 <std::int32_t, std::uint64_t>);
1389 <std::uint32_t, std::uint32_t>);
1392 <std::uint32_t, std::int8_t>);
1394 <std::uint32_t, std::uint8_t>);
1396 <std::uint32_t, std::int16_t>);
1398 <std::uint32_t, std::uint16_t>);
1400 <std::uint32_t, std::int32_t>);
1402 <std::uint32_t, std::int64_t>);
1404 <std::uint32_t, std::uint64_t>);
1406 <std::uint32_t, std::int16_t>);
1408 <std::uint32_t, std::uint16_t>);
1410 <std::uint32_t, std::int32_t>);
1412 <std::uint32_t, std::int64_t>);
1414 <std::uint32_t, std::uint64_t>);
1419 <std::int64_t, std::int64_t>);
1422 <std::int64_t, std::int8_t>);
1424 <std::int64_t, std::uint8_t>);
1426 <std::int64_t, std::int16_t>);
1428 <std::int64_t, std::uint16_t>);
1430 <std::int64_t, std::int32_t>);
1432 <std::int64_t, std::uint32_t>);
1434 <std::int64_t, std::uint64_t>);
1436 <std::int64_t, std::int16_t>);
1438 <std::int64_t, std::uint16_t>);
1440 <std::int64_t, std::int32_t>);
1442 <std::int64_t, std::uint32_t>);
1444 <std::int64_t, std::uint64_t>);
1449 <std::uint64_t, std::uint64_t>);
1452 <std::uint64_t, std::int8_t>);
1454 <std::uint64_t, std::uint8_t>);
1456 <std::uint64_t, std::int16_t>);
1458 <std::uint64_t, std::uint16_t>);
1460 <std::uint64_t, std::int32_t>);
1462 <std::uint64_t, std::uint32_t>);
1464 <std::uint64_t, std::int64_t>);
1466 <std::uint64_t, std::int16_t>);
1468 <std::uint64_t, std::uint16_t>);
1470 <std::uint64_t, std::int32_t>);
1472 <std::uint64_t, std::uint32_t>);
1474 <std::uint64_t, std::int64_t>);
1490 <std::uint64_t, std::uint32_t>);
1492 RColumnElementDeltaSplitLE, <std::uint64_t, std::uint64_t>);
1494 RColumnElementDeltaSplitLE, <std::uint64_t, std::uint32_t>);
1499 static constexpr bool kIsMappable =
false;
1501 static constexpr std::size_t kBitsOnStorage =
kSize * 8;
1504 bool IsMappable() const final {
return kIsMappable; }
1505 void Pack(
void *,
const void *, std::size_t)
const final {}
1506 void Unpack(
void *,
const void *, std::size_t)
const final {}
1508 RIdentifier GetIdentifier() const final
1510 return RIdentifier{
typeid(ROOT::Internal::RTestFutureColumn), kTestFutureColumnType};
1515RColumnElement<bool, ROOT::ENTupleColumnType::kBit>::Pack(
void *dst,
const void *src, std::size_t count)
const
1517 const bool *boolArray =
reinterpret_cast<const bool *
>(src);
1518 char *charArray =
reinterpret_cast<char *
>(dst);
1519 std::bitset<8> bitSet;
1521 for (; i < count; ++i) {
1522 bitSet.set(i % 8, boolArray[i]);
1524 char packed = bitSet.to_ulong();
1525 charArray[i / 8] = packed;
1529 char packed = bitSet.to_ulong();
1530 charArray[i / 8] = packed;
1535RColumnElement<bool, ROOT::ENTupleColumnType::kBit>::Unpack(
void *dst,
const void *src, std::size_t count)
const
1537 bool *boolArray =
reinterpret_cast<bool *
>(dst);
1538 const char *charArray =
reinterpret_cast<const char *
>(src);
1539 std::bitset<8> bitSet;
1540 for (std::size_t i = 0; i < count; i += 8) {
1541 bitSet = charArray[i / 8];
1542 for (std::size_t j = i; j < std::min(count, i + 8); ++j) {
1543 boolArray[j] = bitSet[j % 8];
#define DECLARE_RCOLUMNELEMENT_SPEC_SIMPLE(CppT, ColumnT, BitsOnStorage)
#define DECLARE_RCOLUMNELEMENT_SPEC(CppT, ColumnT, BitsOnStorage, BaseT,...)
These macros are used to declare RColumnElement template specializations below.
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
std::unique_ptr< T[]> MakeUninitArray(std::size_t size)
Make an array of default-initialized elements.
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
A column element encapsulates the translation between basic C++ types and their column representation...
virtual void Unpack(void *destination, const void *source, std::size_t count) const
If the on-storage layout and the in-memory layout differ, unpacking creates a memory page from an on-...
virtual void Pack(void *destination, const void *source, std::size_t count) const
If the on-storage layout and the in-memory layout differ, packing creates an on-disk page from an in-...
The available trivial, native content types of a column.
A column element encapsulates the translation between basic C++ types and their column representation...
The in-memory representation of a 32bit or 64bit on-disk index column.
Base class for all ROOT issued exceptions.
void PackBits(void *dst, const void *src, std::size_t count, std::size_t sizeofSrc, std::size_t nDstBits)
Tightly packs count items of size sizeofSrc contained in src into dst using nDstBits per item.
constexpr std::size_t MinBufSize(std::size_t count, std::size_t nDstBits)
Returns the minimum safe size (in bytes) of a buffer that is intended to be used as a destination for...
void UnpackBits(void *dst, const void *src, std::size_t count, std::size_t sizeofDst, std::size_t nSrcBits)
Undoes the effect of PackBits.
constexpr std::size_t kBitsPerWord
std::size_t LeadingZeroes(T x)
Given an integer x, returns the number of leading 0-bits starting at the most significant bit positio...
std::uint16_t FloatToHalf(float value)
Convert an IEEE single-precision float to half-precision.
std::unique_ptr< T[]> MakeUninitArray(std::size_t size)
Make an array of default-initialized elements.
std::size_t TrailingZeroes(T x)
Given an integer x, returns the number of trailing 0-bits starting at the least significant bit posit...
constexpr ENTupleColumnType kTestFutureColumnType
float HalfToFloat(std::uint16_t value)
Convert an IEEE half-precision float to single-precision.
__device__ AFloat max(AFloat x, AFloat y)
static value_type bswap(value_type x)
static value_type bswap(value_type x)
Helper templated class for swapping bytes; specializations for N={2,4,8} are provided below.