Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
TBasket.cxx
Go to the documentation of this file.
1// @(#)root/tree:$Id: 4e77188fbf1e7fd026a984989de66663c49b12fc $
2// Author: Rene Brun 19/01/96
3/*************************************************************************
4 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#include <chrono>
12
13#include "TBasket.h"
14#include "TBuffer.h"
15#include "TBufferFile.h"
16#include "TTree.h"
17#include "TBranch.h"
18#include "TFile.h"
19#include "TLeaf.h"
20#include "TMath.h"
21#include "TROOT.h"
22#include "TTreeCache.h"
23#include "TVirtualMutex.h"
24#include "TVirtualPerfStats.h"
25#include "TTimeStamp.h"
26#include "ROOT/TIOFeatures.hxx"
27#include "RZip.h"
28
29#include <bitset>
30
31const UInt_t kDisplacementMask = 0xFF000000; // In the streamer the two highest bytes of
32 // the fEntryOffset are used to stored displacement.
33
35
36/** \class TBasket
37\ingroup tree
38
39Manages buffers for branches of a Tree.
40
41See picture in TTree.
42*/
43
44////////////////////////////////////////////////////////////////////////////////
45/// Default constructor.
46
48{
49}
50
51////////////////////////////////////////////////////////////////////////////////
52/// Constructor used during reading.
53
54TBasket::TBasket(TDirectory *motherDir) : TKey(motherDir)
55{
56}
57
58////////////////////////////////////////////////////////////////////////////////
59/// Basket normal constructor, used during writing.
60
61TBasket::TBasket(const char *name, const char *title, TBranch *branch)
62 : TKey(branch->GetDirectory()), fBufferSize(branch->GetBasketSize()), fNevBufSize(branch->GetEntryOffsetLen()),
63 fHeaderOnly(true), fIOBits(branch->GetIOFeatures().GetFeatures())
64{
66 SetTitle(title);
67 fClassName = "TBasket";
68 fBuffer = nullptr;
70 fVersion += 1000;
71 if (branch->GetDirectory()) {
72 TFile *file = branch->GetFile();
73 fBufferRef->SetParent(file);
74 }
75 if (branch->GetTree()) {
76#ifdef R__USE_IMT
78#else
80#endif
85 }
86 }
87 fBranch = branch;
91 fLast = fKeylen;
92 fBuffer = nullptr;
93 fHeaderOnly = false;
94 if (fNevBufSize) {
96 for (Int_t i=0;i<fNevBufSize;i++) fEntryOffset[i] = 0;
97 }
99}
100
101////////////////////////////////////////////////////////////////////////////////
102/// Basket destructor.
103
105{
106 if (fDisplacement) delete [] fDisplacement;
108 if (fBufferRef) delete fBufferRef;
109 fBufferRef = nullptr;
110 fBuffer = nullptr;
111 fDisplacement= nullptr;
112 // Note we only delete the compressed buffer if we own it
115 fCompressedBufferRef = nullptr;
116 }
117 // TKey::~TKey will use fMotherDir to attempt to remove they key
118 // from the directory's list of key. A basket is never in that list
119 // and in some cases (eg. f = new TFile(); TTree t; delete f;) the
120 // directory is gone before the TTree.
121 fMotherDir = nullptr;
122}
123
124////////////////////////////////////////////////////////////////////////////////
125/// Increase the size of the current fBuffer up to newsize.
126
128{
129 if (fBuffer == fBufferRef->Buffer()) {
130 fBufferRef->Expand(newsize);
132 } else {
133 fBufferRef->Expand(newsize);
134 }
136 fBufferSize = newsize;
137 fLastWriteBufferSize[0] = newsize;
141}
142
143////////////////////////////////////////////////////////////////////////////////
144/// Copy the basket of this branch onto the file to.
145
147{
149 Int_t nout = fNbytes - fKeylen;
151 Create(nout, to);
153 fHeaderOnly = true;
155 fHeaderOnly = false;
156 Int_t nBytes = WriteFileKeepBuffer(to);
157
158 return nBytes>0 ? nBytes : -1;
159}
160
161////////////////////////////////////////////////////////////////////////////////
162/// Delete fEntryOffset array.
163
165{
167 fNevBufSize = 0;
168}
169
170////////////////////////////////////////////////////////////////////////////////
171/// Drop buffers of this basket if it is not the current basket.
172
174{
175 if (!fBuffer && !fBufferRef) return 0;
176
177 if (fDisplacement) delete [] fDisplacement;
179 if (fBufferRef) delete fBufferRef;
181 fBufferRef = nullptr;
182 fCompressedBufferRef = nullptr;
183 fBuffer = nullptr;
184 fDisplacement= nullptr;
185 fEntryOffset = nullptr;
187 return fBufferSize;
188}
189
190////////////////////////////////////////////////////////////////////////////////
191/// Calculates the entry offset array, if possible.
192///
193/// Result is cached, meaning that this should only be invoked once per basket.
194
196{
197 if (fEntryOffset != reinterpret_cast<Int_t *>(-1)) {
198 return fEntryOffset;
199 }
200
201 if (R__unlikely(!fBranch)) {
202 Error("GetCalculatedEntryOffset", "Basket entry offset calculation requested, but no associated TBranch!");
203 return nullptr;
204 }
205 if (R__unlikely(fBranch->GetNleaves() != 1)) {
206 Error("GetCalculatedEntryOffset", "Branch contains multiple leaves - unable to calculated entry offsets!");
207 return nullptr;
208 }
209 TLeaf *leaf = static_cast<TLeaf *>((*fBranch->GetListOfLeaves())[0]);
211 return fEntryOffset;
212}
213
214////////////////////////////////////////////////////////////////////////////////
215/// Determine whether we can generate the offset array when this branch is read.
216///
217
219{
220 if (fBranch->GetNleaves() != 1) {
221 return false;
222 }
223 TLeaf *leaf = static_cast<TLeaf *>((*fBranch->GetListOfLeaves())[0]);
224 return leaf->CanGenerateOffsetArray();
225}
226
227////////////////////////////////////////////////////////////////////////////////
228/// Get pointer to buffer for internal entry.
229
231{
233 Int_t *entryOffset = GetEntryOffset();
234 if (entryOffset) offset = entryOffset[entry];
235 else offset = fKeylen + entry*fNevBufSize;
237 return offset;
238}
239
240////////////////////////////////////////////////////////////////////////////////
241/// Load basket buffers in memory without unziping.
242/// This function is called by TTreeCloner.
243/// The function returns 0 in case of success, 1 in case of error.
244
246{
247 if (fBufferRef) {
248 // Reuse the buffer if it exist.
249 fBufferRef->Reset();
250
251 // We use this buffer both for reading and writing, we need to
252 // make sure it is properly sized for writing.
254 if (fBufferRef->BufferSize() < len) {
256 }
258 } else {
260 }
261 fBufferRef->SetParent(file);
262 char *buffer = fBufferRef->Buffer();
263 file->Seek(pos);
264 TFileCacheRead *pf = tree->GetReadCache(file);
265 if (pf) {
267 if (tree->GetPerfStats()) gPerfStats = tree->GetPerfStats();
268 Int_t st = pf->ReadBuffer(buffer,pos,len);
269 if (st < 0) {
270 return 1;
271 } else if (st == 0) {
272 // fOffset might have been changed via TFileCacheRead::ReadBuffer(), reset it
273 file->Seek(pos);
274 // If we are using a TTreeCache, disable reading from the default cache
275 // temporarily, to force reading directly from file
276 TTreeCache *fc = dynamic_cast<TTreeCache*>(file->GetCacheRead());
277 if (fc) fc->Disable();
278 Int_t ret = file->ReadBuffer(buffer,len);
279 if (fc) fc->Enable();
281 pf->AddNoCacheReadCalls(1);
282 if (ret) {
283 return 1;
284 }
285 }
286 gPerfStats = temp;
287 // fOffset might have been changed via TFileCacheRead::ReadBuffer(), reset it
288 file->SetOffset(pos + len);
289 } else {
291 if (tree->GetPerfStats() != nullptr) gPerfStats = tree->GetPerfStats();
292 if (file->ReadBuffer(buffer,len)) {
293 gPerfStats = temp;
294 return 1; //error while reading
295 }
296 else gPerfStats = temp;
297 }
298
302
303 return 0;
304}
305
306////////////////////////////////////////////////////////////////////////////////
307/// Remove the first dentries of this basket, moving entries at
308/// dentries to the start of the buffer.
309
311{
312 Int_t i;
313
314 if (dentries >= fNevBuf) return;
315 Int_t bufbegin;
316 Int_t moved;
317
318 Int_t *entryOffset = GetEntryOffset();
319 if (entryOffset) {
320 bufbegin = entryOffset[dentries];
321 moved = bufbegin-GetKeylen();
322
323 // First store the original location in the fDisplacement array
324 // and record the new start offset
325
326 if (!fDisplacement) {
328 }
329 for (i = 0; i<(fNevBufSize-dentries); ++i) {
330 fDisplacement[i] = entryOffset[i + dentries];
331 entryOffset[i] = entryOffset[i + dentries] - moved;
332 }
333 for (i = fNevBufSize-dentries; i<fNevBufSize; ++i) {
334 fDisplacement[i] = 0;
335 entryOffset[i] = 0;
336 }
337
338 } else {
339 // If there is no EntryOffset array, this means
340 // that each entry has the same size and that
341 // it does not point to other objects (hence there
342 // is no need for a displacement array).
343 bufbegin = GetKeylen() + dentries*fNevBufSize;
344 moved = bufbegin-GetKeylen();
345 }
346 TBuffer *buf = GetBufferRef();
347 char *buffer = buf->Buffer();
348 memmove(buffer+GetKeylen(),buffer+bufbegin,buf->Length()-bufbegin);
349 buf->SetBufferOffset(buf->Length()-moved);
350 fNevBuf -= dentries;
351}
352
353#define OLD_CASE_EXPRESSION fObjlen==fNbytes-fKeylen && GetBranch()->GetCompressionLevel()!=0 && file->GetVersion()<=30401
354////////////////////////////////////////////////////////////////////////////////
355/// By-passing buffer unzipping has been requested and is
356/// possible (only 1 entry in this basket).
357
359{
361
362 // Make sure that the buffer is set at the END of the data
364
365 // Indicate that this buffer is weird.
367
368 // Usage of this mode assume the existence of only ONE
369 // entry in this basket.
371 delete [] fDisplacement; fDisplacement = nullptr;
372
374 return 0;
375}
376
377////////////////////////////////////////////////////////////////////////////////
378/// We always create the TBuffer for the basket but it hold the buffer from the cache.
379
380Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, bool mustFree, TFile* file)
381{
382 if (fBufferRef) {
383 fBufferRef->SetBuffer(buffer, size, mustFree);
385 fBufferRef->Reset();
386 } else {
387 fBufferRef = new TBufferFile(TBuffer::kRead, size, buffer, mustFree);
388 }
389 fBufferRef->SetParent(file);
390
392
393 if (IsZombie()) {
394 return -1;
395 }
396
397 bool oldCase = OLD_CASE_EXPRESSION;
398
399 if ((fObjlen > fNbytes-fKeylen || oldCase) && TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1)) {
401 }
402
404 return fObjlen+fKeylen;
405}
406
407////////////////////////////////////////////////////////////////////////////////
408/// Initialize a buffer for reading if it is not already initialized
409
410static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t len, TFile* file)
411{
413 if (R__likely(bufferRef)) {
414 bufferRef->SetReadMode();
415 Int_t curBufferSize = bufferRef->BufferSize();
416 if (curBufferSize < len) {
417 // Experience shows that giving 5% "wiggle-room" decreases churn.
418 bufferRef->Expand(Int_t(len*1.05));
419 }
420 bufferRef->Reset();
421 result = bufferRef;
422 } else {
424 }
425 result->SetParent(file);
426 return result;
427}
428
429////////////////////////////////////////////////////////////////////////////////
430/// Initialize the compressed buffer; either from the TTree or create a local one.
431
433{
434 bool compressedBufferExists = fCompressedBufferRef != nullptr;
436 if (R__unlikely(!compressedBufferExists)) {
438 }
439}
440
442{
443 if (fEntryOffset != reinterpret_cast<Int_t *>(-1)) {
444 delete[] fEntryOffset;
445 }
446 fEntryOffset = nullptr;
447}
448
449////////////////////////////////////////////////////////////////////////////////
450/// Read basket buffers in memory and cleanup.
451///
452/// Read a basket buffer. Check if buffers of previous ReadBasket
453/// should not be dropped. Remember, we keep buffers in memory up to
454/// fMaxVirtualSize.
455/// The function returns 0 in case of success, 1 in case of error
456/// This function was modified with the addition of the parallel
457/// unzipping, it will try to get the unzipped file from the cache
458/// receiving only a pointer to that buffer (so we shall not
459/// delete that pointer), although we get a new buffer in case
460/// it's not found in the cache.
461/// There is a lot of code duplication but it was necessary to assure
462/// the expected behavior when there is no cache.
463
465{
466 if(!fBranch->GetDirectory()) {
467 return -1;
468 }
469
470 bool oldCase;
471 char *rawUncompressedBuffer, *rawCompressedBuffer;
472 Int_t uncompressedBufferLen;
473
474 // See if the cache has already unzipped the buffer for us.
475 TFileCacheRead *pf = nullptr;
476 {
477 R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
478 pf = fBranch->GetTree()->GetReadCache(file);
479 }
480 if (pf) {
481 Int_t res = -1;
482 bool free = true;
483 char *buffer = nullptr;
484 res = pf->GetUnzipBuffer(&buffer, pos, len, &free);
485 if (R__unlikely(res >= 0)) {
486 len = ReadBasketBuffersUnzip(buffer, res, free, file);
487 // Note that in the kNotDecompressed case, the above function will return 0;
488 // In such a case, we should stop processing
489 if (len <= 0) return -len;
490 goto AfterBuffer;
491 }
492 }
493
494 // Determine which buffer to use, so that we can avoid a memcpy in case of
495 // the basket was not compressed.
496 TBuffer* readBufferRef;
498 // Initialize the buffer to hold the uncompressed data.
500 readBufferRef = fBufferRef;
501 } else {
502 // Initialize the buffer to hold the compressed data.
504 readBufferRef = fCompressedBufferRef;
505 }
506
507 // fBufferSize is likely to be change in the Streamer call (below)
508 // and we will re-add the new size later on.
510
511 if (!readBufferRef) {
512 Error("ReadBasketBuffers", "Unable to allocate buffer.");
513 return 1;
514 }
515
516 if (pf) {
518 if (fBranch->GetTree()->GetPerfStats() != nullptr) gPerfStats = fBranch->GetTree()->GetPerfStats();
519 Int_t st = 0;
520 {
521 R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
522 st = pf->ReadBuffer(readBufferRef->Buffer(),pos,len);
523 }
524 if (st < 0) {
525 return 1;
526 } else if (st == 0) {
527 // Read directly from file, not from the cache
528 // If we are using a TTreeCache, disable reading from the default cache
529 // temporarily, to force reading directly from file
530 R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
531 TTreeCache *fc = dynamic_cast<TTreeCache*>(file->GetCacheRead());
532 if (fc) fc->Disable();
533 Int_t ret = file->ReadBuffer(readBufferRef->Buffer(),pos,len);
534 if (fc) fc->Enable();
536 pf->AddNoCacheReadCalls(1);
537 if (ret) {
538 return 1;
539 }
540 }
541 gPerfStats = temp;
542 } else {
543 // Read from the file and unstream the header information.
545 if (fBranch->GetTree()->GetPerfStats() != nullptr) gPerfStats = fBranch->GetTree()->GetPerfStats();
546 R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
547 if (file->ReadBuffer(readBufferRef->Buffer(),pos,len)) {
548 gPerfStats = temp;
549 return 1;
550 }
551 else gPerfStats = temp;
552 }
553 Streamer(*readBufferRef);
554 if (IsZombie()) {
555 return 1;
556 }
557
558 rawCompressedBuffer = readBufferRef->Buffer();
559
560 // Are we done?
561 if (R__unlikely(readBufferRef == fBufferRef)) // We expect most basket to be compressed.
562 {
564 // The basket was really not compressed as expected.
565 goto AfterBuffer;
566 } else {
567 // Well, somehow the buffer was compressed anyway, we have the compressed data in the uncompressed buffer
568 // Make sure the compressed buffer is initialized, and memcpy.
571 Error("ReadBasketBuffers", "Unable to allocate buffer.");
572 return 1;
573 }
574 fBufferRef->Reset();
575 rawCompressedBuffer = fCompressedBufferRef->Buffer();
576 memcpy(rawCompressedBuffer, fBufferRef->Buffer(), len);
577 }
578 }
579
580 // Initialize buffer to hold the uncompressed data
581 // Note that in previous versions we didn't allocate buffers until we verified
582 // the zip headers; this is no longer beforehand as the buffer lifetime is scoped
583 // to the TBranch.
584 uncompressedBufferLen = len > fObjlen+fKeylen ? len : fObjlen+fKeylen;
585 fBufferRef = R__InitializeReadBasketBuffer(fBufferRef, uncompressedBufferLen, file);
586 rawUncompressedBuffer = fBufferRef->Buffer();
587 fBuffer = rawUncompressedBuffer;
588
589 oldCase = OLD_CASE_EXPRESSION;
590 // Case where ROOT thinks the buffer is compressed. Copy over the key and uncompress the object
591 if (fObjlen > fNbytes-fKeylen || oldCase) {
594 }
595
596 // Optional monitor for zip time profiling.
597 Double_t start = 0;
598 if (R__unlikely(gPerfStats)) {
599 start = TTimeStamp();
600 }
601
602 memcpy(rawUncompressedBuffer, rawCompressedBuffer, fKeylen);
603 char *rawUncompressedObjectBuffer = rawUncompressedBuffer+fKeylen;
604 UChar_t *rawCompressedObjectBuffer = (UChar_t*)rawCompressedBuffer+fKeylen;
605 Int_t nin, nbuf;
606 Int_t nout = 0, noutot = 0, nintot = 0;
607
608 // Unzip all the compressed objects in the compressed object buffer.
609 while (true) {
610 // Check the header for errors.
611 if (R__unlikely(R__unzip_header(&nin, rawCompressedObjectBuffer, &nbuf) != 0)) {
612 Error("ReadBasketBuffers", "Inconsistency found in header (nin=%d, nbuf=%d)", nin, nbuf);
613 break;
614 }
615 if (R__unlikely(oldCase && (nin > fObjlen || nbuf > fObjlen))) {
616 //buffer was very likely not compressed in an old version
617 memcpy(rawUncompressedBuffer+fKeylen, rawCompressedObjectBuffer+fKeylen, fObjlen);
618 goto AfterBuffer;
619 }
620
621 R__unzip(&nin, rawCompressedObjectBuffer, &nbuf, (unsigned char*) rawUncompressedObjectBuffer, &nout);
622 if (!nout) break;
623 noutot += nout;
624 nintot += nin;
625 if (noutot >= fObjlen) break;
626 rawCompressedObjectBuffer += nin;
627 rawUncompressedObjectBuffer += nout;
628 }
629
630 // Make sure the uncompressed numbers are consistent with header.
631 if (R__unlikely(noutot != fObjlen)) {
632 Error("ReadBasketBuffers", "fNbytes = %d, fKeylen = %d, fObjlen = %d, noutot = %d, nout=%d, nin=%d, nbuf=%d", fNbytes,fKeylen,fObjlen, noutot,nout,nin,nbuf);
634 return 1;
635 }
638 if (fBranch->GetTree()->GetPerfStats() != nullptr) gPerfStats = fBranch->GetTree()->GetPerfStats();
639 if (R__unlikely(gPerfStats)) {
640 gPerfStats->UnzipEvent(fBranch->GetTree(),pos,start,nintot,fObjlen);
641 }
642 gPerfStats = temp;
643 } else {
644 // Nothing is compressed - copy over wholesale.
645 memcpy(rawUncompressedBuffer, rawCompressedBuffer, len);
646 }
647
648AfterBuffer:
649
651
652 // Read offsets table if needed.
653 // If there's no EntryOffsetLen in the branch -- or the fEntryOffset is marked to be calculated-on-demand --
654 // then we skip reading out.
655 if (!fBranch->GetEntryOffsetLen() || (fEntryOffset == reinterpret_cast<Int_t *>(-1))) {
656 return 0;
657 }
658 // At this point, we're required to read out an offset array.
659 ResetEntryOffset(); // TODO: every basket, we reset the offset array. Is this necessary?
660 // Could we instead switch to std::vector?
664 fEntryOffset = new Int_t[fNevBuf+1];
666 Warning("ReadBasketBuffers","basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%lld, len=%d, fNbytes=%d, fObjlen=%d, trying to repair",GetName(),fNevBuf,pos,len,fNbytes,fObjlen);
667 return 0;
668 }
670 // In this case, we cannot regenerate the offset array at runtime -- but we wrote out an array of
671 // sizes instead of offsets (as sizes compress much better).
673 for (Int_t idx = 1; idx < fNevBuf + 1; idx++) {
674 fEntryOffset[idx] += fEntryOffset[idx - 1];
675 }
676 }
677 fReadEntryOffset = true;
678 // Read the array of displacement if any.
679 delete [] fDisplacement;
680 fDisplacement = nullptr;
681 if (fBufferRef->Length() != len) {
682 // There is more data in the buffer! It is the displacement
683 // array. If len is less than TBuffer::kMinimalSize the actual
684 // size of the buffer is too large, so we can not use the
685 // fBufferRef->BufferSize()
687 }
688
689 return 0;
690}
691
692////////////////////////////////////////////////////////////////////////////////
693/// Read basket buffers in memory and cleanup
694///
695/// Read first bytes of a logical record starting at position pos
696/// return record length (first 4 bytes of record).
697
699{
700 const Int_t len = 128;
701 char buffer[len];
702 Int_t keylen;
703 file->GetRecordHeader(buffer, pos,len, fNbytes, fObjlen, keylen);
704 fKeylen = keylen;
705 return fNbytes;
706}
707
708////////////////////////////////////////////////////////////////////////////////
709/// Disown all references to the internal buffer - some other object likely now
710/// owns it.
711///
712/// This TBasket is now useless and invalid until it is told to adopt a buffer.
714{
715 fBufferRef = nullptr;
716}
717
718
719////////////////////////////////////////////////////////////////////////////////
720/// Adopt a buffer from an external entity
722{
723 delete fBufferRef;
724 fBufferRef = user_buffer;
725}
726
727////////////////////////////////////////////////////////////////////////////////
728/// Reset the read basket TBuffer memory allocation if needed.
729///
730/// This allows to reduce the number of memory allocation while avoiding to
731/// always use the maximum size.
732
734{
735 // By default, we don't reallocate.
736 fResetAllocation = false;
737#ifdef R__TRACK_BASKET_ALLOC_TIME
738 fResetAllocationTime = 0;
739#endif
740
741 // Downsize the buffer if needed.
742
743 const auto maxbaskets = fBranch->GetMaxBaskets();
744 if (!fBufferRef || basketnumber >= maxbaskets)
745 return;
746
747 Int_t curSize = fBufferRef->BufferSize();
748
749 const Float_t target_mem_ratio = fBranch->GetTree()->GetTargetMemoryRatio();
750 const auto basketbytes = fBranch->GetBasketBytes();
751
752 Int_t max_size = basketbytes[basketnumber];
753 for(Int_t b = basketnumber + 1; (b < maxbaskets) && (b < (basketnumber+10)); ++b) {
754 max_size = std::max(max_size, basketbytes[b]);
755 }
756
757 Float_t cx = 1;
758 if (fBranch->GetZipBytes())
760
761 Int_t target_size = static_cast<Int_t>(cx * target_mem_ratio * Float_t(max_size));
762
763 if (target_size && (curSize > target_size)) {
764 /// Only reduce the size if significant enough?
765 Int_t newSize = max_size + 512 - max_size % 512; // Wiggle room and alignment, as above.
766 // We only bother with a resize if it saves 8KB (two normal memory pages).
767 if ((newSize <= curSize - 8 * 1024) &&
768 (static_cast<Float_t>(curSize) / static_cast<Float_t>(newSize) > target_mem_ratio))
769 {
770 if (gDebug > 0) {
771 Info("ReadResetBuffer",
772 "Resizing %d to %d bytes (was %d); next 10 sizes are [%d, %d, %d, %d, %d, %d, %d, %d, %d, %d]. cx=%f ratio=%f max_size = %d ",
773 basketnumber, newSize, curSize,
774 basketbytes[basketnumber],
775 (basketnumber + 1) < maxbaskets ? basketbytes[basketnumber + 1] : 0,
776 (basketnumber + 2) < maxbaskets ? basketbytes[basketnumber + 2] : 0,
777 (basketnumber + 3) < maxbaskets ? basketbytes[basketnumber + 3] : 0,
778 (basketnumber + 4) < maxbaskets ? basketbytes[basketnumber + 4] : 0,
779 (basketnumber + 5) < maxbaskets ? basketbytes[basketnumber + 5] : 0,
780 (basketnumber + 6) < maxbaskets ? basketbytes[basketnumber + 6] : 0,
781 (basketnumber + 7) < maxbaskets ? basketbytes[basketnumber + 7] : 0,
782 (basketnumber + 8) < maxbaskets ? basketbytes[basketnumber + 8] : 0,
783 (basketnumber + 9) < maxbaskets ? basketbytes[basketnumber + 9] : 0,
784 cx, target_mem_ratio, max_size);
785 }
786 fResetAllocation = true;
787#ifdef R__TRACK_BASKET_ALLOC_TIME
788 std::chrono::time_point<std::chrono::system_clock> start, end;
789 start = std::chrono::high_resolution_clock::now();
790#endif
791 fBufferRef->Expand(newSize, false); // Expand without copying the existing data.
792#ifdef R__TRACK_BASKET_ALLOC_TIME
793 end = std::chrono::high_resolution_clock::now();
794 auto us = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
795 fResetAllocationTime = us.count();
796#endif
797 }
798 }
799}
800
801////////////////////////////////////////////////////////////////////////////////
802/// Reset the write basket to the starting state. i.e. as it was after calling
803/// the constructor (and potentially attaching a TBuffer.)
804/// Reduce memory used by fEntryOffset and the TBuffer if needed ..
805
807{
808 // By default, we don't reallocate.
809 fResetAllocation = false;
810#ifdef R__TRACK_BASKET_ALLOC_TIME
811 fResetAllocationTime = 0;
812#endif
813
814 // Name, Title, fClassName, fBranch
815 // stay the same.
816
817 // Downsize the buffer if needed.
818 // See if our current buffer size is significantly larger (>2x) than the historical average.
819 // If so, try decreasing it at this flush boundary to closer to the size from OptimizeBaskets
820 // (or this historical average).
821 Int_t curSize = fBufferRef->BufferSize();
822 // fBufferLen at this point is already reset, so use indirect measurements
823 Int_t curLen = (GetObjlen() + GetKeylen());
824 Longptr_t newSize = -1;
825 if (curSize > 2*curLen)
826 {
827 Longptr_t curBsize = fBranch->GetBasketSize();
828 if (curSize > 2*curBsize ) {
829 Longptr_t avgSize = (Longptr_t)(fBranch->GetTotBytes() / (1+fBranch->GetWriteBasket())); // Average number of bytes per basket so far
830 if (curSize > 2*avgSize) {
831 newSize = curBsize;
832 if (curLen > newSize) {
833 newSize = curLen;
834 }
835 if (avgSize > newSize) {
836 newSize = avgSize;
837 }
838 newSize = newSize + 512 - newSize%512; // Wiggle room and alignment (512 is same as in OptimizeBaskets)
839 }
840 }
841 }
842 // If fBufferRef grew since we last saw it, shrink it to "target memory ratio" of the occupied size
843 // This discourages us from having poorly-occupied buffers on branches with little variability.
844 //
845 // Does not help protect against a burst in event sizes, but does help in the cases where the basket
846 // size jumps from 4MB to 8MB while filling the basket, but we only end up utilizing 4.1MB.
847 //
848 // The above code block is meant to protect against extremely large events.
849
850 Float_t target_mem_ratio = fBranch->GetTree()->GetTargetMemoryRatio();
852 Int_t target_size = static_cast<Int_t>(target_mem_ratio * Float_t(max_size));
853 if (max_size && (curSize > target_size) && (newSize == -1)) {
854 newSize = target_size;
855 newSize = newSize + 512 - newSize % 512; // Wiggle room and alignment, as above.
856 // We only bother with a resize if it saves 8KB (two normal memory pages).
857 if ((newSize > curSize - 8 * 1024) ||
858 (static_cast<Float_t>(curSize) / static_cast<Float_t>(newSize) < target_mem_ratio)) {
859 newSize = -1;
860 } else if (gDebug > 0) {
861 Info("Reset", "Resizing to %ld bytes (was %d); last three sizes were [%d, %d, %d].", newSize, curSize,
863 }
864 }
865
866 if (newSize != -1) {
867 fResetAllocation = true;
868#ifdef R__TRACK_BASKET_ALLOC_TIME
869 std::chrono::time_point<std::chrono::system_clock> start, end;
870 start = std::chrono::high_resolution_clock::now();
871#endif
872 fBufferRef->Expand(newSize,false); // Expand without copying the existing data.
873#ifdef R__TRACK_BASKET_ALLOC_TIME
874 end = std::chrono::high_resolution_clock::now();
875 auto us = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
876 fResetAllocationTime = us.count();
877#endif
878 }
879
880 // Record the actual occupied size of the buffer.
883
884 TKey::Reset();
885
886 Int_t newNevBufSize = fBranch->GetEntryOffsetLen();
887 if (newNevBufSize==0) {
889 } else if ((newNevBufSize != fNevBufSize) || (fEntryOffset <= reinterpret_cast<Int_t *>(-1))) {
891 fEntryOffset = new Int_t[newNevBufSize];
892 }
893 fNevBufSize = newNevBufSize;
894
895 fNevBuf = 0;
896 Int_t *storeEntryOffset = fEntryOffset;
897 fEntryOffset = nullptr;
898 Int_t *storeDisplacement = fDisplacement;
899 fDisplacement= nullptr;
900 fBuffer = nullptr;
901
902 fBufferRef->Reset();
904
905 fHeaderOnly = true;
906 fLast = 0; //Must initialize before calling Streamer()
907
909
912 fLast = fKeylen;
913 fBuffer = nullptr;
914 fHeaderOnly = false;
915 fDisplacement= storeDisplacement;
916 fEntryOffset = storeEntryOffset;
917 if (fNevBufSize) {
918 for (Int_t i=0;i<fNevBufSize;i++) fEntryOffset[i] = 0;
919 }
920}
921
922////////////////////////////////////////////////////////////////////////////////
923/// Set read mode of basket.
924
926{
929}
930
931////////////////////////////////////////////////////////////////////////////////
932/// Set write mode of basket.
933
935{
938}
939
940////////////////////////////////////////////////////////////////////////////////
941/// Stream a class object.
942
944{
945 // As in TBranch::GetBasket, this is used as a half-hearted measure to suppress
946 // the error reporting when many failures occur.
947 static std::atomic<Int_t> nerrors(0);
948
949 char flag;
950 if (b.IsReading()) {
951 TKey::Streamer(b); //this must be first
952 Version_t v = b.ReadVersion();
953 b >> fBufferSize;
954 // NOTE: we now use the upper-bit of the fNevBufSize to see if we have serialized any of the
955 // optional IOBits. If that bit is set, we immediately read out the IOBits; to replace this
956 // (minimal) safeguard against corruption, we will set aside the upper-bit of fIOBits to do
957 // the same thing (the fact this bit is reserved is tested in the unit tests). If there is
958 // someday a need for more than 7 IOBits, we'll widen the field using the same trick.
959 //
960 // We like to keep this safeguard because we immediately will allocate a buffer based on
961 // the value of fNevBufSize -- and would like to avoid wildly inappropriate allocations.
962 b >> fNevBufSize;
963 if (fNevBufSize < 0) {
965 b >> fIOBits;
966 if (!fIOBits || (fIOBits & (1 << 7))) {
967 Error("TBasket::Streamer",
968 "The value of fNevBufSize (%d) or fIOBits (%d) is incorrect ; setting the buffer to a zombie.",
970 MakeZombie();
971 fNevBufSize = 0;
972 } else if (fIOBits && (fIOBits & ~static_cast<Int_t>(EIOBits::kSupported))) {
973 nerrors++;
974 if (nerrors < 10) {
975 Error("Streamer", "The value of fIOBits (%s) contains unknown flags (supported flags "
976 "are %s), indicating this was written with a newer version of ROOT "
977 "utilizing critical IO features this version of ROOT does not support."
978 " Refusing to deserialize.",
979 std::bitset<32>(static_cast<Int_t>(fIOBits)).to_string().c_str(),
980 std::bitset<32>(static_cast<Int_t>(EIOBits::kSupported)).to_string().c_str());
981 } else if (nerrors == 10) {
982 Error("Streamer", "Maximum number of errors has been reported; disabling further messages"
983 "from this location until the process exits.");
984 }
985 fNevBufSize = 0;
986 MakeZombie();
987 }
988 }
989 b >> fNevBuf;
990 b >> fLast;
991 b >> flag;
993 bool mustGenerateOffsets = false;
994 if (flag >= 80) {
995 mustGenerateOffsets = true;
996 flag -= 80;
997 }
998 if (!mustGenerateOffsets && flag && (flag % 10 != 2)) {
1001 if (fNevBuf) b.ReadArray(fEntryOffset);
1002 if (20<flag && flag<40) {
1003 for(int i=0; i<fNevBuf; i++){
1004 fEntryOffset[i] &= ~kDisplacementMask;
1005 }
1006 }
1007 if (flag>40) {
1009 b.ReadArray(fDisplacement);
1010 }
1011 } else if (mustGenerateOffsets) {
1012 // We currently believe that in all cases when offsets can be generated, then the
1013 // displacement array must be zero.
1014 assert(flag <= 40);
1015 fEntryOffset = reinterpret_cast<Int_t *>(-1);
1016 }
1017 if (flag == 1 || flag > 10) {
1019 fBufferRef->SetParent(b.GetParent());
1020 char *buf = fBufferRef->Buffer();
1021 if (v > 1) b.ReadFastArray(buf,fLast);
1022 else b.ReadArray(buf);
1024 // This is now done in the TBranch streamer since fBranch might not
1025 // yet be set correctly.
1026 // fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
1027 }
1028 } else {
1029
1030 TKey::Streamer(b); //this must be first
1031 b.WriteVersion(TBasket::IsA());
1032 if (fBufferRef) {
1033 Int_t curLast = fBufferRef->Length();
1034 if (!fHeaderOnly && !fSeekKey && curLast > fLast) fLast = curLast;
1035 }
1037
1038 b << fBufferSize;
1039 if (fIOBits) {
1040 b << -fNevBufSize;
1041 b << fIOBits;
1042 } else {
1043 b << fNevBufSize;
1044 }
1045 b << fNevBuf;
1046 b << fLast;
1047 bool mustGenerateOffsets = fEntryOffset && fNevBuf &&
1050 // We currently believe that in all cases when offsets can be generated, then the
1051 // displacement array must be zero.
1052 assert(!mustGenerateOffsets || fDisplacement == nullptr);
1053 if (fHeaderOnly) {
1054 flag = mustGenerateOffsets ? 80 : 0;
1055 b << flag;
1056 } else {
1057 // On return from this function, we are guaranteed that fEntryOffset
1058 // is either a valid pointer or nullptr.
1059 if (fNevBuf) {
1061 }
1062 flag = 1;
1063 if (!fNevBuf || !fEntryOffset)
1064 flag = 2;
1065 if (fBufferRef) flag += 10;
1066 if (fDisplacement) flag += 40;
1067 // Test if we can skip writing out the offset map.
1068 if (mustGenerateOffsets) {
1069 flag += 80;
1070 }
1071 b << flag;
1072
1073 if (!mustGenerateOffsets && fEntryOffset && fNevBuf) {
1074 b.WriteArray(fEntryOffset, fNevBuf);
1075 if (fDisplacement) b.WriteArray(fDisplacement, fNevBuf);
1076 }
1077 if (fBufferRef) {
1078 char *buf = fBufferRef->Buffer();
1079 b.WriteFastArray(buf, fLast);
1080 }
1081 }
1082 }
1083}
1084
1085////////////////////////////////////////////////////////////////////////////////
1086/// Update basket header and EntryOffset table.
1087
1089{
1090 Int_t *entryOffset = GetEntryOffset();
1091 if (entryOffset) {
1092 if (fNevBuf+1 >= fNevBufSize) {
1093 Int_t newsize = TMath::Max(10,2*fNevBufSize);
1094 Int_t *newoff = TStorage::ReAllocInt(fEntryOffset, newsize,
1095 fNevBufSize);
1096 if (fDisplacement) {
1097 Int_t *newdisp = TStorage::ReAllocInt(fDisplacement, newsize,
1098 fNevBufSize);
1099 fDisplacement = newdisp;
1100 }
1101 fEntryOffset = newoff;
1102 fNevBufSize = newsize;
1103
1104 //Update branch only for the first 10 baskets
1105 if (fBranch->GetWriteBasket() < 10) {
1106 fBranch->SetEntryOffsetLen(newsize);
1107 }
1108 }
1110
1111 if (skipped!=offset && !fDisplacement){
1113 for (Int_t i = 0; i<fNevBufSize; i++) fDisplacement[i] = fEntryOffset[i];
1114 }
1115 if (fDisplacement) {
1116 fDisplacement[fNevBuf] = skipped;
1118 }
1119 }
1120
1121 fNevBuf++;
1122}
1123
1124////////////////////////////////////////////////////////////////////////////////
1125/// Write buffer of this basket on the current file.
1126///
1127/// The function returns the number of bytes committed to the memory.
1128/// If a write error occurs, the number of bytes returned is -1.
1129/// If no data are written, the number of bytes returned is 0.
1130
1132{
1133 constexpr Int_t kWrite = 1;
1134
1135 TFile *file = fBranch->GetFile(kWrite);
1136 if (!file) return 0;
1137 if (!file->IsWritable()) {
1138 return -1;
1139 }
1140 fMotherDir = file; // fBranch->GetDirectory();
1141
1142 // This mutex prevents multiple TBasket::WriteBuffer invocations from interacting
1143 // with the underlying TFile at once - TFile is assumed to *not* be thread-safe.
1144 //
1145 // The only parallelism we'd like to exploit (right now!) is the compression
1146 // step - everything else should be serialized at the TFile level.
1147#ifdef R__USE_IMT
1148 std::unique_lock<std::mutex> sentry(file->fWriteMutex);
1149#endif // R__USE_IMT
1150
1152 // Read the basket information that was saved inside the buffer.
1153 bool writing = fBufferRef->IsWriting();
1156
1158 if (writing) fBufferRef->SetWriteMode();
1159 Int_t nout = fNbytes - fKeylen;
1160
1162
1163 Create(nout,file);
1165 fHeaderOnly = true;
1166
1167 Streamer(*fBufferRef); //write key itself again
1168 int nBytes = WriteFileKeepBuffer();
1169 fHeaderOnly = false;
1170 return nBytes>0 ? fKeylen+nout : -1;
1171 }
1172
1173 // Transfer fEntryOffset table at the end of fBuffer.
1174 fLast = fBufferRef->Length();
1175 Int_t *entryOffset = GetEntryOffset();
1176 if (entryOffset) {
1177 bool hasOffsetBit = fIOBits & static_cast<UChar_t>(TBasket::EIOBits::kGenerateOffsetMap);
1178 if (!CanGenerateOffsetArray()) {
1179 // If we have set the offset map flag, but cannot dynamically generate the map, then
1180 // we should at least convert the offset array to a size array. Note that we always
1181 // write out (fNevBuf+1) entries to match the original case.
1182 if (hasOffsetBit) {
1183 for (Int_t idx = fNevBuf; idx > 0; idx--) {
1184 entryOffset[idx] -= entryOffset[idx - 1];
1185 }
1186 entryOffset[0] = 0;
1187 }
1188 fBufferRef->WriteArray(entryOffset, fNevBuf + 1);
1189 // Convert back to offset format: keeping both sizes and offsets in-memory were considered,
1190 // but it seems better to use CPU than memory.
1191 if (hasOffsetBit) {
1192 entryOffset[0] = fKeylen;
1193 for (Int_t idx = 1; idx < fNevBuf + 1; idx++) {
1194 entryOffset[idx] += entryOffset[idx - 1];
1195 }
1196 }
1197 } else if (!hasOffsetBit) { // In this case, write out as normal
1198 fBufferRef->WriteArray(entryOffset, fNevBuf + 1);
1199 }
1200 if (fDisplacement) {
1202 delete[] fDisplacement;
1203 fDisplacement = nullptr;
1204 }
1205 }
1206
1207 Int_t nout, noutot, bufmax, nzip;
1208
1210
1211 fHeaderOnly = true;
1213 Int_t cxlevel = fBranch->GetCompressionLevel();
1215 cxlevel = file->GetCompressionLevel();
1219 if (cxlevel > 0) {
1220 Int_t nbuffers = 1 + (fObjlen - 1) / kMAXZIPBUF;
1221 Int_t buflen = fKeylen + fObjlen + 9 * nbuffers + 28; //add 28 bytes in case object is placed in a deleted gap
1222 InitializeCompressedBuffer(buflen, file);
1223 if (!fCompressedBufferRef) {
1224 Warning("WriteBuffer", "Unable to allocate the compressed buffer");
1225 return -1;
1226 }
1229 char *objbuf = fBufferRef->Buffer() + fKeylen;
1230 char *bufcur = &fBuffer[fKeylen];
1231 noutot = 0;
1232 nzip = 0;
1233 for (Int_t i = 0; i < nbuffers; ++i) {
1234 if (i == nbuffers - 1) bufmax = fObjlen - nzip;
1235 else bufmax = kMAXZIPBUF;
1236 // Compress the buffer. Note that we allow multiple TBasket compressions to occur at once
1237 // for a given TFile: that's because the compression buffer when we use IMT is no longer
1238 // shared amongst several threads.
1239#ifdef R__USE_IMT
1240 sentry.unlock();
1241#endif // R__USE_IMT
1242 // NOTE this is declared with C linkage, so it shouldn't except. Also, when
1243 // USE_IMT is defined, we are guaranteed that the compression buffer is unique per-branch.
1244 // (see fCompressedBufferRef in constructor).
1245 R__zipMultipleAlgorithm(cxlevel, &bufmax, objbuf, &bufmax, bufcur, &nout, cxAlgorithm);
1246#ifdef R__USE_IMT
1247 sentry.lock();
1248#endif // R__USE_IMT
1249
1250 // test if buffer has really been compressed. In case of small buffers
1251 // when the buffer contains random data, it may happen that the compressed
1252 // buffer is larger than the input. In this case, we write the original uncompressed buffer
1253 if (nout == 0 || nout >= fObjlen) {
1254 nout = fObjlen;
1255 // We used to delete fBuffer here, we no longer want to since
1256 // the buffer (held by fCompressedBufferRef) might be re-used later.
1258 Create(fObjlen,file);
1260
1261 Streamer(*fBufferRef); //write key itself again
1262 if ((nout+fKeylen)>buflen) {
1263 Warning("WriteBuffer","Possible memory corruption due to compression algorithm, wrote %d bytes past the end of a block of %d bytes. fNbytes=%d, fObjLen=%d, fKeylen=%d",
1264 (nout+fKeylen-buflen),buflen,fNbytes,fObjlen,fKeylen);
1265 }
1266 goto WriteFile;
1267 }
1268 bufcur += nout;
1269 noutot += nout;
1270 objbuf += kMAXZIPBUF;
1271 nzip += kMAXZIPBUF;
1272 }
1273 nout = noutot;
1274 Create(noutot,file);
1276
1277 Streamer(*fBufferRef); //write key itself again
1278 memcpy(fBuffer,fBufferRef->Buffer(),fKeylen);
1279 } else {
1281 Create(fObjlen,file);
1283
1284 Streamer(*fBufferRef); //write key itself again
1285 nout = fObjlen;
1286 }
1287
1288WriteFile:
1289 Int_t nBytes = WriteFileKeepBuffer();
1290 fHeaderOnly = false;
1291 return nBytes>0 ? fKeylen+nout : -1;
1292}
#define R__likely(expr)
Definition RConfig.hxx:587
#define R__unlikely(expr)
Definition RConfig.hxx:586
#define b(i)
Definition RSha256.hxx:100
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
int Int_t
Definition RtypesCore.h:45
long Longptr_t
Definition RtypesCore.h:82
short Version_t
Definition RtypesCore.h:65
unsigned char UChar_t
Definition RtypesCore.h:38
unsigned int UInt_t
Definition RtypesCore.h:46
float Float_t
Definition RtypesCore.h:57
long long Long64_t
Definition RtypesCore.h:80
#define ClassImp(name)
Definition Rtypes.h:377
static TBuffer * R__InitializeReadBasketBuffer(TBuffer *bufferRef, Int_t len, TFile *file)
Initialize a buffer for reading if it is not already initialized.
Definition TBasket.cxx:410
#define OLD_CASE_EXPRESSION
Definition TBasket.cxx:353
const UInt_t kDisplacementMask
Definition TBasket.cxx:31
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
char name[80]
Definition TGX11.cxx:110
Int_t gDebug
Definition TROOT.cxx:597
R__EXTERN TVirtualMutex * gROOTMutex
Definition TROOT.h:63
void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout)
int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout)
#define R__LOCKGUARD_IMT(mutex)
#define gPerfStats
#define free
Definition civetweb.c:1539
Manages buffers for branches of a Tree.
Definition TBasket.h:34
Int_t * fEntryOffset
[fNevBuf] Offset of entries in fBuffer(TKey); generated at runtime.
Definition TBasket.h:71
Int_t GetEntryPointer(Int_t Entry)
Get pointer to buffer for internal entry.
Definition TBasket.cxx:230
void AdoptBuffer(TBuffer *user_buffer)
Adopt a buffer from an external entity.
Definition TBasket.cxx:721
bool fHeaderOnly
True when only the basket header must be read/written.
Definition TBasket.h:66
TBranch * fBranch
Pointer to the basket support branch.
Definition TBasket.h:73
Long64_t CopyTo(TFile *to)
Copy the basket of this branch onto the file to.
Definition TBasket.cxx:146
Int_t * GetEntryOffset()
Definition TBasket.h:124
void InitializeCompressedBuffer(Int_t len, TFile *file)
Initialize the compressed buffer; either from the TTree or create a local one.
Definition TBasket.cxx:432
UChar_t fNextBufferSizeRecord
! Index into fLastWriteBufferSize of the last buffer written to disk
Definition TBasket.h:77
Int_t ReadBasketBuffersUnzip(char *, Int_t, bool, TFile *)
We always create the TBuffer for the basket but it hold the buffer from the cache.
Definition TBasket.cxx:380
void DisownBuffer()
Disown all references to the internal buffer - some other object likely now owns it.
Definition TBasket.cxx:713
Int_t fLastWriteBufferSize[3]
! Size of the buffer last three buffers we wrote it to disk
Definition TBasket.h:75
Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
Read basket buffers in memory and cleanup.
Definition TBasket.cxx:464
UChar_t fIOBits
!IO feature flags. Serialized in custom portion of streamer to avoid forward compat issues unless nee...
Definition TBasket.h:67
virtual Int_t DropBuffers()
Drop buffers of this basket if it is not the current basket.
Definition TBasket.cxx:173
virtual void MoveEntries(Int_t dentries)
Remove the first dentries of this basket, moving entries at dentries to the start of the buffer.
Definition TBasket.cxx:310
bool fOwnsCompressedBuffer
! Whether or not we own the compressed buffer.
Definition TBasket.h:68
void ResetEntryOffset()
Definition TBasket.cxx:441
TBasket()
Default constructor.
Definition TBasket.cxx:47
virtual void AdjustSize(Int_t newsize)
Increase the size of the current fBuffer up to newsize.
Definition TBasket.cxx:127
virtual void SetReadMode()
Set read mode of basket.
Definition TBasket.cxx:925
virtual void SetWriteMode()
Set write mode of basket.
Definition TBasket.cxx:934
Int_t ReadBasketBuffersUncompressedCase()
By-passing buffer unzipping has been requested and is possible (only 1 entry in this basket).
Definition TBasket.cxx:358
Int_t fBufferSize
fBuffer length in bytes
Definition TBasket.h:62
bool fReadEntryOffset
!Set to true if offset array was read from a file.
Definition TBasket.h:69
Int_t ReadBasketBytes(Long64_t pos, TFile *file)
Read basket buffers in memory and cleanup.
Definition TBasket.cxx:698
virtual void ReadResetBuffer(Int_t basketnumber)
Reset the read basket TBuffer memory allocation if needed.
Definition TBasket.cxx:733
bool fResetAllocation
! True if last reset re-allocated the memory
Definition TBasket.h:76
virtual Int_t WriteBuffer()
Write buffer of this basket on the current file.
Definition TBasket.cxx:1131
void Streamer(TBuffer &) override
Stream a class object.
Definition TBasket.cxx:943
Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree=nullptr)
Load basket buffers in memory without unziping.
Definition TBasket.cxx:245
~TBasket() override
Basket destructor.
Definition TBasket.cxx:104
void Update(Int_t newlast)
Definition TBasket.h:152
Int_t fNevBufSize
Length in Int_t of fEntryOffset OR fixed length of each entry if fEntryOffset is null!
Definition TBasket.h:63
Int_t fNevBuf
Number of entries in basket.
Definition TBasket.h:64
Int_t * GetCalculatedEntryOffset()
Calculates the entry offset array, if possible.
Definition TBasket.cxx:195
TClass * IsA() const override
Definition TBasket.h:156
virtual void DeleteEntryOffset()
Delete fEntryOffset array.
Definition TBasket.cxx:164
TBuffer * fCompressedBufferRef
! Compressed buffer.
Definition TBasket.h:74
Int_t fLast
Pointer to last used byte in basket.
Definition TBasket.h:65
Int_t * fDisplacement
![fNevBuf] Displacement of entries in fBuffer(TKey)
Definition TBasket.h:70
bool CanGenerateOffsetArray()
Determine whether we can generate the offset array when this branch is read.
Definition TBasket.cxx:218
virtual void WriteReset()
Reset the write basket to the starting state.
Definition TBasket.cxx:806
A TTree is a list of TBranches.
Definition TBranch.h:93
Int_t GetCompressionLevel() const
Definition TBranch.h:305
Int_t GetCompressionAlgorithm() const
Definition TBranch.h:299
TTree * GetTree() const
Definition TBranch.h:252
Int_t GetWriteBasket() const
Definition TBranch.h:238
Int_t GetNleaves() const
Definition TBranch.h:249
virtual void SetEntryOffsetLen(Int_t len, bool updateSubBranches=false)
Update the default value for the branch's fEntryOffsetLen if and only if it was already non zero (and...
Definition TBranch.cxx:2821
Int_t GetMaxBaskets() const
Definition TBranch.h:248
Int_t * GetBasketBytes() const
Definition TBranch.h:214
virtual TFile * GetFile(Int_t mode=0)
Return pointer to the file where branch buffers reside, returns 0 in case branch buffers reside in th...
Definition TBranch.cxx:1853
Long64_t GetTotBytes(Option_t *option="") const
Return total number of bytes in the branch (excluding current buffer) if option ="*" includes all sub...
Definition TBranch.cxx:2220
virtual Int_t GetBasketSize() const
Definition TBranch.h:217
Long64_t GetZipBytes(Option_t *option="") const
Return total number of zip bytes in the branch if option ="*" includes all sub-branches of this branc...
Definition TBranch.cxx:2238
TObjArray * GetListOfLeaves()
Definition TBranch.h:247
TDirectory * GetDirectory() const
Definition TBranch.h:224
TBuffer * GetTransientBuffer(Int_t size)
Returns the transient buffer currently used by this TBranch for reading/writing baskets.
Definition TBranch.cxx:523
Int_t GetEntryOffsetLen() const
Definition TBranch.h:227
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition TBufferFile.h:47
@ kNotDecompressed
Definition TBufferIO.h:66
Buffer base class used for serializing objects.
Definition TBuffer.h:43
void SetWriteMode()
Set buffer in write mode.
Definition TBuffer.cxx:315
virtual void Reset()=0
void SetParent(TObject *parent)
Set parent owning this buffer.
Definition TBuffer.cxx:270
void Expand(Int_t newsize, Bool_t copy=kTRUE)
Expand (or shrink) the I/O buffer to newsize bytes.
Definition TBuffer.cxx:223
Int_t BufferSize() const
Definition TBuffer.h:98
void SetBuffer(void *buf, UInt_t bufsiz=0, Bool_t adopt=kTRUE, ReAllocCharFun_t reallocfunc=nullptr)
Sets a new buffer in an existing TBuffer object.
Definition TBuffer.cxx:187
virtual void WriteArray(const Bool_t *b, Int_t n)=0
virtual Int_t ReadArray(Bool_t *&b)=0
@ kWrite
Definition TBuffer.h:73
@ kRead
Definition TBuffer.h:73
Bool_t IsWriting() const
Definition TBuffer.h:87
void SetBufferOffset(Int_t offset=0)
Definition TBuffer.h:93
void SetReadMode()
Set buffer in read mode.
Definition TBuffer.cxx:302
virtual void SetBufferDisplacement()=0
Int_t Length() const
Definition TBuffer.h:100
char * Buffer() const
Definition TBuffer.h:96
Bool_t IsWritable() const override
Describe directory structure in memory.
Definition TDirectory.h:45
A cache when reading files over the network.
virtual Int_t GetUnzipBuffer(char **, Long64_t, Int_t, Bool_t *)
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos.
virtual void AddNoCacheReadCalls(Int_t reads)
virtual void AddNoCacheBytesRead(Long64_t len)
A ROOT file is an on-disk file, usually with extension .root, that stores objects in a file-system-li...
Definition TFile.h:53
virtual void Seek(Long64_t offset, ERelativeTo pos=kBeg)
Seek to a specific position in the file. Pos it either kBeg, kCur or kEnd.
Definition TFile.cxx:2274
Int_t GetCompressionLevel() const
Definition TFile.h:391
std::mutex fWriteMutex
!Lock for writing baskets / keys into the file.
Definition TFile.h:118
virtual void SetOffset(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
Definition TFile.cxx:2253
Int_t GetCompressionAlgorithm() const
Definition TFile.h:385
Int_t GetRecordHeader(char *buf, Long64_t first, Int_t maxbytes, Int_t &nbytes, Int_t &objlen, Int_t &keylen)
Read the logical record header starting at a certain postion.
Definition TFile.cxx:1301
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read a buffer from the file.
Definition TFile.cxx:1770
TFileCacheRead * GetCacheRead(const TObject *tree=nullptr) const
Return a pointer to the current read cache.
Definition TFile.cxx:1262
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition TKey.h:28
Int_t GetKeylen() const
Definition TKey.h:84
Int_t GetObjlen() const
Definition TKey.h:87
TBuffer * fBufferRef
Pointer to the TBuffer object.
Definition TKey.h:50
Int_t fVersion
Key version identifier.
Definition TKey.h:39
Short_t fKeylen
Number of bytes for the key itself.
Definition TKey.h:43
virtual Int_t WriteFileKeepBuffer(TFile *f=nullptr)
Write the encoded object supported by this key.
Definition TKey.cxx:1503
Long64_t fSeekKey
Location of object on file.
Definition TKey.h:45
char * fBuffer
Object buffer.
Definition TKey.h:49
Int_t fNbytes
Number of bytes for the object on file.
Definition TKey.h:40
Int_t fObjlen
Length of uncompressed object in bytes.
Definition TKey.h:41
void Reset()
Reset the key as it had not been 'filled' yet.
Definition TKey.cxx:1314
Short_t fCycle
Cycle number.
Definition TKey.h:44
virtual void Create(Int_t nbytes, TFile *f=nullptr)
Create a TKey object of specified size.
Definition TKey.cxx:463
virtual Int_t WriteFile(Int_t cycle=1, TFile *f=nullptr)
Write the encoded object supported by this key.
Definition TKey.cxx:1462
TDirectory * fMotherDir
!pointer to mother directory
Definition TKey.h:52
TString fClassName
Object Class name.
Definition TKey.h:47
void Streamer(TBuffer &) override
Stream a class object.
Definition TKey.cxx:1365
TBuffer * GetBufferRef() const
Definition TKey.h:79
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
Definition TLeaf.h:57
virtual bool CanGenerateOffsetArray()
Definition TLeaf.h:112
virtual Int_t * GenerateOffsetArray(Int_t base, Int_t events)
Definition TLeaf.h:115
virtual void SetTitle(const char *title="")
Set the title of the TNamed.
Definition TNamed.cxx:164
const char * GetName() const override
Returns name of object.
Definition TNamed.h:47
virtual void SetName(const char *name)
Set the name of the TNamed.
Definition TNamed.cxx:140
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition TObject.h:199
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition TObject.cxx:973
R__ALWAYS_INLINE Bool_t IsZombie() const
Definition TObject.h:153
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition TObject.cxx:780
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition TObject.cxx:987
void MakeZombie()
Definition TObject.h:53
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition TObject.cxx:961
static Int_t * ReAllocInt(Int_t *vp, size_t size, size_t oldsize)
Reallocate (i.e.
Definition TStorage.cxx:258
The TTimeStamp encapsulates seconds and ns since EPOCH.
Definition TTimeStamp.h:45
A cache to speed-up the reading of ROOT datasets.
Definition TTreeCache.h:32
virtual void Disable()
Definition TTreeCache.h:136
virtual void Enable()
Definition TTreeCache.h:137
A TTree represents a columnar dataset.
Definition TTree.h:79
virtual TVirtualPerfStats * GetPerfStats() const
Definition TTree.h:506
virtual void IncrementTotalBuffers(Int_t nbytes)
Definition TTree.h:546
TTreeCache * GetReadCache(TFile *file) const
Find and return the TTreeCache registered with the file and which may contain branches for us.
Definition TTree.cxx:6318
Float_t GetTargetMemoryRatio() const
Definition TTree.h:520
TBuffer * GetTransientBuffer(Int_t size)
Returns the transient buffer currently used by this TTree for reading/writing baskets.
Definition TTree.cxx:1037
Provides the interface for the PROOF internal performance measurement and event tracing.
virtual void UnzipEvent(TObject *tree, Long64_t pos, Double_t start, Int_t complen, Int_t objlen)=0
Short_t Max(Short_t a, Short_t b)
Returns the largest of a and b.
Definition TMathBase.h:250
EValues
Note: this is only temporarily a struct and will become a enum class hence the name.
Definition Compression.h:85
@ kInherit
Some objects use this value to denote that the compression algorithm should be inherited from the par...
Definition Compression.h:88
@ kInherit
Some objects use this value to denote that the compression algorithm should be inherited from the par...
Definition Compression.h:66