Logo ROOT  
Reference Guide
TBasket.cxx
Go to the documentation of this file.
1 // @(#)root/tree:$Id: 4e77188fbf1e7fd026a984989de66663c49b12fc $
2 // Author: Rene Brun 19/01/96
3 /*************************************************************************
4  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
5  * All rights reserved. *
6  * *
7  * For the licensing terms see $ROOTSYS/LICENSE. *
8  * For the list of contributors see $ROOTSYS/README/CREDITS. *
9  *************************************************************************/
10 
11 #include <chrono>
12 
13 #include "TBasket.h"
14 #include "TBuffer.h"
15 #include "TBufferFile.h"
16 #include "TTree.h"
17 #include "TBranch.h"
18 #include "TFile.h"
19 #include "TLeaf.h"
20 #include "TMath.h"
21 #include "TROOT.h"
22 #include "TTreeCache.h"
23 #include "TVirtualMutex.h"
24 #include "TVirtualPerfStats.h"
25 #include "TTimeStamp.h"
26 #include "ROOT/TIOFeatures.hxx"
27 #include "RZip.h"
28 
29 #include <bitset>
30 
31 const UInt_t kDisplacementMask = 0xFF000000; // In the streamer the two highest bytes of
32  // the fEntryOffset are used to stored displacement.
33 
35 
36 /** \class TBasket
37 \ingroup tree
38 
39 Manages buffers for branches of a Tree.
40 
41 See picture in TTree.
42 */
43 
44 ////////////////////////////////////////////////////////////////////////////////
45 /// Default contructor.
46 
48 {
49 }
50 
51 ////////////////////////////////////////////////////////////////////////////////
52 /// Constructor used during reading.
53 
54 TBasket::TBasket(TDirectory *motherDir) : TKey(motherDir)
55 {
56 }
57 
58 ////////////////////////////////////////////////////////////////////////////////
59 /// Basket normal constructor, used during writing.
60 
61 TBasket::TBasket(const char *name, const char *title, TBranch *branch)
62  : TKey(branch->GetDirectory()), fBufferSize(branch->GetBasketSize()), fNevBufSize(branch->GetEntryOffsetLen()),
63  fHeaderOnly(kTRUE), fIOBits(branch->GetIOFeatures().GetFeatures())
64 {
65  SetName(name);
66  SetTitle(title);
67  fClassName = "TBasket";
68  fBuffer = nullptr;
70  fVersion += 1000;
71  if (branch->GetDirectory()) {
72  TFile *file = branch->GetFile();
74  }
75  if (branch->GetTree()) {
76 #ifdef R__USE_IMT
78 #else
80 #endif
82  if (!fCompressedBufferRef) {
85  }
86  }
87  fBranch = branch;
88  Streamer(*fBufferRef);
91  fLast = fKeylen;
92  fBuffer = 0;
94  if (fNevBufSize) {
96  for (Int_t i=0;i<fNevBufSize;i++) fEntryOffset[i] = 0;
97  }
99 }
100 
101 ////////////////////////////////////////////////////////////////////////////////
102 /// Basket destructor.
103 
105 {
106  if (fDisplacement) delete [] fDisplacement;
108  if (fBufferRef) delete fBufferRef;
109  fBufferRef = 0;
110  fBuffer = 0;
111  fDisplacement= 0;
112  // Note we only delete the compressed buffer if we own it
114  delete fCompressedBufferRef;
116  }
117  // TKey::~TKey will use fMotherDir to attempt to remove they key
118  // from the directory's list of key. A basket is never in that list
119  // and in some cases (eg. f = new TFile(); TTree t; delete f;) the
120  // directory is gone before the TTree.
121  fMotherDir = nullptr;
122 }
123 
124 ////////////////////////////////////////////////////////////////////////////////
125 /// Increase the size of the current fBuffer up to newsize.
126 
128 {
129  if (fBuffer == fBufferRef->Buffer()) {
130  fBufferRef->Expand(newsize);
132  } else {
133  fBufferRef->Expand(newsize);
134  }
136  fBufferSize = newsize;
137  fLastWriteBufferSize[0] = newsize;
138  fLastWriteBufferSize[1] = 0;
139  fLastWriteBufferSize[2] = 0;
141 }
142 
143 ////////////////////////////////////////////////////////////////////////////////
144 /// Copy the basket of this branch onto the file to.
145 
147 {
149  Int_t nout = fNbytes - fKeylen;
151  Create(nout, to);
153  fHeaderOnly = kTRUE;
154  Streamer(*fBufferRef);
156  Int_t nBytes = WriteFileKeepBuffer(to);
157 
158  return nBytes>0 ? nBytes : -1;
159 }
160 
161 ////////////////////////////////////////////////////////////////////////////////
162 /// Delete fEntryOffset array.
163 
165 {
167  fNevBufSize = 0;
168 }
169 
170 ////////////////////////////////////////////////////////////////////////////////
171 /// Drop buffers of this basket if it is not the current basket.
172 
174 {
175  if (!fBuffer && !fBufferRef) return 0;
176 
177  if (fDisplacement) delete [] fDisplacement;
179  if (fBufferRef) delete fBufferRef;
181  fBufferRef = 0;
183  fBuffer = 0;
184  fDisplacement= 0;
185  fEntryOffset = 0;
187  return fBufferSize;
188 }
189 
190 ////////////////////////////////////////////////////////////////////////////////
191 /// Calculates the entry offset array, if possible.
192 ///
193 /// Result is cached, meaning that this should only be invoked once per basket.
194 
196 {
197  if (fEntryOffset != reinterpret_cast<Int_t *>(-1)) {
198  return fEntryOffset;
199  }
200 
201  if (R__unlikely(!fBranch)) {
202  Error("GetCalculatedEntryOffset", "Basket entry offset calculation requested, but no associated TBranch!");
203  return nullptr;
204  }
205  if (R__unlikely(fBranch->GetNleaves() != 1)) {
206  Error("GetCalculatedEntryOffset", "Branch contains multiple leaves - unable to calculated entry offsets!");
207  return nullptr;
208  }
209  TLeaf *leaf = static_cast<TLeaf *>((*fBranch->GetListOfLeaves())[0]);
211  return fEntryOffset;
212 }
213 
214 ////////////////////////////////////////////////////////////////////////////////
215 /// Determine whether we can generate the offset array when this branch is read.
216 ///
217 
219 {
220  if (fBranch->GetNleaves() != 1) {
221  return kFALSE;
222  }
223  TLeaf *leaf = static_cast<TLeaf *>((*fBranch->GetListOfLeaves())[0]);
224  return leaf->CanGenerateOffsetArray();
225 }
226 
227 ////////////////////////////////////////////////////////////////////////////////
228 /// Get pointer to buffer for internal entry.
229 
231 {
232  Int_t offset;
233  Int_t *entryOffset = GetEntryOffset();
234  if (entryOffset) offset = entryOffset[entry];
235  else offset = fKeylen + entry*fNevBufSize;
236  fBufferRef->SetBufferOffset(offset);
237  return offset;
238 }
239 
240 ////////////////////////////////////////////////////////////////////////////////
241 /// Load basket buffers in memory without unziping.
242 /// This function is called by TTreeCloner.
243 /// The function returns 0 in case of success, 1 in case of error.
244 
246 {
247  if (fBufferRef) {
248  // Reuse the buffer if it exist.
249  fBufferRef->Reset();
250 
251  // We use this buffer both for reading and writing, we need to
252  // make sure it is properly sized for writing.
254  if (fBufferRef->BufferSize() < len) {
255  fBufferRef->Expand(len);
256  }
258  } else {
260  }
262  char *buffer = fBufferRef->Buffer();
263  file->Seek(pos);
264  TFileCacheRead *pf = tree->GetReadCache(file);
265  if (pf) {
267  if (tree->GetPerfStats()) gPerfStats = tree->GetPerfStats();
268  Int_t st = pf->ReadBuffer(buffer,pos,len);
269  if (st < 0) {
270  return 1;
271  } else if (st == 0) {
272  // fOffset might have been changed via TFileCacheRead::ReadBuffer(), reset it
273  file->Seek(pos);
274  // If we are using a TTreeCache, disable reading from the default cache
275  // temporarily, to force reading directly from file
276  TTreeCache *fc = dynamic_cast<TTreeCache*>(file->GetCacheRead());
277  if (fc) fc->Disable();
278  Int_t ret = file->ReadBuffer(buffer,len);
279  if (fc) fc->Enable();
280  pf->AddNoCacheBytesRead(len);
281  pf->AddNoCacheReadCalls(1);
282  if (ret) {
283  return 1;
284  }
285  }
286  gPerfStats = temp;
287  // fOffset might have been changed via TFileCacheRead::ReadBuffer(), reset it
288  file->SetOffset(pos + len);
289  } else {
291  if (tree->GetPerfStats() != 0) gPerfStats = tree->GetPerfStats();
292  if (file->ReadBuffer(buffer,len)) {
293  gPerfStats = temp;
294  return 1; //error while reading
295  }
296  else gPerfStats = temp;
297  }
298 
301  Streamer(*fBufferRef);
302 
303  return 0;
304 }
305 
306 ////////////////////////////////////////////////////////////////////////////////
307 /// Remove the first dentries of this basket, moving entries at
308 /// dentries to the start of the buffer.
309 
311 {
312  Int_t i;
313 
314  if (dentries >= fNevBuf) return;
315  Int_t bufbegin;
316  Int_t moved;
317 
318  Int_t *entryOffset = GetEntryOffset();
319  if (entryOffset) {
320  bufbegin = entryOffset[dentries];
321  moved = bufbegin-GetKeylen();
322 
323  // First store the original location in the fDisplacement array
324  // and record the new start offset
325 
326  if (!fDisplacement) {
328  }
329  for (i = 0; i<(fNevBufSize-dentries); ++i) {
330  fDisplacement[i] = entryOffset[i + dentries];
331  entryOffset[i] = entryOffset[i + dentries] - moved;
332  }
333  for (i = fNevBufSize-dentries; i<fNevBufSize; ++i) {
334  fDisplacement[i] = 0;
335  entryOffset[i] = 0;
336  }
337 
338  } else {
339  // If there is no EntryOffset array, this means
340  // that each entry has the same size and that
341  // it does not point to other objects (hence there
342  // is no need for a displacement array).
343  bufbegin = GetKeylen() + dentries*fNevBufSize;
344  moved = bufbegin-GetKeylen();
345  }
346  TBuffer *buf = GetBufferRef();
347  char *buffer = buf->Buffer();
348  memmove(buffer+GetKeylen(),buffer+bufbegin,buf->Length()-bufbegin);
349  buf->SetBufferOffset(buf->Length()-moved);
350  fNevBuf -= dentries;
351 }
352 
353 #define OLD_CASE_EXPRESSION fObjlen==fNbytes-fKeylen && GetBranch()->GetCompressionLevel()!=0 && file->GetVersion()<=30401
354 ////////////////////////////////////////////////////////////////////////////////
355 /// By-passing buffer unzipping has been requested and is
356 /// possible (only 1 entry in this basket).
357 
359 {
361 
362  // Make sure that the buffer is set at the END of the data
364 
365  // Indicate that this buffer is weird.
367 
368  // Usage of this mode assume the existance of only ONE
369  // entry in this basket.
371  delete [] fDisplacement; fDisplacement = 0;
372 
374  return 0;
375 }
376 
377 ////////////////////////////////////////////////////////////////////////////////
378 /// We always create the TBuffer for the basket but it hold the buffer from the cache.
379 
381 {
382  if (fBufferRef) {
383  fBufferRef->SetBuffer(buffer, size, mustFree);
385  fBufferRef->Reset();
386  } else {
387  fBufferRef = new TBufferFile(TBuffer::kRead, size, buffer, mustFree);
388  }
390 
391  Streamer(*fBufferRef);
392 
393  if (IsZombie()) {
394  return -1;
395  }
396 
397  Bool_t oldCase = OLD_CASE_EXPRESSION;
398 
399  if ((fObjlen > fNbytes-fKeylen || oldCase) && TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1)) {
401  }
402 
404  return fObjlen+fKeylen;
405 }
406 
407 ////////////////////////////////////////////////////////////////////////////////
408 /// Initialize a buffer for reading if it is not already initialized
409 
410 static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t len, TFile* file)
411 {
412  TBuffer* result;
413  if (R__likely(bufferRef)) {
414  bufferRef->SetReadMode();
415  Int_t curBufferSize = bufferRef->BufferSize();
416  if (curBufferSize < len) {
417  // Experience shows that giving 5% "wiggle-room" decreases churn.
418  bufferRef->Expand(Int_t(len*1.05));
419  }
420  bufferRef->Reset();
421  result = bufferRef;
422  } else {
423  result = new TBufferFile(TBuffer::kRead, len);
424  }
425  result->SetParent(file);
426  return result;
427 }
428 
429 ////////////////////////////////////////////////////////////////////////////////
430 /// Initialize the compressed buffer; either from the TTree or create a local one.
431 
433 {
434  Bool_t compressedBufferExists = fCompressedBufferRef != NULL;
436  if (R__unlikely(!compressedBufferExists)) {
438  }
439 }
440 
442 {
443  if (fEntryOffset != reinterpret_cast<Int_t *>(-1)) {
444  delete[] fEntryOffset;
445  }
446  fEntryOffset = nullptr;
447 }
448 
449 ////////////////////////////////////////////////////////////////////////////////
450 /// Read basket buffers in memory and cleanup.
451 ///
452 /// Read a basket buffer. Check if buffers of previous ReadBasket
453 /// should not be dropped. Remember, we keep buffers in memory up to
454 /// fMaxVirtualSize.
455 /// The function returns 0 in case of success, 1 in case of error
456 /// This function was modified with the addition of the parallel
457 /// unzipping, it will try to get the unzipped file from the cache
458 /// receiving only a pointer to that buffer (so we shall not
459 /// delete that pointer), although we get a new buffer in case
460 /// it's not found in the cache.
461 /// There is a lot of code duplication but it was necesary to assure
462 /// the expected behavior when there is no cache.
463 
465 {
466  if(!fBranch->GetDirectory()) {
467  return -1;
468  }
469 
470  Bool_t oldCase;
471  char *rawUncompressedBuffer, *rawCompressedBuffer;
472  Int_t uncompressedBufferLen;
473 
474  // See if the cache has already unzipped the buffer for us.
475  TFileCacheRead *pf = nullptr;
476  {
477  R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
478  pf = fBranch->GetTree()->GetReadCache(file);
479  }
480  if (pf) {
481  Int_t res = -1;
482  Bool_t free = kTRUE;
483  char *buffer = nullptr;
484  res = pf->GetUnzipBuffer(&buffer, pos, len, &free);
485  if (R__unlikely(res >= 0)) {
486  len = ReadBasketBuffersUnzip(buffer, res, free, file);
487  // Note that in the kNotDecompressed case, the above function will return 0;
488  // In such a case, we should stop processing
489  if (len <= 0) return -len;
490  goto AfterBuffer;
491  }
492  }
493 
494  // Determine which buffer to use, so that we can avoid a memcpy in case of
495  // the basket was not compressed.
496  TBuffer* readBufferRef;
498  // Initialize the buffer to hold the uncompressed data.
500  readBufferRef = fBufferRef;
501  } else {
502  // Initialize the buffer to hold the compressed data.
504  readBufferRef = fCompressedBufferRef;
505  }
506 
507  // fBufferSize is likely to be change in the Streamer call (below)
508  // and we will re-add the new size later on.
510 
511  if (!readBufferRef) {
512  Error("ReadBasketBuffers", "Unable to allocate buffer.");
513  return 1;
514  }
515 
516  if (pf) {
519  Int_t st = 0;
520  {
521  R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
522  st = pf->ReadBuffer(readBufferRef->Buffer(),pos,len);
523  }
524  if (st < 0) {
525  return 1;
526  } else if (st == 0) {
527  // Read directly from file, not from the cache
528  // If we are using a TTreeCache, disable reading from the default cache
529  // temporarily, to force reading directly from file
530  R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
531  TTreeCache *fc = dynamic_cast<TTreeCache*>(file->GetCacheRead());
532  if (fc) fc->Disable();
533  Int_t ret = file->ReadBuffer(readBufferRef->Buffer(),pos,len);
534  if (fc) fc->Enable();
535  pf->AddNoCacheBytesRead(len);
536  pf->AddNoCacheReadCalls(1);
537  if (ret) {
538  return 1;
539  }
540  }
541  gPerfStats = temp;
542  } else {
543  // Read from the file and unstream the header information.
546  R__LOCKGUARD_IMT(gROOTMutex); // Lock for parallel TTree I/O
547  if (file->ReadBuffer(readBufferRef->Buffer(),pos,len)) {
548  gPerfStats = temp;
549  return 1;
550  }
551  else gPerfStats = temp;
552  }
553  Streamer(*readBufferRef);
554  if (IsZombie()) {
555  return 1;
556  }
557 
558  rawCompressedBuffer = readBufferRef->Buffer();
559 
560  // Are we done?
561  if (R__unlikely(readBufferRef == fBufferRef)) // We expect most basket to be compressed.
562  {
563  if (R__likely(fObjlen+fKeylen == fNbytes)) {
564  // The basket was really not compressed as expected.
565  goto AfterBuffer;
566  } else {
567  // Well, somehow the buffer was compressed anyway, we have the compressed data in the uncompressed buffer
568  // Make sure the compressed buffer is initialized, and memcpy.
570  if (!fCompressedBufferRef) {
571  Error("ReadBasketBuffers", "Unable to allocate buffer.");
572  return 1;
573  }
574  fBufferRef->Reset();
575  rawCompressedBuffer = fCompressedBufferRef->Buffer();
576  memcpy(rawCompressedBuffer, fBufferRef->Buffer(), len);
577  }
578  }
579 
580  // Initialize buffer to hold the uncompressed data
581  // Note that in previous versions we didn't allocate buffers until we verified
582  // the zip headers; this is no longer beforehand as the buffer lifetime is scoped
583  // to the TBranch.
584  uncompressedBufferLen = len > fObjlen+fKeylen ? len : fObjlen+fKeylen;
585  fBufferRef = R__InitializeReadBasketBuffer(fBufferRef, uncompressedBufferLen, file);
586  rawUncompressedBuffer = fBufferRef->Buffer();
587  fBuffer = rawUncompressedBuffer;
588 
589  oldCase = OLD_CASE_EXPRESSION;
590  // Case where ROOT thinks the buffer is compressed. Copy over the key and uncompress the object
591  if (fObjlen > fNbytes-fKeylen || oldCase) {
594  }
595 
596  // Optional monitor for zip time profiling.
597  Double_t start = 0;
598  if (R__unlikely(gPerfStats)) {
599  start = TTimeStamp();
600  }
601 
602  memcpy(rawUncompressedBuffer, rawCompressedBuffer, fKeylen);
603  char *rawUncompressedObjectBuffer = rawUncompressedBuffer+fKeylen;
604  UChar_t *rawCompressedObjectBuffer = (UChar_t*)rawCompressedBuffer+fKeylen;
605  Int_t nin, nbuf;
606  Int_t nout = 0, noutot = 0, nintot = 0;
607 
608  // Unzip all the compressed objects in the compressed object buffer.
609  while (1) {
610  // Check the header for errors.
611  if (R__unlikely(R__unzip_header(&nin, rawCompressedObjectBuffer, &nbuf) != 0)) {
612  Error("ReadBasketBuffers", "Inconsistency found in header (nin=%d, nbuf=%d)", nin, nbuf);
613  break;
614  }
615  if (R__unlikely(oldCase && (nin > fObjlen || nbuf > fObjlen))) {
616  //buffer was very likely not compressed in an old version
617  memcpy(rawUncompressedBuffer+fKeylen, rawCompressedObjectBuffer+fKeylen, fObjlen);
618  goto AfterBuffer;
619  }
620 
621  R__unzip(&nin, rawCompressedObjectBuffer, &nbuf, (unsigned char*) rawUncompressedObjectBuffer, &nout);
622  if (!nout) break;
623  noutot += nout;
624  nintot += nin;
625  if (noutot >= fObjlen) break;
626  rawCompressedObjectBuffer += nin;
627  rawUncompressedObjectBuffer += nout;
628  }
629 
630  // Make sure the uncompressed numbers are consistent with header.
631  if (R__unlikely(noutot != fObjlen)) {
632  Error("ReadBasketBuffers", "fNbytes = %d, fKeylen = %d, fObjlen = %d, noutot = %d, nout=%d, nin=%d, nbuf=%d", fNbytes,fKeylen,fObjlen, noutot,nout,nin,nbuf);
634  return 1;
635  }
636  len = fObjlen+fKeylen;
639  if (R__unlikely(gPerfStats)) {
640  gPerfStats->UnzipEvent(fBranch->GetTree(),pos,start,nintot,fObjlen);
641  }
642  gPerfStats = temp;
643  } else {
644  // Nothing is compressed - copy over wholesale.
645  memcpy(rawUncompressedBuffer, rawCompressedBuffer, len);
646  }
647 
648 AfterBuffer:
649 
651 
652  // Read offsets table if needed.
653  // If there's no EntryOffsetLen in the branch -- or the fEntryOffset is marked to be calculated-on-demand --
654  // then we skip reading out.
655  if (!fBranch->GetEntryOffsetLen() || (fEntryOffset == reinterpret_cast<Int_t *>(-1))) {
656  return 0;
657  }
658  // At this point, we're required to read out an offset array.
659  ResetEntryOffset(); // TODO: every basket, we reset the offset array. Is this necessary?
660  // Could we instead switch to std::vector?
663  if (R__unlikely(!fEntryOffset)) {
664  fEntryOffset = new Int_t[fNevBuf+1];
665  fEntryOffset[0] = fKeylen;
666  Warning("ReadBasketBuffers","basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%lld, len=%d, fNbytes=%d, fObjlen=%d, trying to repair",GetName(),fNevBuf,pos,len,fNbytes,fObjlen);
667  return 0;
668  }
670  // In this case, we cannot regenerate the offset array at runtime -- but we wrote out an array of
671  // sizes instead of offsets (as sizes compress much better).
672  fEntryOffset[0] = fKeylen;
673  for (Int_t idx = 1; idx < fNevBuf + 1; idx++) {
674  fEntryOffset[idx] += fEntryOffset[idx - 1];
675  }
676  }
678  // Read the array of diplacement if any.
679  delete [] fDisplacement;
680  fDisplacement = 0;
681  if (fBufferRef->Length() != len) {
682  // There is more data in the buffer! It is the displacement
683  // array. If len is less than TBuffer::kMinimalSize the actual
684  // size of the buffer is too large, so we can not use the
685  // fBufferRef->BufferSize()
687  }
688 
689  return 0;
690 }
691 
692 ////////////////////////////////////////////////////////////////////////////////
693 /// Read basket buffers in memory and cleanup
694 ///
695 /// Read first bytes of a logical record starting at position pos
696 /// return record length (first 4 bytes of record).
697 
699 {
700  const Int_t len = 128;
701  char buffer[len];
702  Int_t keylen;
703  file->GetRecordHeader(buffer, pos,len, fNbytes, fObjlen, keylen);
704  fKeylen = keylen;
705  return fNbytes;
706 }
707 
708 ////////////////////////////////////////////////////////////////////////////////
709 /// Disown all references to the internal buffer - some other object likely now
710 /// owns it.
711 ///
712 /// This TBasket is now useless and invalid until it is told to adopt a buffer.
714 {
715  fBufferRef = NULL;
716 }
717 
718 
719 ////////////////////////////////////////////////////////////////////////////////
720 /// Adopt a buffer from an external entity
721 void TBasket::AdoptBuffer(TBuffer *user_buffer)
722 {
723  delete fBufferRef;
724  fBufferRef = user_buffer;
725 }
726 
727 
728 ////////////////////////////////////////////////////////////////////////////////
729 /// Reset the basket to the starting state. i.e. as it was after calling
730 /// the constructor (and potentially attaching a TBuffer.)
731 /// Reduce memory used by fEntryOffset and the TBuffer if needed ..
732 
734 {
735  // By default, we don't reallocate.
736  fResetAllocation = false;
737 #ifdef R__TRACK_BASKET_ALLOC_TIME
738  fResetAllocationTime = 0;
739 #endif
740 
741  // Name, Title, fClassName, fBranch
742  // stay the same.
743 
744  // Downsize the buffer if needed.
745  // See if our current buffer size is significantly larger (>2x) than the historical average.
746  // If so, try decreasing it at this flush boundary to closer to the size from OptimizeBaskets
747  // (or this historical average).
748  Int_t curSize = fBufferRef->BufferSize();
749  // fBufferLen at this point is already reset, so use indirect measurements
750  Int_t curLen = (GetObjlen() + GetKeylen());
751  Long_t newSize = -1;
752  if (curSize > 2*curLen)
753  {
754  Long_t curBsize = fBranch->GetBasketSize();
755  if (curSize > 2*curBsize ) {
756  Long_t avgSize = (Long_t)(fBranch->GetTotBytes() / (1+fBranch->GetWriteBasket())); // Average number of bytes per basket so far
757  if (curSize > 2*avgSize) {
758  newSize = curBsize;
759  if (curLen > newSize) {
760  newSize = curLen;
761  }
762  if (avgSize > newSize) {
763  newSize = avgSize;
764  }
765  newSize = newSize + 512 - newSize%512; // Wiggle room and alignment (512 is same as in OptimizeBaskets)
766  }
767  }
768  }
769  // If fBufferRef grew since we last saw it, shrink it to "target memory ratio" of the occupied size
770  // This discourages us from having poorly-occupied buffers on branches with little variability.
771  //
772  // Does not help protect against a burst in event sizes, but does help in the cases where the basket
773  // size jumps from 4MB to 8MB while filling the basket, but we only end up utilizing 4.1MB.
774  //
775  // The above code block is meant to protect against extremely large events.
776 
777  Float_t target_mem_ratio = fBranch->GetTree()->GetTargetMemoryRatio();
779  Int_t target_size = static_cast<Int_t>(target_mem_ratio * Float_t(max_size));
780  if (max_size && (curSize > target_size) && (newSize == -1)) {
781  newSize = target_size;
782  newSize = newSize + 512 - newSize % 512; // Wiggle room and alignment, as above.
783  // We only bother with a resize if it saves 8KB (two normal memory pages).
784  if ((newSize > curSize - 8 * 1024) ||
785  (static_cast<Float_t>(curSize) / static_cast<Float_t>(newSize) < target_mem_ratio)) {
786  newSize = -1;
787  } else if (gDebug > 0) {
788  Info("Reset", "Resizing to %ld bytes (was %d); last three sizes were [%d, %d, %d].", newSize, curSize,
790  }
791  }
792 
793  if (newSize != -1) {
794  fResetAllocation = true;
795 #ifdef R__TRACK_BASKET_ALLOC_TIME
796  std::chrono::time_point<std::chrono::system_clock> start, end;
797  start = std::chrono::high_resolution_clock::now();
798 #endif
799  fBufferRef->Expand(newSize,kFALSE); // Expand without copying the existing data.
800 #ifdef R__TRACK_BASKET_ALLOC_TIME
801  end = std::chrono::high_resolution_clock::now();
802  auto us = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
803  fResetAllocationTime = us.count();
804 #endif
805  }
806 
807  // Record the actual occupied size of the buffer.
810 
811  TKey::Reset();
812 
813  Int_t newNevBufSize = fBranch->GetEntryOffsetLen();
814  if (newNevBufSize==0) {
816  } else if ((newNevBufSize != fNevBufSize) || (fEntryOffset <= reinterpret_cast<Int_t *>(-1))) {
818  fEntryOffset = new Int_t[newNevBufSize];
819  }
820  fNevBufSize = newNevBufSize;
821 
822  fNevBuf = 0;
823  Int_t *storeEntryOffset = fEntryOffset;
824  fEntryOffset = 0;
825  Int_t *storeDisplacement = fDisplacement;
826  fDisplacement= 0;
827  fBuffer = 0;
828 
829  fBufferRef->Reset();
831 
832  fHeaderOnly = kTRUE;
833  fLast = 0; //Must initialize before calling Streamer()
834 
835  Streamer(*fBufferRef);
836 
839  fLast = fKeylen;
840  fBuffer = 0;
842  fDisplacement= storeDisplacement;
843  fEntryOffset = storeEntryOffset;
844  if (fNevBufSize) {
845  for (Int_t i=0;i<fNevBufSize;i++) fEntryOffset[i] = 0;
846  }
847 }
848 
849 ////////////////////////////////////////////////////////////////////////////////
850 /// Set read mode of basket.
851 
853 {
854  fLast = fBufferRef->Length();
856 }
857 
858 ////////////////////////////////////////////////////////////////////////////////
859 /// Set write mode of basket.
860 
862 {
865 }
866 
867 ////////////////////////////////////////////////////////////////////////////////
868 /// Stream a class object.
869 
870 void TBasket::Streamer(TBuffer &b)
871 {
872  // As in TBranch::GetBasket, this is used as a half-hearted measure to suppress
873  // the error reporting when many failures occur.
874  static std::atomic<Int_t> nerrors(0);
875 
876  char flag;
877  if (b.IsReading()) {
878  TKey::Streamer(b); //this must be first
879  Version_t v = b.ReadVersion();
880  b >> fBufferSize;
881  // NOTE: we now use the upper-bit of the fNevBufSize to see if we have serialized any of the
882  // optional IOBits. If that bit is set, we immediately read out the IOBits; to replace this
883  // (minimal) safeguard against corruption, we will set aside the upper-bit of fIOBits to do
884  // the same thing (the fact this bit is reserved is tested in the unit tests). If there is
885  // someday a need for more than 7 IOBits, we'll widen the field using the same trick.
886  //
887  // We like to keep this safeguard because we immediately will allocate a buffer based on
888  // the value of fNevBufSize -- and would like to avoid wildly inappropriate allocations.
889  b >> fNevBufSize;
890  if (fNevBufSize < 0) {
892  b >> fIOBits;
893  if (!fIOBits || (fIOBits & (1 << 7))) {
894  Error("TBasket::Streamer",
895  "The value of fNevBufSize (%d) or fIOBits (%d) is incorrect ; setting the buffer to a zombie.",
896  -fNevBufSize, fIOBits);
897  MakeZombie();
898  fNevBufSize = 0;
899  } else if (fIOBits && (fIOBits & ~static_cast<Int_t>(EIOBits::kSupported))) {
900  nerrors++;
901  if (nerrors < 10) {
902  Error("Streamer", "The value of fIOBits (%s) contains unknown flags (supported flags "
903  "are %s), indicating this was written with a newer version of ROOT "
904  "utilizing critical IO features this version of ROOT does not support."
905  " Refusing to deserialize.",
906  std::bitset<32>(static_cast<Int_t>(fIOBits)).to_string().c_str(),
907  std::bitset<32>(static_cast<Int_t>(EIOBits::kSupported)).to_string().c_str());
908  } else if (nerrors == 10) {
909  Error("Streamer", "Maximum number of errors has been reported; disabling further messages"
910  "from this location until the process exits.");
911  }
912  fNevBufSize = 0;
913  MakeZombie();
914  }
915  }
916  b >> fNevBuf;
917  b >> fLast;
918  b >> flag;
920  Bool_t mustGenerateOffsets = false;
921  if (flag >= 80) {
922  mustGenerateOffsets = true;
923  flag -= 80;
924  }
925  if (!mustGenerateOffsets && flag && (flag % 10 != 2)) {
928  if (fNevBuf) b.ReadArray(fEntryOffset);
929  if (20<flag && flag<40) {
930  for(int i=0; i<fNevBuf; i++){
932  }
933  }
934  if (flag>40) {
936  b.ReadArray(fDisplacement);
937  }
938  } else if (mustGenerateOffsets) {
939  // We currently believe that in all cases when offsets can be generated, then the
940  // displacement array must be zero.
941  assert(flag <= 40);
942  fEntryOffset = reinterpret_cast<Int_t *>(-1);
943  }
944  if (flag == 1 || flag > 10) {
946  fBufferRef->SetParent(b.GetParent());
947  char *buf = fBufferRef->Buffer();
948  if (v > 1) b.ReadFastArray(buf,fLast);
949  else b.ReadArray(buf);
951  // This is now done in the TBranch streamer since fBranch might not
952  // yet be set correctly.
953  // fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
954  }
955  } else {
956 
957  TKey::Streamer(b); //this must be first
958  b.WriteVersion(TBasket::IsA());
959  if (fBufferRef) {
960  Int_t curLast = fBufferRef->Length();
961  if (!fHeaderOnly && !fSeekKey && curLast > fLast) fLast = curLast;
962  }
964 
965  b << fBufferSize;
966  if (fIOBits) {
967  b << -fNevBufSize;
968  b << fIOBits;
969  } else {
970  b << fNevBufSize;
971  }
972  b << fNevBuf;
973  b << fLast;
974  Bool_t mustGenerateOffsets = fEntryOffset && fNevBuf &&
977  // We currently believe that in all cases when offsets can be generated, then the
978  // displacement array must be zero.
979  assert(!mustGenerateOffsets || fDisplacement == nullptr);
980  if (fHeaderOnly) {
981  flag = mustGenerateOffsets ? 80 : 0;
982  b << flag;
983  } else {
984  // On return from this function, we are guaranteed that fEntryOffset
985  // is either a valid pointer or nullptr.
986  if (fNevBuf) {
987  GetEntryOffset();
988  }
989  flag = 1;
990  if (!fNevBuf || !fEntryOffset)
991  flag = 2;
992  if (fBufferRef) flag += 10;
993  if (fDisplacement) flag += 40;
994  // Test if we can skip writing out the offset map.
995  if (mustGenerateOffsets) {
996  flag += 80;
997  }
998  b << flag;
999 
1000  if (!mustGenerateOffsets && fEntryOffset && fNevBuf) {
1001  b.WriteArray(fEntryOffset, fNevBuf);
1002  if (fDisplacement) b.WriteArray(fDisplacement, fNevBuf);
1003  }
1004  if (fBufferRef) {
1005  char *buf = fBufferRef->Buffer();
1006  b.WriteFastArray(buf, fLast);
1007  }
1008  }
1009  }
1010 }
1011 
1012 ////////////////////////////////////////////////////////////////////////////////
1013 /// Update basket header and EntryOffset table.
1014 
1015 void TBasket::Update(Int_t offset, Int_t skipped)
1016 {
1017  Int_t *entryOffset = GetEntryOffset();
1018  if (entryOffset) {
1019  if (fNevBuf+1 >= fNevBufSize) {
1020  Int_t newsize = TMath::Max(10,2*fNevBufSize);
1021  Int_t *newoff = TStorage::ReAllocInt(fEntryOffset, newsize,
1022  fNevBufSize);
1023  if (fDisplacement) {
1024  Int_t *newdisp = TStorage::ReAllocInt(fDisplacement, newsize,
1025  fNevBufSize);
1026  fDisplacement = newdisp;
1027  }
1028  fEntryOffset = newoff;
1029  fNevBufSize = newsize;
1030 
1031  //Update branch only for the first 10 baskets
1032  if (fBranch->GetWriteBasket() < 10) {
1033  fBranch->SetEntryOffsetLen(newsize);
1034  }
1035  }
1036  fEntryOffset[fNevBuf] = offset;
1037 
1038  if (skipped!=offset && !fDisplacement){
1040  for (Int_t i = 0; i<fNevBufSize; i++) fDisplacement[i] = fEntryOffset[i];
1041  }
1042  if (fDisplacement) {
1043  fDisplacement[fNevBuf] = skipped;
1045  }
1046  }
1047 
1048  fNevBuf++;
1049 }
1050 
1051 ////////////////////////////////////////////////////////////////////////////////
1052 /// Write buffer of this basket on the current file.
1053 ///
1054 /// The function returns the number of bytes committed to the memory.
1055 /// If a write error occurs, the number of bytes returned is -1.
1056 /// If no data are written, the number of bytes returned is 0.
1057 
1059 {
1060  const Int_t kWrite = 1;
1061 
1062  TFile *file = fBranch->GetFile(kWrite);
1063  if (!file) return 0;
1064  if (!file->IsWritable()) {
1065  return -1;
1066  }
1067  fMotherDir = file; // fBranch->GetDirectory();
1068 
1069  // This mutex prevents multiple TBasket::WriteBuffer invocations from interacting
1070  // with the underlying TFile at once - TFile is assumed to *not* be thread-safe.
1071  //
1072  // The only parallelism we'd like to exploit (right now!) is the compression
1073  // step - everything else should be serialized at the TFile level.
1074 #ifdef R__USE_IMT
1075  std::unique_lock<std::mutex> sentry(file->fWriteMutex);
1076 #endif // R__USE_IMT
1077 
1079  // Read the basket information that was saved inside the buffer.
1080  Bool_t writing = fBufferRef->IsWriting();
1083 
1084  Streamer(*fBufferRef);
1085  if (writing) fBufferRef->SetWriteMode();
1086  Int_t nout = fNbytes - fKeylen;
1087 
1088  fBuffer = fBufferRef->Buffer();
1089 
1090  Create(nout,file);
1092  fHeaderOnly = kTRUE;
1093 
1094  Streamer(*fBufferRef); //write key itself again
1095  int nBytes = WriteFileKeepBuffer();
1096  fHeaderOnly = kFALSE;
1097  return nBytes>0 ? fKeylen+nout : -1;
1098  }
1099 
1100  // Transfer fEntryOffset table at the end of fBuffer.
1101  fLast = fBufferRef->Length();
1102  Int_t *entryOffset = GetEntryOffset();
1103  if (entryOffset) {
1104  Bool_t hasOffsetBit = fIOBits & static_cast<UChar_t>(TBasket::EIOBits::kGenerateOffsetMap);
1105  if (!CanGenerateOffsetArray()) {
1106  // If we have set the offset map flag, but cannot dynamically generate the map, then
1107  // we should at least convert the offset array to a size array. Note that we always
1108  // write out (fNevBuf+1) entries to match the original case.
1109  if (hasOffsetBit) {
1110  for (Int_t idx = fNevBuf; idx > 0; idx--) {
1111  entryOffset[idx] -= entryOffset[idx - 1];
1112  }
1113  entryOffset[0] = 0;
1114  }
1115  fBufferRef->WriteArray(entryOffset, fNevBuf + 1);
1116  // Convert back to offset format: keeping both sizes and offsets in-memory were considered,
1117  // but it seems better to use CPU than memory.
1118  if (hasOffsetBit) {
1119  entryOffset[0] = fKeylen;
1120  for (Int_t idx = 1; idx < fNevBuf + 1; idx++) {
1121  entryOffset[idx] += entryOffset[idx - 1];
1122  }
1123  }
1124  } else if (!hasOffsetBit) { // In this case, write out as normal
1125  fBufferRef->WriteArray(entryOffset, fNevBuf + 1);
1126  }
1127  if (fDisplacement) {
1129  delete[] fDisplacement;
1130  fDisplacement = 0;
1131  }
1132  }
1133 
1134  Int_t lbuf, nout, noutot, bufmax, nzip;
1135  lbuf = fBufferRef->Length();
1136  fObjlen = lbuf - fKeylen;
1137 
1138  fHeaderOnly = kTRUE;
1140  Int_t cxlevel = fBranch->GetCompressionLevel();
1142  if (cxlevel > 0) {
1143  Int_t nbuffers = 1 + (fObjlen - 1) / kMAXZIPBUF;
1144  Int_t buflen = fKeylen + fObjlen + 9 * nbuffers + 28; //add 28 bytes in case object is placed in a deleted gap
1146  if (!fCompressedBufferRef) {
1147  Warning("WriteBuffer", "Unable to allocate the compressed buffer");
1148  return -1;
1149  }
1152  char *objbuf = fBufferRef->Buffer() + fKeylen;
1153  char *bufcur = &fBuffer[fKeylen];
1154  noutot = 0;
1155  nzip = 0;
1156  for (Int_t i = 0; i < nbuffers; ++i) {
1157  if (i == nbuffers - 1) bufmax = fObjlen - nzip;
1158  else bufmax = kMAXZIPBUF;
1159  // Compress the buffer. Note that we allow multiple TBasket compressions to occur at once
1160  // for a given TFile: that's because the compression buffer when we use IMT is no longer
1161  // shared amongst several threads.
1162 #ifdef R__USE_IMT
1163  sentry.unlock();
1164 #endif // R__USE_IMT
1165  // NOTE this is declared with C linkage, so it shouldn't except. Also, when
1166  // USE_IMT is defined, we are guaranteed that the compression buffer is unique per-branch.
1167  // (see fCompressedBufferRef in constructor).
1168  R__zipMultipleAlgorithm(cxlevel, &bufmax, objbuf, &bufmax, bufcur, &nout, cxAlgorithm);
1169 #ifdef R__USE_IMT
1170  sentry.lock();
1171 #endif // R__USE_IMT
1172 
1173  // test if buffer has really been compressed. In case of small buffers
1174  // when the buffer contains random data, it may happen that the compressed
1175  // buffer is larger than the input. In this case, we write the original uncompressed buffer
1176  if (nout == 0 || nout >= fObjlen) {
1177  nout = fObjlen;
1178  // We used to delete fBuffer here, we no longer want to since
1179  // the buffer (held by fCompressedBufferRef) might be re-used later.
1180  fBuffer = fBufferRef->Buffer();
1181  Create(fObjlen,file);
1183 
1184  Streamer(*fBufferRef); //write key itself again
1185  if ((nout+fKeylen)>buflen) {
1186  Warning("WriteBuffer","Possible memory corruption due to compression algorithm, wrote %d bytes past the end of a block of %d bytes. fNbytes=%d, fObjLen=%d, fKeylen=%d",
1187  (nout+fKeylen-buflen),buflen,fNbytes,fObjlen,fKeylen);
1188  }
1189  goto WriteFile;
1190  }
1191  bufcur += nout;
1192  noutot += nout;
1193  objbuf += kMAXZIPBUF;
1194  nzip += kMAXZIPBUF;
1195  }
1196  nout = noutot;
1197  Create(noutot,file);
1199 
1200  Streamer(*fBufferRef); //write key itself again
1201  memcpy(fBuffer,fBufferRef->Buffer(),fKeylen);
1202  } else {
1203  fBuffer = fBufferRef->Buffer();
1204  Create(fObjlen,file);
1206 
1207  Streamer(*fBufferRef); //write key itself again
1208  nout = fObjlen;
1209  }
1210 
1211 WriteFile:
1212  Int_t nBytes = WriteFileKeepBuffer();
1213  fHeaderOnly = kFALSE;
1214  return nBytes>0 ? fKeylen+nout : -1;
1215 }
TBranch::GetCompressionAlgorithm
Int_t GetCompressionAlgorithm() const
Definition: TBranch.h:295
TBasket::TBasket
TBasket()
Default contructor.
Definition: TBasket.cxx:47
TBasket::InitializeCompressedBuffer
void InitializeCompressedBuffer(Int_t len, TFile *file)
Initialize the compressed buffer; either from the TTree or create a local one.
Definition: TBasket.cxx:432
TBasket::ReadBasketBuffers
Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
Read basket buffers in memory and cleanup.
Definition: TBasket.cxx:464
TVirtualPerfStats.h
TBranch::GetNleaves
Int_t GetNleaves() const
Definition: TBranch.h:245
TKey::GetKeylen
Int_t GetKeylen() const
Definition: TKey.h:85
TBranch::GetEntryOffsetLen
Int_t GetEntryOffsetLen() const
Definition: TBranch.h:223
kTRUE
const Bool_t kTRUE
Definition: RtypesCore.h:91
TBasket::DisownBuffer
void DisownBuffer()
Disown all references to the internal buffer - some other object likely now owns it.
Definition: TBasket.cxx:713
TObject::TestBit
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition: TObject.h:172
Version_t
short Version_t
Definition: RtypesCore.h:65
TBuffer::BufferSize
Int_t BufferSize() const
Definition: TBuffer.h:98
R__InitializeReadBasketBuffer
static TBuffer * R__InitializeReadBasketBuffer(TBuffer *bufferRef, Int_t len, TFile *file)
Initialize a buffer for reading if it is not already initialized.
Definition: TBasket.cxx:410
TNamed::SetName
virtual void SetName(const char *name)
Set the name of the TNamed.
Definition: TNamed.cxx:140
TMath::Max
Short_t Max(Short_t a, Short_t b)
Definition: TMathBase.h:212
TBuffer::kRead
@ kRead
Definition: TBuffer.h:73
TBuffer::SetBuffer
void SetBuffer(void *buf, UInt_t bufsiz=0, Bool_t adopt=kTRUE, ReAllocCharFun_t reallocfunc=0)
Sets a new buffer in an existing TBuffer object.
Definition: TBuffer.cxx:187
TFileCacheRead::AddNoCacheReadCalls
virtual void AddNoCacheReadCalls(Int_t reads)
Definition: TFileCacheRead.h:86
fc
static struct mg_connection * fc(struct mg_context *ctx)
Definition: civetweb.c:3728
TTimeStamp.h
TKey::fVersion
Int_t fVersion
Key version identifier.
Definition: TKey.h:39
TTree::IncrementTotalBuffers
virtual void IncrementTotalBuffers(Int_t nbytes)
Definition: TTree.h:541
R__unzip
void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout)
TBuffer::Expand
void Expand(Int_t newsize, Bool_t copy=kTRUE)
Expand (or shrink) the I/O buffer to newsize bytes.
Definition: TBuffer.cxx:223
ROOT::RCompressionSetting::EAlgorithm::EValues
EValues
Note: this is only temporarily a struct and will become a enum class hence the name.
Definition: Compression.h:95
tree
Definition: tree.py:1
ClassImp
#define ClassImp(name)
Definition: Rtypes.h:364
TBasket::fReadEntryOffset
Bool_t fReadEntryOffset
!Set to true if offset array was read from a file.
Definition: TBasket.h:69
TTree::GetReadCache
TTreeCache * GetReadCache(TFile *file) const
Find and return the TTreeCache registered with the file and which may contain branches for us.
Definition: TTree.cxx:6211
TBranch.h
TObject::Info
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:864
Long64_t
long long Long64_t
Definition: RtypesCore.h:73
TObject::Error
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:890
TTree
Definition: TTree.h:79
TBranch::GetTransientBuffer
TBuffer * GetTransientBuffer(Int_t size)
Returns the transient buffer currently used by this TBranch for reading/writing baskets.
Definition: TBranch.cxx:513
TBasket::CanGenerateOffsetArray
Bool_t CanGenerateOffsetArray()
Determine whether we can generate the offset array when this branch is read.
Definition: TBasket.cxx:218
Float_t
float Float_t
Definition: RtypesCore.h:57
TKey::GetObjlen
Int_t GetObjlen() const
Definition: TKey.h:88
R__unzip_header
int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout)
Int_t
int Int_t
Definition: RtypesCore.h:45
TVirtualPerfStats
Definition: TVirtualPerfStats.h:31
TBuffer::Buffer
char * Buffer() const
Definition: TBuffer.h:96
TBasket::fNevBufSize
Int_t fNevBufSize
Length in Int_t of fEntryOffset OR fixed length of each entry if fEntryOffset is null!
Definition: TBasket.h:63
R__LOCKGUARD_IMT
#define R__LOCKGUARD_IMT(mutex)
Definition: TVirtualMutex.h:111
TKey::WriteFile
virtual Int_t WriteFile(Int_t cycle=1, TFile *f=0)
Write the encoded object supported by this key.
Definition: TKey.cxx:1450
TIOFeatures.hxx
OLD_CASE_EXPRESSION
#define OLD_CASE_EXPRESSION
Definition: TBasket.cxx:353
R__likely
#define R__likely(expr)
Definition: RConfig.hxx:605
TBasket::fBranch
TBranch * fBranch
Pointer to the basket support branch.
Definition: TBasket.h:73
TFileCacheRead::GetUnzipBuffer
virtual Int_t GetUnzipBuffer(char **, Long64_t, Int_t, Bool_t *)
Definition: TFileCacheRead.h:97
TKey::fMotherDir
TDirectory * fMotherDir
!pointer to mother directory
Definition: TKey.h:52
TBasket::CopyTo
Long64_t CopyTo(TFile *to)
Copy the basket of this branch onto the file to.
Definition: TBasket.cxx:146
TBuffer::ReadArray
virtual Int_t ReadArray(Bool_t *&b)=0
TLeaf::GenerateOffsetArray
virtual Int_t * GenerateOffsetArray(Int_t base, Int_t events)
Definition: TLeaf.h:114
TKey::fBuffer
char * fBuffer
Object buffer.
Definition: TKey.h:49
TBuffer
Definition: TBuffer.h:43
TKey::GetBufferRef
TBuffer * GetBufferRef() const
Definition: TKey.h:80
TKey::fBufferRef
TBuffer * fBufferRef
Pointer to the TBuffer object.
Definition: TKey.h:50
TTree.h
TBasket::fLastWriteBufferSize
Int_t fLastWriteBufferSize[3]
! Size of the buffer last three buffers we wrote it to disk
Definition: TBasket.h:75
Bool_t
bool Bool_t
Definition: RtypesCore.h:63
TBuffer::SetBufferOffset
void SetBufferOffset(Int_t offset=0)
Definition: TBuffer.h:93
v
@ v
Definition: rootcling_impl.cxx:3635
b
#define b(i)
Definition: RSha256.hxx:118
TFile.h
bool
TTree::GetTargetMemoryRatio
Float_t GetTargetMemoryRatio() const
Definition: TTree.h:515
TBranch::GetTotBytes
Long64_t GetTotBytes(Option_t *option="") const
Return total number of bytes in the branch (excluding current buffer) if option ="*" includes all sub...
Definition: TBranch.cxx:2053
TROOT.h
TBasket::ResetEntryOffset
void ResetEntryOffset()
Definition: TBasket.cxx:441
TBasket::MoveEntries
virtual void MoveEntries(Int_t dentries)
Remove the first dentries of this basket, moving entries at dentries to the start of the buffer.
Definition: TBasket.cxx:310
TBasket::ReadBasketBytes
Int_t ReadBasketBytes(Long64_t pos, TFile *file)
Read basket buffers in memory and cleanup.
Definition: TBasket.cxx:698
TBasket::fDisplacement
Int_t * fDisplacement
![fNevBuf] Displacement of entries in fBuffer(TKey)
Definition: TBasket.h:70
TBasket::DropBuffers
virtual Int_t DropBuffers()
Drop buffers of this basket if it is not the current basket.
Definition: TBasket.cxx:173
TBranch
Definition: TBranch.h:89
TStorage::ReAllocInt
static Int_t * ReAllocInt(Int_t *vp, size_t size, size_t oldsize)
Reallocate (i.e.
Definition: TStorage.cxx:295
TBuffer::Length
Int_t Length() const
Definition: TBuffer.h:100
TBasket::fIOBits
UChar_t fIOBits
!IO feature flags. Serialized in custom portion of streamer to avoid forward compat issues unless nee...
Definition: TBasket.h:67
TTreeCache.h
TBuffer::SetReadMode
void SetReadMode()
Set buffer in read mode.
Definition: TBuffer.cxx:302
TBuffer.h
TBranch::GetFile
virtual TFile * GetFile(Int_t mode=0)
Return pointer to the file where branch buffers reside, returns 0 in case branch buffers reside in th...
Definition: TBranch.cxx:1729
TLeaf.h
TBufferIO::kNotDecompressed
@ kNotDecompressed
Definition: TBufferIO.h:66
TBasket::DeleteEntryOffset
virtual void DeleteEntryOffset()
Delete fEntryOffset array.
Definition: TBasket.cxx:164
gROOTMutex
R__EXTERN TVirtualMutex * gROOTMutex
Definition: TROOT.h:61
TFileCacheRead::ReadBuffer
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos.
Definition: TFileCacheRead.cxx:363
TGeant4Unit::us
static constexpr double us
Definition: TGeant4SystemOfUnits.h:170
TLeaf
Definition: TLeaf.h:57
TObject::SetBit
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:696
TBuffer::IsWriting
Bool_t IsWriting() const
Definition: TBuffer.h:87
TTree::GetPerfStats
virtual TVirtualPerfStats * GetPerfStats() const
Definition: TTree.h:501
TKey::fClassName
TString fClassName
Object Class name.
Definition: TKey.h:47
TBasket::SetWriteMode
virtual void SetWriteMode()
Set write mode of basket.
Definition: TBasket.cxx:861
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:92
TBasket::fLast
Int_t fLast
Pointer to last used byte in basket.
Definition: TBasket.h:65
gDebug
R__EXTERN Int_t gDebug
Definition: RtypesCore.h:119
Long_t
long Long_t
Definition: RtypesCore.h:54
R__unlikely
#define R__unlikely(expr)
Definition: RConfig.hxx:604
TBranch::GetCompressionLevel
Int_t GetCompressionLevel() const
Definition: TBranch.h:301
TBasket::LoadBasketBuffers
Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree=0)
Load basket buffers in memory without unziping.
Definition: TBasket.cxx:245
TBuffer::SetBufferDisplacement
virtual void SetBufferDisplacement()=0
TBasket::Reset
virtual void Reset()
Reset the basket to the starting state.
Definition: TBasket.cxx:733
UInt_t
unsigned int UInt_t
Definition: RtypesCore.h:46
TBasket::GetEntryOffset
Int_t * GetEntryOffset()
Definition: TBasket.h:122
TFileCacheRead::AddNoCacheBytesRead
virtual void AddNoCacheBytesRead(Long64_t len)
Definition: TFileCacheRead.h:85
TObject::MakeZombie
void MakeZombie()
Definition: TObject.h:49
TLeaf::CanGenerateOffsetArray
virtual Bool_t CanGenerateOffsetArray()
Definition: TLeaf.h:111
TBasket::fEntryOffset
Int_t * fEntryOffset
[fNevBuf] Offset of entries in fBuffer(TKey); generated at runtime.
Definition: TBasket.h:71
TBasket::GetCalculatedEntryOffset
Int_t * GetCalculatedEntryOffset()
Calculates the entry offset array, if possible.
Definition: TBasket.cxx:195
TBasket::fBufferSize
Int_t fBufferSize
fBuffer length in bytes
Definition: TBasket.h:62
TNamed::SetTitle
virtual void SetTitle(const char *title="")
Set the title of the TNamed.
Definition: TNamed.cxx:164
TBranch::GetBasketSize
virtual Int_t GetBasketSize() const
Definition: TBranch.h:213
TVirtualMutex.h
TObject::Warning
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:876
TBasket::fNevBuf
Int_t fNevBuf
Number of entries in basket.
Definition: TBasket.h:64
TFile
Definition: TFile.h:54
TFileCacheRead
Definition: TFileCacheRead.h:22
TKey::fKeylen
Short_t fKeylen
Number of bytes for the key itself.
Definition: TKey.h:43
TObject::IsZombie
R__ALWAYS_INLINE Bool_t IsZombie() const
Definition: TObject.h:134
TKey
Definition: TKey.h:28
TBasket::fHeaderOnly
Bool_t fHeaderOnly
True when only the basket header must be read/written.
Definition: TBasket.h:66
TBasket::WriteBuffer
virtual Int_t WriteBuffer()
Write buffer of this basket on the current file.
Definition: TBasket.cxx:1058
TBranch::GetDirectory
TDirectory * GetDirectory() const
Definition: TBranch.h:220
TBasket
Definition: TBasket.h:34
TTreeCache
A cache to speed-up the reading of ROOT datasets.
Definition: TTreeCache.h:32
TKey::fCycle
Short_t fCycle
Cycle number.
Definition: TKey.h:44
TBasket::SetReadMode
virtual void SetReadMode()
Set read mode of basket.
Definition: TBasket.cxx:852
Double_t
double Double_t
Definition: RtypesCore.h:59
TKey::WriteFileKeepBuffer
virtual Int_t WriteFileKeepBuffer(TFile *f=0)
Write the encoded object supported by this key.
Definition: TKey.cxx:1491
TKey::Reset
virtual void Reset()
Reset the key as it had not been 'filled' yet.
Definition: TKey.cxx:1302
TKey::fNbytes
Int_t fNbytes
Number of bytes for the object on file.
Definition: TKey.h:40
TBasket::fResetAllocation
Bool_t fResetAllocation
! True if last reset re-allocated the memory
Definition: TBasket.h:76
file
Definition: file.py:1
TBasket::ReadBasketBuffersUnzip
Int_t ReadBasketBuffersUnzip(char *, Int_t, Bool_t, TFile *)
We always create the TBuffer for the basket but it hold the buffer from the cache.
Definition: TBasket.cxx:380
UChar_t
unsigned char UChar_t
Definition: RtypesCore.h:38
TBasket::AdjustSize
virtual void AdjustSize(Int_t newsize)
Increase the size of the current fBuffer up to newsize.
Definition: TBasket.cxx:127
TBasket::ReadBasketBuffersUncompressedCase
Int_t ReadBasketBuffersUncompressedCase()
By-passing buffer unzipping has been requested and is possible (only 1 entry in this basket).
Definition: TBasket.cxx:358
TBufferFile.h
TBasket::GetEntryPointer
Int_t GetEntryPointer(Int_t Entry)
Get pointer to buffer for internal entry.
Definition: TBasket.cxx:230
TBasket::fNextBufferSizeRecord
UChar_t fNextBufferSizeRecord
! Index into fLastWriteBufferSize of the last buffer written to disk
Definition: TBasket.h:77
TBranch::GetWriteBasket
Int_t GetWriteBasket() const
Definition: TBranch.h:234
TBasket::~TBasket
virtual ~TBasket()
Basket destructor.
Definition: TBasket.cxx:104
TKey::fObjlen
Int_t fObjlen
Length of uncompressed object in bytes.
Definition: TKey.h:41
name
char name[80]
Definition: TGX11.cxx:110
TDirectory
Definition: TDirectory.h:40
TBuffer::WriteArray
virtual void WriteArray(const Bool_t *b, Int_t n)=0
TBuffer::SetWriteMode
void SetWriteMode()
Set buffer in write mode.
Definition: TBuffer.cxx:315
TTree::GetTransientBuffer
TBuffer * GetTransientBuffer(Int_t size)
Returns the transient buffer currently used by this TTree for reading/writing baskets.
Definition: TTree.cxx:1016
TBuffer::Reset
virtual void Reset()=0
TNamed::GetName
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:53
free
#define free
Definition: civetweb.c:1539
TTimeStamp
Definition: TTimeStamp.h:71
TBuffer::SetParent
void SetParent(TObject *parent)
Set parent owning this buffer.
Definition: TBuffer.cxx:270
TBranch::GetListOfLeaves
TObjArray * GetListOfLeaves()
Definition: TBranch.h:243
TBranch::GetTree
TTree * GetTree() const
Definition: TBranch.h:248
TBasket::fOwnsCompressedBuffer
Bool_t fOwnsCompressedBuffer
! Whether or not we own the compressed buffer.
Definition: TBasket.h:68
TBasket::EIOBits::kGenerateOffsetMap
@ kGenerateOffsetMap
TKey::fSeekKey
Long64_t fSeekKey
Location of object on file.
Definition: TKey.h:45
TBasket::Update
void Update(Int_t newlast)
Definition: TBasket.h:150
TKey::Create
virtual void Create(Int_t nbytes, TFile *f=0)
Create a TKey object of specified size.
Definition: TKey.cxx:459
TBasket::fCompressedBufferRef
TBuffer * fCompressedBufferRef
! Compressed buffer.
Definition: TBasket.h:74
gPerfStats
#define gPerfStats
Definition: TVirtualPerfStats.h:92
TBufferFile
Definition: TBufferFile.h:47
TBasket::AdoptBuffer
void AdoptBuffer(TBuffer *user_buffer)
Adopt a buffer from an external entity.
Definition: TBasket.cxx:721
TBasket.h
TBranch::SetEntryOffsetLen
virtual void SetEntryOffsetLen(Int_t len, Bool_t updateSubBranches=kFALSE)
Update the default value for the branch's fEntryOffsetLen if and only if it was already non zero (and...
Definition: TBranch.cxx:2654
TBuffer::kWrite
@ kWrite
Definition: TBuffer.h:73
kDisplacementMask
const UInt_t kDisplacementMask
Definition: TBasket.cxx:31
TMath.h
TBasket::EIOBits::kSupported
@ kSupported
int