62 :
TKey(branch->GetDirectory()), fBufferSize(branch->GetBasketSize()), fNevBufSize(branch->GetEntryOffsetLen()),
63 fHeaderOnly(true), fIOBits(branch->GetIOFeatures().GetFeatures())
158 return nBytes>0 ? nBytes : -1;
202 Error(
"GetCalculatedEntryOffset",
"Basket entry offset calculation requested, but no associated TBranch!");
206 Error(
"GetCalculatedEntryOffset",
"Branch contains multiple leaves - unable to calculated entry offsets!");
234 if (entryOffset)
offset = entryOffset[entry];
267 if (tree->GetPerfStats())
gPerfStats = tree->GetPerfStats();
271 }
else if (st == 0) {
291 if (tree->GetPerfStats() !=
nullptr)
gPerfStats = tree->GetPerfStats();
314 if (dentries >=
fNevBuf)
return;
320 bufbegin = entryOffset[dentries];
331 entryOffset[i] = entryOffset[i + dentries] - moved;
347 char *buffer = buf->
Buffer();
353#define OLD_CASE_EXPRESSION fObjlen==fNbytes-fKeylen && GetBranch()->GetCompressionLevel()!=0 && file->GetVersion()<=30401
416 if (curBufferSize <
len) {
471 char *rawUncompressedBuffer, *rawCompressedBuffer;
472 Int_t uncompressedBufferLen;
483 char *buffer =
nullptr;
489 if (
len <= 0)
return -
len;
511 if (!readBufferRef) {
512 Error(
"ReadBasketBuffers",
"Unable to allocate buffer.");
526 }
else if (st == 0) {
558 rawCompressedBuffer = readBufferRef->
Buffer();
571 Error(
"ReadBasketBuffers",
"Unable to allocate buffer.");
587 fBuffer = rawUncompressedBuffer;
602 memcpy(rawUncompressedBuffer, rawCompressedBuffer,
fKeylen);
603 char *rawUncompressedObjectBuffer = rawUncompressedBuffer+
fKeylen;
606 Int_t nout = 0, noutot = 0, nintot = 0;
612 Error(
"ReadBasketBuffers",
"Inconsistency found in header (nin=%d, nbuf=%d)", nin, nbuf);
621 R__unzip(&nin, rawCompressedObjectBuffer, &nbuf, (
unsigned char*) rawUncompressedObjectBuffer, &nout);
626 rawCompressedObjectBuffer += nin;
627 rawUncompressedObjectBuffer += nout;
632 Error(
"ReadBasketBuffers",
"fNbytes = %d, fKeylen = %d, fObjlen = %d, noutot = %d, nout=%d, nin=%d, nbuf=%d",
fNbytes,
fKeylen,
fObjlen, noutot,nout,nin,nbuf);
645 memcpy(rawUncompressedBuffer, rawCompressedBuffer,
len);
666 Warning(
"ReadBasketBuffers",
"basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%lld, len=%d, fNbytes=%d, fObjlen=%d, trying to repair",
GetName(),
fNevBuf,pos,
len,
fNbytes,
fObjlen);
737#ifdef R__TRACK_BASKET_ALLOC_TIME
738 fResetAllocationTime = 0;
744 if (!
fBufferRef || basketnumber >= maxbaskets)
752 Int_t max_size = basketbytes[basketnumber];
753 for(
Int_t b = basketnumber + 1; (
b < maxbaskets) && (
b < (basketnumber+10)); ++
b) {
754 max_size = std::max(max_size, basketbytes[
b]);
761 Int_t target_size =
static_cast<Int_t>(cx * target_mem_ratio *
Float_t(max_size));
763 if (target_size && (curSize > target_size)) {
765 Int_t newSize = max_size + 512 - max_size % 512;
767 if ((newSize <= curSize - 8 * 1024) &&
768 (
static_cast<Float_t>(curSize) /
static_cast<Float_t>(newSize) > target_mem_ratio))
771 Info(
"ReadResetBuffer",
772 "Resizing %d to %d bytes (was %d); next 10 sizes are [%d, %d, %d, %d, %d, %d, %d, %d, %d, %d]. cx=%f ratio=%f max_size = %d ",
773 basketnumber, newSize, curSize,
774 basketbytes[basketnumber],
775 (basketnumber + 1) < maxbaskets ? basketbytes[basketnumber + 1] : 0,
776 (basketnumber + 2) < maxbaskets ? basketbytes[basketnumber + 2] : 0,
777 (basketnumber + 3) < maxbaskets ? basketbytes[basketnumber + 3] : 0,
778 (basketnumber + 4) < maxbaskets ? basketbytes[basketnumber + 4] : 0,
779 (basketnumber + 5) < maxbaskets ? basketbytes[basketnumber + 5] : 0,
780 (basketnumber + 6) < maxbaskets ? basketbytes[basketnumber + 6] : 0,
781 (basketnumber + 7) < maxbaskets ? basketbytes[basketnumber + 7] : 0,
782 (basketnumber + 8) < maxbaskets ? basketbytes[basketnumber + 8] : 0,
783 (basketnumber + 9) < maxbaskets ? basketbytes[basketnumber + 9] : 0,
784 cx, target_mem_ratio, max_size);
787#ifdef R__TRACK_BASKET_ALLOC_TIME
788 std::chrono::time_point<std::chrono::system_clock> start, end;
789 start = std::chrono::high_resolution_clock::now();
792#ifdef R__TRACK_BASKET_ALLOC_TIME
793 end = std::chrono::high_resolution_clock::now();
794 auto us = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
795 fResetAllocationTime = us.count();
810#ifdef R__TRACK_BASKET_ALLOC_TIME
811 fResetAllocationTime = 0;
825 if (curSize > 2*curLen)
828 if (curSize > 2*curBsize ) {
830 if (curSize > 2*avgSize) {
832 if (curLen > newSize) {
835 if (avgSize > newSize) {
838 newSize = newSize + 512 - newSize%512;
853 if (max_size && (curSize > target_size) && (newSize == -1)) {
854 newSize = target_size;
855 newSize = newSize + 512 - newSize % 512;
857 if ((newSize > curSize - 8 * 1024) ||
858 (
static_cast<Float_t>(curSize) /
static_cast<Float_t>(newSize) < target_mem_ratio)) {
861 Info(
"Reset",
"Resizing to %ld bytes (was %d); last three sizes were [%d, %d, %d].", newSize, curSize,
868#ifdef R__TRACK_BASKET_ALLOC_TIME
869 std::chrono::time_point<std::chrono::system_clock> start, end;
870 start = std::chrono::high_resolution_clock::now();
873#ifdef R__TRACK_BASKET_ALLOC_TIME
874 end = std::chrono::high_resolution_clock::now();
875 auto us = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
876 fResetAllocationTime = us.count();
887 if (newNevBufSize==0) {
947 static std::atomic<Int_t> nerrors(0);
967 Error(
"TBasket::Streamer",
968 "The value of fNevBufSize (%d) or fIOBits (%d) is incorrect ; setting the buffer to a zombie.",
975 Error(
"Streamer",
"The value of fIOBits (%s) contains unknown flags (supported flags "
976 "are %s), indicating this was written with a newer version of ROOT "
977 "utilizing critical IO features this version of ROOT does not support."
978 " Refusing to deserialize.",
979 std::bitset<32>(
static_cast<Int_t>(
fIOBits)).to_string().c_str(),
981 }
else if (nerrors == 10) {
982 Error(
"Streamer",
"Maximum number of errors has been reported; disabling further messages"
983 "from this location until the process exits.");
993 bool mustGenerateOffsets =
false;
995 mustGenerateOffsets =
true;
998 if (!mustGenerateOffsets && flag && (flag % 10 != 2)) {
1002 if (20<flag && flag<40) {
1011 }
else if (mustGenerateOffsets) {
1017 if (flag == 1 || flag > 10) {
1021 if (
v > 1)
b.ReadFastArray(buf,
fLast);
1022 else b.ReadArray(buf);
1054 flag = mustGenerateOffsets ? 80 : 0;
1068 if (mustGenerateOffsets) {
1079 b.WriteFastArray(buf,
fLast);
1133 constexpr Int_t kWrite = 1;
1136 if (!file)
return 0;
1148 std::unique_lock<std::mutex> sentry(file->
fWriteMutex);
1170 return nBytes>0 ?
fKeylen+nout : -1;
1184 entryOffset[idx] -= entryOffset[idx - 1];
1194 entryOffset[idx] += entryOffset[idx - 1];
1197 }
else if (!hasOffsetBit) {
1207 Int_t nout, noutot, bufmax, nzip;
1224 Warning(
"WriteBuffer",
"Unable to allocate the compressed buffer");
1233 for (
Int_t i = 0; i < nbuffers; ++i) {
1234 if (i == nbuffers - 1) bufmax =
fObjlen - nzip;
1235 else bufmax = kMAXZIPBUF;
1245 R__zipMultipleAlgorithm(cxlevel, &bufmax, objbuf, &bufmax, bufcur, &nout, cxAlgorithm);
1253 if (nout == 0 || nout >=
fObjlen) {
1263 Warning(
"WriteBuffer",
"Possible memory corruption due to compression algorithm, wrote %d bytes past the end of a block of %d bytes. fNbytes=%d, fObjLen=%d, fKeylen=%d",
1270 objbuf += kMAXZIPBUF;
1291 return nBytes>0 ?
fKeylen+nout : -1;
#define R__unlikely(expr)
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
static TBuffer * R__InitializeReadBasketBuffer(TBuffer *bufferRef, Int_t len, TFile *file)
Initialize a buffer for reading if it is not already initialized.
#define OLD_CASE_EXPRESSION
const UInt_t kDisplacementMask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
R__EXTERN TVirtualMutex * gROOTMutex
void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout)
int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout)
#define R__LOCKGUARD_IMT(mutex)
Manages buffers for branches of a Tree.
Int_t * fEntryOffset
[fNevBuf] Offset of entries in fBuffer(TKey); generated at runtime.
Int_t GetEntryPointer(Int_t Entry)
Get pointer to buffer for internal entry.
void AdoptBuffer(TBuffer *user_buffer)
Adopt a buffer from an external entity.
bool fHeaderOnly
True when only the basket header must be read/written.
TBranch * fBranch
Pointer to the basket support branch.
Long64_t CopyTo(TFile *to)
Copy the basket of this branch onto the file to.
void InitializeCompressedBuffer(Int_t len, TFile *file)
Initialize the compressed buffer; either from the TTree or create a local one.
UChar_t fNextBufferSizeRecord
! Index into fLastWriteBufferSize of the last buffer written to disk
Int_t ReadBasketBuffersUnzip(char *, Int_t, bool, TFile *)
We always create the TBuffer for the basket but it hold the buffer from the cache.
void DisownBuffer()
Disown all references to the internal buffer - some other object likely now owns it.
Int_t fLastWriteBufferSize[3]
! Size of the buffer last three buffers we wrote it to disk
Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
Read basket buffers in memory and cleanup.
UChar_t fIOBits
!IO feature flags. Serialized in custom portion of streamer to avoid forward compat issues unless nee...
virtual Int_t DropBuffers()
Drop buffers of this basket if it is not the current basket.
virtual void MoveEntries(Int_t dentries)
Remove the first dentries of this basket, moving entries at dentries to the start of the buffer.
bool fOwnsCompressedBuffer
! Whether or not we own the compressed buffer.
TBasket()
Default constructor.
virtual void AdjustSize(Int_t newsize)
Increase the size of the current fBuffer up to newsize.
virtual void SetReadMode()
Set read mode of basket.
virtual void SetWriteMode()
Set write mode of basket.
Int_t ReadBasketBuffersUncompressedCase()
By-passing buffer unzipping has been requested and is possible (only 1 entry in this basket).
Int_t fBufferSize
fBuffer length in bytes
bool fReadEntryOffset
!Set to true if offset array was read from a file.
Int_t ReadBasketBytes(Long64_t pos, TFile *file)
Read basket buffers in memory and cleanup.
virtual void ReadResetBuffer(Int_t basketnumber)
Reset the read basket TBuffer memory allocation if needed.
bool fResetAllocation
! True if last reset re-allocated the memory
virtual Int_t WriteBuffer()
Write buffer of this basket on the current file.
void Streamer(TBuffer &) override
Stream a class object.
Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree=nullptr)
Load basket buffers in memory without unziping.
~TBasket() override
Basket destructor.
void Update(Int_t newlast)
Int_t fNevBufSize
Length in Int_t of fEntryOffset OR fixed length of each entry if fEntryOffset is null!
Int_t fNevBuf
Number of entries in basket.
Int_t * GetCalculatedEntryOffset()
Calculates the entry offset array, if possible.
TClass * IsA() const override
virtual void DeleteEntryOffset()
Delete fEntryOffset array.
TBuffer * fCompressedBufferRef
! Compressed buffer.
Int_t fLast
Pointer to last used byte in basket.
Int_t * fDisplacement
![fNevBuf] Displacement of entries in fBuffer(TKey)
bool CanGenerateOffsetArray()
Determine whether we can generate the offset array when this branch is read.
virtual void WriteReset()
Reset the write basket to the starting state.
A TTree is a list of TBranches.
Int_t GetCompressionLevel() const
Int_t GetCompressionAlgorithm() const
Int_t GetWriteBasket() const
virtual void SetEntryOffsetLen(Int_t len, bool updateSubBranches=false)
Update the default value for the branch's fEntryOffsetLen if and only if it was already non zero (and...
Int_t GetMaxBaskets() const
Int_t * GetBasketBytes() const
virtual TFile * GetFile(Int_t mode=0)
Return pointer to the file where branch buffers reside, returns 0 in case branch buffers reside in th...
Long64_t GetTotBytes(Option_t *option="") const
Return total number of bytes in the branch (excluding current buffer) if option ="*" includes all sub...
virtual Int_t GetBasketSize() const
Long64_t GetZipBytes(Option_t *option="") const
Return total number of zip bytes in the branch if option ="*" includes all sub-branches of this branc...
TObjArray * GetListOfLeaves()
TDirectory * GetDirectory() const
TBuffer * GetTransientBuffer(Int_t size)
Returns the transient buffer currently used by this TBranch for reading/writing baskets.
Int_t GetEntryOffsetLen() const
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Buffer base class used for serializing objects.
void SetWriteMode()
Set buffer in write mode.
void SetParent(TObject *parent)
Set parent owning this buffer.
void Expand(Int_t newsize, Bool_t copy=kTRUE)
Expand (or shrink) the I/O buffer to newsize bytes.
void SetBuffer(void *buf, UInt_t bufsiz=0, Bool_t adopt=kTRUE, ReAllocCharFun_t reallocfunc=nullptr)
Sets a new buffer in an existing TBuffer object.
virtual void WriteArray(const Bool_t *b, Int_t n)=0
virtual Int_t ReadArray(Bool_t *&b)=0
void SetBufferOffset(Int_t offset=0)
void SetReadMode()
Set buffer in read mode.
virtual void SetBufferDisplacement()=0
Bool_t IsWritable() const override
Describe directory structure in memory.
A cache when reading files over the network.
virtual Int_t GetUnzipBuffer(char **, Long64_t, Int_t, Bool_t *)
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos.
virtual void AddNoCacheReadCalls(Int_t reads)
virtual void AddNoCacheBytesRead(Long64_t len)
A ROOT file is an on-disk file, usually with extension .root, that stores objects in a file-system-li...
virtual void Seek(Long64_t offset, ERelativeTo pos=kBeg)
Seek to a specific position in the file. Pos it either kBeg, kCur or kEnd.
Int_t GetCompressionLevel() const
std::mutex fWriteMutex
!Lock for writing baskets / keys into the file.
virtual void SetOffset(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
Int_t GetCompressionAlgorithm() const
Int_t GetRecordHeader(char *buf, Long64_t first, Int_t maxbytes, Int_t &nbytes, Int_t &objlen, Int_t &keylen)
Read the logical record header starting at a certain postion.
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read a buffer from the file.
TFileCacheRead * GetCacheRead(const TObject *tree=nullptr) const
Return a pointer to the current read cache.
Book space in a file, create I/O buffers, to fill them, (un)compress them.
TBuffer * fBufferRef
Pointer to the TBuffer object.
Int_t fVersion
Key version identifier.
Short_t fKeylen
Number of bytes for the key itself.
virtual Int_t WriteFileKeepBuffer(TFile *f=nullptr)
Write the encoded object supported by this key.
Long64_t fSeekKey
Location of object on file.
char * fBuffer
Object buffer.
Int_t fNbytes
Number of bytes for the object on file.
Int_t fObjlen
Length of uncompressed object in bytes.
void Reset()
Reset the key as it had not been 'filled' yet.
Short_t fCycle
Cycle number.
virtual void Create(Int_t nbytes, TFile *f=nullptr)
Create a TKey object of specified size.
virtual Int_t WriteFile(Int_t cycle=1, TFile *f=nullptr)
Write the encoded object supported by this key.
TDirectory * fMotherDir
!pointer to mother directory
TString fClassName
Object Class name.
void Streamer(TBuffer &) override
Stream a class object.
TBuffer * GetBufferRef() const
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
virtual bool CanGenerateOffsetArray()
virtual Int_t * GenerateOffsetArray(Int_t base, Int_t events)
virtual void SetTitle(const char *title="")
Set the title of the TNamed.
const char * GetName() const override
Returns name of object.
virtual void SetName(const char *name)
Set the name of the TNamed.
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
R__ALWAYS_INLINE Bool_t IsZombie() const
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
static Int_t * ReAllocInt(Int_t *vp, size_t size, size_t oldsize)
Reallocate (i.e.
The TTimeStamp encapsulates seconds and ns since EPOCH.
A cache to speed-up the reading of ROOT datasets.
A TTree represents a columnar dataset.
virtual TVirtualPerfStats * GetPerfStats() const
virtual void IncrementTotalBuffers(Int_t nbytes)
TTreeCache * GetReadCache(TFile *file) const
Find and return the TTreeCache registered with the file and which may contain branches for us.
Float_t GetTargetMemoryRatio() const
TBuffer * GetTransientBuffer(Int_t size)
Returns the transient buffer currently used by this TTree for reading/writing baskets.
Provides the interface for the PROOF internal performance measurement and event tracing.
virtual void UnzipEvent(TObject *tree, Long64_t pos, Double_t start, Int_t complen, Int_t objlen)=0
Short_t Max(Short_t a, Short_t b)
Returns the largest of a and b.
EValues
Note: this is only temporarily a struct and will become a enum class hence the name.
@ kInherit
Some objects use this value to denote that the compression algorithm should be inherited from the par...
@ kInherit
Some objects use this value to denote that the compression algorithm should be inherited from the par...