320 :
TFileCacheRead(tree->GetCurrentFile(),
bufsize, tree), fEntryMax(tree->GetEntriesFast()), fEntryNext(0),
321 fBrNames(
new TList), fTree(tree), fPrefillType(GetConfiguredPrefillType())
324 Int_t nleaves = tree->GetListOfLeaves()->GetEntriesFast();
388 const char *bname =
b->GetName();
396 const char *
mothername =
b->GetMother()->GetName();
400 if (
bem->GetType() < 3) {
406 bname = build.
Data();
413 if (
gDebug > 0)
printf(
"Entry: %lld, registering branch: %s\n",
b->GetTree()->GetReadEntry(),
b->GetName());
483 if (
nb==0 &&
strchr(bname,
'*')==
nullptr) {
501 if (t==
nullptr)
continue;
521 if (
gDebug > 0)
printf(
"AddBranch: unknown branch -> %s \n", bname);
522 Error(
"AddBranch",
"unknown branch -> %s", bname);
552 if (
gDebug > 0)
printf(
"Entry: %lld, un-registering branch: %s\n",
b->GetTree()->GetReadEntry(),
b->GetName());
623 if (
nb==0 &&
strchr(bname,
'*')==
nullptr) {
641 if (t==
nullptr)
continue;
661 if (
gDebug > 0)
printf(
"DropBranch: unknown branch -> %s \n", bname);
662 Error(
"DropBranch",
"unknown branch -> %s", bname);
806 for (
int i = 0; i < count; i++) {
811 if (
iopos.fLen == 0) {
826 if (
iopos.fPos ==
b->GetBasketSeek(
bn)) {
937 if (
len > iter->fIO.fLen) {
941 auto offset = iter->fIndex;
962 auto offset = iter->fIndex;
985 Range() : fMin(-1), fMax(-1) {}
989 if (fMin == -1 || min < fMin)
995 if (fMax == -1 || fMax < max)
1002 std::vector<Range> fRanges;
1003 std::map<Long64_t,size_t> fMinimums;
1004 std::map<Long64_t,size_t> fMaximums;
1013 range.UpdateMin(min);
1014 range.UpdateMax(max);
1016 if (old.fMax !=
range.fMax) {
1017 if (old.fMax != -1) {
1018 auto maxIter = fMaximums.find(old.fMax);
1019 if (
maxIter != fMaximums.end()) {
1041 for (
const auto &
r : fRanges) {
1070 if (!fMinimums.empty())
1071 result.fMin = fMinimums.rbegin()->first;
1072 if (!fMaximums.empty())
1073 result.fMax = fMaximums.begin()->first;
1081 for (
const auto &
r : fRanges) {
1082 if (
r.fMin != -1 &&
r.fMax != -1)
1092 for (
const auto &
r : fRanges) {
1093 if (
r.fMin != -1 &&
r.fMax != -1)
1102 for (
size_t i = 0; i < fRanges.size(); ++i) {
1103 if (fRanges[i].fMin != -1 || fRanges[i].fMax != -1)
1104 Printf(
"Range #%zu : %lld to %lld", i, fRanges[i].fMin, fRanges[i].fMax);
1169 if (
entry < 0)
return false;
1195 static constexpr bool showMore =
false;
1200 b->PrintCacheInfo();
1205 Info(
"FillBuffer",
"***** Called for entry %lld",
entry);
1213 if (!
b->fCacheInfo.AllUsed()) {
1221 Info(
"FillBuffer",
"All baskets used already, so refresh the cache early at entry %lld",
entry);
1242 Info(
"FillBuffer",
"*** Will reset the branch information about baskets");
1244 Info(
"FillBuffer",
"*** Info we have on the set of baskets");
1285 Error(
"FillBuffer",
"Inconsistency: fCurrentClusterStart=%lld fEntryCurrent=%lld fNextClusterStart=%lld "
1286 "but fEntryCurrent should not be in between the two",
1304 Int_t t = chain->GetTreeNumber();
1359 bool progress =
true;
1386 bool filled =
false;
1391 if (
b->GetDirectory()->GetFile() !=
fFile)
1406 Info(
"FillBuffer",
"*** Potential Vetos for branch #%d: %s", i,
vetolist.Data());
1408 b->fCacheInfo.Reset();
1412 Long64_t *entries =
b->GetBasketEntry();
1420 return ((
j < (
nb - 1)) ? (entries[
j + 1] - 1) :
fEntryMax - 1);
1425 else if (
cursor[i].fCurrent == -1) {
1427 cursor[i].fCurrent = (start < 0) ? 0 : start;
1439 b->fCacheInfo.SetIsInCache(
j);
1440 b->fCacheInfo.SetUsed(
j);
1461 if (pos <= 0 ||
len <= 0)
1467 Info(
"FillBuffer",
"Skipping branch %s basket %d is too large for the cache: %d > %d",
1483 Info(
"FillBuffer",
"Skipping for now due to gap %d/%d with %lld > %lld", i,
j, entries[
j],
1484 (
reqRanges.AllIncludedRange().fMax + 1));
1498 emax = entries[
j + 1] - 1;
1503 if (
b->fCacheInfo.HasBeenUsed(
j) ||
b->fCacheInfo.IsInCache(
j) ||
b->fCacheInfo.IsVetoed(
j)) {
1507 Info(
"FillBuffer",
"Skipping basket to avoid redo: %d/%d veto: %d", i,
j,
b->fCacheInfo.IsVetoed(
j));
1518 b->fCacheInfo.Veto(
j);
1520 Info(
"FillBuffer",
"Veto-ing cluster %d [%lld,%lld[ in branch %s #%d",
j, entries[
j],
1526 if ((((entries[
j] >
entry)) || (
j <
nb - 1 && entries[
j + 1] <=
entry))) {
1546 "Breaking early because %lld is greater than %d at cluster iteration %d will restart at %lld",
1562 Info(
"FillBuffer",
"Breaking early because %lld is greater than 4*%d at cluster iteration "
1563 "%d pass %d will restart at %lld",
1576 Info(
"FillBuffer",
"Breaking early because %lld is greater than 2*%d at cluster iteration "
1577 "%d pass %d will restart at %lld",
1593 b->fCacheInfo.SetIsInCache(
j);
1596 Info(
"FillBuffer",
"*** Registering branch %d basket %d %s", i,
j,
b->GetName());
1599 cursor[i].fLoadedOnce =
true;
1628 Warning(
"FillBuffer",
"There is more data in this cluster (starting at entry %lld to %lld, "
1629 "current=%lld) than usual ... with %d %.3f%% of the branches we already have "
1630 "%d bytes (instead of %d)",
1634 if (
pass == kStart) {
1658 Info(
"CollectBaskets",
1659 "Entry: %lld, registering baskets branch %s, fEntryNext=%lld, fNseek=%d, ntotCurrentBuf=%d",
1685 full = CollectBaskets(kRegular, kFull,
fEntryNext);
1693 full = CollectBaskets(kRegular, kFull,
fEntryNext);
1719 Info(
"FillBuffer",
"Breaking because %d <= %lld || (%d >= %d) || %lld >= %lld",
fBufferSizeMin,
1738 Info(
"FillBuffer",
"Mem ranges");
1740 Info(
"FillBuffer",
"Combined ranges");
1742 Info(
"FillBuffer",
"Requested ranges");
1752 Info(
"FillBuffer",
"For entry %lld, nothing was added to the cache.",
entry);
1767 Error(
"FillBuffer",
"Reset the next entry because the currently loaded range does not contains the request "
1768 "entry: %lld. fEntryNext updated from %lld to %lld. %d",
1776 Info(
"FillBuffer",
"Complete adding %d baskets from %d branches taking in memory %d out of %d",
1910 if ( opt.
Contains(
"cachedbranches") ) {
1912 printf(
"Cached branches....................:\n");
1917 printf(
"Branch name........................: %s\n",
branch->GetName());
1936 ::Info(
"TTreeCache::ReadBufferNormal",
"Cache miss after an %s FillBuffer: pos=%lld",
1944 ::Info(
"TTreeCache::ReadBufferNormal",
" Missing basket: %d for %s",
j,
b->GetName());
1958 else if (res == 0) {
2045 if (
b->GetDirectory()->GetFile() !=
fFile)
2047 b->fCacheInfo.Reset();
2115 Info(
"SetEntryRange",
"fEntryMin=%lld, fEntryMax=%lld, fEntryNext=%lld",
#define R__unlikely(expr)
int Int_t
Signed integer 4 bytes (int)
unsigned int UInt_t
Unsigned integer 4 bytes (unsigned int)
constexpr Ssiz_t kNPOS
The equivalent of std::string::npos for the ROOT class TString.
long long Long64_t
Portable signed long integer 8 bytes.
const char Option_t
Option string (const char)
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t cursor
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
void Print(GNN_Data &d, std::string txt="")
Int_t gDebug
Global variable setting the debug level. Set to 0 to disable, increase it in steps of 1 to increase t...
void Printf(const char *fmt,...)
Formats a string in a circular formatting buffer and prints the string.
R__EXTERN TSystem * gSystem
A Branch for the case of an object.
A TTree is a list of TBranches.
@ kDoNotProcess
Active bit for branches.
A chain is a collection of files containing TTree objects.
virtual Int_t GetEntries() const
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
<div class="legacybox"><h2>Legacy Code</h2> TEventList is a legacy interface: there will be no bug fi...
virtual bool ContainsRange(Long64_t entrymin, Long64_t entrymax)
Return TRUE if list contains entries from entrymin to entrymax included.
A cache when reading files over the network.
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos.
virtual void SecondPrefetch(Long64_t, Int_t)
Bool_t fEnablePrefetching
reading by prefetching asynchronously
Int_t fNtot
Total size of prefetched blocks.
virtual Int_t SetBufferSize(Long64_t buffersize)
Sets the buffer size.
virtual void Prefetch(Long64_t pos, Int_t len)
Add block of length len at position pos in the list of blocks to be prefetched.
Int_t fBufferSizeMin
Original size of fBuffer.
Bool_t fIsTransferred
True when fBuffer contains something valid.
TFile * fFile
Pointer to file.
Int_t fNseek
Number of blocks to be prefetched.
void Print(Option_t *option="") const override
Print cache statistics.
virtual Int_t GetBufferSize() const
virtual void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Set the file using this cache and reset the current blocks (if any).
A ROOT file is an on-disk file, usually with extension .root, that stores objects in a file-system-li...
virtual void SetCacheRead(TFileCacheRead *cache, TObject *tree=nullptr, ECacheAction action=kDisconnect)
Set a pointer to the read cache.
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
ECacheAction
TTreeCache flushing semantics.
A TFriendElement TF describes a TTree object TF in a file.
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
TObject * FindObject(const char *name) const override
Find an object in this list using its name.
void Add(TObject *obj) override
TObject * Remove(TObject *obj) override
Remove object from the list.
void Delete(Option_t *option="") override
Remove all objects from the list AND delete all heap based objects.
const char * GetName() const override
Returns name of object.
void AddAt(TObject *obj, Int_t idx) override
Add object at position ids.
virtual void AddAtAndExpand(TObject *obj, Int_t idx)
Add object at position idx.
TObject * UncheckedAt(Int_t i) const
TObject * Remove(TObject *obj) override
Remove object from array.
Collectable string class.
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Regular expression class.
void ToLower()
Change string to lower-case.
Int_t Atoi() const
Return integer value of string.
const char * Data() const
TString & ReplaceAll(const TString &s1, const TString &s2)
TString & Append(const char *cs)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Ssiz_t Index(const char *pat, Ssiz_t i=0, ECaseCompare cmp=kExact) const
virtual const char * Getenv(const char *env)
Get environment variable.
virtual void SetLearnPrefill(EPrefillType type=kNoPrefill)
Set whether the learning period is started with a prefilling of the cache and which type of prefillin...
virtual void UpdateBranches(TTree *tree)
Update pointer to current Tree and recompute pointers to the branches in the cache.
Double_t GetEfficiencyRel() const
This will indicate a sort of relative efficiency... a ratio of the reads found in the cache to the nu...
bool fEnabled
! cache enabled for cached reading
Long64_t fLastMiss
! set to the event # of the last miss.
Int_t fNMissReadPref
Number of blocks read into the secondary ("miss") cache.
~TTreeCache() override
Destructor. (in general called by the TFile destructor)
Int_t fNMissReadOk
Number of blocks read, not found in the primary cache, and found in the secondary cache.
Int_t SetBufferSize(Long64_t buffersize) override
Change the underlying buffer size of the cache.
Long64_t fEntryMin
! first entry in the cache
void SetOptimizeMisses(bool opt)
Start of methods for the miss cache.
Int_t LearnBranch(TBranch *b, bool subgbranches=false) override
Add a branch discovered by actual usage to the list of branches to be stored in the cache this functi...
static void SetLearnEntries(Int_t n=10)
Static function to set the number of entries to be used in learning mode The default value for n is 1...
bool CheckMissCache(char *buf, Long64_t pos, int len)
Check the miss cache for a particular buffer, fetching if deemed necessary.
Long64_t fEntryNext
! next entry number where cache must be filled
Int_t fNReadMiss
Number of blocks read and not found in the cache.
bool fReverseRead
! reading in reverse mode
Double_t GetEfficiency() const
Give the total efficiency of the primary cache... defined as the ratio of blocks found in the cache v...
virtual bool FillBuffer()
Fill the cache buffer with the branches in the cache.
Long64_t fNextClusterStart
! End+1 of the cluster(s) where the current content was picked out
bool ProcessMiss(Long64_t pos, int len)
! Given a file read not in the miss cache, handle (possibly) loading the data.
const TObjArray * GetCachedBranches() const
bool fIsLearning
! true if cache is in learning mode
Int_t fNMissReadMiss
Number of blocks read and not found in either cache.
IOPos FindBranchBasketPos(TBranch &, Long64_t entry)
Given a branch and an entry, determine the file location (offset / size) of the corresponding basket.
virtual void SetEntryRange(Long64_t emin, Long64_t emax)
Set the minimum and maximum entry number to be processed this information helps to optimize the numbe...
TTreeCache()
Default Constructor.
std::unique_ptr< MissCache > fMissCache
! Cache contents for misses
bool fFirstBuffer
! true if first buffer is used for prefetching
bool fOptimizeMisses
! true if we should optimize cache misses.
void StartLearningPhase()
The name should be enough to explain the method.
TBranch * CalculateMissEntries(Long64_t, int, bool)
Given an file read, try to determine the corresponding branch.
Long64_t fCurrentClusterStart
! Start of the cluster(s) where the current content was picked out
Double_t GetMissEfficiency() const
The total efficiency of the 'miss cache' - defined as the ratio of blocks found in the cache versus t...
virtual Int_t ReadBufferNormal(char *buf, Long64_t pos, Int_t len)
Old method ReadBuffer before the addition of the prefetch mechanism.
virtual void ResetCache()
This will simply clear the cache.
bool fOneTime
! used in the learning phase
EPrefillType fPrefillType
Whether a pre-filling is enabled (and if applicable which type)
bool fLearnPrefilling
! true if we are in the process of executing LearnPrefill
virtual void LearnPrefill()
Perform an initial prefetch, attempting to read as much of the learning phase baskets for all branche...
virtual Int_t DropBranch(TBranch *b, bool subbranches=false)
Remove a branch to the list of branches to be stored in the cache this function is called by TBranch:...
Long64_t fEntryMax
! last entry in the cache
static Int_t fgLearnEntries
number of entries used for learning mode
virtual Int_t ReadBufferPrefetch(char *buf, Long64_t pos, Int_t len)
Used to read a chunk from a block previously fetched.
Long64_t fEntryCurrent
! current lowest entry number in the cache
void ResetMissCache()
Reset all the miss cache training.
Long64_t fFirstMiss
! set to the event # of the first miss.
Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len) override
Read buffer at position pos if the request is in the list of prefetched blocks read from fBuffer.
bool fReadDirectionSet
! read direction established
bool fIsManual
! true if cache is StopLearningPhase was used
Double_t GetMissEfficiencyRel() const
Relative efficiency of the 'miss cache' - ratio of the reads found in cache to the number of reads so...
Long64_t fFirstEntry
! save the value of the first entry
Int_t fFillTimes
! how many times we can fill the current buffer
Int_t fNReadPref
Number of blocks that were prefetched.
TTree * fTree
! pointer to the current Tree
EPrefillType GetConfiguredPrefillType() const
Return the desired prefill type from the environment or resource variable.
static Int_t GetLearnEntries()
Static function returning the number of entries used to train the cache see SetLearnEntries.
virtual void StopLearningPhase()
This is the counterpart of StartLearningPhase() and can be used to stop the learning phase.
void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect) override
Change the file that is being cached.
Int_t fNbranches
! Number of branches in the cache
Int_t fNReadOk
Number of blocks read and found in the cache.
void Print(Option_t *option="") const override
Print cache statistics.
Int_t AddBranch(TBranch *b, bool subgbranches=false) override
Add a branch to the list of branches to be stored in the cache this function is called by the user vi...
TObjArray * fBranches
! List of branches to be stored in the cache
TList * fBrNames
! list of branch names in the cache
bool fFirstTime
! save the fact that we processes the first entry
Helper class to iterate over cluster of baskets.
A TTree represents a columnar dataset.
virtual TVirtualPerfStats * GetPerfStats() const
virtual TBranch * GetBranch(const char *name)
Return pointer to the branch with the given name in this tree or its friends.
virtual TObjArray * GetListOfLeaves()
virtual Long64_t GetEntries() const
virtual Long64_t GetReadEntry() const
virtual TTree * GetTree() const
TEventList * GetEventList() const
TClass * IsA() const override
virtual TList * GetListOfFriends() const
Provides the interface for the an internal performance measurement and event tracing.
Long64_t BinarySearch(Long64_t n, const T *array, T value)
Binary search in an array of n values to locate value.