Logo ROOT   6.10/09
Reference Guide
TFileCacheRead.cxx
Go to the documentation of this file.
1 // @(#)root/io:$Id$
2 // Author: Rene Brun 18/05/2006
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /**
13  \class TFileCacheRead
14  \ingroup IO
15 
16  A cache when reading files over the network.
17 
18  A caching system to speed up network I/O, i.e. when there is
19  no operating system caching support (like the buffer cache for
20  local disk I/O). The cache makes sure that every I/O is done with
21  a (large) fixed length buffer thereby avoiding many small I/O's.
22  Currently the read cache system is used by the classes TNetFile,
23  TXNetFile and TWebFile (via TFile::ReadBuffers()).
24  When processing TTree, TChain, a specialized class TTreeCache that
25  derives from this class is automatically created.
26 */
27 
28 #include "TEnv.h"
29 #include "TFile.h"
30 #include "TFileCacheRead.h"
31 #include "TFileCacheWrite.h"
32 #include "TFilePrefetch.h"
33 #include "TMath.h"
34 
36 
37 ////////////////////////////////////////////////////////////////////////////////
38 /// Default Constructor.
39 
41 {
42  fBufferSizeMin = 0;
43  fBufferSize = 0;
44  fBufferLen = 0;
45  fBytesRead = 0;
46  fNoCacheBytesRead = 0;
47  fBytesReadExtra = 0;
48  fReadCalls = 0;
49  fNoCacheReadCalls = 0;
50  fNseek = 0;
51  fNtot = 0;
52  fNb = 0;
53  fSeekSize = 0;
54  fSeek = 0;
55  fSeekIndex = 0;
56  fSeekSort = 0;
57  fPos = 0;
58  fSeekLen = 0;
59  fSeekSortLen = 0;
60  fSeekPos = 0;
61  fLen = 0;
62  fFile = 0;
63  fBuffer = 0;
64  fIsSorted = kFALSE;
65  fIsTransferred = kFALSE;
66 
67  //values for the second prefetched block
68  fBNseek = 0;
69  fBNtot = 0;
70  fBNb = 0;
71  fBSeekSize = 0;
72  fBSeek = 0;
73  fBSeekSort = 0;
74  fBSeekIndex = 0;
75  fBPos = 0;
76  fBSeekLen = 0;
77  fBSeekSortLen = 0;
78  fBSeekPos = 0;
79  fBLen = 0;
80  fBIsSorted = kFALSE;
81  fBIsTransferred=kFALSE;
82 
83  fAsyncReading = kFALSE;
84  fEnablePrefetching = kFALSE;
85  fPrefetch = 0;
86  fPrefetchedBlocks= 0;
87 }
88 
89 ////////////////////////////////////////////////////////////////////////////////
90 /// Creates a TFileCacheRead data structure.
91 
93  : TObject()
94 {
95  if (buffersize <=10000) fBufferSize = 100000;
96  else fBufferSize = buffersize;
97 
99  fBufferLen = 0;
100  fBytesRead = 0;
101  fNoCacheBytesRead = 0;
102  fBytesReadExtra = 0;
103  fReadCalls = 0;
104  fNoCacheReadCalls = 0;
105  fNseek = 0;
106  fNtot = 0;
107  fNb = 0;
108  fSeekSize = 10000;
109  fSeek = new Long64_t[fSeekSize];
110  fSeekIndex = new Int_t[fSeekSize];
112  fPos = new Long64_t[fSeekSize];
113  fSeekLen = new Int_t[fSeekSize];
115  fSeekPos = new Int_t[fSeekSize];
116  fLen = new Int_t[fSeekSize];
117  fFile = file;
118 
119  //initialisation for the second block
120  fBNseek = 0;
121  fBNtot = 0;
122  fBNb = 0;
123  fBSeekSize = 10000;
124  fBSeek = new Long64_t[fBSeekSize];
127  fBPos = new Long64_t[fBSeekSize];
128  fBSeekLen = new Int_t[fBSeekSize];
130  fBSeekPos = new Int_t[fBSeekSize];
131  fBLen = new Int_t[fBSeekSize];
132 
133  fBuffer = 0;
134  fPrefetch = 0;
135  fPrefetchedBlocks = 0;
136 
137  //initialise the prefetch object and set the cache directory
138  // start the thread only if the file is not local
139  fEnablePrefetching = gEnv->GetValue("TFile.AsyncPrefetching", 0);
140 
141  if (fEnablePrefetching && strcmp(file->GetEndpointUrl()->GetProtocol(), "file")){
143  }
144  else { //disable the async pref for local files
146  }
147 
148  fIsSorted = kFALSE;
150  fBIsSorted = kFALSE;
152 
153  if (file) file->SetCacheRead(this, tree);
154 }
155 
156 ////////////////////////////////////////////////////////////////////////////////
157 /// Destructor.
158 
160 {
162  delete [] fSeek;
163  delete [] fSeekIndex;
164  delete [] fSeekSort;
165  delete [] fPos;
166  delete [] fSeekLen;
167  delete [] fSeekSortLen;
168  delete [] fSeekPos;
169  delete [] fLen;
170  if (fBuffer)
171  delete [] fBuffer;
172  delete [] fBSeek;
173  delete [] fBSeekIndex;
174  delete [] fBSeekSort;
175  delete [] fBPos;
176  delete [] fBSeekLen;
177  delete [] fBSeekSortLen;
178  delete [] fBSeekPos;
179  delete [] fBLen;
180 }
181 
182 ////////////////////////////////////////////////////////////////////////////////
183 /// Close out any threads or asynchronous fetches used by the underlying
184 /// implementation.
185 /// This is called by TFile::Close to prevent usage of the file handles
186 /// after the closing of the file.
187 
188 void TFileCacheRead::Close(Option_t * /* opt = "" */)
189 {
190  if (fPrefetch) {
191  delete fPrefetch;
192  fPrefetch = 0;
193  }
194 
195 }
196 
197 ////////////////////////////////////////////////////////////////////////////////
198 /// Add block of length len at position pos in the list of blocks to
199 /// be prefetched. If pos <= 0 the current blocks (if any) are reset.
200 
202 {
203  fIsSorted = kFALSE;
205  if (pos <= 0) {
206  fNseek = 0;
207  fNtot = 0;
208  return;
209  }
210  if (fNseek >= fSeekSize) {
211  //reallocate buffers
212  fSeekSize *= 2;
213  Long64_t *aSeek = new Long64_t[fSeekSize];
214  Int_t *aSeekIndex = new Int_t[fSeekSize];
215  Long64_t *aSeekSort = new Long64_t[fSeekSize];
216  Long64_t *aPos = new Long64_t[fSeekSize];
217  Int_t *aSeekLen = new Int_t[fSeekSize];
218  Int_t *aSeekSortLen = new Int_t[fSeekSize];
219  Int_t *aSeekPos = new Int_t[fSeekSize];
220  Int_t *aLen = new Int_t[fSeekSize];
221  for (Int_t i=0;i<fNseek;i++) {
222  aSeek[i] = fSeek[i];
223  aSeekIndex[i] = fSeekIndex[i];
224  aSeekSort[i] = fSeekSort[i];
225  aPos[i] = fPos[i];
226  aSeekLen[i] = fSeekLen[i];
227  aSeekSortLen[i] = fSeekSortLen[i];
228  aSeekPos[i] = fSeekPos[i];
229  aLen[i] = fLen[i];
230  }
231  delete [] fSeek;
232  delete [] fSeekIndex;
233  delete [] fSeekSort;
234  delete [] fPos;
235  delete [] fSeekLen;
236  delete [] fSeekSortLen;
237  delete [] fSeekPos;
238  delete [] fLen;
239  fSeek = aSeek;
240  fSeekIndex = aSeekIndex;
241  fSeekSort = aSeekSort;
242  fPos = aPos;
243  fSeekLen = aSeekLen;
244  fSeekSortLen = aSeekSortLen;
245  fSeekPos = aSeekPos;
246  fLen = aLen;
247  }
248 
249  fSeek[fNseek] = pos;
250  fSeekLen[fNseek] = len;
251  fNseek++;
252  fNtot += len;
253 }
254 
255 
256 ////////////////////////////////////////////////////////////////////////////////
257 
259  //add a new element and increase the size if necessary
260  fBIsSorted = kFALSE;
261  if (pos <= 0) {
262  fBNseek = 0;
263  fBNtot = 0;
264  return;
265  }
266  if (fBNseek >= fBSeekSize) {
267  //reallocate buffers
268  fBSeekSize *= 2;
269  Long64_t *aSeek = new Long64_t[fBSeekSize];
270  Int_t *aSeekIndex = new Int_t[fBSeekSize];
271  Long64_t *aSeekSort = new Long64_t[fBSeekSize];
272  Long64_t *aPos = new Long64_t[fBSeekSize];
273  Int_t *aSeekLen = new Int_t[fBSeekSize];
274  Int_t *aSeekSortLen = new Int_t[fBSeekSize];
275  Int_t *aSeekPos = new Int_t[fBSeekSize];
276  Int_t *aLen = new Int_t[fBSeekSize];
277  for (Int_t i=0;i<fBNseek;i++) {
278  aSeek[i] = fBSeek[i];
279  aSeekIndex[i] = fBSeekIndex[i];
280  aSeekSort[i] = fBSeekSort[i];
281  aPos[i] = fBPos[i];
282  aSeekLen[i] = fBSeekLen[i];
283  aSeekSortLen[i] = fBSeekSortLen[i];
284  aSeekPos[i] = fBSeekPos[i];
285  aLen[i] = fBLen[i];
286  }
287  delete [] fBSeek;
288  delete [] fBSeekIndex;
289  delete [] fBSeekSort;
290  delete [] fBPos;
291  delete [] fBSeekLen;
292  delete [] fBSeekSortLen;
293  delete [] fBSeekPos;
294  delete [] fBLen;
295  fBSeek = aSeek;
296  fBSeekIndex = aSeekIndex;
297  fBSeekSort = aSeekSort;
298  fBPos = aPos;
299  fBSeekLen = aSeekLen;
300  fBSeekSortLen = aSeekSortLen;
301  fBSeekPos = aSeekPos;
302  fBLen = aLen;
303  }
304 
305  fBSeek[fBNseek] = pos;
306  fBSeekLen[fBNseek] = len;
307  fBNseek++;
308  fBNtot += len;
309 }
310 
311 
312 ////////////////////////////////////////////////////////////////////////////////
313 /// Print cache statistics.
314 ///
315 /// The format is:
316 /// ******TreeCache statistics for file: cms2.root ******
317 /// Reading............................: 72761843 bytes in 7 transactions
318 /// Readahead..........................: 256000 bytes with overhead = 0 bytes
319 /// Average transaction................: 10394.549000 Kbytes
320 /// Number of blocks in current cache..: 210, total size: 6280352
321 ///
322 /// If option = "a" the list of blocks in the cache is printed
323 /// NB: this function is automatically called by TTreeCache::Print
324 
325 void TFileCacheRead::Print(Option_t *option) const
326 {
327  TString opt = option;
328  opt.ToLower();
329  printf("Cached Reading.....................: %lld bytes in %d transactions\n",this->GetBytesRead(), this->GetReadCalls());
330  printf("Reading............................: %lld bytes in %d uncached transactions\n",this->GetNoCacheBytesRead(), this->GetNoCacheReadCalls());
331  printf("Readahead..........................: %d bytes with overhead = %lld bytes\n",TFile::GetReadaheadSize(),this->GetBytesReadExtra());
332  if (this->GetReadCalls() > 0)
333  printf("Average transaction................: %f Kbytes\n",0.001*Double_t(this->GetBytesRead())/Double_t(this->GetReadCalls()));
334  else
335  printf("Average transaction................: No read calls yet\n");
336  printf("Number of blocks in current cache..: %d, total size: %d\n",fNseek,fNtot);
337  if (fPrefetch){
338  printf("Prefetching .......................: %lli blocks\n", fPrefetchedBlocks);
339  printf("Prefetching Wait Time..............: %f seconds\n", fPrefetch->GetWaitTime() / 1e+6);
340  }
341 
342  if (!opt.Contains("a")) return;
343  for (Int_t i=0;i<fNseek;i++) {
344  if (fIsSorted && !opt.Contains("s")) {
345  printf("block: %5d, from: %lld to %lld, len = %d bytes\n",i,fSeekSort[i],fSeekSort[i]+fSeekSortLen[i],fSeekSortLen[i]);
346  } else {
347  printf("block: %5d, from: %lld to %lld, len = %d bytes\n",i,fSeek[i],fSeek[i]+fSeekLen[i],fSeekLen[i]);
348  }
349  }
350  printf ("Number of long buffers = %d\n",fNb);
351  for (Int_t j=0;j<fNb;j++) {
352  printf("fPos[%d] = %lld, fLen = %d\n",j,fPos[j],fLen[j]);
353  }
354 }
355 
356 ////////////////////////////////////////////////////////////////////////////////
357 /// Read buffer at position pos.
358 ///
359 /// If pos is in the list of prefetched blocks read from fBuffer,
360 /// otherwise need to make a normal read from file. Returns -1 in case of
361 /// read error, 0 in case not in cache, 1 in case read from cache.
362 
364 {
365  Long64_t fileBytesRead0 = fFile->GetBytesRead();
366  Long64_t fileBytesReadExtra0 = fFile->GetBytesReadExtra();
367  Int_t fileReadCalls0 = fFile->GetReadCalls();
368 
369  Int_t loc = -1;
370  Int_t rc = ReadBufferExt(buf, pos, len, loc);
371 
372  fBytesRead += fFile->GetBytesRead() - fileBytesRead0;
373  fBytesReadExtra += fFile->GetBytesReadExtra() - fileBytesReadExtra0;
374  fReadCalls += fFile->GetReadCalls() - fileReadCalls0;
375 
376  return rc;
377 }
378 
379 ////////////////////////////////////////////////////////////////////////////////
380 
382 {
383  if (fEnablePrefetching)
384  return ReadBufferExtPrefetch(buf, pos, len, loc);
385  else
386  return ReadBufferExtNormal(buf, pos, len, loc);
387 }
388 
389 
390 ////////////////////////////////////////////////////////////////////////////////
391 ///prefetch the first block
392 
394 {
395  if (fNseek > 0 && !fIsSorted) {
396  Sort();
397  loc = -1;
401  }
402 
403  //try to prefetch the second block
404  if (fBNseek > 0 && !fBIsSorted) {
405  SecondSort();
406  loc = -1;
409  }
410 
411  // in case we are writing and reading to/from this file, we must check
412  // if this buffer is in the write cache (not yet written to the file)
413  if (TFileCacheWrite *cachew = fFile->GetCacheWrite()) {
414  if (cachew->ReadBuffer(buf,pos,len) == 0) {
415  fFile->SetOffset(pos+len);
416  return 1;
417  }
418  }
419 
420  // try to prefetch from the first block
421  if (loc < 0) {
423  }
424 
425  if (loc >= 0 && loc < fNseek && pos == fSeekSort[loc]) {
426  if (buf && fPrefetch){
427  // prefetch with the new method
428  fPrefetch->ReadBuffer(buf, pos, len);
429  return 1;
430  }
431  }
432  else if (buf && fPrefetch){
433  // try to preferch from the second block
435 
436  if (loc >= 0 && loc < fBNseek && pos == fBSeekSort[loc]){
437  if (fPrefetch->ReadBuffer(buf, pos, len)) {
438  return 1;
439  }
440  }
441  }
442 
443  return 0;
444 }
445 
446 
447 ////////////////////////////////////////////////////////////////////////////////
448 /// Base function for ReadBuffer.
449 ///
450 /// Also gives out the position of the block in the internal buffer.
451 /// This helps TTreeCacheUnzip to avoid doing twice the binary search.
452 
454 {
455  if (fNseek > 0 && !fIsSorted) {
456  Sort();
457  loc = -1;
458 
459  // If ReadBufferAsync is not supported by this implementation...
460  if (!fAsyncReading) {
461  // Then we use the vectored read to read everything now
463  return -1;
464  }
466  } else {
467  // In any case, we'll start to request the chunks.
468  // This implementation simply reads all the chunks in advance
469  // in the async way.
470 
471  // Use the async readv instead of single reads
472  fFile->ReadBuffers(0, 0, 0, 0); //Clear the XrdClient cache
473  if (fFile->ReadBuffers(0,fPos,fLen,fNb)) {
474  return -1;
475  }
477  }
478  }
479 
480  // in case we are writing and reading to/from this file, we much check
481  // if this buffer is in the write cache (not yet written to the file)
482  if (TFileCacheWrite *cachew = fFile->GetCacheWrite()) {
483  if (cachew->ReadBuffer(buf,pos,len) == 0) {
484  fFile->SetOffset(pos+len);
485  return 1;
486  }
487  }
488 
489  // If asynchronous reading is supported by this implementation...
490  if (fAsyncReading) {
491 
492  // Now we dont have to look for it in the local buffer
493  // if it's async, we expect that the communication library
494  // will handle it more efficiently than we can do here
495 
496  Int_t retval;
497  if (loc < 0)
499 
500  // We use the internal list just to notify if the list is to be reconstructed
501  if (loc >= 0 && loc < fNseek && pos == fSeekSort[loc]) {
502  // Block found, the caller will get it
503 
504  if (buf) {
505  // disable cache to avoid infinite recursion
506  if (fFile->ReadBuffer(buf, pos, len)) {
507  return -1;
508  }
509  fFile->SetOffset(pos+len);
510  }
511 
512  retval = 1;
513  } else {
514  // Block not found in the list, we report it as a miss
515  retval = 0;
516  }
517 
518  if (gDebug > 0)
519  Info("ReadBuffer","pos=%lld, len=%d, retval=%d, loc=%d, "
520  "fseekSort[loc]=%lld, fSeekLen[loc]=%d",
521  pos, len, retval, loc, fSeekSort[loc], fSeekLen[loc]);
522 
523  return retval;
524  } else {
525 
526  if (loc < 0)
528 
529  if (loc >= 0 && loc <fNseek && pos == fSeekSort[loc]) {
530  if (buf) {
531  memcpy(buf,&fBuffer[fSeekPos[loc]],len);
532  fFile->SetOffset(pos+len);
533  }
534  return 1;
535  }
536  }
537 
538  return 0;
539 }
540 
541 ////////////////////////////////////////////////////////////////////////////////
542 /// Set the file using this cache and reset the current blocks (if any).
543 
545 {
546  fFile = file;
547 
548  if (fAsyncReading) {
549  // If asynchronous reading is not supported by this TFile specialization
550  // we use sync primitives, hence we need the local buffer
551  if (file && file->ReadBufferAsync(0, 0)) {
553  fBuffer = new char[fBufferSize];
554  }
555  }
556 
557  if (action == TFile::kDisconnect)
558  Prefetch(0,0);
559 
560  if (fPrefetch) {
561  if (action == TFile::kDisconnect)
562  SecondPrefetch(0, 0);
563  fPrefetch->SetFile(file, action);
564  }
565 }
566 
567 ////////////////////////////////////////////////////////////////////////////////
568 /// Sort buffers to be prefetched in increasing order of positions.
569 /// Merge consecutive blocks if necessary.
570 
572 {
573  if (!fNseek) return;
575  Int_t i;
576  Int_t nb = 0;
577  Int_t effectiveNseek = 0;
578  for (i=0;i<fNseek;i++) {
579  // Skip duplicates
580  Int_t ind = fSeekIndex[i];
581  if (effectiveNseek!=0 && fSeek[ind]==fSeekSort[effectiveNseek-1])
582  {
583  if (fSeekSortLen[effectiveNseek-1] < fSeekLen[ind]) {
584  fSeekSortLen[effectiveNseek-1] = fSeekLen[ind];
585  }
586  continue;
587  }
588  fSeekSort[effectiveNseek] = fSeek[ind];
589  fSeekSortLen[effectiveNseek] = fSeekLen[ind];
590  ++effectiveNseek;
591  }
592  fNseek = effectiveNseek;
593  if (fNtot > fBufferSizeMin) {
594  fBufferSize = fNtot + 100;
595  delete [] fBuffer;
596  fBuffer = 0;
597  // If ReadBufferAsync is not supported by this implementation
598  // it means that we are using sync primitives, hence we need the local buffer
599  if (!fAsyncReading)
600  fBuffer = new char[fBufferSize];
601  }
602  fPos[0] = fSeekSort[0];
603  fLen[0] = fSeekSortLen[0];
604  fSeekPos[0] = 0;
605  for (i=1;i<fNseek;i++) {
606  fSeekPos[i] = fSeekPos[i-1] + fSeekSortLen[i-1];
607  //in the test below 16 MBytes is pure empirirical and may depend on the file system.
608  //increasing this number must be done with care, as it may increase
609  //the job real time (mismatch with OS buffers)
610  if ((fSeekSort[i] != fSeekSort[i-1]+fSeekSortLen[i-1]) ||
611  (fLen[nb] > 16000000)) {
612  nb++;
613  fPos[nb] = fSeekSort[i];
614  fLen[nb] = fSeekSortLen[i];
615  } else {
616  fLen[nb] += fSeekSortLen[i];
617  }
618  }
619  fNb = nb+1;
620  fIsSorted = kTRUE;
621 }
622 
623 
624 ////////////////////////////////////////////////////////////////////////////////
625 /// Sort buffers to be prefetched in increasing order of positions.
626 ///
627 /// Merge consecutive blocks if necessary.
628 
630 {
631  if (!fBNseek) return;
633  Int_t i;
634  Int_t nb = 0;
635  Int_t effectiveNseek = 0;
636  for (i=0;i<fBNseek;i++) {
637  // Skip duplicates
638  Int_t ind = fBSeekIndex[i];
639  if (effectiveNseek!=0 && fBSeek[ind]==fBSeekSort[effectiveNseek-1])
640  {
641  if (fBSeekSortLen[effectiveNseek-1] < fBSeekLen[ind]) {
642  fBSeekSortLen[effectiveNseek-1] = fBSeekLen[ind];
643  }
644  continue;
645  }
646  fBSeekSort[effectiveNseek] = fBSeek[ind];
647  fBSeekSortLen[effectiveNseek] = fBSeekLen[ind];
648  ++effectiveNseek;
649  }
650  fBNseek = effectiveNseek;
651  if (fBNtot > fBufferSizeMin) {
652  fBufferSize = fBNtot + 100;
653  delete [] fBuffer;
654  fBuffer = 0;
655  // If ReadBufferAsync is not supported by this implementation
656  // it means that we are using sync primitives, hence we need the local buffer
657  if (!fAsyncReading)
658  fBuffer = new char[fBufferSize];
659  }
660  fBPos[0] = fBSeekSort[0];
661  fBLen[0] = fBSeekSortLen[0];
662  fBSeekPos[0] = 0;
663  for (i=1;i<fBNseek;i++) {
664  fBSeekPos[i] = fBSeekPos[i-1] + fBSeekSortLen[i-1];
665  //in the test below 16 MBytes is pure empirirical and may depend on the file system.
666  //increasing this number must be done with care, as it may increase
667  //the job real time (mismatch with OS buffers)
668  if ((fBSeekSort[i] != fBSeekSort[i-1]+fBSeekSortLen[i-1]) ||
669  (fBLen[nb] > 16000000)) {
670  nb++;
671  fBPos[nb] = fBSeekSort[i];
672  fBLen[nb] = fBSeekSortLen[i];
673  } else {
674  fBLen[nb] += fBSeekSortLen[i];
675  }
676  }
677  fBNb = nb+1;
678  fBIsSorted = kTRUE;
679 }
680 
681 ////////////////////////////////////////////////////////////////////////////////
682 
684  return this->fPrefetch;
685 }
686 
687 
688 ////////////////////////////////////////////////////////////////////////////////
689 
691 {
692  if ( fEnablePrefetching && fPrefetch ) {
694  }
695 }
696 
697 
698 ////////////////////////////////////////////////////////////////////////////////
699 /// Sets the buffer size.
700 ///
701 /// If the current prefetch list is too large to fit in
702 /// the new buffer some or all of the prefetch blocks are dropped. The
703 /// requested buffersize must be greater than zero.
704 /// Return values:
705 /// - 0 if the prefetch block lists remain unchanged
706 /// - 1 if some or all blocks have been removed from the prefetch list
707 /// - -1 on error
708 
710 {
711  if (buffersize <= 0) return -1;
712  if (buffersize <=10000) buffersize = 100000;
713 
714  if (buffersize == fBufferSize) {
715  fBufferSizeMin = buffersize;
716  return 0;
717  }
718 
719  Bool_t inval = kFALSE;
720 
721  // the cached data is too large to fit in the new buffer size mark data unavailable
722  if (fNtot > buffersize) {
723  Prefetch(0, 0);
724  inval = kTRUE;
725  }
726  if (fBNtot > buffersize) {
727  SecondPrefetch(0, 0);
728  inval = kTRUE;
729  }
730 
731  char *np = 0;
733  char *pres = 0;
734  if (fIsTransferred) {
735  // will need to preserve buffer data
736  pres = fBuffer;
737  fBuffer = 0;
738  }
739  delete [] fBuffer;
740  fBuffer = 0;
741  np = new char[buffersize];
742  if (pres) {
743  memcpy(np, pres, fNtot);
744  }
745  delete [] pres;
746  }
747 
748  delete [] fBuffer;
749  fBuffer = np;
750  fBufferSizeMin = buffersize;
751  fBufferSize = buffersize;
752 
753  if (inval) {
754  return 1;
755  }
756 
757  return 0;
758 }
759 
760 
761 ////////////////////////////////////////////////////////////////////////////////
762 /// Set the prefetching mode of this file.
763 ///
764 /// If 'setPrefetching', enable the asynchronous prefetching
765 /// (using TFilePrefetch) and if the gEnv and rootrc
766 /// variable Cache.Directory is set, also enable the local
767 /// caching of the prefetched blocks.
768 /// if 'setPrefetching', the old prefetcher is enabled is
769 /// the gEnv and rootrc variable is TFile.AsyncReading
770 
772 {
773  SetEnablePrefetchingImpl(setPrefetching);
774 }
775 
776 ////////////////////////////////////////////////////////////////////////////////
777 /// TFileCacheRead implementation of SetEnablePrefetching.
778 ///
779 /// This function is called from the constructor and should not be virtual.
780 
782 {
783  fEnablePrefetching = setPrefetching;
784 
785  if (!fPrefetch && fEnablePrefetching) {
787  const char* cacheDir = gEnv->GetValue("Cache.Directory", "");
788  if (strcmp(cacheDir, ""))
789  if (!fPrefetch->SetCache((char*) cacheDir))
790  fprintf(stderr, "Error while trying to set the cache directory: %s.\n", cacheDir);
791  if (fPrefetch->ThreadStart()){
792  fprintf(stderr,"Error stating prefetching thread. Disabling prefetching.\n");
793  fEnablePrefetching = 0;
794  }
795  } else if (fPrefetch && !fEnablePrefetching) {
797  fPrefetch = NULL;
798  }
799 
800  //environment variable used to switch to the new method of reading asynchronously
801  if (fEnablePrefetching) {
803  }
804  else {
805  fAsyncReading = gEnv->GetValue("TFile.AsyncReading", 0);
806  if (fAsyncReading) {
807  // Check if asynchronous reading is supported by this TFile specialization
809  if (fFile && !(fFile->ReadBufferAsync(0, 0)))
811  }
812  if (!fAsyncReading && fBuffer == 0) {
813  // we use sync primitives, hence we need the local buffer
814  fBuffer = new char[fBufferSize];
815  }
816  }
817 }
818 
Int_t fNtot
Total size of prefetched blocks.
Long64_t * fBSeek
[fBNseek]
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:847
TFile * fFile
Pointer to file.
long long Long64_t
Definition: RtypesCore.h:69
char * fBuffer
[fBufferSize] buffer of contiguous prefetched blocks
virtual void SetEnablePrefetching(Bool_t setPrefetching=kFALSE)
Set the prefetching mode of this file.
virtual Int_t GetNoCacheReadCalls() const
A cache when reading files over the network.
const char Option_t
Definition: RtypesCore.h:62
virtual void SecondSort()
Sort buffers to be prefetched in increasing order of positions.
void SetEnablePrefetchingImpl(Bool_t setPrefetching=kFALSE)
TFileCacheRead implementation of SetEnablePrefetching.
virtual void SetCacheRead(TFileCacheRead *cache, TObject *tree=0, ECacheAction action=kDisconnect)
Set a pointer to the read cache.
Definition: TFile.cxx:2201
virtual void SetOffset(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
Definition: TFile.cxx:2088
Int_t fBufferSize
Allocated size of fBuffer (at a given time)
const char * GetProtocol() const
Definition: TUrl.h:67
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:46
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read a buffer from the file.
Definition: TFile.cxx:1606
Long64_t fPrefetchedBlocks
Number of blocks prefetched.
Long64_t GetWaitTime()
Return the time spent wating for buffer to be read in microseconds.
virtual void Sort()
Sort buffers to be prefetched in increasing order of positions.
Basic string class.
Definition: TString.h:129
Bool_t fIsSorted
True if fSeek array is sorted.
void ToLower()
Change string to lower-case.
Definition: TString.cxx:1099
virtual void Prefetch(Long64_t pos, Int_t len)
Add block of length len at position pos in the list of blocks to be prefetched.
int Int_t
Definition: RtypesCore.h:41
bool Bool_t
Definition: RtypesCore.h:59
virtual Long64_t GetBytesReadExtra() const
Int_t fReadCalls
Number of read calls for this cache.
#define NULL
Definition: RtypesCore.h:88
Int_t ThreadStart()
Used to start the consumer thread.
virtual ~TFileCacheRead()
Destructor.
virtual TFilePrefetch * GetPrefetchObj()
#define SafeDelete(p)
Definition: RConfig.h:499
virtual void SecondPrefetch(Long64_t, Int_t)
Int_t * fBSeekPos
[fBNseek]
virtual void Print(Option_t *option="") const
Print cache statistics.
Long64_t * fBSeekSort
[fBNseek]
Int_t * fSeekLen
[fNseek] Length of buffers to be prefetched
Bool_t fIsTransferred
True when fBuffer contains something valid.
Int_t fBufferLen
Current buffer length (<= fBufferSize)
virtual Int_t GetReadCalls() const
Definition: TFile.h:215
void Sort(Index n, const Element *a, Index *index, Bool_t down=kTRUE)
Definition: TMath.h:1151
Long64_t fBytesRead
Number of bytes read for this cache.
Int_t * fBSeekSortLen
[fBNseek]
virtual Int_t ReadBufferExt(char *buf, Long64_t pos, Int_t len, Int_t &loc)
Int_t * fBSeekIndex
[fBNseek]
TFileCacheRead()
Default Constructor.
virtual Int_t ReadBufferExtNormal(char *buf, Long64_t pos, Int_t len, Int_t &loc)
Base function for ReadBuffer.
Int_t fSeekSize
Allocated size of fSeek.
virtual Long64_t GetBytesRead() const
Int_t * fSeekIndex
[fNseek] sorted index table of fSeek
void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Change the file.
void WaitFinishPrefetch()
Killing the async prefetching thread.
virtual Int_t GetValue(const char *name, Int_t dflt)
Returns the integer value for a resource.
Definition: TEnv.cxx:482
virtual void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Set the file using this cache and reset the current blocks (if any).
Bool_t ReadBuffer(char *, Long64_t, Int_t)
Return a prefetched element.
Long64_t fBytesReadExtra
Number of extra bytes (overhead) read by the readahead buffer.
Long64_t fNoCacheBytesRead
Number of bytes read by basket to fill cached tree.
Bool_t fAsyncReading
virtual Int_t SetBufferSize(Int_t buffersize)
Sets the buffer size.
Long64_t * fPos
[fNb] start of long buffers
const Bool_t kFALSE
Definition: RtypesCore.h:92
virtual Long64_t GetBytesRead() const
Definition: TFile.h:212
Int_t * fBSeekLen
[fBNseek]
Int_t fNb
Number of long buffers.
virtual Long64_t GetNoCacheBytesRead() const
Int_t fBufferSizeMin
Original size of fBuffer.
#define ClassImp(name)
Definition: Rtypes.h:336
double Double_t
Definition: RtypesCore.h:55
static Int_t GetReadaheadSize()
Static function returning the readahead buffer size.
Definition: TFile.cxx:4402
virtual void WaitFinishPrefetch()
TFileCacheWrite * GetCacheWrite() const
Return a pointer to the current write cache.
Definition: TFile.cxx:1219
R__EXTERN TEnv * gEnv
Definition: TEnv.h:170
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:572
you should not use this method at all Int_t Int_t Double_t Double_t Double_t e
Definition: TRolke.cxx:630
The prefetching mechanism uses two classes (TFilePrefetch and TFPBlock) to prefetch in advance a bloc...
Definition: TFilePrefetch.h:31
virtual Long64_t GetBytesReadExtra() const
Definition: TFile.h:213
Int_t * fSeekPos
[fNseek] Position of sorted blocks in fBuffer
Mother of all ROOT objects.
Definition: TObject.h:37
virtual const TUrl * GetEndpointUrl() const
Definition: TFile.h:206
Long64_t * fSeekSort
[fNseek] Position on file of buffers to be prefetched (sorted)
virtual Int_t ReadBufferExtPrefetch(char *buf, Long64_t pos, Int_t len, Int_t &loc)
prefetch the first block
Bool_t fEnablePrefetching
reading by prefetching asynchronously
TFilePrefetch * fPrefetch
! Object that does the asynchronous reading in another thread
Definition: file.py:1
Bool_t SetCache(const char *)
Set the path of the cache directory.
Int_t * fLen
[fNb] Length of long buffers
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
Definition: TFile.cxx:1657
virtual Int_t GetReadCalls() const
Long64_t * fSeek
[fNseek] Position on file of buffers to be prefetched
void ReadBlock(Long64_t *, Int_t *, Int_t)
Create a TFPBlock object or recycle one and add it to the prefetchBlocks list.
Long64_t * fBPos
[fBNb]
R__EXTERN Int_t gDebug
Definition: Rtypes.h:83
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos.
Definition: tree.py:1
Int_t * fBLen
[fBNb]
virtual void Close(Option_t *option="")
Close out any threads or asynchronous fetches used by the underlying implementation.
const Bool_t kTRUE
Definition: RtypesCore.h:91
Int_t * fSeekSortLen
[fNseek] Length of buffers to be prefetched (sorted)
Bool_t fBIsTransferred
Long64_t BinarySearch(Long64_t n, const T *array, T value)
Definition: TMath.h:1093
Int_t fNoCacheReadCalls
Number of read calls by basket to fill cached tree.
A cache when writing files over the network.
virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len)
Definition: TFile.cxx:5013
Int_t fNseek
Number of blocks to be prefetched.
ECacheAction
TTreeCache flushing semantics.
Definition: TFile.h:63