Logo ROOT   6.10/09
Reference Guide
TFileMerger.cxx
Go to the documentation of this file.
1 // @(#)root/io:$Id$
2 // Author: Andreas Peters + Fons Rademakers + Rene Brun 26/5/2005
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /**
13 \class TFileMerger TFileMerger.cxx
14 \ingroup IO
15 
16 This class provides file copy and merging services.
17 
18 It can be used to copy files (not only ROOT files), using TFile or
19 any of its remote file access plugins. It is therefore useful in
20 a Grid environment where the files might be accessible via Castor,
21 rfio, dcap, etc.
22 The merging interface allows files containing histograms and trees
23 to be merged, like the standalone hadd program.
24 */
25 
26 #include "TFileMerger.h"
27 #include "TDirectory.h"
28 #include "TUrl.h"
29 #include "TFile.h"
30 #include "TUUID.h"
31 #include "TSystem.h"
32 #include "TKey.h"
33 #include "THashList.h"
34 #include "TObjString.h"
35 #include "TClass.h"
36 #include "TMethodCall.h"
37 #include "Riostream.h"
38 #include "TFileMergeInfo.h"
39 #include "TClassRef.h"
40 #include "TROOT.h"
41 #include "TMemFile.h"
42 #include "TVirtualMutex.h"
43 
44 #ifdef WIN32
45 // For _getmaxstdio
46 #include <stdio.h>
47 #else
48 // For getrlimit
49 #include <sys/time.h>
50 #include <sys/resource.h>
51 #endif
52 
54 
55 TClassRef R__TH1_Class("TH1");
56 TClassRef R__TTree_Class("TTree");
57 
58 static const Int_t kCpProgress = BIT(14);
59 static const Int_t kCintFileNumber = 100;
60 ////////////////////////////////////////////////////////////////////////////////
61 /// Return the maximum number of allowed opened files minus some wiggle room
62 /// for CINT or at least of the standard library (stdio).
63 
65 {
66  int maxfiles;
67 #ifdef WIN32
68  maxfiles = _getmaxstdio();
69 #else
70  rlimit filelimit;
71  if (getrlimit(RLIMIT_NOFILE,&filelimit)==0) {
72  maxfiles = filelimit.rlim_cur;
73  } else {
74  // We could not get the value from getrlimit, let's return a reasonable default.
75  maxfiles = 512;
76  }
77 #endif
78  if (maxfiles > kCintFileNumber) {
79  return maxfiles - kCintFileNumber;
80  } else if (maxfiles > 5) {
81  return maxfiles - 5;
82  } else {
83  return maxfiles;
84  }
85 }
86 
87 ////////////////////////////////////////////////////////////////////////////////
88 /// Create file merger object.
89 
91  : fOutputFile(0), fFastMethod(kTRUE), fNoTrees(kFALSE), fExplicitCompLevel(kFALSE), fCompressionChange(kFALSE),
92  fPrintLevel(0), fMsgPrefix("TFileMerger"), fMaxOpenedFiles( R__GetSystemMaxOpenedFiles() ),
93  fLocal(isLocal), fHistoOneGo(histoOneGo), fObjectNames()
94 {
95  fFileList = new TList;
96 
97  fMergeList = new TList;
99 
100  fExcessFiles = new TList;
102 
104  gROOT->GetListOfCleanups()->Add(this);
105 }
106 
107 ////////////////////////////////////////////////////////////////////////////////
108 /// Cleanup.
109 
111 {
112  {
114  gROOT->GetListOfCleanups()->Remove(this);
115  }
120 }
121 
122 ////////////////////////////////////////////////////////////////////////////////
123 /// Reset merger file list.
124 
126 {
127  fFileList->Clear();
128  fMergeList->Clear();
129  fExcessFiles->Clear();
131 }
132 
133 ////////////////////////////////////////////////////////////////////////////////
134 /// Add file to file merger.
135 
136 Bool_t TFileMerger::AddFile(const char *url, Bool_t cpProgress)
137 {
138  if (fPrintLevel > 0) {
139  Printf("%s Source file %d: %s",fMsgPrefix.Data(),fFileList->GetEntries()+fExcessFiles->GetEntries()+1,url);
140  }
141 
142  TFile *newfile = 0;
143  TString localcopy;
144 
145  if (fFileList->GetEntries() >= (fMaxOpenedFiles-1)) {
146 
147  TObjString *urlObj = new TObjString(url);
148  fMergeList->Add(urlObj);
149 
150  urlObj = new TObjString(url);
151  urlObj->SetBit(kCpProgress);
152  fExcessFiles->Add(urlObj);
153  return kTRUE;
154  }
155 
156  // We want gDirectory untouched by anything going on here
158 
159  if (fLocal) {
160  TUUID uuid;
161  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
162  if (!TFile::Cp(url, localcopy, cpProgress)) {
163  Error("AddFile", "cannot get a local copy of file %s", url);
164  return kFALSE;
165  }
166  newfile = TFile::Open(localcopy, "READ");
167  } else {
168  newfile = TFile::Open(url, "READ");
169  }
170 
171  // Zombie files should also be skipped
172  if (newfile && newfile->IsZombie()) {
173  delete newfile;
174  newfile = 0;
175  }
176 
177  if (!newfile) {
178  if (fLocal)
179  Error("AddFile", "cannot open local copy %s of URL %s",
180  localcopy.Data(), url);
181  else
182  Error("AddFile", "cannot open file %s", url);
183  return kFALSE;
184  } else {
185  if (fOutputFile && fOutputFile->GetCompressionLevel() != newfile->GetCompressionLevel()) fCompressionChange = kTRUE;
186 
187  newfile->SetBit(kCanDelete);
188  fFileList->Add(newfile);
189 
190  TObjString *urlObj = new TObjString(url);
191  fMergeList->Add(urlObj);
192 
193  return kTRUE;
194  }
195 }
196 
197 ////////////////////////////////////////////////////////////////////////////////
198 /// Add the TFile to this file merger and *do not* give ownership of the TFile to this
199 /// object.
200 ///
201 /// Return kTRUE if the addition was successful.
202 
204 {
205  return AddFile(source,kFALSE,cpProgress);
206 }
207 
208 ////////////////////////////////////////////////////////////////////////////////
209 /// Add the TFile to this file merger and give ownership of the TFile to this
210 /// object (unless kFALSE is returned).
211 ///
212 /// Return kTRUE if the addition was successful.
213 
215 {
216  return AddFile(source,kTRUE,cpProgress);
217 }
218 
219 ////////////////////////////////////////////////////////////////////////////////
220 /// Add the TFile to this file merger and give ownership of the TFile to this
221 /// object (unless kFALSE is returned).
222 ///
223 /// Return kTRUE if the addition was successful.
224 
225 Bool_t TFileMerger::AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
226 {
227  if (source == 0 || source->IsZombie()) {
228  return kFALSE;
229  }
230 
231  if (fPrintLevel > 0) {
232  Printf("%s Source file %d: %s",fMsgPrefix.Data(),fFileList->GetEntries()+1,source->GetName());
233  }
234 
235  TFile *newfile = 0;
236  TString localcopy;
237 
238  // We want gDirectory untouched by anything going on here
240  if (fLocal && !source->InheritsFrom(TMemFile::Class())) {
241  TUUID uuid;
242  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
243  if (!source->Cp(localcopy, cpProgress)) {
244  Error("AddFile", "cannot get a local copy of file %s", source->GetName());
245  return kFALSE;
246  }
247  newfile = TFile::Open(localcopy, "READ");
248  // Zombie files should also be skipped
249  if (newfile && newfile->IsZombie()) {
250  delete newfile;
251  newfile = 0;
252  }
253  } else {
254  newfile = source;
255  }
256 
257  if (!newfile) {
258  if (fLocal)
259  Error("AddFile", "cannot open local copy %s of URL %s",
260  localcopy.Data(), source->GetName());
261  else
262  Error("AddFile", "cannot open file %s", source->GetName());
263  return kFALSE;
264  } else {
265  if (fOutputFile && fOutputFile->GetCompressionLevel() != newfile->GetCompressionLevel()) fCompressionChange = kTRUE;
266 
267  if (own || newfile != source) {
268  newfile->SetBit(kCanDelete);
269  } else {
270  newfile->ResetBit(kCanDelete);
271  }
272  fFileList->Add(newfile);
273 
274  if (!fMergeList) {
275  fMergeList = new TList;
276  }
277  TObjString *urlObj = new TObjString(source->GetName());
278  fMergeList->Add(urlObj);
279 
280  if (newfile != source && own) {
281  delete source;
282  }
283  return kTRUE;
284  }
285 }
286 
287 ////////////////////////////////////////////////////////////////////////////////
288 /// Open merger output file.
289 
290 Bool_t TFileMerger::OutputFile(const char *outputfile, Bool_t force, Int_t compressionLevel)
291 {
292  return OutputFile(outputfile,(force?"RECREATE":"CREATE"),compressionLevel);
293 }
294 
295 ////////////////////////////////////////////////////////////////////////////////
296 /// Open merger output file.
297 
298 Bool_t TFileMerger::OutputFile(const char *outputfile, Bool_t force)
299 {
300  Bool_t res = OutputFile(outputfile,(force?"RECREATE":"CREATE"),1); // 1 is the same as the default from the TFile constructor.
302  return res;
303 }
304 
305 ////////////////////////////////////////////////////////////////////////////////
306 /// Open merger output file.
307 ///
308 /// The 'mode' parameter is passed to the TFile constructor as the option, it
309 /// should be one of 'NEW','CREATE','RECREATE','UPDATE'
310 /// 'UPDATE' is usually used in conjunction with IncrementalMerge.
311 
312 Bool_t TFileMerger::OutputFile(const char *outputfile, const char *mode, Int_t compressionLevel)
313 {
315 
316  TFile *oldfile = fOutputFile;
317  fOutputFile = 0; // This avoids the complaint from RecursiveRemove about the file being deleted which is here spurrious. (see RecursiveRemove).
318  SafeDelete(oldfile);
319 
320  fOutputFilename = outputfile;
321 
322  // We want gDirectory untouched by anything going on here
324  if (!(fOutputFile = TFile::Open(outputfile, mode, "", compressionLevel)) || fOutputFile->IsZombie()) {
325  Error("OutputFile", "cannot open the MERGER output file %s", fOutputFilename.Data());
326  return kFALSE;
327  }
328  return kTRUE;
329 }
330 
331 ////////////////////////////////////////////////////////////////////////////////
332 /// Open merger output file. 'mode' is passed to the TFile constructor as the option, it should
333 /// be one of 'NEW','CREATE','RECREATE','UPDATE'
334 /// 'UPDATE' is usually used in conjunction with IncrementalMerge.
335 
336 Bool_t TFileMerger::OutputFile(const char *outputfile, const char *mode /* = "RECREATE" */)
337 {
338  Bool_t res = OutputFile(outputfile,mode,1); // 1 is the same as the default from the TFile constructor.
340  return res;
341 }
342 
343 ////////////////////////////////////////////////////////////////////////////////
344 /// Print list of files being merged.
345 
347 {
348  fFileList->Print(options);
349  fExcessFiles->Print(options);
350 }
351 
352 ////////////////////////////////////////////////////////////////////////////////
353 /// Merge the files.
354 ///
355 /// If no output file was specified it will write into
356 /// the file "FileMerger.root" in the working directory. Returns true
357 /// on success, false in case of error.
358 
360 {
361  return PartialMerge(kAll | kRegular);
362 }
363 
364 ////////////////////////////////////////////////////////////////////////////////
365 /// Merge all objects in a directory
366 ///
367 /// The type is defined by the bit values in TFileMerger::EPartialMergeType.
368 
369 Bool_t TFileMerger::MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type /* = kRegular | kAll */)
370 {
371  Bool_t status = kTRUE;
372  Bool_t onlyListed = kFALSE;
373  if (fPrintLevel > 0) {
374  Printf("%s Target path: %s",fMsgPrefix.Data(),target->GetPath());
375  }
376 
377  // Get the dir name
378  TString path(target->GetPath());
379  // coverity[unchecked_value] 'target' is from a file so GetPath always returns path starting with filename:
380  path.Remove(0, path.Last(':') + 2);
381 
382  Int_t nguess = sourcelist->GetSize()+1000;
383  THashList allNames(nguess);
384  allNames.SetOwner(kTRUE);
385  // If the mode is set to skipping list objects, add names to the allNames list
386  if (type & kSkipListed) {
387  TObjArray *arr = fObjectNames.Tokenize(" ");
388  arr->SetOwner(kFALSE);
389  for (Int_t iname=0; iname<arr->GetEntriesFast(); iname++)
390  allNames.Add(arr->At(iname));
391  delete arr;
392  }
393  ((THashList*)target->GetList())->Rehash(nguess);
394  ((THashList*)target->GetListOfKeys())->Rehash(nguess);
395 
396  TFileMergeInfo info(target);
397  info.fOptions = fMergeOptions;
398  if (fFastMethod && ((type&kKeepCompression) || !fCompressionChange) ) {
399  info.fOptions.Append(" fast");
400  }
401 
402  TFile *current_file;
403  TDirectory *current_sourcedir;
404  if (type & kIncremental) {
405  current_file = 0;
406  current_sourcedir = target;
407  } else {
408  current_file = (TFile*)sourcelist->First();
409  current_sourcedir = current_file->GetDirectory(path);
410  }
411  while (current_file || current_sourcedir) {
412  // When current_sourcedir != 0 and current_file == 0 we are going over the target
413  // for an incremental merge.
414  if (current_sourcedir && (current_file == 0 || current_sourcedir != target)) {
415 
416  // loop over all keys in this directory
417  TIter nextkey( current_sourcedir->GetListOfKeys() );
418  TKey *key;
419  TString oldkeyname;
420 
421  while ( (key = (TKey*)nextkey())) {
422 
423  // Keep only the highest cycle number for each key for mergeable objects. They are stored
424  // in the (hash) list consecutively and in decreasing order of cycles, so we can continue
425  // until the name changes. We flag the case here and we act consequently later.
426  Bool_t alreadyseen = (oldkeyname == key->GetName()) ? kTRUE : kFALSE;
427 
428  // Read in but do not copy directly the processIds.
429  if (strcmp(key->GetClassName(),"TProcessID") == 0) { key->ReadObj(); continue;}
430 
431  // If we have already seen this object [name], we already processed
432  // the whole list of files for this objects and we can just skip it
433  // and any related cycles.
434  if (allNames.FindObject(key->GetName())) {
435  oldkeyname = key->GetName();
436  continue;
437  }
438 
439  TClass *cl = TClass::GetClass(key->GetClassName());
440  if (!cl) {
441  Info("MergeRecursive", "cannot indentify object type (%s), name: %s title: %s",
442  key->GetClassName(), key->GetName(), key->GetTitle());
443  continue;
444  }
445  // For mergeable objects we add the names in a local hashlist handling them
446  // again (see above)
447  if (cl->GetMerge() || cl->InheritsFrom(TDirectory::Class()) ||
448  (cl->IsTObject() &&
449  (cl->GetMethodWithPrototype("Merge", "TCollection*,TFileMergeInfo*") ||
450  cl->GetMethodWithPrototype("Merge", "TCollection*"))))
451  allNames.Add(new TObjString(key->GetName()));
452 
453  if (fNoTrees && cl->InheritsFrom(R__TTree_Class)) {
454  // Skip the TTree objects and any related cycles.
455  oldkeyname = key->GetName();
456  continue;
457  }
458  // Check if only the listed objects are to be merged
459  if (type & kOnlyListed) {
460  onlyListed = kFALSE;
461  oldkeyname = key->GetName();
462  oldkeyname += " ";
463  onlyListed = fObjectNames.Contains(oldkeyname);
464  oldkeyname = key->GetName();
465  if ((!onlyListed) && (!cl->InheritsFrom(TDirectory::Class()))) continue;
466  }
467 
468  if (!(type&kResetable && type&kNonResetable)) {
469  // If neither or both are requested at the same time, we merger both types.
470  if (!(type&kResetable)) {
471  if (cl->GetResetAfterMerge()) {
472  // Skip the object with a reset after merge routine (TTree and other incrementally mergeable objects)
473  oldkeyname = key->GetName();
474  continue;
475  }
476  }
477  if (!(type&kNonResetable)) {
478  if (!cl->GetResetAfterMerge()) {
479  // Skip the object without a reset after merge routine (Histograms and other non incrementally mergeable objects)
480  oldkeyname = key->GetName();
481  continue;
482  }
483  }
484  }
485  // read object from first source file
486  TObject *obj;
487  if (type & kIncremental) {
488  obj = current_sourcedir->GetList()->FindObject(key->GetName());
489  if (!obj) {
490  obj = key->ReadObj();
491  }
492  } else {
493  obj = key->ReadObj();
494  }
495  if (!obj) {
496  Info("MergeRecursive", "could not read object for key {%s, %s}",
497  key->GetName(), key->GetTitle());
498  continue;
499  }
500  if (cl->IsTObject() && cl != obj->IsA()) {
501  Error("MergeRecursive", "TKey and object retrieve disagree on type (%s vs %s). Continuing with %s.",
502  key->GetClassName(), obj->IsA()->GetName(), obj->IsA()->GetName());
503  cl = obj->IsA();
504  }
505  Bool_t canBeMerged = kTRUE;
506 
507  if ( cl->InheritsFrom( TDirectory::Class() ) ) {
508  // it's a subdirectory
509 
510  target->cd();
511  TDirectory *newdir;
512 
513  // For incremental or already seen we may have already a directory created
514  if (type & kIncremental || alreadyseen) {
515  newdir = target->GetDirectory(obj->GetName());
516  if (!newdir) {
517  newdir = target->mkdir( obj->GetName(), obj->GetTitle() );
518  }
519  } else {
520  newdir = target->mkdir( obj->GetName(), obj->GetTitle() );
521  }
522 
523  // newdir is now the starting point of another round of merging
524  // newdir still knows its depth within the target file via
525  // GetPath(), so we can still figure out where we are in the recursion
526 
527  // If this folder is a onlyListed object, merge everything inside.
528  if (onlyListed) type &= ~kOnlyListed;
529  status = MergeRecursive(newdir, sourcelist, type);
530  if (onlyListed) type |= kOnlyListed;
531  if (!status) return status;
532  } else if (cl->GetMerge()) {
533 
534  // Check if already treated
535  if (alreadyseen) continue;
536 
537  TList inputs;
539 
540  // Loop over all source files and merge same-name object
541  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
542  if (nextsource == 0) {
543  // There is only one file in the list
545  func(obj, &inputs, &info);
546  info.fIsFirst = kFALSE;
547  } else {
548  do {
549  // make sure we are at the correct directory level by cd'ing to path
550  TDirectory *ndir = nextsource->GetDirectory(path);
551  if (ndir) {
552  ndir->cd();
553  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
554  if (key2) {
555  TObject *hobj = key2->ReadObj();
556  if (!hobj) {
557  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
558  key->GetName(), key->GetTitle(), nextsource->GetName());
559  nextsource = (TFile*)sourcelist->After(nextsource);
560  continue;
561  }
562  // Set ownership for collections
563  if (hobj->InheritsFrom(TCollection::Class())) {
564  ((TCollection*)hobj)->SetOwner();
565  }
566  hobj->ResetBit(kMustCleanup);
567  inputs.Add(hobj);
568  if (!oneGo) {
570  Long64_t result = func(obj, &inputs, &info);
571  info.fIsFirst = kFALSE;
572  if (result < 0) {
573  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
574  obj->GetName(), nextsource->GetName());
575  }
576  inputs.Delete();
577  }
578  }
579  }
580  nextsource = (TFile*)sourcelist->After( nextsource );
581  } while (nextsource);
582  // Merge the list, if still to be done
583  if (oneGo || info.fIsFirst) {
585  func(obj, &inputs, &info);
586  info.fIsFirst = kFALSE;
587  inputs.Delete();
588  }
589  }
590  } else if (cl->IsTObject() &&
591  cl->GetMethodWithPrototype("Merge", "TCollection*,TFileMergeInfo*") ) {
592  // Object implements Merge(TCollection*,TFileMergeInfo*) and has a reflex dictionary ...
593 
594  // Check if already treated
595  if (alreadyseen) continue;
596 
597  TList listH;
598  TString listHargs;
599  listHargs.Form("(TCollection*)0x%lx,(TFileMergeInfo*)0x%lx", (ULong_t)&listH,(ULong_t)&info);
600 
601  // Loop over all source files and merge same-name object
602  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
603  if (nextsource == 0) {
604  // There is only one file in the list
605  Int_t error = 0;
606  obj->Execute("Merge", listHargs.Data(), &error);
607  info.fIsFirst = kFALSE;
608  if (error) {
609  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
610  obj->GetName(), key->GetName());
611  }
612  } else {
613  while (nextsource) {
614  // make sure we are at the correct directory level by cd'ing to path
615  TDirectory *ndir = nextsource->GetDirectory(path);
616  if (ndir) {
617  ndir->cd();
618  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
619  if (key2) {
620  TObject *hobj = key2->ReadObj();
621  if (!hobj) {
622  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
623  key->GetName(), key->GetTitle(), nextsource->GetName());
624  nextsource = (TFile*)sourcelist->After(nextsource);
625  continue;
626  }
627  // Set ownership for collections
628  if (hobj->InheritsFrom(TCollection::Class())) {
629  ((TCollection*)hobj)->SetOwner();
630  }
631  hobj->ResetBit(kMustCleanup);
632  listH.Add(hobj);
633  Int_t error = 0;
634  obj->Execute("Merge", listHargs.Data(), &error);
635  info.fIsFirst = kFALSE;
636  if (error) {
637  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
638  obj->GetName(), nextsource->GetName());
639  }
640  listH.Delete();
641  }
642  }
643  nextsource = (TFile*)sourcelist->After( nextsource );
644  }
645  // Merge the list, if still to be done
646  if (info.fIsFirst) {
647  Int_t error = 0;
648  obj->Execute("Merge", listHargs.Data(), &error);
649  info.fIsFirst = kFALSE;
650  listH.Delete();
651  }
652  }
653  } else if (cl->IsTObject() &&
654  cl->GetMethodWithPrototype("Merge", "TCollection*") ) {
655  // Object implements Merge(TCollection*) and has a reflex dictionary ...
656 
657  // Check if already treated
658  if (alreadyseen) continue;
659 
660  TList listH;
661  TString listHargs;
662  listHargs.Form("((TCollection*)0x%lx)", (ULong_t)&listH);
663 
664  // Loop over all source files and merge same-name object
665  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
666  if (nextsource == 0) {
667  // There is only one file in the list
668  Int_t error = 0;
669  obj->Execute("Merge", listHargs.Data(), &error);
670  if (error) {
671  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
672  obj->GetName(), key->GetName());
673  }
674  } else {
675  while (nextsource) {
676  // make sure we are at the correct directory level by cd'ing to path
677  TDirectory *ndir = nextsource->GetDirectory(path);
678  if (ndir) {
679  ndir->cd();
680  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
681  if (key2) {
682  TObject *hobj = key2->ReadObj();
683  if (!hobj) {
684  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
685  key->GetName(), key->GetTitle(), nextsource->GetName());
686  nextsource = (TFile*)sourcelist->After(nextsource);
687  continue;
688  }
689  // Set ownership for collections
690  if (hobj->InheritsFrom(TCollection::Class())) {
691  ((TCollection*)hobj)->SetOwner();
692  }
693  hobj->ResetBit(kMustCleanup);
694  listH.Add(hobj);
695  Int_t error = 0;
696  obj->Execute("Merge", listHargs.Data(), &error);
697  info.fIsFirst = kFALSE;
698  if (error) {
699  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
700  obj->GetName(), nextsource->GetName());
701  }
702  listH.Delete();
703  }
704  }
705  nextsource = (TFile*)sourcelist->After( nextsource );
706  }
707  // Merge the list, if still to be done
708  if (info.fIsFirst) {
709  Int_t error = 0;
710  obj->Execute("Merge", listHargs.Data(), &error);
711  info.fIsFirst = kFALSE;
712  listH.Delete();
713  }
714  }
715  } else {
716  // Object is of no type that we can merge
717  canBeMerged = kFALSE;
718  }
719 
720  // now write the merged histogram (which is "in" obj) to the target file
721  // note that this will just store obj in the current directory level,
722  // which is not persistent until the complete directory itself is stored
723  // by "target->SaveSelf()" below
724  target->cd();
725 
726  oldkeyname = key->GetName();
727  //!!if the object is a tree, it is stored in globChain...
728  if(cl->InheritsFrom( TDirectory::Class() )) {
729  //printf("cas d'une directory\n");
730  // Do not delete the directory if it is part of the output
731  // and we are in incremental mode (because it will be reuse
732  // and has not been written to disk (for performance reason).
733  // coverity[var_deref_model] the IsA()->InheritsFrom guarantees that the dynamic_cast will succeed.
734  if (!(type&kIncremental) || dynamic_cast<TDirectory*>(obj)->GetFile() != target) {
735  delete obj;
736  }
737  } else if (cl->InheritsFrom( TCollection::Class() )) {
738  // Don't overwrite, if the object were not merged.
739  if ( obj->Write( oldkeyname, canBeMerged ? TObject::kSingleKey | TObject::kOverwrite : TObject::kSingleKey) <= 0 ) {
740  status = kFALSE;
741  }
742  ((TCollection*)obj)->SetOwner();
743  delete obj;
744  } else {
745  // Don't overwrite, if the object were not merged.
746  // NOTE: this is probably wrong for emulated objects.
747  if (cl->IsTObject()) {
748  if ( obj->Write( oldkeyname, canBeMerged ? TObject::kOverwrite : 0) <= 0) {
749  status = kFALSE;
750  }
751  } else {
752  if ( target->WriteObjectAny( (void*)obj, cl, oldkeyname, canBeMerged ? "OverWrite" : "" ) <= 0) {
753  status = kFALSE;
754  }
755  }
756  cl->Destructor(obj); // just in case the class is not loaded.
757  }
758  info.Reset();
759  } // while ( ( TKey *key = (TKey*)nextkey() ) )
760  }
761  current_file = current_file ? (TFile*)sourcelist->After(current_file) : (TFile*)sourcelist->First();
762  if (current_file) {
763  current_sourcedir = current_file->GetDirectory(path);
764  } else {
765  current_sourcedir = 0;
766  }
767  }
768  // save modifications to the target directory.
769  if (!(type&kIncremental)) {
770  // In case of incremental build, we will call Write on the top directory/file, so we do not need
771  // to call SaveSelf explicilty.
772  target->SaveSelf(kTRUE);
773  }
774  return status;
775 }
776 
777 ////////////////////////////////////////////////////////////////////////////////
778 /// Merge the files. If no output file was specified it will write into
779 /// the file "FileMerger.root" in the working directory. Returns true
780 /// on success, false in case of error.
781 /// The type is defined by the bit values in EPartialMergeType:
782 /// kRegular : normal merge, overwritting the output file
783 /// kIncremental : merge the input file with the content of the output file (if already exising) (default)
784 /// kAll : merge all type of objects (default)
785 /// kResetable : merge only the objects with a MergeAfterReset member function.
786 /// kNonResetable : merge only the objects without a MergeAfterReset member function.
787 ///
788 /// If the type is set to kIncremental the output file is done deleted at the end of
789 /// this operation. If the type is not set to kIncremental, the output file is closed.
790 
792 {
793  if (!fOutputFile) {
794  TString outf(fOutputFilename);
795  if (outf.IsNull()) {
796  outf.Form("file:%s/FileMerger.root", gSystem->TempDirectory());
797  Info("PartialMerge", "will merge the results to the file %s\n"
798  "since you didn't specify a merge filename",
799  TUrl(outf).GetFile());
800  }
801  if (!OutputFile(outf.Data())) {
802  return kFALSE;
803  }
804  }
805 
806  // Special treament for the single file case ...
807  if ((fFileList->GetEntries() == 1) && !fExcessFiles->GetEntries() &&
808  !(in_type & kIncremental) && !fCompressionChange && !fExplicitCompLevel) {
809  fOutputFile->Close();
811 
812  TFile *file = (TFile *) fFileList->First();
813  if (!file || (file && file->IsZombie())) {
814  Error("PartialMerge", "one-file case: problem attaching to file");
815  return kFALSE;
816  }
817  Bool_t result = kTRUE;
818  if (!(result = file->Cp(fOutputFilename))) {
819  Error("PartialMerge", "one-file case: could not copy '%s' to '%s'",
820  file->GetPath(), fOutputFilename.Data());
821  return kFALSE;
822  }
823  if (file->TestBit(kCanDelete)) file->Close();
824 
825  // Remove the temporary file
826  if (fLocal && !file->InheritsFrom(TMemFile::Class())) {
827  TUrl u(file->GetPath(), kTRUE);
828  if (gSystem->Unlink(u.GetFile()) != 0)
829  Warning("PartialMerge", "problems removing temporary local file '%s'", u.GetFile());
830  }
831  fFileList->Clear();
832  return result;
833  }
834 
836 
838 
839  Bool_t result = kTRUE;
840  Int_t type = in_type;
841  while (result && fFileList->GetEntries()>0) {
842  result = MergeRecursive(fOutputFile, fFileList, type);
843 
844  // Remove local copies if there are any
845  TIter next(fFileList);
846  TFile *file;
847  while ((file = (TFile*) next())) {
848  // close the files
849  if (file->TestBit(kCanDelete)) file->Close();
850  // remove the temporary files
851  if(fLocal && !file->InheritsFrom(TMemFile::Class())) {
852  TString p(file->GetPath());
853  // coverity[unchecked_value] Index is return a value with range or NPos to select the whole name.
854  p = p(0, p.Index(':',0));
855  gSystem->Unlink(p);
856  }
857  }
858  fFileList->Clear();
859  if (result && fExcessFiles->GetEntries() > 0) {
860  // We merge the first set of files in the output,
861  // we now need to open the next set and make
862  // sure we accumulate into the output, so we
863  // switch to incremental merging (if not already set)
864  type = type | kIncremental;
865  result = OpenExcessFiles();
866  }
867  }
868  if (!result) {
869  Error("Merge", "error during merge of your ROOT files");
870  } else {
871  // Close or write is required so the file is complete.
872  if (in_type & kIncremental) {
874  } else {
875  fOutputFile->Close();
876  }
877  }
878 
879  // Cleanup
880  if (in_type & kIncremental) {
881  Clear();
882  } else {
885  }
886  return result;
887 }
888 
889 ////////////////////////////////////////////////////////////////////////////////
890 /// Open up to fMaxOpenedFiles of the excess files.
891 
893 {
894  if (fPrintLevel > 0) {
895  Printf("%s Opening the next %d files",fMsgPrefix.Data(),TMath::Min(fExcessFiles->GetEntries(),(fMaxOpenedFiles-1)));
896  }
897  Int_t nfiles = 0;
898  TIter next(fExcessFiles);
899  TObjString *url = 0;
900  TString localcopy;
901  // We want gDirectory untouched by anything going on here
903  while( nfiles < (fMaxOpenedFiles-1) && ( url = (TObjString*)next() ) ) {
904  TFile *newfile = 0;
905  if (fLocal) {
906  TUUID uuid;
907  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
908  if (!TFile::Cp(url->GetName(), localcopy, url->TestBit(kCpProgress))) {
909  Error("OpenExcessFiles", "cannot get a local copy of file %s", url->GetName());
910  return kFALSE;
911  }
912  newfile = TFile::Open(localcopy, "READ");
913  } else {
914  newfile = TFile::Open(url->GetName(), "READ");
915  }
916 
917  if (!newfile) {
918  if (fLocal)
919  Error("OpenExcessFiles", "cannot open local copy %s of URL %s",
920  localcopy.Data(), url->GetName());
921  else
922  Error("OpenExcessFiles", "cannot open file %s", url->GetName());
923  return kFALSE;
924  } else {
926 
927  newfile->SetBit(kCanDelete);
928  fFileList->Add(newfile);
929  ++nfiles;
930  fExcessFiles->Remove(url);
931  }
932  }
933  return kTRUE;
934 }
935 
936 ////////////////////////////////////////////////////////////////////////////////
937 /// Intercept the case where the output TFile is deleted!
938 
940 {
941  if (obj == fOutputFile) {
942  Fatal("RecursiveRemove","Output file of the TFile Merger (targeting %s) has been deleted (likely due to a TTree larger than 100Gb)", fOutputFilename.Data());
943  }
944 
945 }
946 
947 ////////////////////////////////////////////////////////////////////////////////
948 /// Set a limit to the number of files that TFileMerger will open simultaneously.
949 ///
950 /// If the request is higher than the system limit, we reset it to the system limit.
951 /// If the request is less than two, we reset it to 2 (one for the output file and one for the input file).
952 
954 {
956  if (newmax < sysmax) {
957  fMaxOpenedFiles = newmax;
958  } else {
959  fMaxOpenedFiles = sysmax;
960  }
961  if (fMaxOpenedFiles < 2) {
962  fMaxOpenedFiles = 2;
963  }
964 }
965 
966 ////////////////////////////////////////////////////////////////////////////////
967 /// Set the prefix to be used when printing informational message.
968 
969 void TFileMerger::SetMsgPrefix(const char *prefix)
970 {
971  fMsgPrefix = prefix;
972 }
973 
virtual void Clear(Option_t *="")
Definition: TObject.h:91
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Definition: TObject.cxx:778
Int_t fPrintLevel
How much information to print out at run time.
Definition: TFileMerger.h:38
An array of TObjects.
Definition: TObjArray.h:37
virtual TList * GetListOfKeys() const
Definition: TDirectory.h:148
virtual void Delete(Option_t *option="")
Remove all objects from the list AND delete all heap based objects.
Definition: TList.cxx:409
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:847
long long Long64_t
Definition: RtypesCore.h:69
Collectable string class.
Definition: TObjString.h:28
Long64_t(* MergeFunc_t)(void *, TCollection *, TFileMergeInfo *)
Definition: Rtypes.h:108
const char Option_t
Definition: RtypesCore.h:62
This class represents a WWW compatible URL.
Definition: TUrl.h:35
Bool_t TestBit(UInt_t f) const
Definition: TObject.h:159
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
#define BIT(n)
Definition: Rtypes.h:75
virtual Bool_t Merge(Bool_t=kTRUE)
Merge the files.
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:46
virtual Int_t GetEntries() const
Definition: TCollection.h:86
Normal merge, overwritting the output file.
Definition: TFileMerger.h:56
Bool_t OpenExcessFiles()
Open up to fMaxOpenedFiles of the excess files.
#define gROOT
Definition: TROOT.h:375
Basic string class.
Definition: TString.h:129
Short_t Min(Short_t a, Short_t b)
Definition: TMathBase.h:168
int Int_t
Definition: RtypesCore.h:41
virtual TDirectory * mkdir(const char *name, const char *title="")
Create a sub-directory "a" or a hierarchy of sub-directories "a/b/c/...".
Definition: TDirectory.cxx:958
bool Bool_t
Definition: RtypesCore.h:59
R__EXTERN TVirtualMutex * gROOTMutex
Definition: TROOT.h:57
TString fMergeOptions
Options (in string format) to be passed down to the Merge functions.
Definition: TFileMerger.h:39
TObject * At(Int_t idx) const
Definition: TObjArray.h:165
TObject * FindObject(const char *name) const
Find object using its name.
Definition: THashList.cxx:213
Bool_t fCompressionChange
True if the output and input have different compression level (default kFALSE)
Definition: TFileMerger.h:37
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:687
overwrite existing object with same name
Definition: TObject.h:79
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Definition: TList.cxx:501
if object in a list can be deleted
Definition: TObject.h:58
Only the objects specified in fObjectNames list.
Definition: TFileMerger.h:64
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=1, Int_t netopt=0)
Create / open a file.
Definition: TFile.cxx:3909
virtual Bool_t Cp(const char *dst, Bool_t progressbar=kTRUE, UInt_t buffersize=1000000)
Allows to copy this file to the dst URL.
Definition: TFile.cxx:4807
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition: TUUID.h:42
virtual void RecursiveRemove(TObject *obj)
Intercept the case where the output TFile is deleted!
#define SafeDelete(p)
Definition: RConfig.h:499
virtual int Unlink(const char *name)
Unlink, i.e. remove, a file.
Definition: TSystem.cxx:1353
Merge all type of objects (default)
Definition: TFileMerger.h:61
void Class()
Definition: Class.C:29
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition: THashList.h:34
TList * fExcessFiles
! List of TObjString containing the name of the files not yet added to fFileList due to user or syste...
Definition: TFileMerger.h:47
void Clear()
Clear string without changing its capacity.
Definition: TString.cxx:1150
Skip objects specified in fObjectNames list.
Definition: TFileMerger.h:65
TString & Append(const char *cs)
Definition: TString.h:497
virtual Int_t WriteObjectAny(const void *, const char *, const char *, Option_t *="", Int_t=0)
Definition: TDirectory.h:198
virtual TList * GetList() const
Definition: TDirectory.h:147
Merge the input file with the content of the output file (if already exising).
Definition: TFileMerger.h:57
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
Definition: TSystem.cxx:1454
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition: TKey.h:24
Bool_t fHistoOneGo
Merger histos in one go (default is kTRUE)
Definition: TFileMerger.h:44
ROOT::MergeFunc_t GetMerge() const
void SetMsgPrefix(const char *prefix)
Set the prefix to be used when printing informational message.
virtual Bool_t AddAdoptFile(TFile *source, Bool_t cpProgress=kTRUE)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
TFileMerger(const TFileMerger &)
virtual Bool_t OutputFile(const char *url, Bool_t force)
Open merger output file.
TFile * fOutputFile
The outputfile for merging.
Definition: TFileMerger.h:32
TClassRef R__TTree_Class("TTree")
TList * fMergeList
list of TObjString containing the name of the files need to be merged
Definition: TFileMerger.h:46
A doubly linked list.
Definition: TList.h:43
const char * GetName() const
Returns name of object.
Definition: TObjString.h:39
virtual Bool_t AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
TClassRef R__TH1_Class("TH1")
static const Int_t kCpProgress
Definition: TFileMerger.cxx:58
TString fOutputFilename
The name of the outputfile for merging.
Definition: TFileMerger.h:33
virtual TObject * First() const
Return the first object in the list. Returns 0 when list is empty.
Definition: TList.cxx:561
R__EXTERN TSystem * gSystem
Definition: TSystem.h:539
static Int_t R__GetSystemMaxOpenedFiles()
Return the maximum number of allowed opened files minus some wiggle room for CINT or at least of the ...
Definition: TFileMerger.cxx:64
virtual Int_t Write(const char *name=0, Int_t opt=0, Int_t bufsiz=0)
Write memory objects to this file.
Definition: TFile.cxx:2272
virtual void Execute(const char *method, const char *params, Int_t *error=0)
Execute method on this object with the given parameter string, e.g.
Definition: TObject.cxx:284
This class provides file copy and merging services.
Definition: TFileMerger.h:24
virtual TObject * Remove(TObject *obj)
Remove object from the list.
Definition: TList.cxx:679
virtual Bool_t InheritsFrom(const char *classname) const
Returns kTRUE if object inherits from class "classname".
Definition: TObject.cxx:436
Collection abstract base class.
Definition: TCollection.h:42
Keep compression level unchanged for each input files.
Definition: TFileMerger.h:66
void Destructor(void *obj, Bool_t dtorOnly=kFALSE)
Explicitly call destructor for object.
Definition: TClass.cxx:5063
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2332
Int_t GetEntriesFast() const
Definition: TObjArray.h:64
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:873
TString fMsgPrefix
Prefix to be used when printing informational message (default TFileMerger)
Definition: TFileMerger.h:40
The ROOT global object gROOT contains a list of all defined classes.
Definition: TClass.h:71
virtual TObject * After(const TObject *obj) const
Returns the object after object obj.
Definition: TList.cxx:293
Bool_t InheritsFrom(const char *cl) const
Return kTRUE if this class inherits from a class with name "classname".
Definition: TClass.cxx:4602
write collection with single key
Definition: TObject.h:78
virtual TDirectory * GetDirectory(const char *apath, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory named "apath".
virtual const char * GetPath() const
Returns the full path of the directory.
Definition: TDirectory.cxx:911
TString fObjectNames
List of object names to be either merged exclusively or skipped.
Definition: TFileMerger.h:45
if object destructor must call RecursiveRemove()
Definition: TObject.h:59
#define Printf
Definition: TGeoToOCC.h:18
void SetMaxOpenedFiles(Int_t newmax)
Set a limit to the number of files that TFileMerger will open simultaneously.
#define R__LOCKGUARD2(mutex)
const Bool_t kFALSE
Definition: RtypesCore.h:92
virtual void SaveSelf(Bool_t=kFALSE)
Definition: TDirectory.h:179
TString & Remove(Ssiz_t pos)
Definition: TString.h:621
virtual Bool_t MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type=kRegular|kAll)
Merge all objects in a directory.
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition: TString.cxx:2251
#define ClassImp(name)
Definition: Rtypes.h:336
Bool_t IsZombie() const
Definition: TObject.h:122
ROOT::ResetAfterMergeFunc_t GetResetAfterMerge() const
Describe directory structure in memory.
Definition: TDirectory.h:34
int type
Definition: TGX11.cxx:120
unsigned long ULong_t
Definition: RtypesCore.h:51
double func(double *x, double *p)
Definition: stressTF1.cxx:213
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:572
static const Int_t kCintFileNumber
Definition: TFileMerger.cxx:59
Bool_t fLocal
Makes local copies of merging files if True (default is kTRUE)
Definition: TFileMerger.h:43
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Definition: TClass.cxx:2885
virtual void Clear(Option_t *option="")
Remove all objects from the list.
Definition: TList.cxx:353
const char * AsString() const
Return UUID as string. Copy string immediately since it will be reused.
Definition: TUUID.cxx:537
Bool_t IsNull() const
Definition: TString.h:385
Mother of all ROOT objects.
Definition: TObject.h:37
Bool_t fFastMethod
True if using Fast merging algorithm (default)
Definition: TFileMerger.h:34
Only the objects without a MergeAfterReset member function.
Definition: TFileMerger.h:59
TClassRef is used to implement a permanent reference to a TClass object.
Definition: TClassRef.h:29
virtual const char * GetTitle() const
Returns title of object.
Definition: TObject.cxx:408
Bool_t IsTObject() const
Return kTRUE is the class inherits from TObject.
Definition: TClass.cxx:5575
virtual TObject * ReadObj()
To read a TObject* from the file.
Definition: TKey.cxx:730
Bool_t fNoTrees
True if Trees should not be merged (default is kFALSE)
Definition: TFileMerger.h:35
virtual Bool_t cd(const char *path=0)
Change current directory to "this" directory.
Definition: TDirectory.cxx:435
virtual Bool_t PartialMerge(Int_t type=kAll|kIncremental)
Merge the files.
virtual void Add(TObject *obj)
Definition: TList.h:77
Definition: file.py:1
virtual TDirectory * GetDirectory(const char *namecycle, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory using apath.
Definition: TDirectory.cxx:338
virtual ~TFileMerger()
Cleanup.
double result[121]
void ResetBit(UInt_t f)
Definition: TObject.h:158
virtual void PrintFiles(Option_t *options)
Print list of files being merged.
virtual void Fatal(const char *method, const char *msgfmt,...) const
Issue fatal error message.
Definition: TObject.cxx:901
virtual void Print(Option_t *option="") const
Default print for collections, calls Print(option, 1).
Only the objects with a MergeAfterReset member function.
Definition: TFileMerger.h:58
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:364
virtual Int_t GetSize() const
Definition: TCollection.h:89
TMethod * GetMethodWithPrototype(const char *method, const char *proto, Bool_t objectIsConst=kFALSE, ROOT::EFunctionMatchMode mode=ROOT::kConversionMatch)
Find the method with a given prototype.
Definition: TClass.cxx:4217
virtual void Reset()
Reset merger file list.
const Bool_t kTRUE
Definition: RtypesCore.h:91
Int_t GetCompressionLevel() const
Definition: TFile.h:368
Int_t fMaxOpenedFiles
Maximum number of files opened at the same time by the TFileMerger.
Definition: TFileMerger.h:42
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:859
TList * fFileList
A list the file (TFile*) which shall be merged.
Definition: TFileMerger.h:31
virtual void Close(Option_t *option="")
Close a file.
Definition: TFile.cxx:904
Bool_t fExplicitCompLevel
True if the user explicitly requested a compressio level change (default kFALSE)
Definition: TFileMerger.h:36
const char * Data() const
Definition: TString.h:347