49#include <sys/resource.h>
71 maxfiles = _getmaxstdio();
74 if (getrlimit(RLIMIT_NOFILE,&filelimit)==0) {
75 maxfiles = filelimit.rlim_cur;
83 }
else if (maxfiles > 5) {
95 fLocal(isLocal), fHistoOneGo(histoOneGo)
101 gROOT->GetListOfCleanups()->Add(
this);
111 gROOT->GetListOfCleanups()->Remove(
this);
145 TFile *newfile =
nullptr;
165 if (!
TFile::Cp(url, localcopy, cpProgress)) {
166 Error(
"AddFile",
"cannot get a local copy of file %s", url);
175 if (newfile && newfile->
IsZombie()) {
182 Error(
"AddFile",
"cannot open local copy %s of URL %s",
183 localcopy.
Data(), url);
185 Error(
"AddFile",
"cannot open file %s", url);
231 if (source == 0 || source->
IsZombie()) {
247 if (!source->
Cp(localcopy, cpProgress)) {
248 Error(
"AddFile",
"cannot get a local copy of file %s", source->
GetName());
253 if (newfile && newfile->
IsZombie()) {
263 Error(
"AddFile",
"cannot open local copy %s of URL %s",
266 Error(
"AddFile",
"cannot open file %s", source->
GetName());
271 if (own || newfile != source) {
281 if (newfile != source && own) {
293 return OutputFile(outputfile,(force?
"RECREATE":
"CREATE"),compressionLevel);
319 return OutputFile(std::unique_ptr<TFile>(outputFile));
330 if (!outputfile || outputfile->IsZombie()) {
331 Error(
"OutputFile",
"cannot open the MERGER output file %s", (outputfile) ? outputfile->GetName() :
"");
335 if (!outputfile->IsWritable()) {
336 Error(
"OutputFile",
"output file %s is not writable", outputfile->GetName());
421 if (
target->WriteObjectAny((
void *)obj, cl,
name, canBeMerged ?
"OverWrite" :
"") <= 0) {
467 if (strcmp(keyclassname,
"TProcessID") == 0 && key) {
476 oldkeyname = keyname;
482 Info(
"MergeRecursive",
"cannot indentify object type (%s), name: %s title: %s",
483 keyclassname, keyname, keytitle);
493 oldkeyname = keyname;
498 oldkeyname = keyname;
501 oldkeyname = keyname;
510 oldkeyname = keyname;
517 oldkeyname = keyname;
541 Info(
"MergeRecursive",
"could not read object for key {%s, %s}",
550 Error(
"MergeRecursive",
"TKey and object retrieve disagree on type (%s vs %s). Continuing with %s.",
556 std::map<std::tuple<std::string, std::string, std::string>,
TDirectory*> dirtodelete;
558 auto mapkey = std::make_tuple(parent->
GetName(),
name, pathname.Data());
559 auto result = dirtodelete.find(mapkey);
560 if (
result != dirtodelete.end()) {
566 dirtodelete[mapkey] = dir;
594 if (onlyListed)
type &= ~kOnlyListed;
599 if (!status)
return kFALSE;
603 Warning(
"MergeRecursive",
"Merging RNTuples is experimental");
612 TIter nextFile(sourcelist);
613 while (
const auto &inFile = nextFile()) {
614 mergeData.
Add(inFile);
619 mergeData.
Clear(
"nodelete");
621 Error(
"MergeRecursive",
"Could NOT merge RNTuples!");
626 Error(
"MergeRecursive",
"Merging objects that don't inherit from TObject is unimplemented (key: %s of type %s in file %s)",
627 keyname, keyclassname, nextsource->
GetName());
632 if (alreadyseen)
return kTRUE;
640 if (nextsource == 0) {
643 func(obj, &inputs, &info);
660 Info(
"MergeRecursive",
"could not read object for key {%s, %s}; skipping file %s",
661 keyname, keytitle, nextsource->
GetName());
662 nextsource = (
TFile*)sourcelist->
After(nextsource);
680 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
681 keyname, nextsource->
GetName());
688 nextsource = (
TFile*)sourcelist->
After( nextsource );
689 }
while (nextsource);
693 func(obj, &inputs, &info);
704 listHargs.
Form(
"(TCollection*)0x%zx,(TFileMergeInfo*)0x%zx",
705 (
size_t)&listH, (
size_t)&info);
707 listHargs.
Form(
"((TCollection*)0x%zx)", (
size_t)&listH);
719 if (nextsource == 0) {
725 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
738 Info(
"MergeRecursive",
"could not read object for key {%s, %s}; skipping file %s",
739 keyname, keytitle, nextsource->
GetName());
740 nextsource = (
TFile*)sourcelist->
After(nextsource);
753 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
759 nextsource = (
TFile*)sourcelist->
After( nextsource );
781 oldkeyname = keyname;
786 auto dirobj =
dynamic_cast<TDirectory *
>(obj);
787 TString dirpath(dirobj->GetPath());
789 dirpath.
Remove(0, std::strlen(dirobj->GetFile()->GetPath()));
802 for (
const auto &[
_, ndir] : dirtodelete) {
808 }
else if (!canBeFound) {
811 TIter peeknextkey(nextkey);
812 status = WriteCycleInOrder(oldkeyname, nextkey, peeknextkey,
target) && status;
813 status = WriteOneAndDelete(oldkeyname, cl, obj,
kFALSE, ownobj,
target) && status;
815 status = WriteOneAndDelete(oldkeyname, cl, obj,
kTRUE, ownobj,
target) && status;
838 path.
Remove(0, std::strlen(
target->GetFile()->GetPath()));
848 allNames.
Add(arr->
At(iname));
865 current_sourcedir =
target;
870 while (current_file || current_sourcedir) {
873 if (current_sourcedir && (current_file == 0 || current_sourcedir !=
target)) {
879 while ( (obj = (
TKey*)nextobj())) {
881 info, oldkeyname, allNames, status, onlyListed, path,
882 current_sourcedir, current_file,
883 nullptr, obj, nextobj);
892 while ( (key = (
TKey*)nextkey())) {
894 info, oldkeyname, allNames, status, onlyListed, path,
895 current_sourcedir, current_file,
896 key,
nullptr, nextkey);
901 current_file = current_file ? (
TFile*)sourcelist->
After(current_file) : (
TFile*)sourcelist->
First();
905 current_sourcedir = 0;
942 Info(
"PartialMerge",
"will merge the results to the file %s\n"
943 "since you didn't specify a merge filename",
944 TUrl(outf).GetFile());
958 if (!file || (file && file->
IsZombie())) {
959 Error(
"PartialMerge",
"one-file case: problem attaching to file");
964 Error(
"PartialMerge",
"one-file case: could not copy '%s' to '%s'",
974 Warning(
"PartialMerge",
"problems removing temporary local file '%s'", u.
GetFile());
992 while ((file = (
TFile*) next())) {
999 p =
p(0,
p.Index(
':',0));
1014 Error(
"Merge",
"error during merge of your ROOT files");
1067 Error(
"OpenExcessFiles",
"cannot get a local copy of file %s", url->
GetName());
1077 Error(
"OpenExcessFiles",
"cannot open local copy %s of URL %s",
1080 Error(
"OpenExcessFiles",
"cannot open file %s", url->
GetName());
1100 Fatal(
"RecursiveRemove",
"Output file of the TFile Merger (targeting %s) has been deleted (likely due to a TTree larger than 100Gb)",
fOutputFilename.
Data());
1114 if (newmax < sysmax) {
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
TClassRef R__TH1_Class("TH1")
static Int_t R__GetSystemMaxOpenedFiles()
Return the maximum number of allowed opened files minus some wiggle room for CINT or at least of the ...
TClassRef R__RNTuple_Class("ROOT::RNTuple")
TClassRef R__TTree_Class("TTree")
static const Int_t kCpProgress
static const Int_t kCintFileNumber
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t target
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char mode
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
R__EXTERN TVirtualMutex * gROOTMutex
void Printf(const char *fmt,...)
Formats a string in a circular formatting buffer and prints the string.
R__EXTERN TSystem * gSystem
#define R__LOCKGUARD(mutex)
TClassRef is used to implement a permanent reference to a TClass object.
TClass instances represent classes, structs and namespaces in the ROOT type system.
TMethod * GetMethodWithPrototype(const char *method, const char *proto, Bool_t objectIsConst=kFALSE, ROOT::EFunctionMatchMode mode=ROOT::kConversionMatch)
Find the method with a given prototype.
void Destructor(void *obj, Bool_t dtorOnly=kFALSE)
Explicitly call destructor for object.
ROOT::ResetAfterMergeFunc_t GetResetAfterMerge() const
Return the wrapper around Merge.
Bool_t IsLoaded() const
Return true if the shared library of this class is currently in the a process's memory.
Bool_t IsTObject() const
Return kTRUE is the class inherits from TObject.
Bool_t InheritsFrom(const char *cl) const override
Return kTRUE if this class inherits from a class with name "classname".
ROOT::MergeFunc_t GetMerge() const
Return the wrapper around Merge.
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Collection abstract base class.
virtual Int_t GetEntries() const
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
void Print(Option_t *option="") const override
Default print for collections, calls Print(option, 1).
virtual Int_t GetSize() const
Return the capacity of the collection, i.e.
TDirectory * GetDirectory(const char *apath, Bool_t printError=false, const char *funcname="GetDirectory") override
Find a directory named "apath".
TDirectory::TContext keeps track and restore the current directory.
Describe directory structure in memory.
virtual TList * GetList() const
virtual TDirectory * GetDirectory(const char *namecycle, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory using apath.
virtual const char * GetPath() const
Returns the full path of the directory.
virtual Bool_t cd()
Change current directory to "this" directory.
virtual TList * GetListOfKeys() const
TIOFeatures * fIOFeatures
This class provides file copy and merging services.
TString fObjectNames
List of object names to be either merged exclusively or skipped.
virtual Bool_t OutputFile(const char *url, Bool_t force)
Open merger output file.
TList fMergeList
list of TObjString containing the name of the files need to be merged
virtual Bool_t AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
virtual void PrintFiles(Option_t *options)
Print list of files being merged.
Bool_t fHistoOneGo
Merger histos in one go (default is kTRUE)
virtual Bool_t MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type=kRegular|kAll)
Merge all objects in a directory.
void RecursiveRemove(TObject *obj) override
Intercept the case where the output TFile is deleted!
TList fFileList
A list the file (TFile*) which shall be merged.
virtual Bool_t Merge(Bool_t=kTRUE)
Merge the files.
virtual Bool_t MergeOne(TDirectory *target, TList *sourcelist, Int_t type, TFileMergeInfo &info, TString &oldkeyname, THashList &allNames, Bool_t &status, Bool_t &onlyListed, const TString &path, TDirectory *current_sourcedir, TFile *current_file, TKey *key, TObject *obj, TIter &nextkey)
TString fOutputFilename
The name of the outputfile for merging.
TString fMsgPrefix
Prefix to be used when printing informational message (default TFileMerger)
TIOFeatures * fIOFeatures
IO features to use in the output file.
TFileMerger(const TFileMerger &)=delete
void SetMsgPrefix(const char *prefix)
Set the prefix to be used when printing informational message.
Bool_t fNoTrees
True if Trees should not be merged (default is kFALSE)
bool fOutFileWasExplicitlyClosed
! the user has called CloseOutputFile(), so we shouldn't error out in RecursiveRemove
@ kAll
Merge all type of objects (default)
@ kIncremental
Merge the input file with the content of the output file (if already existing).
@ kKeepCompression
Keep compression level unchanged for each input files.
@ kSkipListed
Skip objects specified in fObjectNames list.
@ kNonResetable
Only the objects without a MergeAfterReset member function.
@ kResetable
Only the objects with a MergeAfterReset member function.
@ kOnlyListed
Only the objects specified in fObjectNames list.
@ kRegular
Normal merge, overwriting the output file.
@ kDelayWrite
Delay the TFile write (to reduce the number of write when reusing the file)
Bool_t fExplicitCompLevel
True if the user explicitly requested a compression level change (default kFALSE)
Bool_t fCompressionChange
True if the output and input have different compression level (default kFALSE)
Int_t fPrintLevel
How much information to print out at run time.
void SetMaxOpenedFiles(Int_t newmax)
Set a limit to the number of files that TFileMerger will open simultaneously.
TString fMergeOptions
Options (in string format) to be passed down to the Merge functions.
void CloseOutputFile()
Closes output file.
~TFileMerger() override
Cleanup.
Bool_t OpenExcessFiles()
Open up to fMaxOpenedFiles of the excess files.
TList fExcessFiles
! List of TObjString containing the name of the files not yet added to fFileList due to user or syste...
TFile * fOutputFile
The outputfile for merging.
virtual Bool_t PartialMerge(Int_t type=kAll|kIncremental)
Merge the files.
Bool_t fLocal
Makes local copies of merging files if True (default is kTRUE)
virtual void Reset()
Reset merger file list.
Int_t fMaxOpenedFiles
Maximum number of files opened at the same time by the TFileMerger.
virtual Bool_t AddAdoptFile(TFile *source, Bool_t cpProgress=kTRUE)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
Bool_t fFastMethod
True if using Fast merging algorithm (default)
A ROOT file is an on-disk file, usually with extension .root, that stores objects in a file-system-li...
Int_t GetCompressionSettings() const
Int_t GetCompressionLevel() const
virtual Bool_t Cp(const char *dst, Bool_t progressbar=kTRUE, UInt_t buffersize=1000000)
Allows to copy this file to the dst URL.
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Int_t Write(const char *name=nullptr, Int_t opt=0, Int_t bufsiz=0) override
Write memory objects to this file.
void Close(Option_t *option="") override
Close a file.
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
TObject * FindObject(const char *name) const override
Find object using its name.
Book space in a file, create I/O buffers, to fill them, (un)compress them.
const char * GetTitle() const override
Returns title (title can contain 32x32 xpm thumbnail/icon).
virtual const char * GetClassName() const
virtual TObject * ReadObj()
To read a TObject* from the file.
TObject * After(const TObject *obj) const override
Returns the object after object obj.
void Clear(Option_t *option="") override
Remove all objects from the list.
TObject * FindObject(const char *name) const override
Find an object in this list using its name.
void Add(TObject *obj) override
TObject * Remove(TObject *obj) override
Remove object from the list.
TObject * First() const override
Return the first object in the list. Returns 0 when list is empty.
void Delete(Option_t *option="") override
Remove all objects from the list AND delete all heap based objects.
const char * GetName() const override
Returns name of object.
Int_t GetEntriesFast() const
TObject * At(Int_t idx) const override
Collectable string class.
const char * GetName() const override
Returns name of object.
Mother of all ROOT objects.
virtual void Clear(Option_t *="")
@ kOverwrite
overwrite existing object with same name
@ kSingleKey
write collection with single key
virtual const char * GetName() const
Returns name of object.
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
virtual TObject * Clone(const char *newname="") const
Make a clone of an object using the Streamer facility.
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
virtual void Execute(const char *method, const char *params, Int_t *error=nullptr)
Execute method on this object with the given parameter string, e.g.
R__ALWAYS_INLINE Bool_t IsZombie() const
virtual Int_t Write(const char *name=nullptr, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
virtual Bool_t InheritsFrom(const char *classname) const
Returns kTRUE if object inherits from class "classname".
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
virtual void Fatal(const char *method, const char *msgfmt,...) const
Issue fatal error message.
virtual const char * GetTitle() const
Returns title of object.
virtual TClass * IsA() const
@ kCanDelete
if object in a list can be deleted
@ kMustCleanup
if object destructor must call RecursiveRemove()
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
void Clear()
Clear string without changing its capacity.
const char * Data() const
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
TString & Remove(Ssiz_t pos)
TString & Append(const char *cs)
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
virtual int Unlink(const char *name)
Unlink, i.e.
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
const char * AsString() const
Return UUID as string. Copy string immediately since it will be reused.
This class represents a WWW compatible URL.
const char * GetFile() const
void(off) SmallVectorTemplateBase< T
Long64_t(* MergeFunc_t)(void *, TCollection *, TFileMergeInfo *)
Short_t Min(Short_t a, Short_t b)
Returns the smallest of a and b.