Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
TFileCollection.cxx
Go to the documentation of this file.
1// @(#)root/base:$Id$
2// Author: Gerhard Erich Bruckner, Jan Fiete Grosse-Oetringhaus 04/06/07
3
4/*************************************************************************
5 * Copyright (C) 1995-2007, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12/** \class TFileCollection
13\ingroup Base
14
15Class that contains a list of TFileInfo's and accumulated meta
16data information about its entries. This class is used to describe
17file sets as stored by Grid file catalogs, by any
18collection of TFile names.
19*/
20
21#include "TFileCollection.h"
22#include "THashList.h"
23#include "TFileInfo.h"
24#include "TIterator.h"
25#include "TMap.h"
26#include "TObjString.h"
27#include "TUri.h"
28#include "TUrl.h"
29#include "TUUID.h"
30#include "TMD5.h"
31#include "TSystem.h"
32#include "TRegexp.h"
33#include "TPRegexp.h"
34#include "TError.h"
35
36#include <iostream>
37#include <fstream>
38
39
40
41////////////////////////////////////////////////////////////////////////////////
42/// TFileCollection constructor. Specify a name and title describing
43/// the list. If textfile is specified the file is opened and a
44/// TFileCollection is created containing the files in the textfile.
45
46TFileCollection::TFileCollection(const char *name, const char *title,
47 const char *textfile, Int_t nfiles, Int_t firstfile)
48 : TNamed(name, title), fList(nullptr), fMetaDataList(nullptr), fDefaultTree(),
49 fTotalSize(0), fNFiles(0), fNStagedFiles(0), fNCorruptFiles(0)
50{
51 fList = new THashList();
52 fList->SetOwner();
53
54 fMetaDataList = new TList;
56
58}
59
60////////////////////////////////////////////////////////////////////////////////
61/// Cleanup.
62
68
69////////////////////////////////////////////////////////////////////////////////
70/// Add TFileInfo to the collection.
71
73{
74 if (fList && info) {
75 if (!fList->FindObject(info->GetName())) {
76 fList->Add(info);
77 if (info->GetIndex() < 0) info->SetIndex(fList->GetSize());
78 Update();
79 return 1;
80 } else {
81 Warning("Add", "file: '%s' already in the list - ignoring",
82 info->GetCurrentUrl()->GetUrl());
83 }
84 }
85 return 0;
86}
87
88////////////////////////////////////////////////////////////////////////////////
89/// Add content of the TFileCollection to this collection.
90
92{
93 if (fList && coll && coll->GetList()) {
94 TIter nxfi(coll->GetList());
95 TFileInfo *fi = nullptr;
96 while ((fi = (TFileInfo *) nxfi())) {
97 TFileInfo *info = new TFileInfo(*fi);
98 fList->Add(info);
99 if (fi->GetIndex() < 0) info->SetIndex(fList->GetSize());
100 }
101 Update();
102 return 1;
103 } else {
104 return 0;
105 }
106}
107
108////////////////////////////////////////////////////////////////////////////////
109/// Add file names contained in the specified text file.
110/// The file should contain one url per line; empty lines or lines starting with '#'
111/// (commented lines) are ignored.
112/// If nfiles > 0 only nfiles files are added, starting from file 'firstfile' (>= 1).
113/// The method returns the number of added files.
114
116{
117 if (!fList)
118 return 0;
119
120 Int_t nf = 0;
122 if (!fn.IsNull() && !gSystem->ExpandPathName(fn)) {
123 std::ifstream f;
124 f.open(fn);
125 if (f.is_open()) {
126 Bool_t all = (nfiles <= 0) ? kTRUE : kFALSE;
127 Int_t ff = (!all && (firstfile < 1)) ? 1 : firstfile;
128 Int_t nn = 0;
129 while (f.good() && (all || nf < nfiles)) {
131 line.ReadToDelim(f);
132 // Skip commented or empty lines
133 if (!line.IsWhitespace() && !line.BeginsWith("#")) {
134 nn++;
135 if (all || nn >= ff) {
137 fList->Add(info);
138 if (info->GetIndex() < 0) info->SetIndex(fList->GetSize());
139 nf++;
140 }
141 }
142 }
143 f.close();
144 Update();
145 } else
146 Error("AddFromFile", "unable to open file %s (%s)", textfile, fn.Data());
147 }
148 return nf;
149}
150
151////////////////////////////////////////////////////////////////////////////////
152/// Add all files matching the specified pattern to the collection.
153/// 'dir' can include wildcards after the last slash, which causes all
154/// matching files in that directory to be added.
155/// If dir is the full path of a file, only one element is added.
156/// Return value is the number of added files.
157
159{
160 Int_t nf = 0;
161
162 if (!fList)
163 return nf;
164
165 if (!dir || !*dir) {
166 Error("Add", "input dir undefined");
167 return nf;
168 }
169
171 FileStat_t tmp;
173 // if the 'dir' or its base dir exist
174 if (gSystem->GetPathInfo(dir, st) == 0 ||
175 gSystem->GetPathInfo(baseDir, tmp) == 0) {
176 // If 'dir' points to a single file, add to the list and exit
177 if (R_ISREG(st.fMode)) {
178 // regular, single file
179 TFileInfo *info = new TFileInfo(dir);
180 info->SetBit(TFileInfo::kStaged);
181 Add(info);
182 nf++;
183 Update();
184 return nf;
185 } else {
186 void *dataSetDir = gSystem->OpenDirectory(gSystem->GetDirName(dir).Data());
187 if (!dataSetDir) {
188 // directory cannot be opened
189 Error("Add", "directory %s cannot be opened",
190 gSystem->GetDirName(dir).Data());
191 } else {
192 const char *ent;
193 TString filesExp(TString("^") + gSystem->BaseName(dir) + "$");
194 filesExp.ReplaceAll("*",".*");
196 while ((ent = gSystem->GetDirEntry(dataSetDir))) {
198 if (entryString.Index(rg) != kNPOS) {
199 // matching dir entry
201 fn += "/";
202 fn += ent;
204 if (R_ISREG(st.fMode)) {
205 // regular file
206 TFileInfo *info = new TFileInfo(fn);
207 info->SetBit(TFileInfo::kStaged);
208 Add(info);
209 nf++;
210 }
211 }
212 }
213 // close the directory
215 Update();
216 }
217 }
218 }
219 return nf;
220}
221
222////////////////////////////////////////////////////////////////////////////////
223/// Remove duplicates based on the UUID, typically after a verification.
224/// Return the number of entries removed.
225
227{
228 THashList *hl = new THashList;
229 hl->SetOwner();
230
231 Int_t n0 = fList->GetSize();
233 TFileInfo *fi = nullptr;
234 while ((fi = (TFileInfo *)nxfi())) {
235 if (!(hl->FindObject(fi->GetUUID()->AsString()))) {
236 // We hash on the UUID
237 fList->Remove(fi);
238 fi->SetName(fi->GetUUID()->AsString());
239 hl->Add(fi);
240 }
241 }
242 delete fList;
243 fList = hl;
244 // How many removed?
245 Int_t nr = n0 - fList->GetSize();
246 if (nr > 0)
247 Info("RemoveDuplicates", "%d duplicates found and removed", nr);
248 // Done
249 return nr;
250}
251
252////////////////////////////////////////////////////////////////////////////////
253/// Creates a subset of the files that have the kStaged & !kCorrupted bit set.
254
256{
257 if (!fList)
258 return nullptr;
259
261
262 TIter iter(fList);
263 TFileInfo *fileInfo = nullptr;
264 while ((fileInfo = dynamic_cast<TFileInfo*>(iter.Next()))) {
265 if (fileInfo->TestBit(TFileInfo::kStaged) && !fileInfo->TestBit(TFileInfo::kCorrupted))
266 subset->Add(fileInfo);
267 }
268
269 subset->Update();
270
271 return subset;
272}
273
274////////////////////////////////////////////////////////////////////////////////
275/// Merge all TFileCollection objects in li into this TFileCollection object.
276/// Updates counters at the end.
277/// Returns the number of merged collections or -1 in case of error.
278
280{
281
282 if (!li) return 0;
283 if (li->IsEmpty()) return 0;
284
286 TIter next(li);
287 while (TObject *o = next()) {
288 TFileCollection* coll = dynamic_cast<TFileCollection*> (o);
289 if (!coll) {
290 Error("Add", "attempt to add object of class: %s to a %s",
291 o->ClassName(),this->ClassName());
292 return -1;
293 }
294 Add(coll);
295 nentries++;
296 }
297 Update();
298
299 return nentries;
300}
301
302////////////////////////////////////////////////////////////////////////////////
303/// Update accumulated information about the elements of the collection
304/// (e.g. fTotalSize). If 'avgsize' > 0, use an average file size of 'avgsize'
305/// bytes when the size info is not available.
306/// Also updates the meta data information by summarizing
307/// the meta data of the contained objects.
308/// Return -1 in case of any failure, 0 if the total size is exact, 1 if
309/// incomplete, 2 if complete but (at least partially) estimated.
310
312{
313 if (!fList)
314 return -1;
315
316 Int_t rc = 0;
317
318 fTotalSize = 0;
319 fNStagedFiles = 0;
320 fNCorruptFiles = 0;
321
322 // Clear internal meta information which is going to be rebuilt in this
323 // function
325 TFileInfoMeta *m = nullptr;
326 while ((m = (TFileInfoMeta *)nxm())) {
329 delete m;
330 }
331 }
332
334
335 TIter iter(fList);
336 TFileInfo *fileInfo = nullptr;
337 while ((fileInfo = dynamic_cast<TFileInfo*> (iter.Next()))) {
338
339 if (fileInfo->GetSize() > 0) {
340 fTotalSize += fileInfo->GetSize();
341 } else {
342 rc = 1;
343 if (avgsize > 0) {
344 rc = 2;
346 }
347 }
348
349 if (fileInfo->TestBit(TFileInfo::kStaged) && !fileInfo->TestBit(TFileInfo::kCorrupted)) {
351
352 if (fileInfo->GetMetaDataList()) {
353 TIter metaDataIter(fileInfo->GetMetaDataList());
354 // other than TFileInfoMeta is also allowed in list
355 TObject *obj = nullptr;
356 while ((obj = metaDataIter.Next())) {
357 TFileInfoMeta *metaData = dynamic_cast<TFileInfoMeta*>(obj);
358 if (!metaData)
359 continue;
360 if (!metaData->IsTree())
361 continue;
362
363 // find corresponding entry in TFileCollection's meta data
366 if (!metaDataSum) {
367 // create explicitly, there are some values that do not make sense for the sum
368 metaDataSum = new TFileInfoMeta(metaData->GetName(), metaData->GetTitle());
370 newObj = kTRUE;
371 }
372
373 // sum the values
374 if (newObj)
375 metaDataSum->SetEntries(metaData->GetEntries());
376 else
377 metaDataSum->SetEntries(metaDataSum->GetEntries() + metaData->GetEntries());
378 }
379 }
380 }
381 if (fileInfo->TestBit(TFileInfo::kCorrupted))
383 }
384
385 // Done
386 return rc;
387}
388
389////////////////////////////////////////////////////////////////////////////////
390/// Prints the contents of the TFileCollection.
391/// If option contains:
392///
393/// - 'M' print global meta information
394/// - 'F' print all the files in the collection in compact form
395/// (current url, default tree name|class|entries, md5)
396/// - 'L' together with 'F', print all the files in the collection
397/// in long form (uuid, md5, all URLs, all meta objects; on
398/// many lines)
399/// - "filter:[SsCc]" invokes PrintDetailed() which prints out dataset
400/// content in a formatted fashion by filtering on files
401/// which are (S)taged or not (s), (C)orrupted or not (c)
402
404{
405 TString opt(option);
406 TPMERegexp re("(^|;| )filter:([SsCc]+)( |;|$)", 4);
407 if (re.Match(option) == 4) {
408 TString showOnly = re[2];
410 return;
411 }
412
413 Printf("TFileCollection %s - %s contains: %lld files with a size of"
414 " %lld bytes, %.1f %% staged - default tree name: '%s'",
417
418 if (opt.Contains("M", TString::kIgnoreCase)) {
419 Printf("The files contain the following trees:");
420
422 TFileInfoMeta* metaData = nullptr;
423 while ((metaData = dynamic_cast<TFileInfoMeta*>(metaDataIter.Next()))) {
424 if (!metaData->IsTree())
425 continue;
426
427 Printf("Tree %s: %lld events", metaData->GetName(), metaData->GetEntries());
428 }
429 }
430
431 if (fList && opt.Contains("F", TString::kIgnoreCase)) {
432 Printf("The collection contains the following files:");
433 if (!opt.Contains("L") && !fDefaultTree.IsNull())
434 opt += TString::Format(" T:%s", fDefaultTree.Data());
435 fList->Print(opt);
436 }
437}
438
439////////////////////////////////////////////////////////////////////////////////
440/// Print detailed.
441
443{
444 Bool_t bS, bs, bC, bc;
445 bS = bs = bC = bc = kFALSE;
446
447 if (showOnly.Index('S') >= 0) bS = kTRUE;
448 if (showOnly.Index('s') >= 0) bs = kTRUE;
449 if (showOnly.Index('C') >= 0) bC = kTRUE;
450 if (showOnly.Index('c') >= 0) bc = kTRUE;
451
452 // If Ss (or Cc) omitted, show both Ss (or Cc)
453 if (!bc && !bC) bc = bC = kTRUE;
454 if (!bs && !bS) bs = bS = kTRUE;
455
456 TIter it(fList);
458 UInt_t countAll = 0;
459 UInt_t countMatch = 0;
460
461 Printf("\033[1m #. SC | Entries | Size | URL\033[m");
462
463 TString um;
464 Double_t sz;
465
466 while ((info = dynamic_cast<TFileInfo *>(it.Next()))) {
467
468 Bool_t s = info->TestBit(TFileInfo::kStaged);
470
471 TUrl *url;
472
473 countAll++;
474
475 if ( ((s && bS) || (!s && bs)) && ((c && bC) || (!c && bc)) ) {
476
477 TFileInfoMeta *meta = info->GetMetaData(); // gets the first one
478 Int_t entries = -1;
479
480 if (meta) entries = meta->GetEntries();
481
482 FormatSize(info->GetSize(), um, sz);
483
484 // First line: current URL with all information
485 info->ResetUrl();
486 TUrl *curUrl = info->GetCurrentUrl();
487 const char *curUrlStr = curUrl ? curUrl->GetUrl() : "n.a.";
488 Printf("\033[1m%4u.\033[m %c%c | %-7s | %6.1lf %s | %s",
489 ++countMatch,
490 (s ? 'S' : 's'), (c ? 'C' : 'c'),
491 ((entries > 0) ? Form("% 7d", entries) : "n.a."),
492 sz, um.Data(), curUrlStr);
493 info->NextUrl();
494
495 // Every other URL shown below current one
496 while ((url = info->NextUrl())) {
497 Printf(" | | | %s", url->GetUrl());
498 }
499 info->ResetUrl();
500
501 } // end match filters
502
503 } // end loop over entries
504
505 if (countAll) {
506
507 Printf(">> There are \033[1m%u\033[m file(s) in dataset: "
508 "\033[1m%u (%5.1f%%)\033[m matched your criteria (%s)",
511
513 Printf(">> Total size : \033[1m%.1f %s\033[m", sz, um.Data());
514 Printf(">> Staged (S) : \033[1m%5.1f %%\033[m", GetStagedPercentage());
515 Printf(">> Corrupted (C) : \033[1m%5.1f %%\033[m",
517
518 }
519 else {
520 Printf(">> No files in dataset");
521 }
522
523 const char *treeName = GetDefaultTreeName();
524 Printf(">> Default tree : \033[1m%s\033[m",
525 (treeName ? treeName : "(no default tree)"));
526
527}
528
529////////////////////////////////////////////////////////////////////////////////
530/// Format size.
531
533 Double_t &size) const
534{
535 static const char *ums[] = { "byt", "KiB", "MiB", "GiB", "TiB" };
536 Int_t maxDiv = sizeof(ums)/sizeof(const char *);
537 Int_t nDiv = 0;
538 Double_t b = bytes;
539
540 while ((b >= 1024.) && (nDiv+1 < maxDiv)) {
541 b /= 1024.;
542 nDiv++;
543 }
544
545 um = ums[nDiv];
546 size = b;
547}
548
549////////////////////////////////////////////////////////////////////////////////
550/// Calls TUrl::SetAnchor() for all URLs contained in all TFileInfos.
551
553{
554 if (!fList)
555 return;
556
557 TIter iter(fList);
558 TFileInfo *fileInfo = nullptr;
559 while ((fileInfo = dynamic_cast<TFileInfo*>(iter.Next()))) {
560 fileInfo->ResetUrl();
561 TUrl *url = nullptr;
562 while ((url = fileInfo->NextUrl()))
563 url->SetAnchor(anchor);
564 fileInfo->ResetUrl();
565 }
566}
567
568////////////////////////////////////////////////////////////////////////////////
569/// Set the bit for all TFileInfos
570
572{
573 if (!fList)
574 return;
575
576 TIter iter(fList);
577 TFileInfo *fileInfo = nullptr;
578 while ((fileInfo = dynamic_cast<TFileInfo*>(iter.Next())))
579 fileInfo->SetBit(f);
580}
581
582////////////////////////////////////////////////////////////////////////////////
583/// Reset the bit for all TFileInfos
584
586{
587 if (!fList)
588 return;
589
590 TIter iter(fList);
591 TFileInfo *fileInfo = nullptr;
592 while ((fileInfo = dynamic_cast<TFileInfo*>(iter.Next())))
593 fileInfo->ResetBit(f);
594}
595
596////////////////////////////////////////////////////////////////////////////////
597/// Returns the tree set with SetDefaultTreeName if set
598/// Returns the name of the first tree in the meta data list.
599/// Returns 0 in case no trees are found in the meta data list.
600
602{
603 if (fDefaultTree.Length() > 0)
604 return fDefaultTree;
605
607 TFileInfoMeta *metaData = nullptr;
608 while ((metaData = dynamic_cast<TFileInfoMeta*>(metaDataIter.Next()))) {
609 if (!metaData->IsTree())
610 continue;
611 return metaData->GetName();
612 }
613 return nullptr;
614}
615
616////////////////////////////////////////////////////////////////////////////////
617/// Returns the number of entries for the specified tree (retrieved from meta data).
618/// If tree is not specified, use the default tree name.
619/// Returns -1 in case the specified tree is not found.
620
622{
623 if (!tree || !*tree) {
624 tree = GetDefaultTreeName();
625 if (!tree)
626 return -1;
627 }
628
630 if (!metaData)
631 return -1;
632
633 return metaData->GetEntries();
634}
635
636////////////////////////////////////////////////////////////////////////////////
637/// Returns the meta data object with the specified meta name.
638/// Returns 0 in case specified meta data is not found.
639
641{
642 if (!meta || !*meta)
643 return nullptr;
644
645 return dynamic_cast<TFileInfoMeta*>(fMetaDataList->FindObject(meta));
646}
647
648////////////////////////////////////////////////////////////////////////////////
649/// Moves the indicated meta data in the first position, so that
650/// it becomes effectively the default.
651
660
661////////////////////////////////////////////////////////////////////////////////
662/// Removes the indicated meta data object in all TFileInfos and this object
663/// If no name is given all metadata is removed
664
666{
667 if (fList) {
668 TIter iter(fList);
669 TFileInfo *fileInfo = nullptr;
670 while ((fileInfo = dynamic_cast<TFileInfo*>(iter.Next())))
671 fileInfo->RemoveMetaData(meta);
672 }
673
674 if (meta) {
675 TObject* obj = fMetaDataList->FindObject("meta");
676 if (obj) {
677 fMetaDataList->Remove(obj);
678 delete obj;
679 }
680 } else
682}
683
684////////////////////////////////////////////////////////////////////////////////
685/// Sort the collection.
686
688{
689 if (!fList)
690 return;
691
692 // Make sure the relevant bit has the wanted value
693 if (useindex) {
695 } else {
697 }
698
699 fList->Sort();
700}
701
702////////////////////////////////////////////////////////////////////////////////
703/// Export the relevant info as a string; use 'name' as collection name,
704/// if defined, else use GetName().
705/// The output object must be destroyed by the caller
706
708{
710 if (GetDefaultTreeName()) {
712 if (popt == 1) {
714 if (meta)
715 treeInfo += TString::Format(", %lld entries", meta->GetEntries());
716 TFileInfoMeta *frac = GetMetaData("/FractionOfTotal");
717 if (frac)
718 treeInfo += TString::Format(", %3.1f %% of total", frac->GetEntries() / 10.);
719 } else {
720 treeInfo.Form(" %s ", GetDefaultTreeName());
721 if (treeInfo.Length() > 14) treeInfo.Replace(13, 1, '>');
722 treeInfo.Resize(14);
723 if (meta) {
724 if (meta->GetEntries() > 99999999) {
725 treeInfo += TString::Format("| %8lld ", meta->GetEntries());
726 } else {
727 treeInfo += TString::Format("| %8.4g ", (Double_t) meta->GetEntries());
728 }
729 }
730 }
731 } else {
732 treeInfo = " N/A";
733 }
734 if (popt == 0) treeInfo.Resize(25);
735
736 // Renormalize the size to kB, MB or GB
737 const char *unit[4] = {"kB", "MB", "GB", "TB"};
738 Int_t k = 0;
739 Long64_t refsz = 1024;
741 while (xsz > 1024 && k < 3) {
742 k++;
743 refsz *= 1024;
744 xsz = (Long64_t) (GetTotalSize() / refsz);
745 }
746
747 // The name
749 if (dsname.IsNull()) dsname = GetName();
750
751 // Create the output string
752 TObjString *outs = nullptr;
753 if (popt == 1) {
754 outs = new TObjString(Form("%s %lld files, %lld %s, staged %d %%, tree: %s", dsname.Data(),
755 GetNFiles(), xsz, unit[k],
757 } else {
758 outs = new TObjString(Form("%s| %7lld |%s| %5lld %s | %3d %%", dsname.Data(),
759 GetNFiles(), treeInfo.Data(), xsz, unit[k],
761 }
762 // Done
763 return outs;
764}
765
766////////////////////////////////////////////////////////////////////////////////
767/// Return the subset of files served by 'server'. The syntax for 'server' is
768/// the standard URI one, i.e. `[<scheme>://]<host>[:port]`
769
771{
772 TFileCollection *fc = nullptr;
773
774 // Server specification is mandatory
775 if (!server || strlen(server) <= 0) {
776 Info("GetFilesOnServer", "server undefined - do nothing");
777 return fc;
778 }
779
780 // Nothing to do for empty lists
781 if (!fList || fList->GetSize() <= 0) {
782 Info("GetFilesOnServer", "the list is empty - do nothing");
783 return fc;
784 }
785
786 // Define the server reference string
787 TUri uri(server);
788 TString srv, scheme("root"), port;
789 if (uri.GetScheme() != "") scheme = uri.GetScheme();
790 if (uri.GetPort() != "") port.Form(":%s", uri.GetPort().Data());
791 srv.Form("%s://%s%s", scheme.Data(), TUrl(server).GetHostFQDN(), port.Data());
792 if (gDebug > 0)
793 Info("GetFilesOnServer", "searching for files on server: '%s' (input: '%s')",
794 srv.Data(), server);
795
796 // Prepare the output
797 fc = new TFileCollection(GetName());
798 TString title;
799 if (GetTitle() && strlen(GetTitle()) > 0) {
800 title.Form("%s (subset on server %s)", GetTitle(), srv.Data());
801 } else {
802 title.Form("subset of '%s' on server %s", GetName(), srv.Data());
803 }
804 fc->SetTitle(title.Data());
805 // The default tree name
806 fc->SetDefaultTreeName(GetDefaultTreeName());
807
808 // We look for URL starting with srv
809 srv.Insert(0, "^");
810
811 // Go through the list
812 TIter nxf(fList);
813 TFileInfo *fi = nullptr;
814 while ((fi = (TFileInfo *)nxf())) {
815 TUrl *xu = nullptr;
816 if ((xu = fi->FindByUrl(srv.Data()))) {
817 // Create a new TFileInfo object
818 TFileInfo *nfi = new TFileInfo(xu->GetUrl(), fi->GetSize(),
819 fi->GetUUID() ? fi->GetUUID()->AsString() : nullptr,
820 fi->GetMD5() ? fi->GetMD5()->AsString() : nullptr);
821 if (fi->GetMetaDataList()) {
822 TIter nxm(fi->GetMetaDataList());
823 TFileInfoMeta *md = nullptr;
824 while ((md = (TFileInfoMeta *) nxm())) {
825 nfi->AddMetaData(new TFileInfoMeta(*md));
826 }
827 }
828 if (fi->TestBit(TFileInfo::kStaged)) nfi->SetBit(TFileInfo::kStaged);
829 if (fi->TestBit(TFileInfo::kCorrupted)) nfi->SetBit(TFileInfo::kCorrupted);
830 if (gDebug > 1)
831 Info("GetFilesOnServer", "adding: %s", xu->GetUrl());
832 fc->Add(nfi);
833 }
834 }
835
836 // If nothing found, delete the object
837 if (fc->GetList()->GetSize() <= 0) {
838 delete fc;
839 fc = nullptr;
840 Info("GetFilesOnServer", "dataset '%s' has no files on server: '%s' (searched for: '%s')",
841 GetName(), server, srv.Data());
842 }
843
844 // Fill up sums on the sub file collection
845 if (fc) {
846 fc->Update();
847 // Fraction of total in permille
848 Long64_t xf = (fc->GetTotalSize() * 1000) / GetTotalSize();
849 TFileInfoMeta *m = new TFileInfoMeta("FractionOfTotal", "External Info", xf);
851 fc->AddMetaData(m);
852 }
853
854 // Done
855 return fc;
856}
857
858////////////////////////////////////////////////////////////////////////////////
859/// Return a map of TFileCollections with the files on each data server,
860/// excluding servers in the comma-separated list 'exclude'.
861/// If curronly is kTRUE, only the URL flagged as current in the TFileInfo
862/// are considered.
863
865{
866 TMap *dsmap = nullptr;
867
868 // Nothing to do for empty lists
869 if (!fList || fList->GetSize() <= 0) {
870 Info("GetFilesPerServer", "the list is empty - do nothing");
871 return dsmap;
872 }
873
874 // List of servers to be ignored
875 THashList *excl = nullptr;
876 if (exclude && strlen(exclude) > 0) {
877 excl = new THashList;
878 excl->SetOwner();
879 TUri uri;
880 TString srvs(exclude), s, srv, scheme, port;
881 Int_t from = 0;
882 while (srvs.Tokenize(s, from, ",")) {
883 uri.SetUri(s.Data());
884 scheme = "root";
885 port = "";
886 if (uri.GetScheme() != "") scheme = uri.GetScheme();
887 if (uri.GetPort() != "") port.Form(":%s", uri.GetPort().Data());
888 srv.Form("%s://%s%s", scheme.Data(), TUrl(s.Data()).GetHostFQDN(), port.Data());
889 // Add
890 excl->Add(new TObjString(srv.Data()));
891 }
892 }
893
894 // Prepare the output
895 dsmap = new TMap();
896
897 // Go through the list
898 TIter nxf(fList);
899 TFileInfo *fi = nullptr;
900 TUri uri;
901 TString key;
902 TFileCollection *fc = nullptr;
903 while ((fi = (TFileInfo *)nxf())) {
904 // Save current URL
905 TUrl *curl = fi->GetCurrentUrl();
906 // Loop over URLs
907 if (!curronly) fi->ResetUrl();
908 TUrl *xurl = nullptr;
909 while ((xurl = (curronly) ? curl : fi->NextUrl())) {
910 // Find the key for this server
911 key.Form("%s://%s", xurl->GetProtocol(), xurl->GetHostFQDN());
912 // Check if this has to be ignored
913 if (excl && excl->FindObject(key.Data())) {
914 if (curronly) break;
915 continue;
916 } else if (excl && xurl->GetPort() > 0) {
917 // Complete the key, if needed, and recheck
918 key += TString::Format(":%d", xurl->GetPort());
919 if (excl->FindObject(key.Data())) {
920 if (curronly) break;
921 continue;
922 }
923 }
924 // Get the map entry for this key
925 TPair *ent = nullptr;
926 if (!(ent = (TPair *) dsmap->FindObject(key.Data()))) {
927 // Create the TFileCollection
928 fc = new TFileCollection(GetName());
929 TString title;
930 if (GetTitle() && strlen(GetTitle()) > 0) {
931 title.Form("%s (subset on server %s)", GetTitle(), key.Data());
932 } else {
933 title.Form("subset of '%s' on server %s", GetName(), key.Data());
934 }
935 fc->SetTitle(title.Data());
936 // The default tree name
937 fc->SetDefaultTreeName(GetDefaultTreeName());
938 // Add it to the map
939 dsmap->Add(new TObjString(key.Data()), fc);
940 // Notify
941 if (gDebug > 0)
942 Info("GetFilesPerServer", "found server: '%s' (fc: %p)", key.Data(), fc);
943 } else {
944 // Attach to the TFileCollection
945 fc = (TFileCollection *) ent->Value();
946 }
947 // Create a new TFileInfo object
948 TFileInfo *nfi = new TFileInfo(xurl->GetUrl(kTRUE), fi->GetSize(),
949 fi->GetUUID() ? fi->GetUUID()->AsString() : nullptr,
950 fi->GetMD5() ? fi->GetMD5()->AsString() : nullptr);
951 if (fi->GetMetaDataList()) {
952 TIter nxm(fi->GetMetaDataList());
953 TFileInfoMeta *md = nullptr;
954 while ((md = (TFileInfoMeta *) nxm())) {
955 nfi->AddMetaData(new TFileInfoMeta(*md));
956 }
957 }
958 if (fi->TestBit(TFileInfo::kStaged)) nfi->SetBit(TFileInfo::kStaged);
959 if (fi->TestBit(TFileInfo::kCorrupted)) nfi->SetBit(TFileInfo::kCorrupted);
960 fc->Add(nfi);
961 // In current_only mode we are done
962 if (curronly) break;
963 }
964 // Restore current URL
965 fi->SetCurrentUrl(curl);
966 }
967
968 // Fill up sums on the sub file collections
969 TIter nxk(dsmap);
970 TObject *k = nullptr;
971 while ((k = nxk()) && (fc = (TFileCollection *) dsmap->GetValue(k))) {
972 fc->Update();
973 // Fraction of total in permille
974 Long64_t xf = (fc->GetTotalSize() * 1000) / GetTotalSize();
975 TFileInfoMeta *m = new TFileInfoMeta("FractionOfTotal", "External Info", xf);
977 fc->AddMetaData(m);
978 }
979
980 // Cleanup
981 if (excl) delete excl;
982
983 // Done
984 return dsmap;
985}
986
987////////////////////////////////////////////////////////////////////////////////
988/// Add's a meta data object to the file collection object. The object will be
989/// adopted by the TFileCollection and should not be deleted by the user.
990/// Typically objects of class TFileInfoMeta or derivatives should be added,
991/// but any class is accepted.
992///
993/// NB : a call to TFileCollection::Update will remove these objects unless the
994/// bit TFileInfoMeta::kExternal is set.
995/// Returns kTRUE if successful, kFALSE otherwise.
996
998{
999 if (meta) {
1000 if (!fMetaDataList) {
1001 fMetaDataList = new TList;
1003 }
1005 return kTRUE;
1006 }
1007 return kFALSE;
1008}
#define b(i)
Definition RSha256.hxx:100
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
float Float_t
Float 4 bytes (float)
Definition RtypesCore.h:71
constexpr Bool_t kFALSE
Definition RtypesCore.h:108
constexpr Ssiz_t kNPOS
The equivalent of std::string::npos for the ROOT class TString.
Definition RtypesCore.h:131
long long Long64_t
Portable signed long integer 8 bytes.
Definition RtypesCore.h:83
constexpr Bool_t kTRUE
Definition RtypesCore.h:107
const char Option_t
Option string (const char)
Definition RtypesCore.h:80
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
Option_t Option_t option
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
char name[80]
Definition TGX11.cxx:110
int nentries
Int_t gDebug
Global variable setting the debug level. Set to 0 to disable, increase it in steps of 1 to increase t...
Definition TROOT.cxx:627
char * Form(const char *fmt,...)
Formats a string in a circular formatting buffer.
Definition TString.cxx:2495
void Printf(const char *fmt,...)
Formats a string in a circular formatting buffer and prints the string.
Definition TString.cxx:2509
Bool_t R_ISREG(Int_t mode)
Definition TSystem.h:126
R__EXTERN TSystem * gSystem
Definition TSystem.h:572
Collection abstract base class.
Definition TCollection.h:65
virtual Int_t GetEntries() const
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
void Print(Option_t *option="") const override
Default print for collections, calls Print(option, 1).
virtual Int_t GetSize() const
Return the capacity of the collection, i.e.
Class that contains a list of TFileInfo's and accumulated meta data information about its entries.
Bool_t AddMetaData(TObject *meta)
Add's a meta data object to the file collection object.
TFileCollection * GetStagedSubset()
Creates a subset of the files that have the kStaged & !kCorrupted bit set.
TMap * GetFilesPerServer(const char *exclude=nullptr, Bool_t curronly=kFALSE)
Return a map of TFileCollections with the files on each data server, excluding servers in the comma-s...
TObjString * ExportInfo(const char *name=nullptr, Int_t popt=0)
Export the relevant info as a string; use 'name' as collection name, if defined, else use GetName().
Int_t RemoveDuplicates()
Remove duplicates based on the UUID, typically after a verification.
Int_t Update(Long64_t avgsize=-1)
Update accumulated information about the elements of the collection (e.g.
TFileCollection(const TFileCollection &)=delete
Long64_t GetTotalEntries(const char *tree) const
Returns the number of entries for the specified tree (retrieved from meta data).
void SetBitAll(UInt_t f)
Set the bit for all TFileInfos.
void Sort(Bool_t useindex=kFALSE)
Sort the collection.
void SetAnchor(const char *anchor)
Calls TUrl::SetAnchor() for all URLs contained in all TFileInfos.
void ResetBitAll(UInt_t f)
Reset the bit for all TFileInfos.
Float_t GetStagedPercentage() const
void FormatSize(Long64_t bytes, TString &um, Double_t &size) const
Format size.
void SetDefaultMetaData(const char *meta)
Moves the indicated meta data in the first position, so that it becomes effectively the default.
virtual ~TFileCollection()
Cleanup.
Long64_t GetNFiles() const
Long64_t GetTotalSize() const
void RemoveMetaData(const char *meta=nullptr)
Removes the indicated meta data object in all TFileInfos and this object If no name is given all meta...
Long64_t Merge(TCollection *list)
Merge all TFileCollection objects in li into this TFileCollection object.
Int_t AddFromFile(const char *file, Int_t nfiles=-1, Int_t firstfile=1)
Add file names contained in the specified text file.
void Print(Option_t *option="") const override
Prints the contents of the TFileCollection.
Long64_t fNCorruptFiles
const char * GetDefaultTreeName() const
Returns the tree set with SetDefaultTreeName if set Returns the name of the first tree in the meta da...
Float_t GetCorruptedPercentage() const
THashList * fList
Int_t Add(TFileInfo *info)
Add TFileInfo to the collection.
void PrintDetailed(TString &showOnly) const
Print detailed.
TFileCollection * GetFilesOnServer(const char *server)
Return the subset of files served by 'server'.
TFileInfoMeta * GetMetaData(const char *meta=nullptr) const
Returns the meta data object with the specified meta name.
Long64_t GetEntries() const
Definition TFileInfo.h:140
Class describing a generic file including meta information.
Definition TFileInfo.h:39
@ kSortWithIndex
Definition TFileInfo.h:60
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition THashList.h:34
TObject * Remove(TObject *obj) override
Remove object from the list.
TObject * FindObject(const char *name) const override
Find object using its name.
TObject * Next()
A doubly linked list.
Definition TList.h:38
void Clear(Option_t *option="") override
Remove all objects from the list.
Definition TList.cxx:399
TObject * FindObject(const char *name) const override
Find an object in this list using its name.
Definition TList.cxx:575
void Add(TObject *obj) override
Definition TList.h:81
TObject * Remove(TObject *obj) override
Remove object from the list.
Definition TList.cxx:819
void AddFirst(TObject *obj) override
Add object at the beginning of the list.
Definition TList.cxx:97
virtual void Sort(Bool_t order=kSortAscending)
Sort linked list.
Definition TList.cxx:934
TMap implements an associative array of (key,value) pairs using a THashTable for efficient retrieval ...
Definition TMap.h:40
The TNamed class is the base class for all named ROOT classes.
Definition TNamed.h:29
const char * GetName() const override
Returns name of object.
Definition TNamed.h:49
const char * GetTitle() const override
Returns title of object.
Definition TNamed.h:50
Collectable string class.
Definition TObjString.h:28
Mother of all ROOT objects.
Definition TObject.h:41
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition TObject.h:202
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition TObject.cxx:1057
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition TObject.cxx:864
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition TObject.cxx:1071
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition TObject.cxx:1045
Wrapper for PCRE library (Perl Compatible Regular Expressions).
Definition TPRegexp.h:97
Class used by TMap to store (key,value) pairs.
Definition TMap.h:102
Regular expression class.
Definition TRegexp.h:31
Basic string class.
Definition TString.h:138
Ssiz_t Length() const
Definition TString.h:425
const char * Data() const
Definition TString.h:384
@ kIgnoreCase
Definition TString.h:285
Bool_t IsNull() const
Definition TString.h:422
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString.
Definition TString.cxx:2384
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition TString.cxx:2362
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition TString.h:640
virtual Bool_t ExpandPathName(TString &path)
Expand a pathname getting rid of special shell characters like ~.
Definition TSystem.cxx:1285
virtual void FreeDirectory(void *dirp)
Free a directory.
Definition TSystem.cxx:855
virtual void * OpenDirectory(const char *name)
Open a directory.
Definition TSystem.cxx:846
int GetPathInfo(const char *path, Long_t *id, Long_t *size, Long_t *flags, Long_t *modtime)
Get info about a file: id, size, flags, modification time.
Definition TSystem.cxx:1409
virtual const char * GetDirEntry(void *dirp)
Get a directory entry. Returns 0 if no more entries.
Definition TSystem.cxx:863
virtual const char * BaseName(const char *pathname)
Base name of a file name. Base name of /user/root is root.
Definition TSystem.cxx:944
virtual TString GetDirName(const char *pathname)
Return the directory name in pathname.
Definition TSystem.cxx:1042
This class represents a RFC 3986 compatible URI.
Definition TUri.h:35
const TString GetPort() const
Definition TUri.h:86
Bool_t SetUri(const TString &uri)
Parse URI and set the member variables accordingly, returns kTRUE if URI validates,...
Definition TUri.cxx:600
const TString GetScheme() const
Definition TUri.h:80
This class represents a WWW compatible URL.
Definition TUrl.h:33
const char * GetHostFQDN() const
Return fully qualified domain name of url host.
Definition TUrl.cxx:476
TLine * line
TMarker m
Definition textangle.C:8