Logo ROOT  
Reference Guide
TAlienCollection.cxx
Go to the documentation of this file.
1// @(#)root/alien:$Id$
2// Author: Andreas-Joachim Peters 9/5/2005
3
4/*************************************************************************
5 * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12//////////////////////////////////////////////////////////////////////////
13// //
14// TAlienCollection //
15// //
16// Class which manages collection of files on AliEn middleware. //
17// The file collection is in the form of an XML file. //
18// //
19// The internal list is managed as follows: //
20// TList* ===> TMap*(file) ===> TMap*(attributes) //
21// //
22//////////////////////////////////////////////////////////////////////////
23
24#include "TAlienCollection.h"
25#include "TAlienResult.h"
26#include "TAlienFile.h"
27#include "TDSet.h"
28#include "TMap.h"
29#include "TFile.h"
30#include "TXMLEngine.h"
31#include "TObjString.h"
32#include "TEntryList.h"
33#include "TObjArray.h"
34#include "TROOT.h"
35#include "TError.h"
36#include "TFileCollection.h"
37#include "TFileInfo.h"
38#include "snprintf.h"
39#include <cstdlib>
40
42
43////////////////////////////////////////////////////////////////////////////////
44/// Create Alien event collection by reading collection from the specified XML file.
45/// You can restrict the number of importet entries using the maxentries value.
46
47TAlienCollection::TAlienCollection(const char *localcollectionfile,
48 UInt_t maxentries)
49{
50 fXmlFile = localcollectionfile;
51 fFileGroupList = new TList();
54 fCurrent = 0;
55 fNofGroups = 0;
60 fFileStager = 0;
61 fExportUrl = "";
62 fInfoComment = "";
63 fCollectionName = "unnamed";
65 if (localcollectionfile != 0) {
66 ParseXML(maxentries);
67 }
68}
69
70////////////////////////////////////////////////////////////////////////////////
71/// Create Alien event collection using an event list.
72
74 UInt_t nofgroupfiles)
75{
76 fFileGroupList = eventlist;
79 fCurrent = 0;
80 fNofGroups = nofgroups;
81 fNofGroupfiles = nofgroupfiles;
85 fFileStager = 0;
86 fExportUrl = "";
87 fInfoComment = "";
88 fCollectionName = "unnamed";
90}
91
92////////////////////////////////////////////////////////////////////////////////
93/// Clean up event file collection.
94
96{
98 delete fFileGroupList;
99
101 delete fFileGroupListIter;
102
103 if (fTagFilterList)
104 delete fTagFilterList;
105}
106
107////////////////////////////////////////////////////////////////////////////////
108/// Static method used to create an Alien event collection, by reading
109/// an XML collection from the specified url. All ROOT URLs are supported.
110/// You can restrict the number of importet entries using the maxentries value
111
112TGridCollection *TAlienCollection::Open(const char *collectionurl,
113 UInt_t maxentries)
114{
115 //! stage the url to gSystem->TempDirectory() for remote url
116
117 TString coll(collectionurl);
118 Bool_t isRemote = coll.Contains(":/") && !coll.Contains("file:");
119 if (isRemote) {
120 TUUID uuid;
121 coll = gSystem->TempDirectory();
122 coll += "/aliencollection.";
123 coll += uuid.AsString();
124 if (!TFile::Cp(collectionurl, coll.Data())) {
125 ::Error("TAlienCollection::Open", "Cannot make a local copy of collection with url %s",
126 collectionurl);
127 return 0;
128 }
129 }
130
131 TAlienCollection *collection =
132 new TAlienCollection(coll, maxentries);
133
134 if (isRemote && gSystem->Unlink(coll.Data())) {
135 ::Error("TAlienCollection::Open", "Cannot remove the local copy of the collection %s",
136 coll.Data());
137 }
138
139 return dynamic_cast <TAlienCollection * > (collection);
140}
141
142////////////////////////////////////////////////////////////////////////////////
143/// Open the file specified by `<filename>` from the currently active file group in the collection via its TURL.
144
146{
147 const char *turl = GetTURL(filename);
148 if (turl) {
149 return TFile::Open(turl);
150 }
151 return 0;
152}
153
154////////////////////////////////////////////////////////////////////////////////
155/// Static method used to create an Alien event collection, by creating
156/// collection from a TGridResult GetCollection result (TAlien::GetCollection)
157
159 Option_t* /*option*/)
160{
161 if (!queryresult)
162 return 0;
163
164 TList *filelist = new TList();
165
166 TIterator* fileiter = queryresult->MakeIterator();
167 TMap *filemap = 0;
168
169 while ((filemap = ((TMap *) fileiter->Next()))) {
170 if (!filemap->GetValue("origLFN"))
171 continue;
172
173 filemap->Add(new TObjString("lfn"), new TObjString(filemap->GetValue("origLFN")->GetName()));
174 filemap->Add(new TObjString("turl"), new TObjString(Form("alien://%s", filemap->GetValue("origLFN")->GetName())));
175 TString bname = gSystem->BaseName(filemap->GetValue("origLFN")->GetName());
176 filemap->Add(new TObjString("name"), new TObjString(bname.Data()));
177
178 TMap* filegroup = new TMap();
179 filegroup->Add(new TObjString(bname.Data()), filemap);
180 filegroup->Add(new TObjString(""), filemap);
181
182 // store the filegroup
183 filelist->Add(filegroup);
184 }
185 delete fileiter;
186
187 return new TAlienCollection(filelist, filelist->GetEntries(), 1);;
188}
189
190////////////////////////////////////////////////////////////////////////////////
191/// Static method used to create an Alien event collection, by creating
192/// collection from a TGridResult Query result (TAlien::Query)
193/// nogrouping means that files in the same directory are treated as not belonging to a file group
194
196 Bool_t nogrouping)
197{
198 UInt_t nofgroups = 0;
199 UInt_t nofgroupfiles = 0;
200
201 if (!queryresult) {
202 return 0;
203 }
204
205 TIterator *fileiter = queryresult->MakeIterator();
206 TMap *filemap = 0;
207 TString prev_bname = "";
208 TString prev_dname = "";
209 TMap *filegroup = 0;
210
211 TList *filelist = new TList();
212
213 while ((filemap = ((TMap *) fileiter->Next()))) {
214 if (!filemap->GetValue("lfn"))
215 continue;
216 TString dname =
217 gSystem->GetDirName(filemap->GetValue("lfn")->GetName());
218 TString bname =
219 gSystem->BaseName(filemap->GetValue("lfn")->GetName());
220 filemap->Add(new TObjString("name"), new TObjString(bname.Data()));
221 if ((!nogrouping)
222 && ((bname != prev_bname) && (dname == prev_dname)
223 && (filegroup))) {
224 // add to group
225 filegroup->Add(new TObjString(bname.Data()), filemap);
226 if (nofgroups == 0)
227 nofgroupfiles++;
228 } else {
229 // new group
230 if (filegroup) {
231 // store the old filegroup
232 filelist->Add(filegroup);
233 nofgroups++;
234 }
235 if (nofgroups == 0)
236 nofgroupfiles++;
237 filegroup = new TMap();
238 filegroup->Add(new TObjString(bname.Data()), filemap);
239 filegroup->Add(new TObjString(""), filemap);
240 }
241
242 prev_bname = bname;
243 prev_dname = dname;
244 }
245 if (filegroup) {
246 nofgroups++;
247 filelist->Add(filegroup);
248 }
249 // set tag filter list
250 TList *filterlist = new TList();
251 filterlist->Add(new TObjString("type"));
252 filterlist->Add(new TObjString("dir"));
253 filterlist->Add(new TObjString("perm"));
254 filterlist->Add(new TObjString("owner"));
255 filterlist->Add(new TObjString("ctime"));
256 filterlist->Add(new TObjString("seStringlist"));
257 filterlist->Add(new TObjString("aclId"));
258 filterlist->Add(new TObjString("expiretime"));
259 filterlist->Add(new TObjString("replicated"));
260 filterlist->Add(new TObjString("entryId"));
261 filterlist->Add(new TObjString("gowner"));
262 filterlist->Add(new TObjString("selist"));
263 filterlist->Add(new TObjString("select"));
264 filterlist->Add(new TObjString("online"));
265
266
267 TAlienCollection *newcollection =
268 new TAlienCollection(filelist, nofgroups, nofgroupfiles);
269 if (newcollection) {
270 newcollection->SetTagFilterList(filterlist);
271 }
272 return dynamic_cast <TAlienCollection *> (newcollection);
273}
274
275////////////////////////////////////////////////////////////////////////////////
276/// Reset file iterator.
277
279{
281 fCurrent = 0;
282}
283
284////////////////////////////////////////////////////////////////////////////////
285/// Parse event file collection XML file.
286/// `<maxentries>` stops the parsing after `<maxentries>`.
287
289{
290 TXMLEngine xml;
291 UInt_t parsedentries = 0;
292
294 if (!xdoc) {
295 Error("ParseXML", "cannot parse the xml file %s", fXmlFile.Data());
296 return;
297 }
298
299 XMLNodePointer_t xalien = xml.DocGetRootElement(xdoc);
300 if (!xalien) {
301 Error("ParseXML", "cannot find the <alien> tag in %s",
302 fXmlFile.Data());
303 return;
304 }
305
306 XMLNodePointer_t xcollection = xml.GetChild(xalien);
307 if (!xcollection) {
308 Error("ParseXML", "cannot find the <collection> tag in %s",
309 fXmlFile.Data());
310 return;
311 }
312
313 if (xml.GetAttr(xcollection,"name")) {
314 fCollectionName = TString(xml.GetAttr(xcollection,"name"));
315 } else {
316 fCollectionName = ("unnamed");
317 }
318
319 XMLNodePointer_t xevent = xml.GetChild(xcollection);;
320 if (!xevent) {
321 Error("ParseXML", "cannot find the <event> tag in %s",
322 fXmlFile.Data());
323 return;
324 }
325
326 fNofGroups = 0;
327 fNofGroupfiles = 0;
330
331 do {
332 if (TString(xml.GetNodeName(xevent)) == "event") {
333 parsedentries++;
334 fNofGroups++;
335 TMap *files = new TMap();
336
337 // here is our event
338 // printf("Found event: %s\n",xml.GetAttr(xevent,"name"));
339
340 // files
341 XMLNodePointer_t xfile = xml.GetChild(xevent);
342 if (!xfile)
343 continue;
344
345 Bool_t firstfile = kTRUE;
346 do {
347 // here we have an event file
348 // get the attributes;
349 xml.GetAttr(xfile, "lfn");
350 xml.GetAttr(xfile, "turl");
351
352 TMap *attributes = new TMap();
353 TObjString *oname = new TObjString(xml.GetAttr(xfile, "name"));
354 TObjString *oturl = new TObjString(xml.GetAttr(xfile, "turl"));
355 TObjString *olfn = new TObjString(xml.GetAttr(xfile, "lfn"));
356 TObjString *omd5 = new TObjString(xml.GetAttr(xfile, "md5"));
357 TObjString *osize = new TObjString(xml.GetAttr(xfile, "size"));
358 TObjString *oguid = new TObjString(xml.GetAttr(xfile, "guid"));
359 TObjString *osurl = new TObjString(xml.GetAttr(xfile, "surl"));
360 TObjString *osselect =
361 new TObjString(xml.GetAttr(xfile, "select"));
362 TObjString *ossexporturl =
363 new TObjString(xml.GetAttr(xfile, "exporturl"));
364 TObjString *osonline =
365 new TObjString(xml.GetAttr(xfile, "online"));
366
367 TObjString *oseStringlist =
368 new TObjString(xml.GetAttr(xfile, "seStringlist"));
369 TObjString *oevlist =
370 new TObjString(xml.GetAttr(xfile, "evlist"));
371 // if oevlist is defined, we parse it and fill a TEntyList
372 if (oevlist && strlen(oevlist->GetName())) {
373 TEntryList *xmlentrylist =
374 new TEntryList(oturl->GetName(), oguid->GetName());
375 TString stringevlist = oevlist->GetName();
376 TObjArray *evlist = stringevlist.Tokenize(",");
377 for (Int_t n = 0; n < evlist->GetEntries(); n++) {
378 xmlentrylist->
379 Enter(atol
380 (((TObjString *) evlist->At(n))->GetName()));
381 }
382 attributes->Add(new TObjString("evlist"), xmlentrylist);
383 }
384 attributes->Add(new TObjString("name"), oname);
385 attributes->Add(new TObjString("turl"), oturl);
386 attributes->Add(new TObjString("lfn"), olfn);
387 attributes->Add(new TObjString("md5"), omd5);
388 attributes->Add(new TObjString("size"), osize);
389 attributes->Add(new TObjString("guid"), oguid);
390 attributes->Add(new TObjString("seStringlist"), oseStringlist);
391
392 if (osurl && strlen(osurl->GetName())) {
393 attributes->Add(new TObjString("surl"), osurl);
395 }
396 if (osselect && strlen(osselect->GetName())) {
397 attributes->Add(new TObjString("select"), osselect);
399 }
400 if (osonline && strlen(osonline->GetName())) {
401 attributes->Add(new TObjString("online"), osonline);
403 }
404
405 if (ossexporturl && strlen(ossexporturl->GetName())) {
406 attributes->Add(new TObjString("exporturl"), ossexporturl);
407 fExportUrl = ossexporturl->GetName();
408 }
409
410 files->Add(new TObjString(xml.GetAttr(xfile, "name")),
411 attributes);
412
413 // we add the first file always as a file without name to the map
414 if (firstfile) {
415 files->Add(new TObjString(""), attributes);
416 firstfile = kFALSE;
417 }
418 if (fNofGroups == 1)
420 } while ((xfile = xml.GetNext(xfile)));
421 fFileGroupList->Add(files);
422 }
423
424 if (TString(xml.GetNodeName(xevent)) == "info") {
425 if (xml.GetAttr(xevent,"comment")) {
426 fInfoComment = TString(xml.GetAttr(xevent,"comment"));
427 } else {
428 fInfoComment = "";
429 }
430 }
431
432 if (TString(xml.GetNodeName(xevent)) == "export") {
433 if (xml.GetAttr(xevent,"url")) {
434 SetExportUrl(xml.GetAttr(xevent,"url"));
435 } else {
436 fExportUrl = "";
437 }
438 }
439 if (parsedentries >= maxentries)
440 return;
441 } while ((xevent = xml.GetNext(xevent)));
442}
443
444////////////////////////////////////////////////////////////////////////////////
445/// Return next event file map.
446
448{
450 return fCurrent;
451}
452
453////////////////////////////////////////////////////////////////////////////////
454/// Prints statistics, selection and status about the loaded collection.
455
457{
458 TIter *statuslist = new TIter(fFileGroupList);
459 statuslist->Reset();
460 TMap *oldcurrent = fCurrent;
461 TMap *nextgroup = 0;
462 UInt_t gc;
463 UInt_t fc;
464 UInt_t onlinegroups;
465 UInt_t offlinegroups;
466 UInt_t onlinefiles;
467 UInt_t offlinefiles;
468 UInt_t totalfiles;
469 Long64_t totalfilesize;
470 Long64_t onlinefilesize;
471 Long64_t offlinefilesize;
472
473 gc = 0;
474 fc = 0;
475 onlinegroups = offlinegroups = onlinefiles = offlinefiles = totalfiles =
476 totalfilesize = onlinefilesize = offlinefilesize = 0;
477 while ((nextgroup = (TMap *) statuslist->Next())) {
478 gc++;
479 fc = 0;
480 Bool_t online;
481 Bool_t selected;
482 online = kTRUE;
483 selected = kFALSE;
484 TMap *attributes;
485 TIterator *nextfile = nextgroup->MakeIterator();
486 nextfile->Reset();
487 while ((attributes = (TMap *) nextfile->Next())) {
488 fCurrent = nextgroup;
489 if (TString(attributes->GetName()) != "") {
490 totalfiles++;
491 totalfilesize += GetSize(attributes->GetName());
492 }
493 if (IsSelected(attributes->GetName())) {
494 selected = kTRUE;
495 if (TString(attributes->GetName()) != "") {
496 fc++;
497 fCurrent = nextgroup;
498 if (!IsOnline(attributes->GetName())) {
499 online = kFALSE;
500 offlinefiles++;
501 offlinefilesize += GetSize(attributes->GetName());
502 } else {
503 onlinefiles++;
504 onlinefilesize += GetSize(attributes->GetName());
505 }
506 }
507 }
508 }
509 if (selected) {
510 if (online)
511 onlinegroups++;
512 else
513 offlinegroups++;
514 }
515 }
516 fCurrent = oldcurrent;
517 Info("Status", "=========================================");
518 Info("Status", " Tot. Number of files: %u", totalfiles);
519 Info("Status", " Tot. Size: %0.2f GB",
520 totalfilesize / 1024.0 / 1024.0 / 1024.0);
521 Info("Status", " Number of file groups: %u", gc);
522 Info("Status", "Number of files per group: %u", fc);
523 Info("Status", "-----------------------------------------");
524 Info("Status", "Online (staged [selected]):");
525 Info("Status", " Number of file groups: %u", onlinegroups);
526 Info("Status", " Number of files: %u", onlinefiles);
527 Info("Status", " Size: %0.2f GB",
528 onlinefilesize / 1024.0 / 1024.0 / 1024.0);
529 Info("Status", " Fraction avail: %0.2f %%",
530 100.0 * onlinefilesize / (onlinefilesize + offlinefilesize +
531 0.0000001));
532 Info("Status", "-----------------------------------------");
533 Info("Status", "Offline (to be staged [selected]):");
534 Info("Status", " Number of file groups: %u", offlinegroups);
535 Info("Status", " Number of files: %u", offlinefiles);
536 Info("Status", " Size: %0.2f GB",
537 offlinefilesize / 1024.0 / 1024.0 / 1024.0);
538 Info("Status", " Fraction miss: %0.2f %%",
539 100.0 * offlinefilesize / (onlinefilesize + offlinefilesize +
540 0.0000001));
541 Info("Status", "=========================================\n");
542
543 delete statuslist;
544}
545
546////////////////////////////////////////////////////////////////////////////////
547/// Set's a key value pair in a tagmap. If it is existing, the existing tag is overwritten. If not, it is created.
548
549void TAlienCollection::SetTag(const char *tag, const char *value,
550 TMap * tagmap)
551{
552 if ((!tag) || (!value) || (!tagmap)) {
553 return;
554 }
555 TObject *delobject = tagmap->FindObject(tag);
556 if (delobject) {
557 TObject *keyobject = ((TPair *) delobject)->Key();
558 tagmap->Remove(keyobject);;
559 }
560 tagmap->Add(new TObjString(tag), new TObjString(value));
561}
562
563
564////////////////////////////////////////////////////////////////////////////////
565/// Selects all files with name `<filename>` in the collection
566/// All files can be selected using "*" as filename
567
569{
570 Int_t cnt=0;
572 Reset();
573 TMap *nextgroup;
574 while ((nextgroup = (TMap *) Next())) {
575 cnt++;
576 TMap *attributes;
577 TIterator *nextfile = nextgroup->MakeIterator();
578 nextfile->Reset();
579 if ( ((nstart == -1 ) && (nstop == -1)) ||
580 ((nstart != -1 ) && (cnt >= nstart) && (nstop == -1)) ||
581 ((nstart != -1 ) && (cnt >= nstart) && (nstop != -1) && (cnt <= nstop)) ||
582 ((nstop != -1 ) && (cnt <= nstop) && (nstart == -1))) {
583 while ((attributes = (TMap *) nextfile->Next())) {
584 if (TString(attributes->GetName()) != "") {
585 if ((TString(attributes->GetName()) == TString(filename)) ||
586 (TString(filename) == TString("*"))) {
587 SetTag("select", "1",
588 ((TMap *) nextgroup->GetValue(attributes->GetName())));
589 }
590 }
591 }
592 }
593 }
594 return kTRUE;
595}
596
597////////////////////////////////////////////////////////////////////////////////
598/// Deselects the file `<filename>` from the loaded collection
599/// All files can be deselected using "*" as `<filename>`
600
602{
603 Int_t cnt=0;
604 Reset();
605 TMap *nextgroup;
606 while ((nextgroup = (TMap *) Next())) {
607 cnt++;
608 TMap *attributes;
609 TIterator *nextfile = nextgroup->MakeIterator();
610 nextfile->Reset();
611 if ( ((nstart == -1 ) && (nstop == -1)) ||
612 ((nstart != -1 ) && (cnt >= nstart) && (nstop == -1)) ||
613 ((nstart != -1 ) && (cnt >= nstart) && (nstop != -1) && (cnt <= nstop)) ||
614 ((nstop != -1 ) && (cnt <= nstop) && (nstart == -1))) {
615 while ((attributes = (TMap *) nextfile->Next())) {
616 if (TString(attributes->GetName()) != "") {
617 if ((TString(attributes->GetName()) == TString(filename)) ||
618 (TString(filename) == TString("*"))) {
619 SetTag("select", "0",
620 ((TMap *) nextgroup->GetValue(attributes->GetName())));
621 }
622 }
623 }
624 }
625 }
626
627 return kTRUE;
628}
629
630////////////////////////////////////////////////////////////////////////////////
631/// Invert the selection.
632
634{
635 Int_t cnt=0;
637 Reset();
638 TMap *nextgroup;
639 while ((nextgroup = (TMap *) Next())) {
640 cnt++;
641 TMap *attributes;
642 TIterator *nextfile = nextgroup->MakeIterator();
643 nextfile->Reset();
644 while ((attributes = (TMap *) nextfile->Next())) {
645 if (IsSelected(attributes->GetName())) {
646 SetTag("select", "0",
647 ((TMap *) nextgroup->GetValue(attributes->GetName())));
648 } else {
649 SetTag("select", "1",
650 ((TMap *) nextgroup->GetValue(attributes->GetName())));
651 }
652 }
653 }
654 return kTRUE;
655}
656
657////////////////////////////////////////////////////////////////////////////////
658/// downscales the selection with scaler
659
661{
662 Int_t cnt = 0;
663
664 Reset();
665 TMap *nextgroup;
666 while ((nextgroup = (TMap *) Next())) {
667 cnt++;
668 TMap *attributes;
669 TIterator *nextfile = nextgroup->MakeIterator();
670 nextfile->Reset();
671 if (cnt%scaler) {
672 while ((attributes = (TMap *) nextfile->Next())) {
673 if (TString(attributes->GetName()) != "") {
674 SetTag("select", "0",
675 ((TMap *) nextgroup->GetValue(attributes->GetName())));
676 }
677 }
678 }
679 }
680 return kTRUE;
681}
682
683
684////////////////////////////////////////////////////////////////////////////////
685/// Return next event file map.
686
688{
689 if (fFileGroupList->Remove(map)) {
690 return kTRUE;
691 } else {
692 return kFALSE;
693 }
694}
695
696////////////////////////////////////////////////////////////////////////////////
697/// Get a file's transport URL (TURL). Returns 0 in case of error.
698
699const char *TAlienCollection::GetTURL(const char *filename)
700{
701 if (fCurrent) {
702 TMap *obj = (TMap *) fCurrent->GetValue(filename);
703 if (obj) {
704 if (obj->GetValue("turl")) {
705 return (((TObjString *) obj->GetValue("turl"))->GetName());
706 }
707 }
708 }
709 Error("GetTURL", "cannot get TURL of file %s", filename);
710 return 0;
711}
712
713////////////////////////////////////////////////////////////////////////////////
714/// Get a file's storage URL (SURL). Returns 0 in case of error.
715
716const char *TAlienCollection::GetSURL(const char *filename)
717{
718 if (fCurrent) {
719 TMap *obj = (TMap *) fCurrent->GetValue(filename);
720 if (obj) {
721 if (obj->GetValue("surl")) {
722 return (((TObjString *) obj->GetValue("surl"))->GetName());
723 }
724 }
725 }
726 Error("GetSURL", "cannot get SURL of file %s", filename);
727 return 0;
728}
729
730////////////////////////////////////////////////////////////////////////////////
731/// Get a file's online tag. Returns false if not online or CheckIfOnline was never executed, true if online
732
734{
735 if (fCurrent) {
736 TMap *obj = (TMap *) fCurrent->GetValue(filename);
737 if (obj) {
738 if (obj->GetValue("online")) {
739 TString online =
740 (((TObjString *) obj->GetValue("online"))->GetName());
741 if (online == "1") {
742 return kTRUE;
743 } else {
744 return kFALSE;
745 }
746 }
747 }
748 }
749 // Error("IsOnline", "cannot get online tag of file %s", filename);
750 return kFALSE;
751}
752
753////////////////////////////////////////////////////////////////////////////////
754/// Get a file's online tag. Returns false if not online or CheckIfOnline was never executed, true if online
755
757{
758 if (fCurrent) {
759 TMap *obj = (TMap *) fCurrent->GetValue(filename);
760 if (obj) {
761 if (obj->GetValue("select")) {
762 TString selected;
763 selected =
764 (((TObjString *) obj->GetValue("select"))->GetName());
765 if (selected == TString("1")) {
766 return kTRUE;
767 } else {
768 return kFALSE;
769 }
770 }
771 }
772 }
773 return kFALSE;
774}
775
776////////////////////////////////////////////////////////////////////////////////
777/// Get a file's event list. Returns 0 in case of error.
778
780{
781 if (fCurrent) {
782 TMap *obj = (TMap *) fCurrent->GetValue(filename);
783 if (obj) {
784 if (obj->GetValue("evlist")) {
785 return ((TEntryList *) obj->GetValue("evlist"));
786 }
787 }
788 }
789 Error("GetEntryList", "cannot get evelist of file %s", filename);
790 return 0;
791}
792
793////////////////////////////////////////////////////////////////////////////////
794/// Get a file's LFN. Returns 0 in case of error.
795
796const char *TAlienCollection::GetLFN(const char *filename)
797{
798 if (fCurrent) {
799 TMap *obj = (TMap *) fCurrent->GetValue(filename);
800 if (obj) {
801 if (obj->GetValue("lfn")) {
802 return (((TObjString *) obj->GetValue("lfn"))->GetName());
803 }
804 }
805 }
806 Error("GetLFN", "cannot get LFN");
807 return 0;
808}
809
810////////////////////////////////////////////////////////////////////////////////
811/// Get a file's LFN. Returns 0 in case of error.
812
814{
815 if (fCurrent) {
816 TMap *obj = (TMap *) fCurrent->GetValue(filename);
817 if (obj) {
818 if (obj->GetValue("size")) {
819 TString ssize =
820 (((TObjString *) obj->GetValue("size"))->GetName());
821 return ssize.Atoll();
822 }
823 }
824 }
825 Error("GetSize", "cannot get size of %s", filename);
826 return 0;
827}
828
829////////////////////////////////////////////////////////////////////////////////
830/// Print event file collection.
831
833{
834 Info("Print", "dumping %d elements", fFileGroupList->GetSize());
835 TIter next(fFileGroupList);
836 TMap *filemap;
837 Int_t count = 0;
838 while ((filemap = (TMap *) next())) {
839 count++;
840 Info("Print", "printing element %d", count);
841 filemap->Print();
842 }
843}
844
845////////////////////////////////////////////////////////////////////////////////
846/// Return a TDSet from a collection. Filter, Selections and online tags are not
847/// taken into account.
848
849TDSet *TAlienCollection::GetDataset(const char *type, const char *objname,
850 const char *dir)
851{
852 Reset();
853 TDSet *dset = new TDSet(type, objname, dir);
854 if (!dset) {
855 return 0;
856 }
857
858 while (Next()) {
859 if (((TObjString *) fCurrent->GetValue("")))
860 dset->Add(((TMap *) (fCurrent->GetValue("")))->GetValue("turl")->
861 GetName());;
862 }
863 return dset;
864}
865
866////////////////////////////////////////////////////////////////////////////////
867/// Return a TGridResult.
868/// If files have been selected in this collection, only the selected files
869/// are returned. If `<onlyonline>` is set to kTRUE, only files which are
870/// 'online' (staged) are included. If no online check was done, TGridResult
871/// will be empty. `<publicaccess>` adds the publicaccess option to the TGridResult entries
872
874 Bool_t onlyonline,
875 Bool_t publicaccess)
876{
877 Reset();
879
880 while (Next()) {
881 if (((TObjString *) fCurrent->GetValue(filename))) {
882 TMap *attributes = (TMap *) fCurrent->GetValue(filename)->Clone();
883 if (publicaccess) {
884 attributes->Add(new TObjString("options"),
885 new TObjString("&publicaccess=1"));
886 }
887 if ((!fHasSelection) || (IsSelected(filename))) {
888 if ((!onlyonline) || (fHasOnline && IsOnline(filename))) {
889 result->Add(attributes);
890 }
891 }
892 }
893 }
894 return dynamic_cast < TGridResult * >(result);
895}
896
897////////////////////////////////////////////////////////////////////////////////
898/// return kTRUE if comparator overlaps with this
899/// all objects in this collection, which are not defined in the `<comparator>` collection are removed.
900
902{
903 if ((!comparator)) {
904 return kFALSE;
905 }
906
907loopagain:
908 // loop over col1 and try to find it in col2
909 this->Reset();
910 // loop over all elements in reference (=this)
911 TMap *overlapmap;
912 while ((overlapmap = this->Next())) {
913 comparator->Reset();
914 Bool_t found = kFALSE;
915 // try to find in the comparator collection
916 while ((comparator->Next())) {
917 TString s1 = this->GetLFN();
918 TString s2 = comparator->GetLFN();
919 if (s1 == s2) {
920 found = kTRUE;
921 break;
922 }
923 }
924 if (!found) {
925 this->Remove(overlapmap);
926 goto loopagain;
927 }
928 }
929 return kTRUE;
930}
931
932////////////////////////////////////////////////////////////////////////////////
933/// adds `<addcollection>` to this collection - equal elements are skipped
934
936{
937 if ((!addcollection)) {
938 return;
939 }
940 // loop over col1 and try to find it in col2
941 addcollection->Reset();
942 // loop over all elements in reference (=this)
943 TMap *addmap;
944 while ((addmap = addcollection->Next())) {
945 Reset();
946 // try to find in the comparator collection
947 TString s2 = addcollection->GetLFN();
948 while (Next()) {
949 TString s1 = GetLFN();
950 // printf("%s = %s\n", s1.Data(), s2.Data());
951 if (s1 == s2) {
952 Error("Add",
953 "File group with lfn %s exists already in this collection - skipping",
954 GetLFN());
955 goto leaveloop;
956 }
957 }
958 TMap *clonemap;
959 clonemap = (TMap *) addmap->Clone();
960 fFileGroupList->Add(clonemap);
961leaveloop:
962 ;
963 }
964}
965
966////////////////////////////////////////////////////////////////////////////////
967/// adds `<addcollection>` to this collection - NO check for identical elements
968
970{
971 if ((!addcollection)) {
972 return;
973 }
974 addcollection->Reset();
975 TMap *addmap, *clonemap;
976 while ((addmap = addcollection->Next())) {
977 clonemap = (TMap *) addmap->Clone();
978 fFileGroupList->Add(clonemap);
979 }
980}
981
982////////////////////////////////////////////////////////////////////////////////
983/// retrieves all the SURLS for the LFNS
984
986{
987 Bool_t ok = kTRUE;
988 UInt_t lc = 0;
989 Reset();
990 TMap *filemap;
991 while ((filemap = Next())) {
992 // loop over all files in this map
993 TIterator *nextfile = filemap->MakeIterator();
994 TMap *attributes;
995 while ((attributes = (TMap *) nextfile->Next())) {
996 if (TString(attributes->GetName()) != "") {
997 lc++;
998 if (fHasSelection && (!IsSelected(attributes->GetName())))
999 continue;
1000 // there is always an "" entry in the map to point to the first file of a file group
1001 if (verbose)
1002 Info("LookupSUrls", "Lookup SURL for %s [%u/%u]",
1003 GetTURL(attributes->GetName()), lc,
1005 TString surl =
1006 TAlienFile::SUrl(GetTURL(attributes->GetName()));
1007 if (!surl.Data()) {
1008 ok = kFALSE;
1009 } else {
1010 // delete the surl map entry, if it exists already
1011 TObject *delobject =
1012 ((TMap *) filemap->GetValue(attributes->GetName()))->
1013 FindObject("surl");
1014 if (delobject) {
1015 TObject *keyobject = ((TPair *) delobject)->Key();
1016 ((TMap *) filemap->GetValue(attributes->GetName()))->
1017 Remove(keyobject);;
1018 }
1019
1020 ((TMap *) filemap->GetValue(attributes->GetName()))->
1021 Add(new TObjString("surl"),
1022 new TObjString(surl.Data()));
1023 if (verbose)
1024 Info("LookupSUrls", "SURL = %s", surl.Data());
1025
1026 }
1027
1028 }
1029 }
1030 }
1031 if (ok) {
1032 fHasSUrls = kTRUE;
1033 }
1034 return ok;
1035}
1036
1037////////////////////////////////////////////////////////////////////////////////
1038/// Execute the 'stage' method for all files in this collection (trigger staging).
1039/// The `<bulk>` stage method is currently not working.
1040
1042{
1043 if (!fHasSUrls) {
1044 Error("Stage",
1045 "You have to execute LookupSUrls() before you can stage this collection");
1046 return kFALSE;
1047 }
1048 UInt_t fc = 0;
1049 Reset();
1050 TMap *filemap;
1051
1052 if (!bulk) {
1053 while ((filemap = Next())) {
1054 // loop over all files in this map
1055 TIterator *nextfile = filemap->MakeIterator();
1056 TMap *attributes;
1057 while ((attributes = (TMap *) nextfile->Next())) {
1058 if (TString(attributes->GetName()) != "") {
1059 fc++;
1060 if (fHasSelection && (!IsSelected(attributes->GetName())))
1061 continue;
1062
1063 if (!fFileStager) {
1064 fFileStager =
1065 TFileStager::Open(GetSURL(attributes->GetName()));
1066 }
1067 if ((fFileStager)->Stage(GetSURL(attributes->GetName()),option)) {
1068 // file staged
1069 Info("Stage", "[%05u/%05u] <Staged> : %s", fc,
1071 GetLFN(attributes->GetName()));
1072 } else {
1073 // file stage failed
1074 Error("Stage", "[%05u/%05u] <Failed to stage> : %s",
1076 GetLFN(attributes->GetName()));
1077 }
1078 }
1079 }
1080 }
1081 } else {
1082 // bulk request
1083 TList* stagelist = new TList();
1084 stagelist->SetOwner(kTRUE);
1085 Bool_t stageresult=kFALSE;
1086 Reset();
1087 while ((filemap = Next())) {
1088 TIterator *nextfile = filemap->MakeIterator();
1089 TMap *attributes;
1090 while ((attributes = (TMap *) nextfile->Next())) {
1091 if (TString(attributes->GetName()) != "") {
1092 fc++;
1093 stagelist->Add( new TUrl((GetSURL(attributes->GetName()))));
1094 }
1095 }
1096 }
1097
1098 if (fc) {
1099 if (!fFileStager) {
1100 fFileStager = TFileStager::Open(stagelist->First()->GetName());
1101 }
1102
1103 stageresult = (fFileStager)->Stage(stagelist,option);
1104 }
1105 delete stagelist;
1106 return stageresult;
1107 }
1108 return kTRUE;
1109}
1110
1111////////////////////////////////////////////////////////////////////////////////
1112/// Run an online check if files are currently accessible (staged) or offline (to be staged).
1113/// The `<bulk>` check is currently not working.
1114
1116{
1117 if (!fHasSUrls) {
1118 Error("CheckIfOnline",
1119 "You have to execute LookupSUrls() before you can prepare this collection");
1120 return kFALSE;
1121 }
1122
1123 Reset();
1124 TMap *filemap;
1125 UInt_t fc=0;
1126
1127 if (!bulk) {
1128 while ((filemap = Next())) {
1129 // loop over all files in this map
1130 TIterator *nextfile = filemap->MakeIterator();
1131 TMap *attributes;
1132 while ((attributes = (TMap *) nextfile->Next())) {
1133 if (fHasSelection && (!IsSelected(attributes->GetName())))
1134 continue;
1135 if (TString(attributes->GetName()) != "") {
1136 fc++;
1137 // check if we have a fFileStager
1138 if (!fFileStager) {
1139 fFileStager =
1140 TFileStager::Open(GetSURL(attributes->GetName()));
1141 }
1142 Bool_t online = kFALSE;
1143 if ((fFileStager)->IsStaged(GetSURL(attributes->GetName()))) {
1144 // file is online
1145 Info("CheckIfOnline", "[%05u/%05u] <Online> : %s", fc,
1147 GetLFN(attributes->GetName()));
1148 online = kTRUE;
1149 } else {
1150 // file is offline
1151 Info("CheckIfOnline", "[%05u/%05u]<Offline> : %s", fc,
1153 GetLFN(attributes->GetName()));
1154 online = kFALSE;
1155 }
1156
1157 {
1158 // set the online tag in the collection
1159 // delete the online tag entry, if it exists already
1160 TObject *delobject =
1161 ((TMap *) filemap->GetValue(attributes->GetName()))->
1162 FindObject("online");
1163 if (delobject) {
1164 TObject *keyobject = ((TPair *) delobject)->Key();
1165 ((TMap *) filemap->GetValue(attributes->GetName()))->
1166 Remove(keyobject);;
1167 }
1168 if (online)
1169 ((TMap *) filemap->GetValue(attributes->GetName()))->
1170 Add(new TObjString("online"),
1171 new TObjString("1"));
1172 else
1173 ((TMap *) filemap->GetValue(attributes->GetName()))->
1174 Add(new TObjString("online"),
1175 new TObjString("0"));
1176 }
1177 }
1178 }
1179 }
1180 } else {
1181 // bulk lookup
1182 TList *lookuplist = new TList();
1183 if (lookuplist) {
1184 lookuplist->SetOwner(kTRUE);
1185 while ((filemap = Next())) {
1186 // loop over all files in this map
1187 TIterator *nextfile = filemap->MakeIterator();
1188 TMap *attributes;
1189 while ((attributes = (TMap *) nextfile->Next())) {
1190 if (TString(attributes->GetName()) != "") {
1191 fc++;
1192 // check if we have a fFileStager
1193 if (!fFileStager) {
1194 fFileStager =
1195 TFileStager::Open(GetSURL(attributes->GetName()));
1196 }
1197 lookuplist->
1198 Add(new TObjString(GetSURL(attributes->GetName())));
1199 }
1200 }
1201 }
1202
1203 TList *onlinelist = fFileStager->GetStaged(lookuplist);
1204 if (!onlinelist) {
1205 SafeDelete(lookuplist);
1206 return kFALSE;
1207 }
1208
1209 Reset();
1210 fc=0;
1211 while ((filemap = Next())) {
1212 // loop over all files in this map
1213 TIterator *nextfile = filemap->MakeIterator();
1214 TMap *attributes;
1215 while ((attributes = (TMap *) nextfile->Next())) {
1216 if (TString(attributes->GetName()) != "") {
1217 fc++;
1218 Bool_t online;
1219 // check if it is in the online list
1220 if (onlinelist->
1221 FindObject(GetSURL(attributes->GetName()))) {
1222 // this file is online
1223 Info("CheckIfOnline", "[%05u/%05u] <Online> : %s", fc,
1225 GetLFN(attributes->GetName()));
1226 online = kTRUE;
1227 } else {
1228 // this file is offline
1229 Info("CheckIfOnline", "[%05u/%05u]<Offline> : %s", fc,
1231 GetLFN(attributes->GetName()));
1232 online = kFALSE;
1233 }
1234
1235 {
1236 // set the online tag in the collection
1237 // delete the online tag entry, if it exists already
1238 TObject *delobject =
1239 ((TMap *) filemap->
1240 GetValue(attributes->GetName()))->
1241 FindObject("online");
1242 if (delobject) {
1243 TObject *keyobject = ((TPair *) delobject)->Key();
1244 ((TMap *) filemap->
1245 GetValue(attributes->GetName()))->
1246 Remove(keyobject);
1247 }
1248 if (online)
1249 ((TMap *) filemap->
1250 GetValue(attributes->GetName()))->
1251 Add(new TObjString("online"), new TObjString("1"));
1252 else
1253 ((TMap *) filemap->
1254 GetValue(attributes->GetName()))->
1255 Add(new TObjString("online"), new TObjString("0"));
1256 }
1257 }
1258 }
1259 }
1260
1261 SafeDelete(onlinelist);
1262 SafeDelete(lookuplist);
1263 } else {
1265 return kFALSE;
1266 }
1267 }
1268
1269 fHasOnline = kTRUE;
1270 return kTRUE;
1271}
1272
1273////////////////////////////////////////////////////////////////////////////////
1274/// Exports the contens of the TAlienCollection into an XML formatted file.
1275/// By default exports only selected files. By default exports only accessible (online) files.
1276/// You can change this behaviour by specifying online=kFALSE or selected=kFALSE.
1277/// `<name>` specifies a name you want to assign to this collection.
1278/// `<comment>` can be a user comment to this collection.
1279/// If `<exporturl>`="" the collection is exported to the URL which was stored inside the collection or
1280/// was specified by the ExportUrl(const char* url) method.
1281
1282Bool_t TAlienCollection::ExportXML(const char *exporturl, Bool_t selected, Bool_t online,
1283 const char *name , const char *comment)
1284{
1285
1286 TFile *exportfile;
1287
1288 if ((exporturl == 0) || (TString(exporturl) == "")) {
1289 if (!(exporturl = GetExportUrl())) {
1290 Error("ExportXML",
1291 "You used the option to store to the collection export url, but this is not defined!");
1292 return kFALSE;
1293 }
1294 }
1295 TUrl expfn(exporturl);
1296 TString options = expfn.GetOptions();
1297 if (options.Length()) {
1298 options += ",filetype=raw";
1299 } else {
1300 options = "filetype=raw";
1301 }
1302 expfn.SetOptions(options.Data());
1303
1304 if (!(exportfile = TFile::Open(expfn.GetUrl(), "RECREATE"))) {
1305 Error("ExportXML", "Cannot open export URL %s", expfn.GetUrl());
1306 return kFALSE;
1307 }
1308
1309 Bool_t expret = ExportXML(exportfile, selected, online, name, comment);
1310 exportfile->Close();
1311 return expret;
1312}
1313
1314////////////////////////////////////////////////////////////////////////////////
1315/// Internal Export function to write a collection as an XML file. See above.
1316
1318 Bool_t online, const char *name,
1319 const char *comment)
1320{
1321 char outline[4096];
1322
1323 // write headers
1324 snprintf(outline,4096,
1325 "<?xml version=\"1.0\"?>\n<alien>\n <collection name=\"%s\">\n",
1326 name);
1327 if ((exportfile->WriteBuffer(outline, strlen(outline)))) {
1328 Error("ExportXML", "Error writing XML to export file");
1329 exportfile->Close();
1330 return kFALSE;
1331 }
1332 // write events
1333 TMap *oldcurrent = fCurrent;
1334 TMap *nextgroup;
1335 TIter *filegroups = new TIter(fFileGroupList);
1336 filegroups->Reset();
1337 UInt_t groupcnt = 0;
1338 while ((nextgroup = (TMap *) filegroups->Next())) {
1339 Bool_t isselected;
1340 isselected = kFALSE;
1341 TMap *attributes;
1342 TIterator *nextfile = nextgroup->MakeIterator();
1343 nextfile->Reset();
1344 // check if something is selected
1345 while ((attributes = (TMap *) nextfile->Next())) {
1346 if (TString(attributes->GetName()) != "") {
1347 fCurrent = nextgroup;
1348 if (IsSelected(attributes->GetName())) {
1349 isselected = kTRUE;
1350 }
1351
1352 }
1353 }
1354 if ((!selected) || isselected) {
1355 // loop again and export files
1356 nextfile->Reset();
1357 groupcnt++;
1358 // open new event header
1359 snprintf(outline,4096, " <event name=\"%d\">\n", groupcnt);
1360 if ((exportfile->WriteBuffer(outline, strlen(outline)))) {
1361 Error("ExportXML", "Error writing XML to export file");
1362 exportfile->Close();
1363 return kFALSE;
1364 }
1365 while ((attributes = (TMap *) nextfile->Next())) {
1366 if (TString(attributes->GetName()) != "") {
1367 fCurrent = nextgroup;
1368 if ((!selected) || (IsSelected(attributes->GetName()))) {
1369 if ((!online) || (IsOnline(attributes->GetName()))) {
1370 // export this file
1371 /////////////////////////////////////////////////////////////
1372 // open file tag
1373 snprintf(outline,4096, " <file ");
1374 if ((exportfile->
1375 WriteBuffer(outline, strlen(outline)))) {
1376 Error("ExportXML",
1377 "Error writing XML to export file");
1378 exportfile->Close();
1379 return kFALSE;
1380 }
1381 // loop over map keys
1382 TIterator *mkeys =
1383 ((TMap *) nextgroup->
1384 GetValue(attributes->GetName()))->MakeIterator();
1385 mkeys->Reset();
1386 TObjString *tagname = 0;
1387 TObjString *tagval = 0;
1388 while ((tagname = (TObjString *) mkeys->Next())) {
1389 Bool_t filtered = kFALSE;
1390 // check for filtered tags from the filter list
1391 if (fTagFilterList) {
1392 TIter next(fTagFilterList);
1393 TObjString *filtertag;
1394 while ((filtertag = (TObjString *) next())) {
1395 if (TString(tagname->GetName()) ==
1396 TString(filtertag->GetName())) {
1397 filtered = kTRUE;
1398 break;
1399 }
1400 }
1401 }
1402 if (!filtered) {
1403 tagval =
1404 (TObjString *) ((TMap *) nextgroup->
1405 GetValue(attributes->
1406 GetName()))->
1407 GetValue(tagname);
1408 if (TString(tagname->GetName()) != "evlist") {
1409 snprintf(outline,4096, "%s=\"%s\" ",
1410 tagname->GetName(),
1411 tagval->GetName());
1412 } else {
1413 // the eventlist has to be converted from TEventList to a string list with komma separation
1414 TEntryList *xmlentrylist =
1415 (TEntryList *) tagval;
1416 if (!xmlentrylist)
1417 continue;
1418 TString slist = "";
1419 for (Int_t i = 0; i < xmlentrylist->GetN(); i++) {
1420 if (i > 0)
1421 slist += ",";
1422 slist += xmlentrylist->GetEntry(i);
1423 }
1424 snprintf(outline,4096, "%s=\"%s\" ",
1425 tagname->GetName(), slist.Data());
1426 }
1427
1428 if ((exportfile->
1429 WriteBuffer(outline, strlen(outline)))) {
1430 Error("ExportXML",
1431 "Error writing XML to export file");
1432 exportfile->Close();
1433 return kFALSE;
1434 }
1435 }
1436 }
1437
1438 // close file tag
1439 snprintf(outline,4096, "/>\n");
1440 if ((exportfile->
1441 WriteBuffer(outline, strlen(outline)))) {
1442 Error("ExportXML",
1443 "Error writing XML to export file");
1444 exportfile->Close();
1445 return kFALSE;
1446 }
1447 }
1448 }
1449 }
1450 }
1451
1452 // close event
1453 snprintf(outline,4096, " </event>\n");
1454 if ((exportfile->WriteBuffer(outline, strlen(outline)))) {
1455 Error("ExportXML", "Error writing XML to export file");
1456 exportfile->Close();
1457 return kFALSE;
1458 }
1459 }
1460 }
1461
1462 fCurrent = oldcurrent;
1463
1464 // write export url if present
1465 if (GetExportUrl()) {
1466 snprintf(outline,4096, " <export url=\"%s\">\n",GetExportUrl());
1467 if ((exportfile->WriteBuffer(outline, strlen(outline)))) {
1468 Error("ExportXML", "Error writing XML to export file");
1469 exportfile->Close();
1470 return kFALSE;
1471 }
1472 }
1473
1474
1475 // write trailer
1476 snprintf(outline,4096,
1477 " <info comment=\"%s\" />\n</collection>\n</alien>\n",
1478 comment);
1479 if ((exportfile->WriteBuffer(outline, strlen(outline)))) {
1480 Error("ExportXML", "Error writing XML to export file");
1481 exportfile->Close();
1482 return kFALSE;
1483 }
1484
1485 delete filegroups;
1486
1487 return kTRUE;
1488}
1489
1490////////////////////////////////////////////////////////////////////////////////
1491/// Set the 'default' export URL for an XML collection. A collection can be restored to the export URL using ExportXML("");
1492
1494{
1495 if (exporturl)
1496 fExportUrl = exporturl;
1497
1498 if (fExportUrl == "") {
1499 Info("ExportUrl",
1500 "There is no remote url defined in this collection");
1501 return kFALSE;
1502 }
1503 return kTRUE;
1504}
1505
1506////////////////////////////////////////////////////////////////////////////////
1507/// Adds to a file given by infile the collection identification , f.e.
1508/// for collection files sitting in directories like 100/1/AliESD.root
1509/// ...
1510/// 110/1/AliESD.root
1511/// "./histo.root" will be converted to "./histo.100_1-110_1.230.root
1512///
1513/// The name syntax is <oldname>.<first run>_<first event>-<last run>.<last event>.<nevents>.root
1514///
1515/// By default the found files are renamed on the local disk
1516/// example:
1517/// - you specify f.e. as arguments GetOutputFileName("*.root",kTRUE);
1518/// --->> this renames all ROOT files corresponding to the collection contents
1519
1520const char *TAlienCollection::GetOutputFileName(const char *infile,
1521 Bool_t rename, const char* suffix)
1522{
1523 Bool_t first = kTRUE;
1524 TString firstrun;
1525 TString firstevent;
1526 TString lastrun;
1527 TString lastevent;
1528 UInt_t nevents = 0;
1529
1530 Reset();
1531 while (Next()) {
1533 TString s2(gSystem->BaseName(s1.Data()));
1534 TString s3 = gSystem->GetDirName(s1.Data());
1535 TString s4(gSystem->BaseName(s3.Data()));
1536 nevents++;
1537 if (first) {
1538 first = kFALSE;
1539 firstevent = s2;
1540 firstrun = s4;
1541 }
1542 lastevent = s2;
1543 lastrun = s4;
1544 }
1545
1546 // list the matching files
1547 TString pcmd("ls ");
1548 pcmd += infile;
1549 //printf("Pipe is %s\n",pcmd.Data());
1550 FILE *fp = gSystem->OpenPipe(pcmd.Data(), "r");
1551 if (fp) {
1552 char rootfile[4096];
1553 Int_t item;
1554 while ((item = fscanf(fp, "%4095s", rootfile)) == 1) {
1555 TString rootdir = gSystem->GetDirName(rootfile);
1556 TString rootbase(gSystem->BaseName(rootfile));
1557 TString rootbasenosuffix;
1558 rootbasenosuffix = rootbase(0, rootbase.First('.'));;
1559 // build new name like <oldname>.<firstrun>_<firstevent>-<lastrun>_<lastevent>.root
1560 TString newrootname;
1561 if (rootdir.Length()) {
1562 newrootname += rootdir;
1563 newrootname += "/";
1564 }
1565 newrootname += rootbasenosuffix;
1566 newrootname += ".";
1567 newrootname += firstrun;
1568 newrootname += "_";
1569 newrootname += firstevent;
1570 newrootname += "-";
1571 newrootname += lastrun;
1572 newrootname += "_";
1573 newrootname += lastevent;
1574 newrootname += ".";
1575 newrootname += nevents;
1576 newrootname += ".";
1577 newrootname += suffix;
1578 Info("GetOutputFilename", "Renaming File %s to %s", rootfile,
1579 newrootname.Data());
1580 fLastOutFileName = newrootname;
1581 if (rename) {
1582 gSystem->Rename(rootfile, newrootname.Data());
1583 }
1584 }
1585 gSystem->ClosePipe(fp);
1586 }
1587 return fLastOutFileName.Data();
1588}
1589
1590////////////////////////////////////////////////////////////////////////////////
1591/// creates a TFileCollection objects and fills it with the information from this collection
1592/// note that TFileCollection has a flat structure and no groups --> all files are filles on a flat level
1593/// the TFileInfo of each file in the TFileCollection is filled with turl, size, md5, guid
1594///
1595/// the object has to be deleted by the user
1596
1597TFileCollection *TAlienCollection::GetFileCollection(const char* name, const char* title) const
1598{
1599 TFileCollection* collection = new TFileCollection(name, title);
1600
1601 TIter next(fFileGroupList);
1602 TMap* group = 0;
1603 while ((group = dynamic_cast<TMap*>(next()))) {
1604 TIter next2(group);
1605 TObjString* key = 0;
1606 while ((key = dynamic_cast<TObjString*> (next2()))) {
1607 if (key->String().Length() == 0)
1608 continue;
1609
1610 TMap* file = dynamic_cast<TMap*> (group->GetValue(key));
1611 if (!file)
1612 continue;
1613
1614 TObjString* turl = dynamic_cast<TObjString*> (file->GetValue("turl"));
1615 TObjString* size = dynamic_cast<TObjString*> (file->GetValue("size"));
1616 TObjString* md5 = dynamic_cast<TObjString*> (file->GetValue("md5"));
1617 TObjString* guid = dynamic_cast<TObjString*> (file->GetValue("guid"));
1618
1619 if (!turl || turl->String().Length() == 0)
1620 continue;
1621
1622 TFileInfo* fileInfo = new TFileInfo(turl->String(), size->String().Atoi(), guid->String(), md5->String());
1623 collection->Add(fileInfo);
1624 }
1625 }
1626
1627 collection->Update();
1628
1629 return collection;
1630}
#define SafeDelete(p)
Definition: RConfig.hxx:537
#define s1(x)
Definition: RSha256.hxx:91
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
const Bool_t kFALSE
Definition: RtypesCore.h:101
long long Long64_t
Definition: RtypesCore.h:80
const Bool_t kTRUE
Definition: RtypesCore.h:100
const char Option_t
Definition: RtypesCore.h:66
#define ClassImp(name)
Definition: Rtypes.h:375
const char rootdir[]
Option_t Option_t option
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void gc
char name[80]
Definition: TGX11.cxx:110
char * Form(const char *fmt,...)
Formats a string in a circular formatting buffer.
Definition: TString.cxx:2456
R__EXTERN TSystem * gSystem
Definition: TSystem.h:560
void * XMLNodePointer_t
Definition: TXMLEngine.h:17
void * XMLDocPointer_t
Definition: TXMLEngine.h:20
#define snprintf
Definition: civetweb.c:1540
void Print(Option_t *opt) const
Print event file collection.
const char * GetExportUrl()
Bool_t IsSelected(const char *name="")
Get a file's online tag. Returns false if not online or CheckIfOnline was never executed,...
const char * GetLFN(const char *name="")
Get a file's LFN. Returns 0 in case of error.
const char * GetSURL(const char *name="")
Get a file's storage URL (SURL). Returns 0 in case of error.
const char * GetTURL(const char *name="")
Get a file's transport URL (TURL). Returns 0 in case of error.
void Reset()
Reset file iterator.
Bool_t Remove(TMap *map)
Return next event file map.
Bool_t ExportXML(TFile *file, Bool_t selected, Bool_t online, const char *name, const char *comment)
Internal Export function to write a collection as an XML file. See above.
TMap * Next()
Return next event file map.
void Add(TGridCollection *addcollection)
adds <addcollection> to this collection - equal elements are skipped
Bool_t InvertSelection()
Invert the selection.
UInt_t GetNofGroupfiles() const
static TAlienCollection * OpenAlienCollection(TGridResult *queryresult, Option_t *option="")
Static method used to create an Alien event collection, by creating collection from a TGridResult Get...
void Status()
Prints statistics, selection and status about the loaded collection.
Bool_t LookupSUrls(Bool_t verbose=kTRUE)
retrieves all the SURLS for the LFNS
TDSet * GetDataset(const char *type, const char *objname="*", const char *dir="/")
Return a TDSet from a collection.
TString fExportUrl
pointer to the file stager object
Bool_t OverlapCollection(TGridCollection *comparator)
return kTRUE if comparator overlaps with this all objects in this collection, which are not defined i...
static TGridCollection * OpenQuery(TGridResult *queryresult, Bool_t nogrouping=kFALSE)
Static method used to create an Alien event collection, by creating collection from a TGridResult Que...
TFileCollection * GetFileCollection(const char *name="", const char *title="") const
creates a TFileCollection objects and fills it with the information from this collection note that TF...
TFileStager * fFileStager
virtual void ParseXML(UInt_t maxentries)
Parse event file collection XML file.
TMap * fCurrent
event file list iterator
TEntryList * GetEntryList(const char *name)
Get a file's event list. Returns 0 in case of error.
void SetTagFilterList(TList *filterlist)
Bool_t DownscaleSelection(UInt_t scaler=2)
downscales the selection with scaler
TFile * OpenFile(const char *filename)
Open the file specified by <filename> from the currently active file group in the collection via its ...
Long64_t GetSize(const char *name="")
Get a file's LFN. Returns 0 in case of error.
TGridResult * GetGridResult(const char *filename="", Bool_t onlyonline=kTRUE, Bool_t publicaccess=kFALSE)
Return a TGridResult.
static TGridCollection * Open(const char *collectionurl, UInt_t maxentries=1000000)
Static method used to create an Alien event collection, by reading an XML collection from the specifi...
Bool_t DeselectFile(const char *name, Int_t=-1, Int_t=-1)
Deselects the file <filename> from the loaded collection All files can be deselected using "*" as <fi...
UInt_t GetNofGroups() const
UInt_t fNofGroups
current event file map
Bool_t IsOnline(const char *name="")
Get a file's online tag. Returns false if not online or CheckIfOnline was never executed,...
Bool_t SetExportUrl(const char *exporturl=0)
Set the 'default' export URL for an XML collection. A collection can be restored to the export URL us...
void SetTag(const char *tag, const char *value, TMap *tagmap)
Set's a key value pair in a tagmap. If it is existing, the existing tag is overwritten....
const char * GetOutputFileName(const char *infile, Bool_t rename=kTRUE, const char *suffix="root")
Adds to a file given by infile the collection identification , f.e.
Bool_t CheckIfOnline(Bool_t bulk=kFALSE)
Run an online check if files are currently accessible (staged) or offline (to be staged).
virtual ~TAlienCollection()
Clean up event file collection.
Bool_t SelectFile(const char *name, Int_t=-1, Int_t=-1)
Selects all files with name <filename> in the collection All files can be selected using "*" as filen...
TIter * fFileGroupListIter
void AddFast(TGridCollection *addcollection)
adds <addcollection> to this collection - NO check for identical elements
Bool_t Stage(Bool_t bulk=kFALSE, Option_t *option="")
Execute the 'stage' method for all files in this collection (trigger staging).
static TString SUrl(const char *lfn)
Get surl from lfn by asking AliEn catalog.
Definition: TAlienFile.cxx:613
const char * GetName() const override
Return name of this collection.
virtual Int_t GetEntries() const
Definition: TCollection.h:179
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
void Print(Option_t *option="") const override
Default print for collections, calls Print(option, 1).
TObject * Clone(const char *newname="") const override
Make a clone of an collection using the Streamer facility.
virtual Int_t GetSize() const
Return the capacity of the collection, i.e.
Definition: TCollection.h:184
This class implements a data set to be used for PROOF processing.
Definition: TDSet.h:153
virtual Bool_t Add(const char *file, const char *objname=0, const char *dir=0, Long64_t first=0, Long64_t num=-1, const char *msd=0)
Add file to list of files to be analyzed.
Definition: TDSet.cxx:1052
A List of entry numbers in a TTree or TChain.
Definition: TEntryList.h:26
virtual Long64_t GetEntry(Long64_t index)
Return the number of the entry #index of this TEntryList in the TTree or TChain See also Next().
Definition: TEntryList.cxx:753
virtual Long64_t GetN() const
Definition: TEntryList.h:78
Class that contains a list of TFileInfo's and accumulated meta data information about its entries.
Int_t Update(Long64_t avgsize=-1)
Update accumulated information about the elements of the collection (e.g.
Int_t Add(TFileInfo *info)
Add TFileInfo to the collection.
Class describing a generic file including meta information.
Definition: TFileInfo.h:39
static TFileStager * Open(const char *stager)
Open a stager, after having loaded the relevant plug-in.
virtual TList * GetStaged(TCollection *pathlist)
Retrieves the staging (online) status for a list of path names.
Definition: TFileStager.cxx:46
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format.
Definition: TFile.h:54
virtual Bool_t WriteBuffer(const char *buf, Int_t len)
Write a buffer to the file.
Definition: TFile.cxx:2442
virtual Bool_t Cp(const char *dst, Bool_t progressbar=kTRUE, UInt_t buffersize=1000000)
Allows to copy this file to the dst URL.
Definition: TFile.cxx:4973
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Definition: TFile.cxx:4053
void Close(Option_t *option="") override
Close a file.
Definition: TFile.cxx:916
virtual TMap * Next()
virtual const char * GetLFN(const char *="")
virtual void Reset()
TObject * Next()
Definition: TCollection.h:251
void Reset()
Definition: TCollection.h:254
Iterator abstract base class.
Definition: TIterator.h:30
virtual void Reset()=0
virtual TObject * Next()=0
A doubly linked list.
Definition: TList.h:38
void Add(TObject *obj) override
Definition: TList.h:81
TObject * Remove(TObject *obj) override
Remove object from the list.
Definition: TList.cxx:822
TObject * First() const override
Return the first object in the list. Returns 0 when list is empty.
Definition: TList.cxx:659
TIterator * MakeIterator(Bool_t dir=kIterForward) const override
Return a list iterator.
Definition: TList.cxx:722
TMap implements an associative array of (key,value) pairs using a THashTable for efficient retrieval ...
Definition: TMap.h:40
void Add(TObject *obj) override
This function may not be used (but we need to provide it since it is a pure virtual in TCollection).
Definition: TMap.cxx:54
TObject * FindObject(const char *keyname) const override
Check if a (key,value) pair exists with keyname as name of the key.
Definition: TMap.cxx:215
TObject * Remove(TObject *key) override
Remove the (key,value) pair with key from the map.
Definition: TMap.cxx:296
TIterator * MakeIterator(Bool_t dir=kIterForward) const override
Create an iterator for TMap.
Definition: TMap.cxx:258
TObject * GetValue(const char *keyname) const
Returns a pointer to the value associated with keyname as name of the key.
Definition: TMap.cxx:236
An array of TObjects.
Definition: TObjArray.h:31
Int_t GetEntries() const override
Return the number of objects in array (i.e.
Definition: TObjArray.cxx:523
TObject * At(Int_t idx) const override
Definition: TObjArray.h:164
Collectable string class.
Definition: TObjString.h:28
TString & String()
Definition: TObjString.h:48
const char * GetName() const override
Returns name of object.
Definition: TObjString.h:38
Mother of all ROOT objects.
Definition: TObject.h:41
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:440
virtual TObject * Clone(const char *newname="") const
Make a clone of an object using the Streamer facility.
Definition: TObject.cxx:223
virtual TObject * FindObject(const char *name) const
Must be redefined in derived classes.
Definition: TObject.cxx:404
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:970
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:944
Class used by TMap to store (key,value) pairs.
Definition: TMap.h:102
Basic string class.
Definition: TString.h:136
Ssiz_t Length() const
Definition: TString.h:410
Ssiz_t First(char c) const
Find first occurrence of a character c.
Definition: TString.cxx:532
const char * Data() const
Definition: TString.h:369
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition: TString.cxx:2231
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:624
Long64_t Atoll() const
Return long long value of string.
Definition: TString.cxx:1981
virtual FILE * OpenPipe(const char *command, const char *mode)
Open a pipe.
Definition: TSystem.cxx:665
virtual int Rename(const char *from, const char *to)
Rename a file.
Definition: TSystem.cxx:1353
virtual int ClosePipe(FILE *pipe)
Close the pipe.
Definition: TSystem.cxx:674
virtual const char * BaseName(const char *pathname)
Base name of a file name. Base name of /user/root is root.
Definition: TSystem.cxx:937
virtual TString GetDirName(const char *pathname)
Return the directory name in pathname.
Definition: TSystem.cxx:1035
virtual int Unlink(const char *name)
Unlink, i.e.
Definition: TSystem.cxx:1384
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
Definition: TSystem.cxx:1485
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition: TUUID.h:42
const char * AsString() const
Return UUID as string. Copy string immediately since it will be reused.
Definition: TUUID.cxx:571
This class represents a WWW compatible URL.
Definition: TUrl.h:33
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition: TUrl.cxx:390
void SetOptions(const char *opt)
Definition: TUrl.h:87
const char * GetOptions() const
Definition: TUrl.h:71
XMLNodePointer_t GetChild(XMLNodePointer_t xmlnode, Bool_t realnode=kTRUE)
returns first child of xmlnode
XMLNodePointer_t DocGetRootElement(XMLDocPointer_t xmldoc)
returns root node of document
const char * GetNodeName(XMLNodePointer_t xmlnode)
returns name of xmlnode
const char * GetAttr(XMLNodePointer_t xmlnode, const char *name)
returns value of attribute for xmlnode
Definition: TXMLEngine.cxx:555
XMLDocPointer_t ParseFile(const char *filename, Int_t maxbuf=100000)
Parses content of file and tries to produce xml structures.
XMLNodePointer_t GetNext(XMLNodePointer_t xmlnode, Bool_t realnode=kTRUE)
return next to xmlnode node if realnode==kTRUE, any special nodes in between will be skipped
const Int_t n
Definition: legend1.C:16
static const std::string comment("comment")
Definition: file.py:1
Definition: first.py:1
const char * cnt
Definition: TXMLSetup.cxx:75