Logo ROOT   6.12/07
Reference Guide
TPerfStats.cxx
Go to the documentation of this file.
1 // @(#)root/proofplayer:$Id$
2 // Author: Kristjan Gulbrandsen 11/05/04
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2004, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /** \class TPerfStats
13 \ingroup proofkernel
14 
15 Provides the interface for the PROOF internal performance measurement
16 and event tracing.
17 
18 */
19 
20 #include "TPerfStats.h"
21 
22 #include "Riostream.h"
23 #include "TCollection.h"
24 #include "TEnv.h"
25 #include "TError.h"
26 #include "TFile.h"
27 #include "TH1.h"
28 #include "TH2.h"
29 #include "TDSet.h"
30 #include "TProofDebug.h"
31 #include "TProof.h"
32 #include "TProofServ.h"
33 #include "TSlave.h"
34 #include "TStatus.h"
35 #include "TTree.h"
36 #include "TSQLServer.h"
37 #include "TSQLResult.h"
38 #include "TParameter.h"
39 #include "TPluginManager.h"
40 #include "TROOT.h"
41 #include "TTimeStamp.h"
42 #include "TProofMonSender.h"
43 
46 
47 
48 //------------------------------------------------------------------------------
49 
50 ////////////////////////////////////////////////////////////////////////////////
51 /// Constructor
52 
54  : fEvtNode("-3"), fType(TVirtualPerfStats::kUnDefined), fSlave(),
55  fEventsProcessed(0), fBytesRead(0), fLen(0), fLatency(0.0), fProcTime(0.0), fCpuTime(0.0),
56  fIsStart(kFALSE), fIsOk(kFALSE)
57 {
58  if (gProofServ != 0) {
60  } else {
61  if (gProof && gProof->IsLite())
62  fEvtNode = "0";
63  else
64  fEvtNode = "-2"; // not on a PROOF server
65  }
66 
67  if (offset != 0) {
69  fTimeStamp.GetNanoSec() - offset->GetNanoSec());
70  }
71 }
72 
73 ////////////////////////////////////////////////////////////////////////////////
74 /// Compare method. Must return -1 if this is smaller than obj,
75 /// 0 if objects are equal and 1 if this is larger than obj.
76 
78 {
79  const TPerfEvent *pe = dynamic_cast<const TPerfEvent*>(obj);
80 
81  if (!pe) {
82  Error("Compare", "input is not a TPerfEvent object");
83  return 0;
84  }
85 
86  if (fTimeStamp < pe->fTimeStamp) {
87  return -1;
88  } else if (fTimeStamp == pe->fTimeStamp) {
89  return 0;
90  } else {
91  return 1;
92  }
93 }
94 
95 ////////////////////////////////////////////////////////////////////////////////
96 /// Dump content of this instance
97 
99 {
100  TString where;
101  if (fEvtNode == -2) {
102  where = "TPerfEvent: StandAlone ";
103  } else if ( fEvtNode == -1 ) {
104  where = "TPerfEvent: Master ";
105  } else {
106  where.Form("TPerfEvent: Worker %s ", fEvtNode.Data());
107  }
108  Printf("%s %s %f", where.Data(),
110 }
111 
114 
115 ////////////////////////////////////////////////////////////////////////////////
116 /// Normal constructor.
117 
119  : fTrace(0), fPerfEvent(0), fPacketsHist(0), fProcPcktHist(0),
120  fEventsHist(0), fNodeHist(0), fLatencyHist(0),
121  fProcTimeHist(0), fCpuTimeHist(0), fBytesRead(0),
122  fTotCpuTime(0.), fTotBytesRead(0), fTotEvents(0), fNumEvents(0),
123  fSlaves(0), fDoHist(kFALSE),
124  fDoTrace(kFALSE), fDoTraceRate(kFALSE), fDoSlaveTrace(kFALSE), fDoQuota(kFALSE),
125  fMonitorPerPacket(kFALSE), fMonSenders(3),
126  fDataSet("+++none+++"), fDataSetSize(-1), fOutput(output)
127 {
128  TProof *proof = (gProofServ) ? gProofServ->GetProof() : gProof;
129 
130  // Master flag
131  Bool_t isMaster = ((proof && proof->TestBit(TProof::kIsMaster)) ||
133  Bool_t isEndMaster = ((gProofServ && gProofServ->IsEndMaster()) ||
134  (proof && proof->IsLite())) ? kTRUE : kFALSE;
135 
136  TList *l = 0;
137  Bool_t deletel = kFALSE;
138  TParameter<Int_t> *dyns = (TParameter<Int_t> *) input->FindObject("PROOF_DynamicStartup");
139  if (dyns) {
140  // When starring up dynamically the number of slots needs to be guessed from the
141  // maximum workers request. There is no way to change this later on.
142  Int_t nwrks = dyns->GetVal();
143  if (nwrks > 0) {
144  l = new TList;
145  for (Int_t i = 0; i < nwrks; i++) {
146  TSlaveInfo *wi = new TSlaveInfo(TString::Format("0.%d", i));
148  l->Add(wi);
149  }
150  l->SetOwner(kTRUE);
151  deletel = kTRUE;
152  }
153  }
154  if (!l) l = proof ? proof->GetListOfSlaveInfos() : 0 ;
155 
156  TIter nextslaveinfo(l);
157  while (TSlaveInfo *si = dynamic_cast<TSlaveInfo*>(nextslaveinfo()))
158  if (si->fStatus == TSlaveInfo::kActive) fSlaves++;
159 
160  PDB(kMonitoring,1) Info("TPerfStats", "Statistics for %d slave(s)", fSlaves);
161 
162  fDoHist = (input->FindObject("PROOF_StatsHist") != 0);
163  fDoTrace = (input->FindObject("PROOF_StatsTrace") != 0);
164  fDoTraceRate = (input->FindObject("PROOF_RateTrace") != 0);
165  fDoSlaveTrace = (input->FindObject("PROOF_SlaveStatsTrace") != 0);
166  PDB(kMonitoring,1)
167  Info("TPerfStats", "master:%d hist:%d,trace:%d,rate:%d,wrktrace:%d",
169 
170  // Check per packet monitoring
171  Int_t perpacket = -1;
172  if (TProof::GetParameter(input, "PROOF_MonitorPerPacket", perpacket) != 0) {
173  // Check if there is a global monitor-per-packet setting
174  perpacket = gEnv->GetValue("Proof.MonitorPerPacket", 0);
175  }
176  fMonitorPerPacket = (perpacket == 1) ? kTRUE : kFALSE;
177  if (fMonitorPerPacket)
178  Info("TPerfStats", "sending full information after each packet");
179 
180  // Extract the name of the dataset
181  TObject *o = 0;
182  TIter nxi(input);
183  while ((o = nxi()))
184  if (!strncmp(o->ClassName(), "TDSet", strlen("TDSet"))) break;
185  if (o) {
186  fDSet = (TDSet *) o;
188  if (fDataSetSize > 0) {
189  fDataSet = "";
190  TString grus = (gProofServ) ? TString::Format("/%s/%s/", gProofServ->GetGroup(),
191  gProofServ->GetUser()) : TString("");
192  TString dss = fDSet->GetName(), ds;
193  Ssiz_t fd = 0, nq = kNPOS;
194  while (dss.Tokenize(ds, fd, "[,| ]")) {
195  if ((nq = ds.Index("?")) != kNPOS) ds.Remove(nq);
196  ds.ReplaceAll(grus, "");
197  if (!fDataSet.IsNull()) fDataSet += ",";
198  fDataSet += ds;
199  }
200  }
201  }
202 
203  // Dataset string limited in length: get the authorized size
204  fDataSetLen = gEnv->GetValue("Proof.Monitor.DataSetLen", 512);
205  if (fDataSetLen != 512)
206  Info("TPerfStats", "dataset string length truncated to %d chars", fDataSetLen);
208  //
209  PDB(kMonitoring,1)
210  Info("TPerfStats", "dataset: '%s', # files: %d", fDataSet.Data(), fDataSetSize);
211 
212  if ((isMaster && (fDoTrace || fDoTraceRate)) || (!isMaster && fDoSlaveTrace)) {
213  // Construct tree
214  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_PerfStats"));
215  fTrace = new TTree("PROOF_PerfStats", "PROOF Statistics");
216  fTrace->SetDirectory(0);
217  fTrace->Bronch("PerfEvents", "TPerfEvent", &fPerfEvent, 64000, 0);
218  output->Add(fTrace);
219  PDB(kMonitoring,1)
220  Info("TPerfStats", "tree '%s' added to the output list", fTrace->GetName());
221  }
222 
223  if (fDoHist && isEndMaster) {
224  // Make Histograms
225  Double_t time_per_bin = 1e-3; // 10ms
226  Double_t min_time = 0;
227  Int_t ntime_bins = 1000;
228 
229  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_PacketsHist"));
230  fPacketsHist = new TH1D("PROOF_PacketsHist", "Packets processed per Worker",
231  fSlaves, 0, fSlaves);
235  output->Add(fPacketsHist);
236  PDB(kMonitoring,1)
237  Info("TPerfStats", "histo '%s' added to the output list", fPacketsHist->GetName());
238 
239  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_ProcPcktHist"));
240  fProcPcktHist = new TH1I("PROOF_ProcPcktHist", "Packets being processed per Worker",
241  fSlaves, 0, fSlaves);
245  output->Add(fProcPcktHist);
246  PDB(kMonitoring,1)
247  Info("TPerfStats", "histo '%s' added to the output list", fProcPcktHist->GetName());
248 
249  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_EventsHist"));
250  fEventsHist = new TH1D("PROOF_EventsHist", "Events processed per Worker",
251  fSlaves, 0, fSlaves);
255  output->Add(fEventsHist);
256  PDB(kMonitoring,1)
257  Info("TPerfStats", "histo '%s' added to the output list", fEventsHist->GetName());
258 
259  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_NodeHist"));
260  fNodeHist = new TH1D("PROOF_NodeHist", "Slaves per Fileserving Node",
261  fSlaves, 0, fSlaves);
263  fNodeHist->SetMinimum(0);
265  output->Add(fNodeHist);
266  PDB(kMonitoring,1)
267  Info("TPerfStats", "histo '%s' added to the output list", fNodeHist->GetName());
268 
269  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_LatencyHist"));
270  fLatencyHist = new TH2D("PROOF_LatencyHist", "GetPacket Latency per Worker",
271  fSlaves, 0, fSlaves,
272  ntime_bins, min_time, time_per_bin);
276  output->Add(fLatencyHist);
277  PDB(kMonitoring,1)
278  Info("TPerfStats", "histo '%s' added to the output list", fLatencyHist->GetName());
279 
280  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_ProcTimeHist"));
281  fProcTimeHist = new TH2D("PROOF_ProcTimeHist", "Packet Processing Time per Worker",
282  fSlaves, 0, fSlaves,
283  ntime_bins, min_time, time_per_bin);
287  output->Add(fProcTimeHist);
288  PDB(kMonitoring,1)
289  Info("TPerfStats", "histo '%s' added to the output list", fProcTimeHist->GetName());
290 
291  gDirectory->RecursiveRemove(gDirectory->FindObject("PROOF_CpuTimeHist"));
292  fCpuTimeHist = new TH2D("PROOF_CpuTimeHist", "Packet CPU Time per Worker",
293  fSlaves, 0, fSlaves,
294  ntime_bins, min_time, time_per_bin);
298  output->Add(fCpuTimeHist);
299  PDB(kMonitoring,1)
300  Info("TPerfStats", "histo '%s' added to the output list", fCpuTimeHist->GetName());
301 
302  nextslaveinfo.Reset();
303  Int_t slavebin=1;
304  while (TSlaveInfo *si = dynamic_cast<TSlaveInfo*>(nextslaveinfo())) {
305  if (si->fStatus == TSlaveInfo::kActive) {
306  fPacketsHist->GetXaxis()->SetBinLabel(slavebin, si->GetOrdinal());
307  fProcPcktHist->GetXaxis()->SetBinLabel(slavebin, si->GetOrdinal());
308  fEventsHist->GetXaxis()->SetBinLabel(slavebin, si->GetOrdinal());
309  fNodeHist->GetXaxis()->SetBinLabel(slavebin, si->GetOrdinal());
310  fLatencyHist->GetXaxis()->SetBinLabel(slavebin, si->GetOrdinal());
311  fProcTimeHist->GetXaxis()->SetBinLabel(slavebin, si->GetOrdinal());
312  fCpuTimeHist->GetXaxis()->SetBinLabel(slavebin, si->GetOrdinal());
313  slavebin++;
314  }
315  }
316  }
317  // Cleanup
318  if (deletel) delete(l);
319 
320  if (isMaster) {
321 
322  // Monitoring for query performances using monitoring system (e.g. Monalisa, SQL, ...)
323  //
324  // We support multiple specifications separated by ',' or '|' or '\' (the latter need
325  // top be escaped three times in the regular experession), e.g.
326  // ProofServ.Monitoring: Monalisa bla bla bla,
327  // +ProofServ.Monitoring: SQL blu blu blu
328 
329  TString mons = gEnv->GetValue("ProofServ.Monitoring", ""), mon;
330  Ssiz_t fmon = 0;
331  TProofMonSender *monSender = 0;
332  while (mons.Tokenize(mon, fmon, "[,|\\\\]")) {
333  if (mon != "") {
334  // Extract arguments (up to 9 'const char *')
335  TString a[10];
336  Int_t from = 0;
337  TString tok, sendopts;
338  Int_t na = 0;
339  while (mon.Tokenize(tok, from, " ")) {
340  if (tok.BeginsWith("sendopts:")) {
341  tok.ReplaceAll("sendopts:", "");
342  sendopts = tok;
343  } else {
344  a[na++] = tok;
345  }
346  }
347  na--;
348  // Get monitor object from the plugin manager
349  TPluginHandler *h = 0;
350  if ((h = gROOT->GetPluginManager()->FindHandler("TProofMonSender", a[0]))) {
351  if (h->LoadPlugin() != -1) {
352  monSender =
353  (TProofMonSender *) h->ExecPlugin(na, a[1].Data(), a[2].Data(), a[3].Data(),
354  a[4].Data(), a[5].Data(), a[6].Data(),
355  a[7].Data(), a[8].Data(), a[9].Data());
356  if (monSender && monSender->TestBit(TObject::kInvalidObject)) SafeDelete(monSender);
357  if (monSender && monSender->SetSendOptions(sendopts) != 0) SafeDelete(monSender);
358  }
359  }
360  }
361 
362  if (monSender) {
363  fMonSenders.Add(monSender);
364  PDB(kMonitoring,1)
365  Info("TPerfStats", "created monitoring object: %s - # of active monitors: %d",
366  mon.Data(), fMonSenders.GetEntries());
367  fDoQuota = kTRUE;
368  }
369  monSender = 0;
370  }
371  }
372 }
373 
374 ////////////////////////////////////////////////////////////////////////////////
375 /// Destructor
376 
378 {
379  // Shutdown the monitor writers, if any
382 }
383 
384 ////////////////////////////////////////////////////////////////////////////////
385 /// Simple event.
386 
388 {
389  if (type == kStop && fPacketsHist != 0) {
391  fPacketsHist->LabelsOption("auv","X");
392  }
393 
394  if (type == kStop && fDoQuota)
395  WriteQueryLog();
396 
397  if (fTrace == 0) return;
398 
399  TPerfEvent pe(&fTzero);
400  pe.fType = type;
401 
402  fPerfEvent = &pe;
403  fTrace->SetBranchAddress("PerfEvents",&fPerfEvent);
404  fTrace->Fill();
405  fPerfEvent = 0;
406 }
407 
408 ////////////////////////////////////////////////////////////////////////////////
409 /// Packet event.
410 /// See WriteQueryLog for the descripition of the structure sent for monitoring
411 /// when fMonitorPerPacket is kTRUE.
412 
413 void TPerfStats::PacketEvent(const char *slave, const char* slavename, const char* filename,
414  Long64_t eventsprocessed, Double_t latency, Double_t proctime,
415  Double_t cputime, Long64_t bytesRead)
416 {
417  if (fDoTrace && fTrace != 0) {
418  TPerfEvent pe(&fTzero);
419 
420  pe.fType = kPacket;
421  pe.fSlaveName = slavename;
422  pe.fFileName = filename;
423  pe.fSlave = slave;
424  pe.fEventsProcessed = eventsprocessed;
425  pe.fBytesRead = bytesRead;
426  pe.fLatency = latency;
427  pe.fProcTime = proctime;
428  pe.fCpuTime = cputime;
429 
430  fPerfEvent = &pe;
431  fTrace->SetBranchAddress("PerfEvents",&fPerfEvent);
432  fTrace->Fill();
433  fPerfEvent = 0;
434  }
435 
436  PDB(kMonitoring,1)
437  Info("PacketEvent","%s: fDoHist: %d, fPacketsHist: %p, eventsprocessed: %lld",
438  slave, fDoHist, fPacketsHist, eventsprocessed);
439 
440  if (fDoHist && fPacketsHist != 0) {
441  fPacketsHist->Fill(slave, 1);
442  fEventsHist->Fill(slave, eventsprocessed);
443  fLatencyHist->Fill(slave, latency, 1);
444  fProcTimeHist->Fill(slave, proctime, 1);
445  fCpuTimeHist->Fill(slave, cputime, 1);
446  }
447 
448  if (fDoQuota) {
449  fTotCpuTime += cputime;
450  fTotBytesRead += bytesRead;
451  fTotEvents += eventsprocessed;
452  }
453 
454  // Write to monitoring system, if requested
458  if (!gProofServ || !gProofServ->GetSessionTag() || !gProofServ->GetProof() || !qr) {
459  Error("PacketEvent", "some required object are undefined (%p %p %p %p)",
461  (gProofServ ? gProofServ->GetProof() : 0),
462  ((gProofServ && gProofServ->GetProof()) ? qr : 0));
463  return;
464  }
465 
466  TTimeStamp stop;
467  TString identifier;
468  identifier.Form("%s-q%d", gProofServ->GetSessionTag(), qr->GetSeqNum());
469 
470  TList values;
471  values.SetOwner();
472  values.Add(new TParameter<int>("id", 0));
473  values.Add(new TNamed("user", gProofServ->GetUser()));
474  values.Add(new TNamed("proofgroup", gProofServ->GetGroup()));
475  values.Add(new TNamed("begin", fTzero.AsString("s")));
476  values.Add(new TNamed("end", stop.AsString("s")));
477  values.Add(new TParameter<int>("walltime", stop.GetSec()-fTzero.GetSec()));
478  values.Add(new TParameter<Long64_t>("bytesread", fTotBytesRead));
479  values.Add(new TParameter<Long64_t>("events", fTotEvents));
480  values.Add(new TParameter<Long64_t>("totevents", fNumEvents));
481  values.Add(new TParameter<int>("workers", fSlaves));
482  values.Add(new TNamed("querytag", identifier.Data()));
483 
484  // Memory usage on workers
485  TStatus *pst = (fOutput) ? (TStatus *) fOutput->FindObject("PROOF_Status") : 0;
486  // This most likely will be always NULL when sending from GetNextPacket ...
487  Long64_t vmxw = (pst) ? (Long64_t) pst->GetVirtMemMax() : -1;
488  Long64_t rmxw = (pst) ? (Long64_t) pst->GetResMemMax() : -1;
489  values.Add(new TParameter<Long64_t>("vmemmxw", vmxw));
490  values.Add(new TParameter<Long64_t>("rmemmxw", rmxw));
491  // Memory usage on master
492  values.Add(new TParameter<Long64_t>("vmemmxm", (Long64_t) fgVirtMemMax));
493  values.Add(new TParameter<Long64_t>("rmemmxm", (Long64_t) fgResMemMax));
494  // Dataset information
495  values.Add(new TNamed("dataset", fDataSet.Data()));
496  values.Add(new TParameter<int>("numfiles", fDataSetSize));
497  // Missing files
498  TList *mfls = (fOutput) ? (TList *) fOutput->FindObject("MissingFiles") : 0;
499  Int_t nmiss = (mfls && mfls->GetSize() > 0) ? mfls->GetSize() : 0;
500  values.Add(new TParameter<int>("missfiles", nmiss));
501  // Query status
502  Int_t est = (pst) ? pst->GetExitStatus() : -1;
503  values.Add(new TParameter<int>("status", est));
504  // Root version
505  TString rver = TString::Format("%s|%s", gROOT->GetVersion(), gROOT->GetGitCommit());
506  values.Add(new TNamed("rootver", rver.Data()));
507 
508  for (Int_t i = 0; i < fMonSenders.GetEntries(); i++) {
510  if (m) {
511  // Send query summary
512  if (m->SendSummary(&values, identifier) != 0)
513  Error("PacketEvent", "sending of summary info failed (%s)", m->GetName());
514  } else {
515  Warning("PacketEvent", "undefined entry found in monitors array for id: %d", i);
516  }
517  }
518  }
519 }
520 
521 ////////////////////////////////////////////////////////////////////////////////
522 /// File event.
523 
524 void TPerfStats::FileEvent(const char *slave, const char *slavename, const char *nodename,
525  const char *filename, Bool_t isStart)
526 {
527  if (fDoTrace && fTrace != 0) {
528  TPerfEvent pe(&fTzero);
529 
530  pe.fType = kFile;
531  pe.fSlaveName = slavename;
532  pe.fNodeName = nodename;
533  pe.fFileName = filename;
534  pe.fSlave = slave;
535  pe.fIsStart = isStart;
536 
537  fPerfEvent = &pe;
538  fTrace->SetBranchAddress("PerfEvents",&fPerfEvent);
539  fTrace->Fill();
540  fPerfEvent = 0;
541  }
542 
543  if (fDoHist && fPacketsHist != 0) {
544  fNodeHist->Fill(nodename, isStart ? 1 : -1);
545  }
546 }
547 
548 ////////////////////////////////////////////////////////////////////////////////
549 /// Open file event.
550 
551 void TPerfStats::FileOpenEvent(TFile *file, const char *filename, Double_t start)
552 {
553  if (fDoTrace && fTrace != 0) {
554  TPerfEvent pe(&fTzero);
555 
556  pe.fType = kFileOpen;
557  pe.fFileName = filename;
558  pe.fFileClass = file != 0 ? file->ClassName() : "none";
559  pe.fProcTime = double(TTimeStamp())-start;
560  pe.fIsOk = (file != 0);
561 
562  fPerfEvent = &pe;
563  fTrace->SetBranchAddress("PerfEvents",&fPerfEvent);
564  fTrace->Fill();
565  fPerfEvent = 0;
566  }
567 }
568 
569 ////////////////////////////////////////////////////////////////////////////////
570 /// Read file event.
571 
573 {
574  if (fDoTrace && fTrace != 0) {
575  TPerfEvent pe(&fTzero);
576 
577  pe.fType = kFileRead;
578  pe.fFileName = file->GetName();
579  pe.fFileClass = file->ClassName();
580  pe.fLen = len;
581  pe.fProcTime = double(TTimeStamp())-start;
582 
583  fPerfEvent = &pe;
584  fTrace->SetBranchAddress("PerfEvents",&fPerfEvent);
585  fTrace->Fill();
586  fPerfEvent = 0;
587  }
588 }
589 
590 ////////////////////////////////////////////////////////////////////////////////
591 /// Record TTree file unzip event.
592 /// start is the TimeStamp before unzip
593 /// pos is where in the file the compressed buffer came from
594 /// complen is the length of the compressed buffer
595 /// objlen is the length of the de-compressed buffer
596 
597 void TPerfStats::UnzipEvent(TObject * /* tree */, Long64_t /* pos */,
598  Double_t /* start */, Int_t /* complen */,
599  Int_t /* objlen */)
600 {
601  // Do nothing for now.
602 }
603 
604 ////////////////////////////////////////////////////////////////////////////////
605 /// Rate event.
606 
607 void TPerfStats::RateEvent(Double_t proctime, Double_t deltatime,
608  Long64_t eventsprocessed, Long64_t bytesRead)
609 {
610  if ((fDoTrace || fDoTraceRate) && fTrace != 0) {
611  TPerfEvent pe(&fTzero);
612 
613  pe.fType = kRate;
614  pe.fEventsProcessed = eventsprocessed;
615  pe.fBytesRead = bytesRead;
616  pe.fProcTime = proctime;
617  pe.fLatency = deltatime;
618 
619  fPerfEvent = &pe;
620  fTrace->SetBranchAddress("PerfEvents",&fPerfEvent);
621  fTrace->Fill();
622  fPerfEvent = 0;
623  }
624 }
625 
626 ////////////////////////////////////////////////////////////////////////////////
627 /// Set number of bytes read.
628 
630 {
631  fBytesRead = num;
632 }
633 
634 ////////////////////////////////////////////////////////////////////////////////
635 /// Get number of bytes read.
636 
638 {
639  return fBytesRead;
640 }
641 
642 ////////////////////////////////////////////////////////////////////////////////
643 /// Send to the connected monitoring servers information related to this query.
644 /// The information is of three types: 'summary', 'dataset' and 'files'.
645 /// Actual 'table' formatting is done by the relevant sender, implementation of
646 /// TProofMonSender, where the details are given.
647 
649 {
650  TTimeStamp stop;
651 
652  // Write to monitoring system
653  if (!fMonSenders.IsEmpty()) {
656  if (!gProofServ || !gProofServ->GetSessionTag() || !gProofServ->GetProof() || !qr) {
657  Error("WriteQueryLog", "some required object are undefined (%p %p %p %p)",
659  (gProofServ ? gProofServ->GetProof() : 0),
660  ((gProofServ && gProofServ->GetProof()) ? qr : 0));
661  return;
662  }
663 
664  TString identifier;
665  identifier.Form("%s-q%d", gProofServ->GetSessionTag(), qr->GetSeqNum());
666 
667  TList values;
668  values.SetOwner();
669  values.Add(new TParameter<int>("id", 0));
670  values.Add(new TNamed("user", gProofServ->GetUser()));
671  values.Add(new TNamed("proofgroup", gProofServ->GetGroup()));
672  values.Add(new TNamed("begin", fTzero.AsString("s")));
673  values.Add(new TNamed("end", stop.AsString("s")));
674  values.Add(new TParameter<int>("walltime", stop.GetSec()-fTzero.GetSec()));
675  values.Add(new TParameter<float>("cputime", fTotCpuTime));
676  values.Add(new TParameter<Long64_t>("bytesread", fTotBytesRead));
677  values.Add(new TParameter<Long64_t>("events", fTotEvents));
678  values.Add(new TParameter<Long64_t>("totevents", fTotEvents));
679  values.Add(new TParameter<int>("workers", fSlaves));
680  values.Add(new TNamed("querytag", identifier.Data()));
681 
682  TList *mfls = (fOutput) ? (TList *) fOutput->FindObject("MissingFiles") : 0;
683  // Memory usage on workers
684  TStatus *pst = (fOutput) ? (TStatus *) fOutput->FindObject("PROOF_Status") : 0;
685  Long64_t vmxw = (pst) ? (Long64_t) pst->GetVirtMemMax() : -1;
686  Long64_t rmxw = (pst) ? (Long64_t) pst->GetResMemMax() : -1;
687  values.Add(new TParameter<Long64_t>("vmemmxw", vmxw));
688  values.Add(new TParameter<Long64_t>("rmemmxw", rmxw));
689  // Memory usage on master
690  values.Add(new TParameter<Long64_t>("vmemmxm", (Long64_t) fgVirtMemMax));
691  values.Add(new TParameter<Long64_t>("rmemmxm", (Long64_t) fgResMemMax));
692  // Dataset information
693  values.Add(new TNamed("dataset", fDataSet.Data()));
694  values.Add(new TParameter<int>("numfiles", fDataSetSize));
695  // Missing files
696  Int_t nmiss = (mfls && mfls->GetSize() > 0) ? mfls->GetSize() : 0;
697  values.Add(new TParameter<int>("missfiles", nmiss));
698  // Query status
699  Int_t est = (pst) ? pst->GetExitStatus() : -1;
700  values.Add(new TParameter<int>("status", est));
701  // Root version
702  TString rver = TString::Format("%s|%s", gROOT->GetVersion(), gROOT->GetGitCommit());
703  values.Add(new TNamed("rootver", rver.Data()));
704 
705  for (Int_t i = 0; i < fMonSenders.GetEntries(); i++) {
707  if (m) {
708  // Send query summary
709  if (m->SendSummary(&values, identifier) != 0)
710  Error("WriteQueryLog", "sending of summary info failed (%s)", m->GetName());
711  // Send dataset information
712  if (m->SendDataSetInfo(fDSet, mfls, fTzero.AsString("s"), identifier) != 0)
713  Error("WriteQueryLog", "sending of dataset info failed (%s)", m->GetName());
714  // Send file information
715  if (m->SendFileInfo(fDSet, mfls, fTzero.AsString("s"), identifier) != 0)
716  Error("WriteQueryLog", "sending of files info failed (%s)", m->GetName());
717  } else {
718  Warning("WriteQueryLog", "undefined entry found in monitors array for id: %d", i);
719  }
720  }
721  }
722 }
723 
724 ////////////////////////////////////////////////////////////////////////////////
725 /// Setup the PROOF input list with requested statistics and tracing options.
726 
728 {
729  const Int_t ntags=3;
730  const char *tags[ntags] = {"StatsHist", "StatsTrace", "SlaveStatsTrace"};
731 
732  TString varname, parname;
733  for (Int_t i=0; i<ntags; i++) {
734  varname.Form("Proof.%s", tags[i]);
735  parname.Form("PROOF_%s", tags[i]);
736  if (!input->FindObject(parname))
737  if (gEnv->GetValue(varname, 0)) input->Add(new TNamed(parname.Data(),""));
738  }
739 }
740 
741 ////////////////////////////////////////////////////////////////////////////////
742 /// Initialize PROOF statistics run.
743 
745 {
746  if (gPerfStats)
747  delete gPerfStats;
748  fgVirtMemMax = -1;
749  fgResMemMax = -1;
751 
752  gPerfStats = new TPerfStats(input, output);
753  if (gPerfStats && !gPerfStats->TestBit(TObject::kInvalidObject)) {
754  // This measures the time taken by the constructor: not negligeable ...
756  } else {
758  }
759 }
760 
761 ////////////////////////////////////////////////////////////////////////////////
762 /// Terminate the PROOF statistics run.
763 
765 {
766  if (!gPerfStats) return;
767 
769  gPerfStats->SimpleEvent(TVirtualPerfStats::kStop);
770 
771  delete gPerfStats;
772  gPerfStats = 0;
773 }
774 
775 ////////////////////////////////////////////////////////////////////////////////
776 /// Record memory usage
777 
779 {
780  ProcInfo_t pi;
781  if (!gSystem->GetProcInfo(&pi)){
784  }
785 }
786 
787 ////////////////////////////////////////////////////////////////////////////////
788 /// Get memory usage
789 
791 {
792  vmax = fgVirtMemMax;
793  rmax = fgResMemMax;
794 }
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
Bool_t fIsOk
Definition: TPerfStats.h:57
virtual Int_t Fill(Double_t x)
Increment bin with abscissa X by 1.
Definition: TH1.cxx:3244
TVirtualPerfStats::EEventType fType
Definition: TPerfStats.h:44
THist< 1, int, THistStatContent > TH1I
Definition: THist.hxx:287
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:854
long long Long64_t
Definition: RtypesCore.h:69
static constexpr double pi
auto * m
Definition: textangle.C:8
virtual void LabelsOption(Option_t *option="h", Option_t *axis="X")
Set option(s) to draw axis with labels.
Definition: TH1.cxx:5059
Bool_t fDoHist
number of active slaves
Definition: TPerfStats.h:92
TH1D * fEventsHist
histogram of packets being processed per slave
Definition: TPerfStats.h:80
TList * fOutput
Saved pointer to the TDSet object.
Definition: TPerfStats.h:106
virtual void SetDirectory(TDirectory *dir)
By default when an histogram is created, it is added to the list of histogram objects in the current ...
Definition: TH1.cxx:8194
Provides the interface for the PROOF internal performance measurement and event tracing.
const char Option_t
Definition: RtypesCore.h:62
virtual void Delete(Option_t *option="")
Remove all objects from the array AND delete all heap based objects.
Definition: TObjArray.cxx:355
Definition: Rtypes.h:59
Int_t SetSendOptions(const char *)
Parse send options from string &#39;sendopts&#39;.
const Ssiz_t kNPOS
Definition: RtypesCore.h:111
TString & ReplaceAll(const TString &s1, const TString &s2)
Definition: TString.h:638
TObject * GetParameter(const char *par) const
Get specified parameter.
Definition: TProof.cxx:9890
virtual Int_t Fill()
Fill all branches.
Definition: TTree.cxx:4364
This class implements a data set to be used for PROOF processing.
Definition: TDSet.h:151
const char * GetGroup() const
Definition: TProofServ.h:242
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
static const char * EventType(EEventType type)
Return the name of the event type.
TH1 * h
Definition: legend2.C:5
TDSet * fDSet
of files in the dataset
Definition: TPerfStats.h:105
Int_t GetNumOfFiles()
Return the number of files in the dataset.
Definition: TDSet.cxx:1992
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:46
PyObject * fTrace
Long_t GetResMemMax(Bool_t master=kFALSE) const
Definition: TStatus.h:63
Bool_t fIsStart
Definition: TPerfStats.h:56
virtual void SetMinimum(Double_t minimum=-1111)
Definition: TH1.h:391
#define gROOT
Definition: TROOT.h:402
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition: TObject.h:172
TPerfStats(TList *input, TList *output)
Max resident memory used by this process.
Definition: TPerfStats.cxx:118
Int_t LoadPlugin()
Load the plugin library for this handler.
TString fSlaveName
Definition: TPerfStats.h:45
Basic string class.
Definition: TString.h:125
Int_t GetExitStatus() const
Definition: TStatus.h:62
Bool_t IsEmpty() const
Definition: TObjArray.h:70
int Int_t
Definition: RtypesCore.h:41
bool Bool_t
Definition: RtypesCore.h:59
time_t GetSec() const
Definition: TTimeStamp.h:135
TTimeStamp fTzero
TTree with trace events.
Definition: TPerfStats.h:76
TString fNodeName
Definition: TPerfStats.h:46
Definition: Rtypes.h:59
Long64_t fNumEvents
total number of events processed
Definition: TPerfStats.h:89
Double_t fCpuTime
Definition: TPerfStats.h:55
Int_t GetSeqNum() const
Definition: TQueryResult.h:115
static Long_t fgVirtMemMax
Saved pointer to the output list.
Definition: TPerfStats.h:108
void Reset()
Definition: TCollection.h:250
Bool_t IsLite() const
Definition: TProof.h:933
Long_t GetVirtMemMax(Bool_t master=kFALSE) const
Definition: TStatus.h:64
virtual TObject * FindObject(const char *name) const
Delete a TObjLink object.
Definition: TList.cxx:574
void WriteQueryLog()
Send to the connected monitoring servers information related to this query.
Definition: TPerfStats.cxx:648
void PacketEvent(const char *slave, const char *slavename, const char *filename, Long64_t eventsprocessed, Double_t latency, Double_t proctime, Double_t cputime, Long64_t bytesRead)
Packet event.
Definition: TPerfStats.cxx:413
#define SafeDelete(p)
Definition: RConfig.h:509
TH1D * fNodeHist
histogram of events processed per slave
Definition: TPerfStats.h:81
void RateEvent(Double_t proctime, Double_t deltatime, Long64_t eventsprocessed, Long64_t bytesRead)
Rate event.
Definition: TPerfStats.cxx:607
virtual const char * ClassName() const
Returns name of class to which the object belongs.
Definition: TObject.cxx:128
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString...
Definition: TString.cxx:2365
#define PDB(mask, level)
Definition: TProofDebug.h:56
virtual Int_t SetBranchAddress(const char *bname, void *add, TBranch **ptr=0)
Change branch address, dealing with clone trees properly.
Definition: TTree.cxx:7898
This class holds the status of an ongoing operation and collects error messages.
Definition: TStatus.h:32
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:29
virtual Int_t SendSummary(TList *, const char *)=0
TH2D * fLatencyHist
histogram of slaves per file serving node
Definition: TPerfStats.h:82
TH1D * fPacketsHist
TPerfEvent used to fill tree.
Definition: TPerfStats.h:78
void SimpleEvent(EEventType type)
Simple event.
Definition: TPerfStats.cxx:387
Bool_t fDoQuota
Full tracing in workers.
Definition: TPerfStats.h:96
const char * AsString(const Option_t *option="") const
Return the date & time as a string.
Definition: TTimeStamp.cxx:271
TString fEvtNode
Definition: TPerfStats.h:42
TObjArray fMonSenders
Whether to send the full entry per each packet.
Definition: TPerfStats.h:100
TQueryResult * GetQueryResult(const char *ref=0)
Return pointer to the full TQueryResult instance owned by the player and referenced by &#39;ref&#39;...
Definition: TProof.cxx:2126
void SetBytesRead(Long64_t num)
Set number of bytes read.
Definition: TPerfStats.cxx:629
static void GetMemValues(Long_t &vmax, Long_t &rmax)
Get memory usage.
Definition: TPerfStats.cxx:790
Double_t fTotCpuTime
track bytes read of main file
Definition: TPerfStats.h:86
Bool_t fDoSlaveTrace
Trace processing rate in master.
Definition: TPerfStats.h:95
A container class for query results.
Definition: TQueryResult.h:36
TList * GetListOfSlaveInfos()
Returns list of TSlaveInfo&#39;s. In case of error return 0.
Definition: TProof.cxx:2299
static void Start(TList *input, TList *output)
Initialize PROOF statistics run.
Definition: TPerfStats.cxx:744
A doubly linked list.
Definition: TList.h:44
TH1I * fProcPcktHist
histogram of packets processed per slave
Definition: TPerfStats.h:79
void UnzipEvent(TObject *tree, Long64_t pos, Double_t start, Int_t complen, Int_t objlen)
Record TTree file unzip event.
Definition: TPerfStats.cxx:597
TString fDataSet
Monitoring engines.
Definition: TPerfStats.h:102
const char * GetSessionTag() const
Definition: TProofServ.h:245
static void Setup(TList *input)
Setup the PROOF input list with requested statistics and tracing options.
Definition: TPerfStats.cxx:727
Named parameter, streamable and storable.
Definition: TParameter.h:37
Long64_t fTotEvents
total bytes read on all slaves
Definition: TPerfStats.h:88
Bool_t IsEndMaster() const
Definition: TProofServ.h:292
Bool_t fMonitorPerPacket
Save stats on SQL server for quota management.
Definition: TPerfStats.h:98
virtual Int_t SendDataSetInfo(TDSet *, TList *, const char *, const char *)=0
R__EXTERN TSystem * gSystem
Definition: TSystem.h:540
if object ctor succeeded but object should not be used
Definition: TObject.h:68
virtual void SetFillColor(Color_t fcolor)
Set the fill area color.
Definition: TAttFill.h:37
TPerfEvent * fPerfEvent
start time of this run
Definition: TPerfStats.h:77
Long64_t fEventsProcessed
Definition: TPerfStats.h:50
auto * a
Definition: textangle.C:12
Double_t fProcTime
Definition: TPerfStats.h:54
Int_t fSlaves
total number of events to be processed
Definition: TPerfStats.h:90
Long_t ExecPlugin(int nargs, const T &... params)
Long_t fMemVirtual
Definition: TSystem.h:197
Bool_t BeginsWith(const char *s, ECaseCompare cmp=kExact) const
Definition: TString.h:561
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2343
virtual void LabelsDeflate(Option_t *axis="X")
Reduce the number of bins for the axis passed in the option to the number of bins having a label...
Definition: TH1.cxx:4928
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:880
Ssiz_t Length() const
Definition: TString.h:386
TString fFileName
Definition: TPerfStats.h:47
Long64_t fBytesRead
histogram of cpu time spent processing packets
Definition: TPerfStats.h:85
virtual void SetMarkerStyle(Style_t mstyle=1)
Set the marker style.
Definition: TAttMarker.h:40
Int_t fDataSetSize
Maximum size of the dataset string fDataSet.
Definition: TPerfStats.h:104
void SetStatus(ESlaveStatus stat)
Definition: TProof.h:234
static Long_t fgResMemMax
Max virtual memory used by this process.
Definition: TPerfStats.h:109
virtual TBranch * Bronch(const char *name, const char *classname, void *addobj, Int_t bufsize=32000, Int_t splitlevel=99)
Create a new TTree BranchElement.
Definition: TTree.cxx:2264
Int_t Compare(const TObject *obj) const
Compare method.
Definition: TPerfStats.cxx:77
#define Printf
Definition: TGeoToOCC.h:18
#define gPerfStats
const Bool_t kFALSE
Definition: RtypesCore.h:88
PyObject * fType
long Long_t
Definition: RtypesCore.h:50
int Ssiz_t
Definition: RtypesCore.h:63
R__EXTERN TProof * gProof
Definition: TProof.h:1077
Long64_t GetBytesRead() const
Get number of bytes read.
Definition: TPerfStats.cxx:637
TTimeStamp fTimeStamp
Definition: TPerfStats.h:43
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition: TString.cxx:2251
virtual void SetDirectory(TDirectory *dir)
Change the tree&#39;s directory.
Definition: TTree.cxx:8464
TString fSlave
Definition: TPerfStats.h:49
#define ClassImp(name)
Definition: Rtypes.h:359
double Double_t
Definition: RtypesCore.h:55
int type
Definition: TGX11.cxx:120
R__EXTERN TEnv * gEnv
Definition: TEnv.h:171
Provides the interface for the PROOF internal performance measurement and event tracing.
Definition: TPerfStats.h:70
The TTimeStamp encapsulates seconds and ns since EPOCH.
Definition: TTimeStamp.h:71
This class controls a Parallel ROOT Facility, PROOF, cluster.
Definition: TProof.h:316
TString fFileClass
Definition: TPerfStats.h:48
TPerfEvent(TTimeStamp *offset=0)
Constructor.
Definition: TPerfStats.cxx:53
you should not use this method at all Int_t Int_t Double_t Double_t Double_t e
Definition: TRolke.cxx:630
virtual Int_t SendFileInfo(TDSet *, TList *, const char *, const char *)=0
Int_t fDataSetLen
Dataset string.
Definition: TPerfStats.h:103
THist< 2, double, THistStatContent, THistStatUncertainty > TH2D
Definition: THist.hxx:290
virtual void SetBinLabel(Int_t bin, const char *label)
Set label for bin.
Definition: TAxis.cxx:809
Long64_t fLen
Definition: TPerfStats.h:52
Bool_t fDoTrace
Fill histos.
Definition: TPerfStats.h:93
Definition: Rtypes.h:59
void FileOpenEvent(TFile *file, const char *filename, Double_t start)
Open file event.
Definition: TPerfStats.cxx:551
Bool_t IsNull() const
Definition: TString.h:383
virtual UInt_t SetCanExtend(UInt_t extendBitMask)
Make the histogram axes extendable / not extendable according to the bit mask returns the previous bi...
Definition: TH1.cxx:6100
Mother of all ROOT objects.
Definition: TObject.h:37
Long64_t fBytesRead
Definition: TPerfStats.h:51
TTree * fTrace
Definition: TPerfStats.h:75
R__EXTERN TProofServ * gProofServ
Definition: TProofServ.h:347
virtual ~TPerfStats()
Destructor.
Definition: TPerfStats.cxx:377
virtual void Add(TObject *obj)
Definition: TList.h:87
auto * l
Definition: textangle.C:4
Definition: file.py:1
Provides the interface for PROOF monitoring to different writers.
Long64_t fTotBytesRead
total cpu time of all slaves
Definition: TPerfStats.h:87
Bool_t fDoTraceRate
Trace details in master.
Definition: TPerfStats.h:94
virtual int GetProcInfo(ProcInfo_t *info) const
Returns cpu and memory used by this process into the ProcInfo_t structure.
Definition: TSystem.cxx:2518
THist< 1, double, THistStatContent, THistStatUncertainty > TH1D
Definition: THist.hxx:284
Double_t fLatency
Definition: TPerfStats.h:53
Int_t GetEntries() const
Return the number of objects in array (i.e.
Definition: TObjArray.cxx:522
TProof * GetProof() const
Definition: TProofServ.h:237
void FileEvent(const char *slave, const char *slavename, const char *nodename, const char *filename, Bool_t isStart)
File event.
Definition: TPerfStats.cxx:524
static void Stop()
Terminate the PROOF statistics run.
Definition: TPerfStats.cxx:764
void Add(TObject *obj)
Definition: TObjArray.h:73
A TTree object has a header with a name and a title.
Definition: TTree.h:70
const AParamType & GetVal() const
Definition: TParameter.h:69
#define gDirectory
Definition: TDirectory.h:213
TH2D * fCpuTimeHist
histogram of real time spent processing packets
Definition: TPerfStats.h:84
virtual Int_t GetSize() const
Definition: TCollection.h:180
const char * GetUser() const
Definition: TProofServ.h:241
Int_t Fill(Double_t)
Invalid Fill method.
Definition: TH2.cxx:292
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
Definition: TEnv.cxx:491
TH2D * fProcTimeHist
histogram of latency due to packet requests
Definition: TPerfStats.h:83
const Bool_t kTRUE
Definition: RtypesCore.h:87
const char * GetOrdinal() const
Definition: TProofServ.h:253
static void SetMemValues()
Record memory usage.
Definition: TPerfStats.cxx:778
Bool_t IsMaster() const
Definition: TProofServ.h:293
void FileReadEvent(TFile *file, Int_t len, Double_t start)
Read file event.
Definition: TPerfStats.cxx:572
Int_t GetNanoSec() const
Definition: TTimeStamp.h:136
Long_t fMemResident
Definition: TSystem.h:196
void Print(Option_t *option="") const
Dump content of this instance.
Definition: TPerfStats.cxx:98
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:866
TAxis * GetXaxis()
Get the behaviour adopted by the object about the statoverflows. See EStatOverflows for more informat...
Definition: TH1.h:315
void Resize(Ssiz_t n)
Resize the string. Truncate or add blanks as necessary.
Definition: TString.cxx:1069
const char * Data() const
Definition: TString.h:345