Logo ROOT   6.10/09
Reference Guide
TProofMonSenderSQL.cxx
Go to the documentation of this file.
1 // @(#)root/proofplayer:$Id$
2 // Author: G.Ganis July 2011
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2004, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 
13 /** \class TProofMonSenderSQL
14 \ingroup proofkernel
15 
16 TProofMonSender implementation for the SQL writers
17 
18 */
19 
20 #include "TProofMonSenderSQL.h"
21 
22 #include "TDSet.h"
23 #include "TFileInfo.h"
24 #include "THashList.h"
25 #include "TList.h"
26 #include "TPluginManager.h"
27 #include "TProofDebug.h"
28 #include "TROOT.h"
29 #include "TSystem.h"
30 #include "TVirtualMonitoring.h"
31 
32 ////////////////////////////////////////////////////////////////////////////////
33 /// Main constructor
34 
35 TProofMonSenderSQL::TProofMonSenderSQL(const char *serv, const char *user,
36  const char *pass, const char *table,
37  const char *dstab, const char *filestab)
38  : TProofMonSender(serv,"ProofMonSenderSQL"),
39  fDSetSendOpts("bulk,table=proofquerydsets"),
40  fFilesSendOpts("bulk,table=proofqueryfiles")
41 {
42  fWriter = 0;
43  // Init the sender instance using the plugin manager
44  TPluginHandler *h = 0;
45  if ((h = gROOT->GetPluginManager()->FindHandler("TVirtualMonitoringWriter", "SQL"))) {
46  if (h->LoadPlugin() != -1) {
47  fWriter = (TVirtualMonitoringWriter *) h->ExecPlugin(4, serv, user, pass, table);
49  }
50  }
51  // Flag this instance as valid if the writer initialization succeeded
53 
54  // Set default send control options
58  fSummaryVrs = 2;
59  fDataSetInfoVrs = 1;
60  fFileInfoVrs = 1;
61 
62  // Transfer verbosity requirements
63  PDB(kMonitoring,1) if (fWriter) fWriter->Verbose(kTRUE);
64 
65  // Reformat the send options strings, if needed
66  if (dstab && strlen(dstab) > 0) fDSetSendOpts.Form("bulk,table=%s", dstab);
67  if (filestab && strlen(filestab) > 0) fFilesSendOpts.Form("bulk,table=%s", filestab);
68 }
69 
70 ////////////////////////////////////////////////////////////////////////////////
71 /// Destructor
72 
74 {
76 }
77 
78 ////////////////////////////////////////////////////////////////////////////////
79 /// Send 'summary' record for the table 'proofquerylog'.
80 ///
81 /// There are three versions of this record, corresponding the evolution
82 /// in time of the monitoring requirements.
83 ///
84 /// The default version 2 corresponds to the table created with the following command:
85 ///
86 /// CREATE TABLE proofquerylog (
87 /// id int(11) NOT NULL auto_increment,
88 /// proofuser varchar(32) NOT NULL,
89 /// proofgroup varchar(32) default NULL,
90 /// querybegin datetime default NULL,
91 /// queryend datetime default NULL,
92 /// walltime int(11) default NULL,
93 /// cputime float default NULL,
94 /// bytesread bigint(20) default NULL,
95 /// events bigint(20) default NULL,
96 /// totevents bigint(20) default NULL,
97 /// workers int(11) default NULL,
98 /// querytag varchar(64) NOT NULL,
99 /// vmemmxw bigint(20) default NULL,
100 /// rmemmxw bigint(20) default NULL,
101 /// vmemmxm bigint(20) default NULL,
102 /// rmemmxm bigint(20) default NULL,
103 /// numfiles int(11) default NULL,
104 /// missfiles int(11) default NULL,
105 /// status int(11) default NULL,
106 /// rootver varchar(32) NOT NULL,
107 /// PRIMARY KEY (id) );
108 ///
109 /// Version 1 corresponds to the table created with the following command:
110 /// ('user','begin','end' instead of 'proofuser', 'querybegin', 'queryend';
111 /// no 'status', 'missfiles', 'rootver'; 'dataset' field with name(s) of
112 /// processed dataset(s))
113 ///
114 /// CREATE TABLE proofquerylog (
115 /// id int(11) NOT NULL auto_increment,
116 /// user varchar(32) NOT NULL,
117 /// proofgroup varchar(32) default NULL,
118 /// begin datetime default NULL,
119 /// end datetime default NULL,
120 /// walltime int(11) default NULL,
121 /// cputime float default NULL,
122 /// bytesread bigint(20) default NULL,
123 /// events bigint(20) default NULL,
124 /// totevents bigint(20) default NULL,
125 /// workers int(11) default NULL,
126 /// querytag varchar(64) NOT NULL,
127 /// vmemmxw bigint(20) default NULL,
128 /// rmemmxw bigint(20) default NULL,
129 /// vmemmxm bigint(20) default NULL,
130 /// rmemmxm bigint(20) default NULL,
131 /// numfiles int(11) default NULL,
132 /// dataset varchar(512) NOT NULL,
133 /// PRIMARY KEY (id) );
134 
135 Int_t TProofMonSenderSQL::SendSummary(TList *recs, const char *dumid)
136 {
137  //
138  // Version 0 corresponds to the table created with the following command:
139  // ('group' instead of 'proofgroup'; no 'querytag', 'vmemmxw',
140  // 'rmemmxw', 'vmemmxm', 'rmemmxm', 'numfiles', 'dataset')
141  //
142  // CREATE TABLE proofquerylog (
143  // id int(11) NOT NULL auto_increment,
144  // user varchar(32) NOT NULL,
145  // group varchar(32) default NULL,
146  // begin datetime default NULL,
147  // end datetime default NULL,
148  // walltime int(11) default NULL,
149  // cputime float default NULL,
150  // bytesread bigint(20) default NULL,
151  // events bigint(20) default NULL,
152  // totevents bigint(20) default NULL,
153  // workers int(11) default NULL,
154  // PRIMARY KEY (id) );
155  //
156  // Return 0 on success, -1 on any failure.
157 
158  if (!IsValid()) {
159  Error("SendSummary", "invalid instance: do nothing!");
160  return -1;
161  }
162 
163  // Are we requested to send this info?
164  if (!TestBit(TProofMonSender::kSendSummary)) return 0;
165 
166  PDB(kMonitoring,1) Info("SendSummary", "preparing (qid: '%s')", dumid);
167 
168  // Make sure we have something to send
169  if (!recs || (recs && recs->GetSize() <= 0)) {
170  Error("SendSummary", "records list undefined or empty!");
171  return -1;
172  }
173  TList *xrecs = recs;
174 
175  TObject *dsn = 0;
176  TNamed *nm = 0;
177  // We may need to correct some variable names first
178  if (fSummaryVrs > 1) {
179  if ((nm = (TNamed *) recs->FindObject("user"))) nm->SetName("proofuser");
180  if ((nm = (TNamed *) recs->FindObject("begin"))) nm->SetName("querybegin");
181  if ((nm = (TNamed *) recs->FindObject("end"))) nm->SetName("queryend");
182  if ((dsn = recs->FindObject("dataset"))) recs->Remove(dsn);
183  } else if (fSummaryVrs == 0) {
184  // Only the first records
185  xrecs = new TList;
186  xrecs->SetOwner(kFALSE);
187  TIter nxr(recs);
188  TObject *o = 0;
189  while ((o = nxr())) {
190  if (!strcmp(o->GetName(), "vmemmxw")) break;
191  xrecs->Add(o);
192  }
193  }
194 
195  PDB(kMonitoring,1) Info("SendSummary", "sending (%d entries)", xrecs->GetSize());
196 
197  // Now we are ready to send
198  Bool_t rc = fWriter->SendParameters(xrecs, dumid);
199 
200  // Restore the "dataset" entry in the list
201  if (fSummaryVrs > 1 && dsn && xrecs == recs) {
202  TObject *num = recs->FindObject("numfiles");
203  if (num)
204  recs->AddBefore(num, dsn);
205  else
206  recs->Add(dsn);
207  }
208  if (xrecs != recs) SafeDelete(xrecs);
209 
210  // Done
211  return (rc ? 0 : -1);
212 }
213 
214 ////////////////////////////////////////////////////////////////////////////////
215 /// Post information about the processed dataset(s). The information is taken
216 /// from the TDSet object 'dset' and integrated with the missing files
217 /// information in the list 'missing'. The string 'qid' is the uninque
218 /// ID of the query; 'begin' the starting time.
219 ///
220 /// The record is formatted for the table 'proofquerydsets'.
221 ///
222 /// There are two versions of this record, with or without the starting time.
223 /// The starting time could be looked up from the summary record, if available.
224 ///
225 /// The default version 1 corresponds to the table created with the following command:
226 ///
227 /// CREATE TABLE proofquerydsets (
228 /// id int(11) NOT NULL auto_increment,
229 /// dsn varchar(512) NOT NULL,
230 /// querytag varchar(64) NOT NULL,
231 /// querybegin datetime default NULL,
232 /// numfiles int(11) default NULL,
233 /// missfiles int(11) default NULL,
234 /// PRIMARY KEY (id),
235 /// KEY ix_querytag (querytag) );
236 ///
237 /// Version 0 corresponds to the table created with the following command:
238 /// (no 'querybegin')
239 ///
240 /// CREATE TABLE proofquerydsets (
241 /// id int(11) NOT NULL auto_increment,
242 /// dsn varchar(512) NOT NULL,
243 /// querytag varchar(64) NOT NULL,
244 /// numfiles int(11) default NULL,
245 /// missfiles int(11) default NULL,
246 /// PRIMARY KEY (id),
247 /// KEY ix_querytag (querytag) );
248 ///
249 /// The information is posted with a bulk insert.
250 ///
251 /// Returns 0 on success, -1 on failure.
252 
254  const char *begin, const char *qid)
255 {
256  if (!IsValid()) {
257  Error("SendDataSetInfo", "invalid instance: do nothing!");
258  return -1;
259  }
260 
261  // Are we requested to send this info?
263 
264  // The query id (tag) must be given
265  if (!qid || (qid && strlen(qid) <= 0)) {
266  Error("SendDataSetInfo", "query id (tag) undefined!");
267  return -1;
268  }
269  // The dataset must be given
270  if (!dset) {
271  Error("SendDataSetInfo", "TDSet object undefined! (qid: '%s')", qid);
272  return -1;
273  }
274 
275  PDB(kMonitoring,1) Info("SendDataSetInfo", "preparing (qid: '%s')", qid);
276 
277  TList plets;
278  // Extract the information and save it into the relevant multiplets
279  TString dss(dset->GetName()), ds;
280  Ssiz_t from = 0;
281  while ((dss.Tokenize(ds, from , "[,| ]"))) {
282  // Create a new TDSetPlet and add it to the list
283  plets.Add(new TDSetPlet(ds.Data(), dset));
284  }
285 
286  // Now try to count the files
287  TDSetPlet *plet = 0;
288  TIter nxpl(&plets);
289  TObject *o = 0;
290  TDSetElement *e = 0, *ee = 0;
291  TDSet *dsete = 0;
292  TIter nxe(dset->GetListOfElements());
293  TString dse;
294  while ((o = nxe())) {
295  if ((e = dynamic_cast<TDSetElement *>(o))) {
296  dse = e->GetDataSet();
297  if (!dse.IsNull()) {
298  nxpl.Reset();
299  while ((plet = (TDSetPlet *) nxpl())) {
300  if (dse == plet->GetName()) {
301  plet->fFiles += 1;
302  break;
303  }
304  }
305  }
306  } else if ((dsete = dynamic_cast<TDSet *>(o))) {
307  PDB(kMonitoring,1)
308  Info("SendDataSetInfo", "dset '%s' (%d files)",
309  o->GetName(), dsete->GetListOfElements()->GetSize());
310  TIter nxee(dsete->GetListOfElements());
311  while ((ee = (TDSetElement *) nxee())) {
312  dse = ee->GetDataSet();
313  if (!dse.IsNull()) {
314  nxpl.Reset();
315  while ((plet = (TDSetPlet *) nxpl())) {
316  if (dse == plet->GetName()) {
317  plet->fFiles += 1;
318  plet->fDSet = dsete;
319  break;
320  }
321  }
322  }
323  }
324  } else {
325  Warning("SendDataSetInfo", "ignoring unknown element type: '%s'", o->ClassName());
326  }
327  }
328 
329  // Now try to include the missing files info
330  if (missing) {
331  TFileInfo *fi = 0;
332  TIter nxm(missing);
333  TString dsfi, fn;
334  while ((fi = (TFileInfo *) nxm())) {
335  dsfi = fi->GetTitle();
336  if (!dsfi.IsNull() && dsfi != "TFileInfo") {
337  nxpl.Reset();
338  while ((plet = (TDSetPlet *) nxpl())) {
339  if (dsfi == plet->GetName()) {
340  fn = fi->GetCurrentUrl()->GetUrl();
341  if (plet->fDSet && plet->fDSet->GetListOfElements() &&
342  !(plet->fDSet->GetListOfElements()->FindObject(fn))) plet->fFiles += 1;
343  plet->fMissing += 1;
344  break;
345  }
346  }
347  }
348  }
349  }
350 
351  // Now we can prepare the input for SendParameters
352  TList values;
353  TString ent("dsn,querytag,querybegin,numfiles,missfiles");
354  if (fDataSetInfoVrs == 0) ent = "dsn,querytag,numfiles,missfiles";
355  values.Add(new TObjString(ent.Data()));
356  nxpl.Reset();
357  while ((plet = (TDSetPlet *) nxpl())) {
358  if (fDataSetInfoVrs == 0)
359  ent.Form("'%s','%s',%d,%d", plet->GetName(), qid, plet->fFiles, plet->fMissing);
360  else
361  ent.Form("'%s','%s','%s',%d,%d", plet->GetName(), qid, begin, plet->fFiles, plet->fMissing);
362  values.Add(new TObjString(ent.Data()));
363  }
364 
365  PDB(kMonitoring,1)
366  Info("SendDataSetInfo", "sending (%d entries)", values.GetSize());
367 
368  // Now we are ready to send
369  Bool_t rc = fWriter->SendParameters(&values, fDSetSendOpts);
370 
371  // Done
372  return (rc ? 0 : -1);
373 }
374 
375 ////////////////////////////////////////////////////////////////////////////////
376 /// Post information about the requested files. The information is taken
377 /// from the TDSet object 'dset' and integrated with the missing files
378 /// information in the list 'missing'. The string 'qid' is the unique
379 /// ID of the query; 'begin' the starting time.
380 ///
381 /// The record is formatted for the table 'proofqueryfiles'.
382 ///
383 /// There are two versions of this record, with or without the starting time.
384 /// The starting time could be looked up from the summary record, if available.
385 ///
386 /// The default version 1 corresponds to the table created with the following command:
387 ///
388 /// CREATE TABLE proofqueryfiles (
389 /// id int(11) NOT NULL auto_increment,
390 /// lfn varchar(255) NOT NULL,
391 /// path varchar(2048) NOT NULL,
392 /// querytag varchar(64) NOT NULL,
393 /// querybegin datetime default NULL,
394 /// status enum('Ok','Failed') NOT NULL default 'Ok',
395 /// PRIMARY KEY (id),
396 /// KEY ix_querytag (querytag) );
397 ///
398 /// Version 0 corresponds to the table created with the following command:
399 /// (no 'querybegin')
400 ///
401 /// CREATE TABLE proofqueryfiles (
402 /// id int(11) NOT NULL auto_increment,
403 /// lfn varchar(255) NOT NULL,
404 /// path varchar(2048) NOT NULL,
405 /// querytag varchar(64) NOT NULL,
406 /// status enum('Ok','Failed') NOT NULL default 'Ok',
407 /// PRIMARY KEY (id),
408 /// KEY ix_querytag (querytag) );
409 ///
410 /// The information is posted with a bulk insert.
411 ///
412 /// Returns 0 on success, -1 on failure.
413 
415  const char *begin, const char *qid)
416 {
417  if (!IsValid()) {
418  Error("SendFileInfo", "invalid instance: do nothing!");
419  return -1;
420  }
421 
422  // Are we requested to send this info?
423  if (!TestBit(TProofMonSender::kSendFileInfo)) return 0;
424 
425  // The query id (tag) must be given
426  if (!qid || (qid && strlen(qid) <= 0)) {
427  Error("SendFileInfo", "query id (tag) undefined!");
428  return -1;
429  }
430  // The dataset must be given
431  if (!dset) {
432  Error("SendFileInfo", "TDSet object undefined! (qid: '%s')", qid);
433  return -1;
434  }
435 
436  PDB(kMonitoring,1) Info("SendFileInfo", "preparing (qid: '%s')", qid);
437  THashList hmiss;
438  if (missing) {
439  TIter nxfm(missing);
440  TFileInfo *fi = 0;
441  while ((fi = (TFileInfo *)nxfm())) {
442  hmiss.Add(new TObjString(fi->GetCurrentUrl()->GetUrl()));
443  }
444  PDB(kMonitoring,2) hmiss.Print();
445  }
446 
447  TList values;
448  TString ent("lfn,path,querytag,querybegin,status");
449  if (fFileInfoVrs == 0) ent = "lfn,path,querytag,status";
450  values.Add(new TObjString(ent.Data()));
451 
452  // Create the file-plets
453  TObject *o = 0;
454  TDSetElement *e = 0, *ee = 0;
455  TDSet *dsete = 0;
456  TIter nxe(dset->GetListOfElements());
457  TString fne, status;
458  while ((o = nxe())) {
459  if ((e = dynamic_cast<TDSetElement *>(o))) {
460  fne = e->GetName();
461  // Try to determine the status
462  status = "Ok";
463  if (hmiss.FindObject(fne)) status = "Failed";
464  if (fFileInfoVrs == 0)
465  ent.Form("'%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->DirName(fne),
466  qid, status.Data());
467  else
468  ent.Form("'%s','%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->DirName(fne),
469  qid, begin, status.Data());
470  values.Add(new TObjString(ent.Data()));
471  } else if ((dsete = dynamic_cast<TDSet *>(o))) {
472  PDB(kMonitoring,1)
473  Info("SendFileInfo", "dset '%s' (%d files)",
474  o->GetName(), dsete->GetListOfElements()->GetSize());
475  TIter nxee(dsete->GetListOfElements());
476  while ((ee = (TDSetElement *) nxee())) {
477  fne = ee->GetName();
478  // Try to determine the status
479  status = "Ok";
480  if (hmiss.FindObject(fne)) status = "Failed";
481  if (fFileInfoVrs == 0)
482  ent.Form("'%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->DirName(fne),
483  qid, status.Data());
484  else
485  ent.Form("'%s','%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->DirName(fne),
486  qid, begin, status.Data());
487  values.Add(new TObjString(ent.Data()));
488  }
489  } else {
490  Warning("SendFileInfo", "ignoring unknown element type: '%s'", o->ClassName());
491  }
492  }
493 
494  PDB(kMonitoring,1) Info("SendFileInfo", "sending (%d entries)", values.GetSize());
495 
496  // Now we are ready to send
498 
499  // Done
500  return (rc ? 0 : -1);
501 }
virtual const char * BaseName(const char *pathname)
Base name of a file name. Base name of /user/root is root.
Definition: TSystem.cxx:931
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:847
virtual void Verbose(Bool_t)
Collectable string class.
Definition: TObjString.h:28
TUrl * GetCurrentUrl() const
Return the current url.
Definition: TFileInfo.cxx:248
Bool_t TestBit(UInt_t f) const
Definition: TObject.h:159
This class implements a data set to be used for PROOF processing.
Definition: TDSet.h:151
virtual void SetName(const char *name)
Set the name of the TNamed.
Definition: TNamed.cxx:131
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
TH1 * h
Definition: legend2.C:5
Int_t SendDataSetInfo(TDSet *, TList *, const char *, const char *)
Post information about the processed dataset(s).
Bool_t IsValid() const
#define gROOT
Definition: TROOT.h:375
Int_t LoadPlugin()
Load the plugin library for this handler.
Basic string class.
Definition: TString.h:129
int Int_t
Definition: RtypesCore.h:41
virtual const char * DirName(const char *pathname)
Return the directory name in pathname.
Definition: TSystem.cxx:1003
bool Bool_t
Definition: RtypesCore.h:59
TObject * FindObject(const char *name) const
Find object using its name.
Definition: THashList.cxx:213
TList * GetListOfElements() const
Definition: TDSet.h:229
void Reset()
Definition: TCollection.h:156
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:687
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition: TUrl.cxx:387
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Definition: TList.cxx:501
Manages an element of a TDSet.
Definition: TDSet.h:66
#define SafeDelete(p)
Definition: RConfig.h:499
virtual const char * ClassName() const
Returns name of class to which the object belongs.
Definition: TObject.cxx:135
#define PDB(mask, level)
Definition: TProofDebug.h:56
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition: THashList.h:34
TVirtualMonitoringWriter * fWriter
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:29
A doubly linked list.
Definition: TList.h:43
R__EXTERN TSystem * gSystem
Definition: TSystem.h:539
if object ctor succeeded but object should not be used
Definition: TObject.h:65
Long_t ExecPlugin(int nargs, const T &... params)
virtual TObject * Remove(TObject *obj)
Remove object from the list.
Definition: TList.cxx:679
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2332
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:873
Int_t SendFileInfo(TDSet *, TList *, const char *, const char *)
Post information about the requested files.
virtual Bool_t SendParameters(TList *, const char *=0)
Int_t SendSummary(TList *, const char *)
Send &#39;summary&#39; record for the table &#39;proofquerylog&#39;.
virtual void AddBefore(const TObject *before, TObject *obj)
Insert object before object before in the list.
Definition: TList.cxx:177
const Bool_t kFALSE
Definition: RtypesCore.h:92
virtual ~TProofMonSenderSQL()
Destructor.
int Ssiz_t
Definition: RtypesCore.h:63
Bool_t IsZombie() const
Definition: TObject.h:122
you should not use this method at all Int_t Int_t Double_t Double_t Double_t e
Definition: TRolke.cxx:630
Bool_t IsNull() const
Definition: TString.h:385
Mother of all ROOT objects.
Definition: TObject.h:37
const char * GetDataSet() const
Definition: TDSet.h:122
virtual void Add(TObject *obj)
Definition: TList.h:77
TProofMonSenderSQL(const char *serv, const char *user, const char *pass, const char *table="proof.proofquerylog", const char *dstab=0, const char *filestab=0)
Main constructor.
Provides the interface for PROOF monitoring to different writers.
Class describing a generic file including meta information.
Definition: TFileInfo.h:38
void ResetBit(UInt_t f)
Definition: TObject.h:158
virtual void Print(Option_t *option="") const
Default print for collections, calls Print(option, 1).
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:364
virtual Int_t GetSize() const
Definition: TCollection.h:89
const Bool_t kTRUE
Definition: RtypesCore.h:91
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:859
virtual const char * GetTitle() const
Returns title of object.
Definition: TNamed.h:48
const char * Data() const
Definition: TString.h:347