Logo ROOT  
Reference Guide
TProofMonSenderSQL.cxx
Go to the documentation of this file.
1// @(#)root/proofplayer:$Id$
2// Author: G.Ganis July 2011
3
4/*************************************************************************
5 * Copyright (C) 1995-2004, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12
13/** \class TProofMonSenderSQL
14\ingroup proofkernel
15
16TProofMonSender implementation for the SQL writers
17
18*/
19
20#include "TProofMonSenderSQL.h"
21
22#include "TDSet.h"
23#include "TFileInfo.h"
24#include "THashList.h"
25#include "TList.h"
26#include "TUrl.h"
27#include "TPluginManager.h"
28#include "TProofDebug.h"
29#include "TROOT.h"
30#include "TSystem.h"
31#include "TObjString.h"
32#include "TVirtualMonitoring.h"
33
34////////////////////////////////////////////////////////////////////////////////
35/// Main constructor
36
37TProofMonSenderSQL::TProofMonSenderSQL(const char *serv, const char *user,
38 const char *pass, const char *table,
39 const char *dstab, const char *filestab)
40 : TProofMonSender(serv,"ProofMonSenderSQL"),
41 fDSetSendOpts("bulk,table=proofquerydsets"),
42 fFilesSendOpts("bulk,table=proofqueryfiles")
43{
44 fWriter = 0;
45 // Init the sender instance using the plugin manager
46 TPluginHandler *h = 0;
47 if ((h = gROOT->GetPluginManager()->FindHandler("TVirtualMonitoringWriter", "SQL"))) {
48 if (h->LoadPlugin() != -1) {
49 fWriter = (TVirtualMonitoringWriter *) h->ExecPlugin(4, serv, user, pass, table);
51 }
52 }
53 // Flag this instance as valid if the writer initialization succeeded
55
56 // Set default send control options
60 fSummaryVrs = 2;
62 fFileInfoVrs = 1;
63
64 // Transfer verbosity requirements
65 PDB(kMonitoring,1) if (fWriter) fWriter->Verbose(kTRUE);
66
67 // Reformat the send options strings, if needed
68 if (dstab && strlen(dstab) > 0) fDSetSendOpts.Form("bulk,table=%s", dstab);
69 if (filestab && strlen(filestab) > 0) fFilesSendOpts.Form("bulk,table=%s", filestab);
70}
71
72////////////////////////////////////////////////////////////////////////////////
73/// Destructor
74
76{
78}
79
80////////////////////////////////////////////////////////////////////////////////
81/// Send 'summary' record for the table 'proofquerylog'.
82///
83/// There are three versions of this record, corresponding the evolution
84/// in time of the monitoring requirements.
85///
86/// The default version 2 corresponds to the table created with the following command:
87///
88/// CREATE TABLE proofquerylog (
89/// id int(11) NOT NULL auto_increment,
90/// proofuser varchar(32) NOT NULL,
91/// proofgroup varchar(32) default NULL,
92/// querybegin datetime default NULL,
93/// queryend datetime default NULL,
94/// walltime int(11) default NULL,
95/// cputime float default NULL,
96/// bytesread bigint(20) default NULL,
97/// events bigint(20) default NULL,
98/// totevents bigint(20) default NULL,
99/// workers int(11) default NULL,
100/// querytag varchar(64) NOT NULL,
101/// vmemmxw bigint(20) default NULL,
102/// rmemmxw bigint(20) default NULL,
103/// vmemmxm bigint(20) default NULL,
104/// rmemmxm bigint(20) default NULL,
105/// numfiles int(11) default NULL,
106/// missfiles int(11) default NULL,
107/// status int(11) default NULL,
108/// rootver varchar(32) NOT NULL,
109/// PRIMARY KEY (id) );
110///
111/// Version 1 corresponds to the table created with the following command:
112/// ('user','begin','end' instead of 'proofuser', 'querybegin', 'queryend';
113/// no 'status', 'missfiles', 'rootver'; 'dataset' field with name(s) of
114/// processed dataset(s))
115///
116/// CREATE TABLE proofquerylog (
117/// id int(11) NOT NULL auto_increment,
118/// user varchar(32) NOT NULL,
119/// proofgroup varchar(32) default NULL,
120/// begin datetime default NULL,
121/// end datetime default NULL,
122/// walltime int(11) default NULL,
123/// cputime float default NULL,
124/// bytesread bigint(20) default NULL,
125/// events bigint(20) default NULL,
126/// totevents bigint(20) default NULL,
127/// workers int(11) default NULL,
128/// querytag varchar(64) NOT NULL,
129/// vmemmxw bigint(20) default NULL,
130/// rmemmxw bigint(20) default NULL,
131/// vmemmxm bigint(20) default NULL,
132/// rmemmxm bigint(20) default NULL,
133/// numfiles int(11) default NULL,
134/// dataset varchar(512) NOT NULL,
135/// PRIMARY KEY (id) );
136
138{
139 //
140 // Version 0 corresponds to the table created with the following command:
141 // ('group' instead of 'proofgroup'; no 'querytag', 'vmemmxw',
142 // 'rmemmxw', 'vmemmxm', 'rmemmxm', 'numfiles', 'dataset')
143 //
144 // CREATE TABLE proofquerylog (
145 // id int(11) NOT NULL auto_increment,
146 // user varchar(32) NOT NULL,
147 // group varchar(32) default NULL,
148 // begin datetime default NULL,
149 // end datetime default NULL,
150 // walltime int(11) default NULL,
151 // cputime float default NULL,
152 // bytesread bigint(20) default NULL,
153 // events bigint(20) default NULL,
154 // totevents bigint(20) default NULL,
155 // workers int(11) default NULL,
156 // PRIMARY KEY (id) );
157 //
158 // Return 0 on success, -1 on any failure.
159
160 if (!IsValid()) {
161 Error("SendSummary", "invalid instance: do nothing!");
162 return -1;
163 }
164
165 // Are we requested to send this info?
167
168 PDB(kMonitoring,1) Info("SendSummary", "preparing (qid: '%s')", dumid);
169
170 // Make sure we have something to send
171 if (!recs || (recs && recs->GetSize() <= 0)) {
172 Error("SendSummary", "records list undefined or empty!");
173 return -1;
174 }
175 TList *xrecs = recs;
176
177 TObject *dsn = 0;
178 TNamed *nm = 0;
179 // We may need to correct some variable names first
180 if (fSummaryVrs > 1) {
181 if ((nm = (TNamed *) recs->FindObject("user"))) nm->SetName("proofuser");
182 if ((nm = (TNamed *) recs->FindObject("begin"))) nm->SetName("querybegin");
183 if ((nm = (TNamed *) recs->FindObject("end"))) nm->SetName("queryend");
184 if ((dsn = recs->FindObject("dataset"))) recs->Remove(dsn);
185 } else if (fSummaryVrs == 0) {
186 // Only the first records
187 xrecs = new TList;
188 xrecs->SetOwner(kFALSE);
189 TIter nxr(recs);
190 TObject *o = 0;
191 while ((o = nxr())) {
192 if (!strcmp(o->GetName(), "vmemmxw")) break;
193 xrecs->Add(o);
194 }
195 }
196
197 PDB(kMonitoring,1) Info("SendSummary", "sending (%d entries)", xrecs->GetSize());
198
199 // Now we are ready to send
200 Bool_t rc = fWriter->SendParameters(xrecs, dumid);
201
202 // Restore the "dataset" entry in the list
203 if (fSummaryVrs > 1 && dsn && xrecs == recs) {
204 TObject *num = recs->FindObject("numfiles");
205 if (num)
206 recs->AddBefore(num, dsn);
207 else
208 recs->Add(dsn);
209 }
210 if (xrecs != recs) SafeDelete(xrecs);
211
212 // Done
213 return (rc ? 0 : -1);
214}
215
216////////////////////////////////////////////////////////////////////////////////
217/// Post information about the processed dataset(s). The information is taken
218/// from the TDSet object 'dset' and integrated with the missing files
219/// information in the list 'missing'. The string 'qid' is the uninque
220/// ID of the query; 'begin' the starting time.
221///
222/// The record is formatted for the table 'proofquerydsets'.
223///
224/// There are two versions of this record, with or without the starting time.
225/// The starting time could be looked up from the summary record, if available.
226///
227/// The default version 1 corresponds to the table created with the following command:
228///
229/// CREATE TABLE proofquerydsets (
230/// id int(11) NOT NULL auto_increment,
231/// dsn varchar(512) NOT NULL,
232/// querytag varchar(64) NOT NULL,
233/// querybegin datetime default NULL,
234/// numfiles int(11) default NULL,
235/// missfiles int(11) default NULL,
236/// PRIMARY KEY (id),
237/// KEY ix_querytag (querytag) );
238///
239/// Version 0 corresponds to the table created with the following command:
240/// (no 'querybegin')
241///
242/// CREATE TABLE proofquerydsets (
243/// id int(11) NOT NULL auto_increment,
244/// dsn varchar(512) NOT NULL,
245/// querytag varchar(64) NOT NULL,
246/// numfiles int(11) default NULL,
247/// missfiles int(11) default NULL,
248/// PRIMARY KEY (id),
249/// KEY ix_querytag (querytag) );
250///
251/// The information is posted with a bulk insert.
252///
253/// Returns 0 on success, -1 on failure.
254
256 const char *begin, const char *qid)
257{
258 if (!IsValid()) {
259 Error("SendDataSetInfo", "invalid instance: do nothing!");
260 return -1;
261 }
262
263 // Are we requested to send this info?
265
266 // The query id (tag) must be given
267 if (!qid || (qid && strlen(qid) <= 0)) {
268 Error("SendDataSetInfo", "query id (tag) undefined!");
269 return -1;
270 }
271 // The dataset must be given
272 if (!dset) {
273 Error("SendDataSetInfo", "TDSet object undefined! (qid: '%s')", qid);
274 return -1;
275 }
276
277 PDB(kMonitoring,1) Info("SendDataSetInfo", "preparing (qid: '%s')", qid);
278
279 TList plets;
280 // Extract the information and save it into the relevant multiplets
281 TString dss(dset->GetName()), ds;
282 Ssiz_t from = 0;
283 while ((dss.Tokenize(ds, from , "[,| ]"))) {
284 // Create a new TDSetPlet and add it to the list
285 plets.Add(new TDSetPlet(ds.Data(), dset));
286 }
287
288 // Now try to count the files
289 TDSetPlet *plet = 0;
290 TIter nxpl(&plets);
291 TObject *o = 0;
292 TDSetElement *e = 0, *ee = 0;
293 TDSet *dsete = 0;
294 TIter nxe(dset->GetListOfElements());
295 TString dse;
296 while ((o = nxe())) {
297 if ((e = dynamic_cast<TDSetElement *>(o))) {
298 dse = e->GetDataSet();
299 if (!dse.IsNull()) {
300 nxpl.Reset();
301 while ((plet = (TDSetPlet *) nxpl())) {
302 if (dse == plet->GetName()) {
303 plet->fFiles += 1;
304 break;
305 }
306 }
307 }
308 } else if ((dsete = dynamic_cast<TDSet *>(o))) {
309 PDB(kMonitoring,1)
310 Info("SendDataSetInfo", "dset '%s' (%d files)",
311 o->GetName(), dsete->GetListOfElements()->GetSize());
312 TIter nxee(dsete->GetListOfElements());
313 while ((ee = (TDSetElement *) nxee())) {
314 dse = ee->GetDataSet();
315 if (!dse.IsNull()) {
316 nxpl.Reset();
317 while ((plet = (TDSetPlet *) nxpl())) {
318 if (dse == plet->GetName()) {
319 plet->fFiles += 1;
320 plet->fDSet = dsete;
321 break;
322 }
323 }
324 }
325 }
326 } else {
327 Warning("SendDataSetInfo", "ignoring unknown element type: '%s'", o->ClassName());
328 }
329 }
330
331 // Now try to include the missing files info
332 if (missing) {
333 TFileInfo *fi = 0;
334 TIter nxm(missing);
335 TString dsfi, fn;
336 while ((fi = (TFileInfo *) nxm())) {
337 dsfi = fi->GetTitle();
338 if (!dsfi.IsNull() && dsfi != "TFileInfo") {
339 nxpl.Reset();
340 while ((plet = (TDSetPlet *) nxpl())) {
341 if (dsfi == plet->GetName()) {
342 fn = fi->GetCurrentUrl()->GetUrl();
343 if (plet->fDSet && plet->fDSet->GetListOfElements() &&
344 !(plet->fDSet->GetListOfElements()->FindObject(fn))) plet->fFiles += 1;
345 plet->fMissing += 1;
346 break;
347 }
348 }
349 }
350 }
351 }
352
353 // Now we can prepare the input for SendParameters
354 TList values;
355 TString ent("dsn,querytag,querybegin,numfiles,missfiles");
356 if (fDataSetInfoVrs == 0) ent = "dsn,querytag,numfiles,missfiles";
357 values.Add(new TObjString(ent.Data()));
358 nxpl.Reset();
359 while ((plet = (TDSetPlet *) nxpl())) {
360 if (fDataSetInfoVrs == 0)
361 ent.Form("'%s','%s',%d,%d", plet->GetName(), qid, plet->fFiles, plet->fMissing);
362 else
363 ent.Form("'%s','%s','%s',%d,%d", plet->GetName(), qid, begin, plet->fFiles, plet->fMissing);
364 values.Add(new TObjString(ent.Data()));
365 }
366
367 PDB(kMonitoring,1)
368 Info("SendDataSetInfo", "sending (%d entries)", values.GetSize());
369
370 // Now we are ready to send
372
373 // Done
374 return (rc ? 0 : -1);
375}
376
377////////////////////////////////////////////////////////////////////////////////
378/// Post information about the requested files. The information is taken
379/// from the TDSet object 'dset' and integrated with the missing files
380/// information in the list 'missing'. The string 'qid' is the unique
381/// ID of the query; 'begin' the starting time.
382///
383/// The record is formatted for the table 'proofqueryfiles'.
384///
385/// There are two versions of this record, with or without the starting time.
386/// The starting time could be looked up from the summary record, if available.
387///
388/// The default version 1 corresponds to the table created with the following command:
389///
390/// CREATE TABLE proofqueryfiles (
391/// id int(11) NOT NULL auto_increment,
392/// lfn varchar(255) NOT NULL,
393/// path varchar(2048) NOT NULL,
394/// querytag varchar(64) NOT NULL,
395/// querybegin datetime default NULL,
396/// status enum('Ok','Failed') NOT NULL default 'Ok',
397/// PRIMARY KEY (id),
398/// KEY ix_querytag (querytag) );
399///
400/// Version 0 corresponds to the table created with the following command:
401/// (no 'querybegin')
402///
403/// CREATE TABLE proofqueryfiles (
404/// id int(11) NOT NULL auto_increment,
405/// lfn varchar(255) NOT NULL,
406/// path varchar(2048) NOT NULL,
407/// querytag varchar(64) NOT NULL,
408/// status enum('Ok','Failed') NOT NULL default 'Ok',
409/// PRIMARY KEY (id),
410/// KEY ix_querytag (querytag) );
411///
412/// The information is posted with a bulk insert.
413///
414/// Returns 0 on success, -1 on failure.
415
417 const char *begin, const char *qid)
418{
419 if (!IsValid()) {
420 Error("SendFileInfo", "invalid instance: do nothing!");
421 return -1;
422 }
423
424 // Are we requested to send this info?
426
427 // The query id (tag) must be given
428 if (!qid || (qid && strlen(qid) <= 0)) {
429 Error("SendFileInfo", "query id (tag) undefined!");
430 return -1;
431 }
432 // The dataset must be given
433 if (!dset) {
434 Error("SendFileInfo", "TDSet object undefined! (qid: '%s')", qid);
435 return -1;
436 }
437
438 PDB(kMonitoring,1) Info("SendFileInfo", "preparing (qid: '%s')", qid);
439 THashList hmiss;
440 if (missing) {
441 TIter nxfm(missing);
442 TFileInfo *fi = 0;
443 while ((fi = (TFileInfo *)nxfm())) {
444 hmiss.Add(new TObjString(fi->GetCurrentUrl()->GetUrl()));
445 }
446 PDB(kMonitoring,2) hmiss.Print();
447 }
448
449 TList values;
450 TString ent("lfn,path,querytag,querybegin,status");
451 if (fFileInfoVrs == 0) ent = "lfn,path,querytag,status";
452 values.Add(new TObjString(ent.Data()));
453
454 // Create the file-plets
455 TObject *o = 0;
456 TDSetElement *e = 0, *ee = 0;
457 TDSet *dsete = 0;
458 TIter nxe(dset->GetListOfElements());
459 TString fne, status;
460 while ((o = nxe())) {
461 if ((e = dynamic_cast<TDSetElement *>(o))) {
462 fne = e->GetName();
463 // Try to determine the status
464 status = "Ok";
465 if (hmiss.FindObject(fne)) status = "Failed";
466 if (fFileInfoVrs == 0)
467 ent.Form("'%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
468 qid, status.Data());
469 else
470 ent.Form("'%s','%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
471 qid, begin, status.Data());
472 values.Add(new TObjString(ent.Data()));
473 } else if ((dsete = dynamic_cast<TDSet *>(o))) {
474 PDB(kMonitoring,1)
475 Info("SendFileInfo", "dset '%s' (%d files)",
476 o->GetName(), dsete->GetListOfElements()->GetSize());
477 TIter nxee(dsete->GetListOfElements());
478 while ((ee = (TDSetElement *) nxee())) {
479 fne = ee->GetName();
480 // Try to determine the status
481 status = "Ok";
482 if (hmiss.FindObject(fne)) status = "Failed";
483 if (fFileInfoVrs == 0)
484 ent.Form("'%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
485 qid, status.Data());
486 else
487 ent.Form("'%s','%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
488 qid, begin, status.Data());
489 values.Add(new TObjString(ent.Data()));
490 }
491 } else {
492 Warning("SendFileInfo", "ignoring unknown element type: '%s'", o->ClassName());
493 }
494 }
495
496 PDB(kMonitoring,1) Info("SendFileInfo", "sending (%d entries)", values.GetSize());
497
498 // Now we are ready to send
500
501 // Done
502 return (rc ? 0 : -1);
503}
#define SafeDelete(p)
Definition: RConfig.hxx:536
#define h(i)
Definition: RSha256.hxx:106
#define e(i)
Definition: RSha256.hxx:103
int Int_t
Definition: RtypesCore.h:45
int Ssiz_t
Definition: RtypesCore.h:67
const Bool_t kFALSE
Definition: RtypesCore.h:101
bool Bool_t
Definition: RtypesCore.h:63
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define PDB(mask, level)
Definition: TProofDebug.h:56
#define gROOT
Definition: TROOT.h:404
R__EXTERN TSystem * gSystem
Definition: TSystem.h:559
virtual void Print(Option_t *option="") const
Default print for collections, calls Print(option, 1).
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
virtual Int_t GetSize() const
Return the capacity of the collection, i.e.
Definition: TCollection.h:184
Manages an element of a TDSet.
Definition: TDSet.h:66
This class implements a data set to be used for PROOF processing.
Definition: TDSet.h:153
TList * GetListOfElements() const
Definition: TDSet.h:231
Class describing a generic file including meta information.
Definition: TFileInfo.h:39
TUrl * GetCurrentUrl() const
Return the current url.
Definition: TFileInfo.cxx:249
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition: THashList.h:34
TObject * FindObject(const char *name) const
Find object using its name.
Definition: THashList.cxx:262
void Reset()
Definition: TCollection.h:254
A doubly linked list.
Definition: TList.h:44
virtual void Add(TObject *obj)
Definition: TList.h:87
virtual TObject * Remove(TObject *obj)
Remove object from the list.
Definition: TList.cxx:822
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Definition: TList.cxx:578
virtual void AddBefore(const TObject *before, TObject *obj)
Insert object before object before in the list.
Definition: TList.cxx:196
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:29
virtual const char * GetTitle() const
Returns title of object.
Definition: TNamed.h:48
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
Collectable string class.
Definition: TObjString.h:28
Mother of all ROOT objects.
Definition: TObject.h:37
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:359
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition: TObject.h:187
virtual const char * ClassName() const
Returns name of class to which the object belongs.
Definition: TObject.cxx:130
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:879
R__ALWAYS_INLINE Bool_t IsZombie() const
Definition: TObject.h:149
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:696
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:893
void ResetBit(UInt_t f)
Definition: TObject.h:186
@ kInvalidObject
if object ctor succeeded but object should not be used
Definition: TObject.h:68
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:867
virtual ~TProofMonSenderSQL()
Destructor.
Int_t SendFileInfo(TDSet *, TList *, const char *, const char *)
Post information about the requested files.
Int_t SendDataSetInfo(TDSet *, TList *, const char *, const char *)
Post information about the processed dataset(s).
Int_t SendSummary(TList *, const char *)
Send 'summary' record for the table 'proofquerylog'.
TProofMonSenderSQL(const char *serv, const char *user, const char *pass, const char *table="proof.proofquerylog", const char *dstab=0, const char *filestab=0)
Main constructor.
TVirtualMonitoringWriter * fWriter
Provides the interface for PROOF monitoring to different writers.
Bool_t IsValid() const
Basic string class.
Definition: TString.h:136
const char * Data() const
Definition: TString.h:369
Bool_t IsNull() const
Definition: TString.h:407
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2314
virtual const char * BaseName(const char *pathname)
Base name of a file name. Base name of /user/root is root.
Definition: TSystem.cxx:935
virtual TString GetDirName(const char *pathname)
Return the directory name in pathname.
Definition: TSystem.cxx:1032
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition: TUrl.cxx:389
virtual Bool_t SendParameters(TList *, const char *=0)
virtual void Verbose(Bool_t)
static constexpr double nm