Logo ROOT  
Reference Guide
TProofCondor.cxx
Go to the documentation of this file.
1// @(#)root/proof:$Id$
2// Author: Fons Rademakers 13/02/97
3
4/*************************************************************************
5 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12//////////////////////////////////////////////////////////////////////////
13// //
14// TProof //
15// //
16// This class controls a Parallel ROOT Facility, PROOF, cluster. //
17// It fires the slave servers, it keeps track of how many slaves are //
18// running, it keeps track of the slaves running status, it broadcasts //
19// messages to all slaves, it collects results, etc. //
20// //
21//////////////////////////////////////////////////////////////////////////
22
23#include "TProofCondor.h"
24
25#include "TCondor.h"
26#include "TList.h"
27#include "TMap.h"
28#include "TMessage.h"
29#include "TMonitor.h"
30#include "TProofNodeInfo.h"
32#include "TProofServ.h"
33#include "TSlave.h"
34#include "TSocket.h"
35#include "TString.h"
36#include "TTimer.h"
37
39
40////////////////////////////////////////////////////////////////////////////////
41/// Start proof using condor
42
43TProofCondor::TProofCondor(const char *masterurl, const char *conffile,
44 const char *confdir, Int_t loglevel,
45 const char *, TProofMgr *mgr)
46 : fCondor(0), fTimer(0)
47{
48 // Default initializations
50
51 // This may be needed during init
52 fManager = mgr;
53
54 fUrl = TUrl(masterurl);
55
56 if (!conffile || !conffile[0]) {
57 conffile = kPROOF_ConfFile;
58 } else if (!strncasecmp(conffile, "condor:", 7)) {
59 conffile+=7;
60 }
61
62 if (!confdir || !confdir[0]) {
63 confdir = kPROOF_ConfDir;
64 }
65
66 Init(masterurl, conffile, confdir, loglevel);
67}
68
69////////////////////////////////////////////////////////////////////////////////
70/// Clean up Condor PROOF environment.
71
73{
76}
77
78////////////////////////////////////////////////////////////////////////////////
79/// Setup Condor workers using dynamic information
80
82{
83 fCondor = new TCondor;
84 TString jobad = GetJobAd();
85
87 if (fImage.Length() == 0) {
88 Error("StartSlaves", "Empty Condor image found for system %s",
89 gSystem->HostName());
90 return kFALSE;
91 }
92
93 TList claims;
94 if (fConfFile.IsNull()) {
95 // startup all slaves if no config file given
96 TList *condorclaims = fCondor->Claim(9999, jobad);
97 TIter nextclaim(condorclaims);
98 while (TObject *o = nextclaim()) claims.Add(o);
99 } else {
100 // parse config file
102 fConfFile = resources->GetFileName(); // Update the global file name (with path)
103 PDB(kGlobal,1) Info("StartSlaves", "using PROOF config file: %s", fConfFile.Data());
104
105 // Get all workers
106 TList *workerList = resources->GetWorkers();
107 if (workerList->GetSize() == 0) {
108 Error("StartSlaves", "Found no condorworkers in %s", fConfFile.Data());
109 return kFALSE;
110 }
111
112 // check for valid slave lines and claim condor nodes
113 Int_t ord = 0;
114
115 // Loop over all workers and start them
116 TListIter next(workerList);
117 TObject *to;
118 TProofNodeInfo *worker;
119 int nSlavesDone = 0;
120 while ((to = next())) {
121 // Get the next worker from the list
122 worker = (TProofNodeInfo *)to;
123
124 // Read back worker node info
125 const Char_t *image = worker->GetImage().Data();
126 const Char_t *workdir = worker->GetWorkDir().Data();
127 Int_t perfidx = worker->GetPerfIndex();
128
129 gSystem->Sleep(10 /* ms */);
130 TCondorSlave* csl = fCondor->Claim(worker->GetNodeName().Data(), jobad);
131 if (csl) {
132 csl->fPerfIdx = perfidx;
133 csl->fImage = image;
134 csl->fWorkDir = workdir;
136 TString fullord = TString(gProofServ->GetOrdinal()) + "." + ((Long_t) ord);
137 csl->fOrdinal = fullord.Data();
138 claims.Add(csl);
139 ord++;
140 }
141
142 // Notify claim creation
143 nSlavesDone++;
145 m << TString("Creating COD Claim") << workerList->GetSize()
146 << nSlavesDone << (csl != 0);
148
149 } // end while (worker loop)
150
151 // Cleanup
152 delete resources;
153 resources = 0;
154 } // end else (parse config file)
155
156 Long_t delay = 500; // timer delay 0.5s
157 Int_t ntries = 20; // allow 20 tries (must be > 1 for algorithm to work)
158 Int_t trial = 1;
159 Int_t idx = 0;
160
161 int nClaims = claims.GetSize();
162 int nClaimsDone = 0;
163 while (claims.GetSize() > 0) {
164 TCondorSlave* c = 0;
165
166 // Get Condor Slave
167 if (trial == 1) {
168 c = dynamic_cast<TCondorSlave*>(claims.At(idx));
169 } else {
170 TPair *p = dynamic_cast<TPair*>(claims.At(idx));
171 if (p) {
172 TTimer *t = dynamic_cast<TTimer*>(p->Value());
173 if (t) {
174 // wait remaining time
175 Long64_t wait = t->GetAbsTime()-gSystem->Now();
176 if (wait > 0) gSystem->Sleep((UInt_t)wait);
177 c = dynamic_cast<TCondorSlave*>(p->Key());
178 }
179 }
180 }
181
182 // create slave
183 TSlave *slave = 0;
184 if (c) slave = CreateSlave(Form("%s:%d", c->fHostname.Data(), c->fPort), c->fOrdinal,
185 c->fPerfIdx, c->fImage, c->fWorkDir);
186
187 // add slave to appropriate list
188 if (trial < ntries) {
189 if (slave && slave->IsValid()) {
190 fSlaves->Add(slave);
191 if (trial == 1) {
192 claims.Remove(c);
193 } else {
194 TPair *p = dynamic_cast<TPair*>(claims.Remove(c));
195 if (p) {
196 TTimer *xt = dynamic_cast<TTimer*>(p->Value());
197 if (xt) delete xt;
198 delete p;
199 }
200 }
201 nClaimsDone++;
203 m << TString("Opening connections to workers") << nClaims
204 << nClaimsDone << kTRUE;
206 } else if (slave) {
207 if (trial == 1) {
208 TTimer* timer = new TTimer(delay);
209 TPair *p = new TPair(c, timer);
210 claims.RemoveAt(idx);
211 claims.AddAt(p, idx);
212 } else {
213 TPair *p = dynamic_cast<TPair*>(claims.At(idx));
214 if (p && p->Value()) {
215 TTimer *xt = dynamic_cast<TTimer*>(p->Value());
216 if (xt) xt->Reset();
217 }
218 }
219 delete slave;
220 idx++;
221 } else {
222 Warning("StartSlaves", "could not create TSlave object!");
223 }
224 } else {
225 if (slave) {
226 fSlaves->Add(slave);
227 TPair *p = dynamic_cast<TPair*>(claims.Remove(c));
228 if (p && p->Value()) {
229 TTimer *xt = dynamic_cast<TTimer*>(p->Value());
230 delete xt;
231 }
232 if (p) delete p;
233
234 nClaimsDone++;
236 m << TString("Opening connections to workers") << nClaims
237 << nClaimsDone << slave->IsValid();
239 } else {
240 Warning("StartSlaves", "could not create TSlave object!");
241 }
242 }
243
244 if (idx>=claims.GetSize()) {
245 trial++;
246 idx = 0;
247 }
248 }
249
250 // Here we finalize the server startup: in this way the bulk
251 // of remote operations are almost parallelized
252 TIter nxsl(fSlaves);
253 TSlave *sl = 0;
254 int nSlavesDone = 0, nSlavesTotal = fSlaves->GetSize();
255 while ((sl = (TSlave *) nxsl())) {
256
257 // Finalize setup of the server
258 if (sl->IsValid()) {
260 }
261
262 if (sl->IsValid()) {
263 fAllMonitor->Add(sl->GetSocket());
264 } else {
265 fBadSlaves->Add(sl);
266 }
267
268 // Notify end of startup operations
269 nSlavesDone++;
271 Bool_t wrkvalid = sl->IsValid() ? kTRUE : kFALSE;
272 m << TString("Setting up worker servers") << nSlavesTotal
273 << nSlavesDone << wrkvalid;
275 }
276
277 return kTRUE;
278}
279
280////////////////////////////////////////////////////////////////////////////////
281/// Suspend or resume PROOF via Condor.
282
284{
285 if (fTimer == 0) {
286 fTimer = new TTimer();
287 }
288 if (active) {
289 PDB(kCondor,1) Info("SetActive","-- Condor Resume --");
290 fTimer->Stop();
292 fCondor->Resume();
293 } else {
294#if 1
295 return; // don't suspend for the moment
296#else
297 Int_t delay = 60000; // milli seconds
298 PDB(kCondor,1) Info("SetActive","-- Delayed Condor Suspend (%d msec / to %lld) --",
299 delay, delay + Long64_t(gSystem->Now()));
300 fTimer->Connect("Timeout()", "TCondor", fCondor, "Suspend()");
301 fTimer->Start(10000, kTRUE); // single shot
302#endif
303 }
304}
305
306////////////////////////////////////////////////////////////////////////////////
307/// Get job Ad
308
310{
311 TString ad;
312
313 ad = "JobUniverse = 5\n"; // vanilla
314 ad += Form("Cmd = \"%s/bin/proofd\"\n", GetConfDir());
315 ad += Form("Iwd = \"%s\"\n", gSystem->TempDirectory());
316 ad += "In = \"/dev/null\"\n";
317 ad += Form("Out = \"%s/proofd.out.$(Port)\"\n", gSystem->TempDirectory());
318 ad += Form("Err = \"%s/proofd.err.$(Port)\"\n", gSystem->TempDirectory());
319 ad += Form("Args = \"-f -p $(Port) -d %d %s\"\n", GetLogLevel(), GetConfDir());
320
321 return ad;
322}
@ kPROOF_SERVERSTARTED
Definition: MessageTypes.h:82
#define SafeDelete(p)
Definition: RConfig.hxx:534
#define c(i)
Definition: RSha256.hxx:101
char Char_t
Definition: RtypesCore.h:37
const Bool_t kFALSE
Definition: RtypesCore.h:101
long Long_t
Definition: RtypesCore.h:54
long long Long64_t
Definition: RtypesCore.h:80
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define ClassImp(name)
Definition: Rtypes.h:375
winID h TVirtualViewer3D TVirtualGLPainter p
#define PDB(mask, level)
Definition: TProofDebug.h:56
R__EXTERN TProofServ * gProofServ
Definition: TProofServ.h:347
const char *const kPROOF_ConfFile
Definition: TProof.h:122
const char *const kPROOF_ConfDir
Definition: TProof.h:123
char * Form(const char *fmt,...)
Formats a string in a circular formatting buffer.
Definition: TString.cxx:2447
R__EXTERN TSystem * gSystem
Definition: TSystem.h:559
virtual Int_t GetSize() const
Return the capacity of the collection, i.e.
Definition: TCollection.h:184
Int_t fPerfIdx
Definition: TCondor.h:38
TString fWorkDir
Definition: TCondor.h:42
TString fImage
Definition: TCondor.h:39
TString fOrdinal
Definition: TCondor.h:41
Bool_t Resume()
Resume worker.
Definition: TCondor.cxx:343
TList * Claim(Int_t n, const char *cmd)
Claim n virtual machines This function figures out the image and performance index before returning t...
Definition: TCondor.cxx:233
@ kSuspended
Definition: TCondor.h:54
EState GetState() const
Definition: TCondor.h:79
TString GetImage(const char *host) const
Get image of the worker.
Definition: TCondor.cxx:440
Iterator of linked list.
Definition: TList.h:191
A doubly linked list.
Definition: TList.h:38
void AddAt(TObject *obj, Int_t idx) override
Insert object at position idx in the list.
Definition: TList.cxx:306
void Add(TObject *obj) override
Definition: TList.h:81
TObject * Remove(TObject *obj) override
Remove object from the list.
Definition: TList.cxx:822
TObject * At(Int_t idx) const override
Returns the object at position idx. Returns 0 if idx is out of range.
Definition: TList.cxx:357
virtual void Add(TSocket *sock, Int_t interest=kRead)
Add socket to the monitor's active list.
Definition: TMonitor.cxx:168
Mother of all ROOT objects.
Definition: TObject.h:37
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:879
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:893
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:867
Class used by TMap to store (key,value) pairs.
Definition: TMap.h:102
virtual ~TProofCondor()
Clean up Condor PROOF environment.
friend class TCondor
Definition: TProofCondor.h:35
Bool_t StartSlaves(Bool_t)
Setup Condor workers using dynamic information.
TProofCondor(const char *masterurl, const char *conffile=kPROOF_ConfFile, const char *confdir=kPROOF_ConfDir, Int_t loglevel=0, const char *alias=0, TProofMgr *mgr=0)
Start proof using condor.
TCondor * fCondor
Definition: TProofCondor.h:38
TTimer * fTimer
Definition: TProofCondor.h:39
TString GetJobAd()
Get job Ad.
virtual void SetActive()
Definition: TProofCondor.h:50
The PROOF manager interacts with the PROOF server coordinator to create or destroy a PROOF session,...
Definition: TProofMgr.h:43
The purpose of this class is to provide a complete node description for masters, submasters and worke...
const TString & GetImage() const
const TString & GetWorkDir() const
Int_t GetPerfIndex() const
const TString & GetNodeName() const
The purpose of this class is to provide a standard interface to static config files.
TList * GetWorkers()
Get the list of worker nodes.
const char * GetOrdinal() const
Definition: TProofServ.h:253
TSocket * GetSocket() const
Definition: TProofServ.h:257
TProofMgr * fManager
Definition: TProof.h:587
TUrl fUrl
Definition: TProof.h:567
void InitMembers()
Default initializations.
Definition: TProof.cxx:527
TList * fBadSlaves
Definition: TProof.h:574
Int_t Init(const char *masterurl, const char *conffile, const char *confdir, Int_t loglevel, const char *alias=0)
Start the PROOF environment.
Definition: TProof.cxx:757
const char * GetConfDir() const
Definition: TProof.h:904
TString fConfDir
Definition: TProof.h:569
TSlave * CreateSlave(const char *url, const char *ord, Int_t perf, const char *image, const char *workdir)
Create a new TSlave of type TSlave::kSlave.
Definition: TProof.cxx:1853
TString fConfFile
Definition: TProof.h:568
Int_t GetLogLevel() const
Definition: TProof.h:916
TList * fSlaves
Definition: TProof.h:572
TMonitor * fAllMonitor
Definition: TProof.h:575
TString fImage
Definition: TProof.h:570
Bool_t Connect(const char *signal, const char *receiver_class, void *receiver, const char *slot)
Non-static method is used to connect from the signal of this object to the receiver slot.
Definition: TQObject.cxx:869
virtual TObject * RemoveAt(Int_t idx)
Class describing a PROOF worker server.
Definition: TSlave.h:46
@ kSlave
Definition: TSlave.h:55
TSocket * GetSocket() const
Definition: TSlave.h:134
virtual Int_t SetupServ(Int_t stype, const char *conffile)
Init a PROOF slave object.
Definition: TSlave.cxx:178
virtual Bool_t IsValid() const
Definition: TSlave.h:150
virtual Int_t Send(const TMessage &mess)
Send a TMessage object.
Definition: TSocket.cxx:522
Basic string class.
Definition: TString.h:136
Ssiz_t Length() const
Definition: TString.h:410
const char * Data() const
Definition: TString.h:369
Bool_t IsNull() const
Definition: TString.h:407
virtual Bool_t ExpandPathName(TString &path)
Expand a pathname getting rid of special shell characters like ~.
Definition: TSystem.cxx:1274
virtual TTime Now()
Get current time in milliseconds since 0:00 Jan 1 1995.
Definition: TSystem.cxx:466
virtual const char * HostName()
Return the system's host name.
Definition: TSystem.cxx:306
virtual void Sleep(UInt_t milliSec)
Sleep milliSec milli seconds.
Definition: TSystem.cxx:440
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
Definition: TSystem.cxx:1482
Handles synchronous and a-synchronous timer events.
Definition: TTimer.h:51
TTime GetAbsTime() const
Definition: TTimer.h:78
virtual void Start(Long_t milliSec=-1, Bool_t singleShot=kFALSE)
Starts the timer with a milliSec timeout.
Definition: TTimer.cxx:213
void Reset()
Reset the timer.
Definition: TTimer.cxx:159
virtual void Stop()
Definition: TTimer.h:94
This class represents a WWW compatible URL.
Definition: TUrl.h:33
auto * m
Definition: textangle.C:8