Logo ROOT   6.10/09
Reference Guide
TDavixFile.cxx
Go to the documentation of this file.
1 // @(#)root/net:$Id$
2 // Author: Adrien Devresse and Tigran Mkrtchyan
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2013, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 //////////////////////////////////////////////////////////////////////////
13 // //
14 // TDavixFile //
15 // //
16 // A TDavixFile is like a normal TFile except that it uses //
17 // libdavix to read/write remote files. //
18 // It supports HTTP and HTTPS in a number of dialects and options //
19 // e.g. S3 is one of them //
20 // Other caracteristics come from the full support of Davix, //
21 // e.g. full redirection support in any circumstance //
22 // //
23 // Authors: Adrien Devresse (CERN IT/SDC) //
24 // Tigran Mkrtchyan (DESY) //
25 // //
26 // Checks and ROOT5 porting: //
27 // Fabrizio Furano (CERN IT/SDC) //
28 // //
29 // September 2013 //
30 // //
31 //////////////////////////////////////////////////////////////////////////
32 
33 
34 #include "TDavixFile.h"
35 #include "TROOT.h"
36 #include "TSocket.h"
37 #include "Bytes.h"
38 #include "TError.h"
39 #include "TSystem.h"
40 #include "TEnv.h"
41 #include "TBase64.h"
42 #include "TVirtualPerfStats.h"
43 #include "TDavixFileInternal.h"
44 #include "TSocket.h"
45 
46 #include <errno.h>
47 #include <stdlib.h>
48 #include <unistd.h>
49 #include <fcntl.h>
50 #include <davix.hpp>
51 #include <sstream>
52 #include <string>
53 #include <cstring>
54 
55 
56 static const std::string VERSION = "0.2.0";
57 
58 static const std::string gUserAgent = "ROOT/" + std::string(gROOT->GetVersion()) +
59 " TDavixFile/" + VERSION + " davix/" + Davix::version();
60 
61 // The prefix that is used to find the variables in the gEnv
62 #define ENVPFX "Davix."
63 
65 
66 using namespace Davix;
67 
68 const char* grid_mode_opt = "grid_mode=yes";
69 const char* ca_check_opt = "ca_check=no";
70 const char* s3_seckey_opt = "s3seckey=";
71 const char* s3_acckey_opt = "s3acckey=";
72 const char* s3_region_opt = "s3region=";
73 const char* s3_token_opt = "s3token=";
74 const char* s3_alternate_opt = "s3alternate=";
75 const char* open_mode_read = "READ";
76 const char* open_mode_create = "CREATE";
77 const char* open_mode_new = "NEW";
78 const char* open_mode_update = "UPDATE";
79 
82 
83 
84 ////////////////////////////////////////////////////////////////////////////////
85 
86 bool isno(const char *str)
87 {
88  if (!str) return false;
89 
90  if (!strcmp(str, "n") || !strcmp(str, "no") || !strcmp(str, "0") || !strcmp(str, "false")) return true;
91 
92  return false;
93 
94 }
95 
96 bool strToBool(const char *str, bool defvalue) {
97  if(!str) return defvalue;
98 
99  if(strcmp(str, "n") == 0 || strcmp(str, "no") == 0 || strcmp(str, "0") == 0 || strcmp(str, "false") == 0) return false;
100  if(strcmp(str, "y") == 0 || strcmp(str, "yes") == 0 || strcmp(str, "1") == 0 || strcmp(str, "true") == 0) return true;
101 
102  return defvalue;
103 }
104 
105 ////////////////////////////////////////////////////////////////////////////////
106 
107 int configure_open_flag(const std::string &str, int old_flag)
108 {
109  if (strcasecmp(str.c_str(), open_mode_read) == 0)
110  old_flag |= O_RDONLY;
111  if ((strcasecmp(str.c_str(), open_mode_create) == 0)
112  || (strcasecmp(str.c_str(), open_mode_new) == 0)) {
113  old_flag |= (O_CREAT | O_WRONLY | O_TRUNC);
114  }
115  if ((strcasecmp(str.c_str(), open_mode_update) == 0)) {
116  old_flag |= (O_RDWR);
117  }
118  return old_flag;
119 }
120 
121 ////////////////////////////////////////////////////////////////////////////////
122 
124 {
125  Int_t log_level = (gEnv) ? gEnv->GetValue("Davix.Debug", 0) : 0;
126 
127  switch (log_level) {
128  case 0:
129  davix_set_log_level(0);
130  break;
131  case 1:
132  davix_set_log_level(DAVIX_LOG_WARNING);
133  break;
134  case 2:
135  davix_set_log_level(DAVIX_LOG_VERBOSE);
136  break;
137  case 3:
138  davix_set_log_level(DAVIX_LOG_DEBUG);
139  break;
140  default:
141  davix_set_log_level(DAVIX_LOG_ALL);
142  break;
143  }
144 }
145 
146 ///////////////////////////////////////////////////////////////////
147 // Authn implementation, Locate and get VOMS cred if exist
148 
149 ////////////////////////////////////////////////////////////////////////////////
150 
151 static void TDavixFile_http_get_ucert(std::string &ucert, std::string &ukey)
152 {
153  char default_proxy[64];
154  const char *genvvar = 0, *genvvar1 = 0;
155  // The gEnv has higher priority, let's look for a proxy cert
156  genvvar = gEnv->GetValue("Davix.GSI.UserProxy", (const char *) NULL);
157  if (genvvar) {
158  ucert = ukey = genvvar;
159  if (gDebug > 0)
160  Info("TDavixFile_http_get_ucert", "Found proxy in gEnv");
161  return;
162  }
163 
164  // Try explicit environment for proxy
165  if (getenv("X509_USER_PROXY")) {
166  if (gDebug > 0)
167  Info("TDavixFile_http_get_ucert", "Found proxy in X509_USER_PROXY");
168  ucert = ukey = getenv("X509_USER_PROXY");
169  return;
170  }
171 
172  // Try with default location
173  snprintf(default_proxy, sizeof(default_proxy), "/tmp/x509up_u%d",
174  geteuid());
175 
176  if (access(default_proxy, R_OK) == 0) {
177  if (gDebug > 0)
178  Info("TDavixFile_http_get_ucert", "Found proxy in /tmp");
179  ucert = ukey = default_proxy;
180  return;
181  }
182 
183  // It seems we got no proxy, let's try to gather the keys
184  genvvar = gEnv->GetValue("Davix.GSI.UserCert", (const char *) NULL);
185  genvvar1 = gEnv->GetValue("Davix.GSI.UserKey", (const char *) NULL);
186  if (genvvar || genvvar1) {
187  if (gDebug > 0)
188  Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv");
189 
190  ucert = genvvar;
191  ukey = genvvar1;
192  return;
193  }
194 
195  // try with X509_* environment
196  if (getenv("X509_USER_CERT"))
197  ucert = getenv("X509_USER_CERT");
198  if (getenv("X509_USER_KEY"))
199  ukey = getenv("X509_USER_KEY");
200 
201  if ((ucert.size() > 0) || (ukey.size() > 0)) {
202  if (gDebug > 0)
203  Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv");
204  }
205  return;
206 
207 }
208 
209 ////////////////////////////////////////////////////////////////////////////////
210 
211 static int TDavixFile_http_authn_cert_X509(void *userdata, const Davix::SessionInfo &info,
212  Davix::X509Credential *cert, Davix::DavixError **err)
213 {
214  (void) userdata; // keep quiete compilation warnings
215  (void) info;
216  std::string ucert, ukey;
217  TDavixFile_http_get_ucert(ucert, ukey);
218 
219  if (ucert.empty() || ukey.empty()) {
220  Davix::DavixError::setupError(err, "TDavixFile",
221  Davix::StatusCode::AuthentificationError,
222  "Could not set the user's proxy or certificate");
223  return -1;
224  }
225  return cert->loadFromFilePEM(ukey, ucert, "", err);
226 }
227 /////////////////////////////////////////////////////////////////////////////////////////////
228 
229 ////////////////////////////////////////////////////////////////////////////////
230 
232 {
233  delete davixPosix;
234  delete davixParam;
235 }
236 
237 ////////////////////////////////////////////////////////////////////////////////
238 
240 {
241  if (davix_context_s == NULL) {
242  TLockGuard guard(&createLock);
243  if (davix_context_s == NULL) {
244  davix_context_s = new Context();
245  }
246  }
247  return davix_context_s;
248 }
249 
250 ////////////////////////////////////////////////////////////////////////////////
251 
253 {
254  DavixError *davixErr = NULL;
255  Davix_fd *fd = davixPosix->open(davixParam, fUrl.GetUrl(), oflags, &davixErr);
256  if (fd == NULL) {
257  // An error has occurred.. We might be able to recover with metalinks.
258  // Try to populate the replicas vector. If successful, TFile will try
259  // the replicas one by one
260 
261  replicas.clear();
262  DavixError *davixErr2 = NULL;
263  try {
264  DavFile file(*davixContext, Davix::Uri(fUrl.GetUrl()));
265  std::vector<DavFile> replicasLocal = file.getReplicas(NULL, &davixErr2);
266  for(size_t i = 0; i < replicasLocal.size(); i++) {
267  replicas.push_back(replicasLocal[i].getUri().getString());
268  }
269  }
270  catch(...) {}
271  DavixError::clearError(&davixErr2);
272 
273  if(replicas.empty()) {
274  // I was unable to retrieve a list of replicas: propagate the original
275  // error.
276  Error("DavixOpen", "can not open file with davix: %s (%d)",
277  davixErr->getErrMsg().c_str(), davixErr->getStatus());
278  }
279  DavixError::clearError(&davixErr);
280  } else {
281  // setup ROOT style read
282  davixPosix->fadvise(fd, 0, 300, Davix::AdviseRandom);
283  }
284 
285  return fd;
286 }
287 
288 ////////////////////////////////////////////////////////////////////////////////
289 
291 {
292  DavixError *davixErr = NULL;
293  if (davixFd != NULL && davixPosix->close(davixFd, &davixErr)) {
294  Error("DavixClose", "can not to close file with davix: %s (%d)",
295  davixErr->getErrMsg().c_str(), davixErr->getStatus());
296  DavixError::clearError(&davixErr);
297  }
298 }
299 
300 ////////////////////////////////////////////////////////////////////////////////
301 
303 {
304  const char *env_var = NULL;
305 
306  if (gDebug > 1)
307  Info("enableGridMode", " grid mode enabled !");
308 
309  if( ( env_var = getenv("X509_CERT_DIR")) == NULL){
310  env_var= "/etc/grid-security/certificates/";
311  }
312  davixParam->addCertificateAuthorityPath(env_var);
313  if (gDebug > 0)
314  Info("enableGridMode", "Adding CAdir %s", env_var);
315 }
316 
317 ////////////////////////////////////////////////////////////////////////////////
318 
319 // Only newer versions of davix support setting the S3 region and STS tokens.
320 // But it's only possible to check the davix version through a #define starting from
321 // 0.6.4.
322 // I have no way to check if setAwsRegion is available, so let's use SFINAE. :-)
323 // The first overload will always take priority - if "substitution" fails, meaning
324 // setAwsRegion is not there, the compiler will pick the second overload with
325 // the ellipses. (...)
326 
327 template<typename TRequestParams = Davix::RequestParams>
328 static auto awsRegion(TRequestParams *parameters, const char *region)
329  -> decltype(parameters->setAwsRegion(region), void())
330 {
331  if (gDebug > 1) Info("awsRegion", "Setting S3 Region to '%s' - v4 signature will be used", region);
332  parameters->setAwsRegion(region);
333 }
334 
335 template<typename TRequestParams = Davix::RequestParams>
336 static void awsRegion(...) {
337  Warning("setAwsRegion", "Unable to set AWS region, not supported by this version of davix");
338 }
339 
340 // Identical SFINAE trick as above for setAwsToken
341 template<typename TRequestParams = Davix::RequestParams>
342 static auto awsToken(TRequestParams *parameters, const char *token)
343  -> decltype(parameters->setAwsToken(token), void())
344 {
345  if (gDebug > 1) Info("awsToken", "Setting S3 STS temporary credentials");
346  parameters->setAwsToken(token);
347 }
348 
349 template<typename TRequestParams = Davix::RequestParams>
350 static void awsToken(...) {
351  Warning("awsToken", "Unable to set AWS token, not supported by this version of davix");
352 }
353 
354 // Identical SFINAE trick as above for setAwsAlternate
355 template<typename TRequestParams = Davix::RequestParams>
356 static auto awsAlternate(TRequestParams *parameters, bool option)
357  -> decltype(parameters->setAwsAlternate(option), void())
358 {
359  if (gDebug > 1) Info("awsAlternate", "Setting S3 path-based bucket option (s3alternate)");
360  parameters->setAwsAlternate(option);
361 }
362 
363 template<typename TRequestParams = Davix::RequestParams>
364 static void awsAlternate(...) {
365  Warning("awsAlternate", "Unable to set AWS path-based bucket option (s3alternate), not supported by this version of davix");
366 }
367 
368 void TDavixFileInternal::setAwsRegion(const std::string & region) {
369  if(!region.empty()) {
370  awsRegion(davixParam, region.c_str());
371  }
372 }
373 
374 void TDavixFileInternal::setAwsToken(const std::string & token) {
375  if(!token.empty()) {
376  awsToken(davixParam, token.c_str());
377  }
378 }
379 
380 void TDavixFileInternal::setAwsAlternate(const bool & option) {
381  awsAlternate(davixParam, option);
382 }
383 
384 
385 void TDavixFileInternal::setS3Auth(const std::string &secret, const std::string &access,
386  const std::string &region, const std::string &token)
387 {
388  if (gDebug > 1) {
389  Info("setS3Auth", " Aws S3 tokens configured");
390  }
391  davixParam->setAwsAuthorizationKeys(secret, access);
392  davixParam->setProtocol(RequestProtocol::AwsS3);
393 
394  setAwsRegion(region);
395  setAwsToken(token);
396 }
397 
398 ////////////////////////////////////////////////////////////////////////////////
399 
401 {
402  const char *env_var = NULL, *env_var2 = NULL;
403  // default opts
404  davixParam->setTransparentRedirectionSupport(true);
405  davixParam->setClientCertCallbackX509(&TDavixFile_http_authn_cert_X509, NULL);
406 
407  // setup CADIR
408  env_var = gEnv->GetValue("Davix.GSI.CAdir", (const char *) NULL);
409  if (env_var) {
410  davixParam->addCertificateAuthorityPath(env_var);
411  if (gDebug > 0)
412  Info("parseConfig", "Add CAdir: %s", env_var);
413  }
414  // CA Check
415  bool ca_check_local = !isno(gEnv->GetValue("Davix.GSI.CACheck", (const char *)"y"));
416  davixParam->setSSLCAcheck(ca_check_local);
417  if (gDebug > 0)
418  Info("parseConfig", "Setting CAcheck to %s", ((ca_check_local) ? ("true") : ("false")));
419 
420  // S3 Auth
421  if (((env_var = gEnv->GetValue("Davix.S3.SecretKey", getenv("S3_SECRET_KEY"))) != NULL)
422  && ((env_var2 = gEnv->GetValue("Davix.S3.AccessKey", getenv("S3_ACCESS_KEY"))) != NULL)) {
423  Info("parseConfig", "Setting S3 SecretKey and AccessKey. Access Key : %s ", env_var2);
424  davixParam->setAwsAuthorizationKeys(env_var, env_var2);
425 
426  // need to set region?
427  if ( (env_var = gEnv->GetValue("Davix.S3.Region", getenv("S3_REGION"))) != NULL) {
428  setAwsRegion(env_var);
429  }
430  // need to set STS token?
431  if( (env_var = gEnv->GetValue("Davix.S3.Token", getenv("S3_TOKEN"))) != NULL) {
432  setAwsToken(env_var);
433  }
434  // need to set aws alternate?
435  if( (env_var = gEnv->GetValue("Davix.S3.Alternate", getenv("S3_ALTERNATE"))) != NULL) {
436  setAwsAlternate(strToBool(env_var, false));
437  }
438  }
439 
440  env_var = gEnv->GetValue("Davix.GSI.GridMode", (const char *)"y");
441  if (!isno(env_var))
442  enableGridMode();
443 }
444 
445 ////////////////////////////////////////////////////////////////////////////////
446 /// intput params
447 
449 {
450  std::stringstream ss(option);
451  std::string item;
452  std::vector<std::string> parsed_options;
453  // parameters
454  std::string s3seckey, s3acckey, s3region, s3token;
455 
456  while (std::getline(ss, item, ' ')) {
457  parsed_options.push_back(item);
458  }
459 
460  for (std::vector<std::string>::iterator it = parsed_options.begin(); it < parsed_options.end(); ++it) {
461  // grid mode option
462  if ((strcasecmp(it->c_str(), grid_mode_opt)) == 0) {
463  enableGridMode();
464  }
465  // ca check option
466  if ((strcasecmp(it->c_str(), ca_check_opt)) == 0) {
467  davixParam->setSSLCAcheck(false);
468  }
469  // s3 sec key
470  if (strncasecmp(it->c_str(), s3_seckey_opt, strlen(s3_seckey_opt)) == 0) {
471  s3seckey = std::string(it->c_str() + strlen(s3_seckey_opt));
472  }
473  // s3 access key
474  if (strncasecmp(it->c_str(), s3_acckey_opt, strlen(s3_acckey_opt)) == 0) {
475  s3acckey = std::string(it->c_str() + strlen(s3_acckey_opt));
476  }
477  // s3 region
478  if (strncasecmp(it->c_str(), s3_region_opt, strlen(s3_region_opt)) == 0) {
479  s3region = std::string(it->c_str() + strlen(s3_region_opt));
480  }
481  // s3 sts token
482  if (strncasecmp(it->c_str(), s3_token_opt, strlen(s3_token_opt)) == 0) {
483  s3token = std::string(it->c_str() + strlen(s3_token_opt));
484  }
485  // s3 alternate option
486  if (strncasecmp(it->c_str(), s3_alternate_opt, strlen(s3_alternate_opt)) == 0) {
487  setAwsAlternate(strToBool(it->c_str() + strlen(s3_alternate_opt), false));
488  }
489  // open mods
490  oflags = configure_open_flag(*it, oflags);
491  }
492 
493  if (s3seckey.size() > 0) {
494  setS3Auth(s3seckey, s3acckey, s3region, s3token);
495  }
496 
497  if (oflags == 0) // default open mode
498  oflags = O_RDONLY;
499 }
500 
501 ////////////////////////////////////////////////////////////////////////////////
502 
504 {
505  davixPosix = new DavPosix(davixContext);
506  davixParam = new RequestParams();
507  davixParam->setUserAgent(gUserAgent);
508  davixParam->setMetalinkMode(Davix::MetalinkMode::Disable);
510  parseConfig();
511  parseParams(opt);
512 }
513 
514 ////////////////////////////////////////////////////////////////////////////////
515 
516 Int_t TDavixFileInternal::DavixStat(const char *url, struct stat *st)
517 {
518  DavixError *davixErr = NULL;
519 
520  if (davixPosix->stat(davixParam, url, st, &davixErr) < 0) {
521 
522  Error("DavixStat", "can not stat the file with davix: %s (%d)",
523  davixErr->getErrMsg().c_str(), davixErr->getStatus());
524  DavixError::clearError(&davixErr);
525  return 0;
526  }
527  return 1;
528 }
529 
530 /////////////////////////////////////////////////////////////////////////////////////////////
531 
532 ////////////////////////////////////////////////////////////////////////////////
533 
534 TDavixFile::TDavixFile(const char *url, Option_t *opt, const char *ftitle, Int_t compress) : TFile(url, "WEB"),
535  d_ptr(new TDavixFileInternal(fUrl, opt))
536 {
537  (void) ftitle;
538  (void) compress;
539  Init(kFALSE);
540 }
541 
542 ////////////////////////////////////////////////////////////////////////////////
543 
545 {
546  d_ptr->Close();
547  delete d_ptr;
548 }
549 
550 ////////////////////////////////////////////////////////////////////////////////
551 
553 {
554  (void) init;
555  //initialize davix
556  d_ptr->init();
557  // pre-open file
558  if ((d_ptr->getDavixFileInstance()) == NULL){
559  MakeZombie();
560  gDirectory = gROOT;
561  return;
562  }
564  fOffset = 0;
565  fD = -2; // so TFile::IsOpen() will return true when in TFile::~TFi */
566 }
567 
569  std::vector<std::string> replicas = d_ptr->getReplicas();
570  TString newUrl;
571  if(!replicas.empty()) {
572  std::stringstream ss;
573  for(size_t i = 0; i < replicas.size(); i++) {
574  ss << replicas[i];
575  if(i != replicas.size()-1) ss << "|";
576  }
577  newUrl = ss.str();
578  }
579  return newUrl;
580 }
581 
582 ////////////////////////////////////////////////////////////////////////////////
583 /// Set position from where to start reading.
584 
586 {
587  TLockGuard guard(&(d_ptr->positionLock));
588  switch (pos) {
589  case kBeg:
590  fOffset = offset + fArchiveOffset;
591  break;
592  case kCur:
593  fOffset += offset;
594  break;
595  case kEnd:
596  // this option is not used currently in the ROOT code
597  if (fArchiveOffset)
598  Error("Seek", "seeking from end in archive is not (yet) supported");
599  fOffset = fEND - offset; // is fEND really EOF or logical EOF?
600  break;
601  }
602 
603  if (gDebug > 1)
604  Info("Seek", " move cursor to %lld"
605  , fOffset);
606 }
607 
608 ////////////////////////////////////////////////////////////////////////////////
609 /// Read specified byte range from remote file via HTTP.
610 /// Returns kTRUE in case of error.
611 
613 {
614  TLockGuard guard(&(d_ptr->positionLock));
615  Davix_fd *fd;
616  if ((fd = d_ptr->getDavixFileInstance()) == NULL)
617  return kTRUE;
618  Long64_t ret = DavixReadBuffer(fd, buf, len);
619  if (ret < 0)
620  return kTRUE;
621 
622  if (gDebug > 1)
623  Info("ReadBuffer", "%lld bytes of data read sequentially"
624  " (%d requested)", ret, len);
625 
626  return kFALSE;
627 }
628 
629 ////////////////////////////////////////////////////////////////////////////////
630 
632 {
633  Davix_fd *fd;
634  if ((fd = d_ptr->getDavixFileInstance()) == NULL)
635  return kTRUE;
636 
637  Long64_t ret = DavixPReadBuffer(fd, buf, pos, len);
638  if (ret < 0)
639  return kTRUE;
640 
641  if (gDebug > 1)
642  Info("ReadBuffer", "%lld bytes of data read from offset"
643  " %lld (%d requested)", ret, pos, len);
644  return kFALSE;
645 }
646 
647 ////////////////////////////////////////////////////////////////////////////////
648 
650 {
651  Davix_fd *fd;
652  if ((fd = d_ptr->getDavixFileInstance()) == NULL)
653  return kFALSE;
654 
655  d_ptr->davixPosix->fadvise(fd, static_cast<dav_off_t>(offs), static_cast<dav_size_t>(len), Davix::AdviseRandom);
656 
657  if (gDebug > 1)
658  Info("ReadBufferAsync", "%d bytes of data prefected from offset"
659  " %lld ", len, offs);
660  return kFALSE;
661 }
662 
663 ////////////////////////////////////////////////////////////////////////////////
664 
665 Bool_t TDavixFile::ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
666 {
667  Davix_fd *fd;
668  if ((fd = d_ptr->getDavixFileInstance()) == NULL)
669  return kTRUE;
670 
671  Long64_t ret = DavixReadBuffers(fd, buf, pos, len, nbuf);
672  if (ret < 0)
673  return kTRUE;
674 
675  if (gDebug > 1)
676  Info("ReadBuffers", "%lld bytes of data read from a list of %d buffers",
677  ret, nbuf);
678 
679  return kFALSE;
680 }
681 
682 ////////////////////////////////////////////////////////////////////////////////
683 
684 Bool_t TDavixFile::WriteBuffer(const char *buf, Int_t len)
685 {
686  Davix_fd *fd;
687  if ((fd = d_ptr->getDavixFileInstance()) == NULL)
688  return kTRUE;
689 
690  Long64_t ret = DavixWriteBuffer(fd, buf, len);
691  if (ret < 0)
692  return kTRUE;
693 
694  if (gDebug > 1)
695  Info("WriteBuffer", "%lld bytes of data write"
696  " %d requested", ret, len);
697  return kFALSE;
698 }
699 
700 ////////////////////////////////////////////////////////////////////////////////
701 
703 {
704  d_ptr->davixParam->setSSLCAcheck((bool)check);
705 }
706 
707 ////////////////////////////////////////////////////////////////////////////////
708 
710 {
712 }
713 
714 ////////////////////////////////////////////////////////////////////////////////
715 
717 {
718  TLockGuard l(&(openLock));
719  std::vector<void *>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd);
720  return (f != dirdVec.end());
721 }
722 
723 ////////////////////////////////////////////////////////////////////////////////
724 
726 {
727  TLockGuard l(&(openLock));
728  dirdVec.push_back(fd);
729 }
730 
731 ////////////////////////////////////////////////////////////////////////////////
732 
734 {
735  TLockGuard l(&(openLock));
736  std::vector<void *>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd);
737  if (f != dirdVec.end())
738  dirdVec.erase(f);
739 }
740 
741 ////////////////////////////////////////////////////////////////////////////////
742 
744 {
745  struct stat st;
746  Int_t ret = d_ptr->DavixStat(fUrl.GetUrl(), &st);
747  if (ret) {
748  if (gDebug > 1)
749  Info("GetSize", "file size requested: %lld", (Long64_t)st.st_size);
750  return st.st_size;
751  }
752  return -1;
753 }
754 
755 ////////////////////////////////////////////////////////////////////////////////
756 
758 {
759  if (gPerfStats)
760  return TTimeStamp();
761  return 0;
762 }
763 
764 ////////////////////////////////////////////////////////////////////////////////
765 /// set TFile state info
766 
767 void TDavixFile::eventStop(Double_t t_start, Long64_t len, bool read)
768 {
769  if(read) {
770  fBytesRead += len;
771  fReadCalls += 1;
772 
775 
776  if (gPerfStats)
777  gPerfStats->FileReadEvent(this, (Int_t) len, t_start);
778  } else {
779  fBytesWrite += len;
781  }
782 }
783 
784 ////////////////////////////////////////////////////////////////////////////////
785 
786 Long64_t TDavixFile::DavixReadBuffer(Davix_fd *fd, char *buf, Int_t len)
787 {
788  DavixError *davixErr = NULL;
789  Double_t start_time = eventStart();
790 
791  Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, fOffset, &davixErr);
792  if (ret < 0) {
793  Error("DavixReadBuffer", "can not read data with davix: %s (%d)",
794  davixErr->getErrMsg().c_str(), davixErr->getStatus());
795  DavixError::clearError(&davixErr);
796  } else {
797  fOffset += ret;
798  eventStop(start_time, ret);
799  }
800 
801  return ret;
802 }
803 
804 ////////////////////////////////////////////////////////////////////////////////
805 
806 Long64_t TDavixFile::DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len)
807 {
808  DavixError *davixErr = NULL;
809  Double_t start_time = eventStart();
810 
811  Long64_t ret = d_ptr->davixPosix->pwrite(fd, buf, len, fOffset, &davixErr);
812  if (ret < 0) {
813  Error("DavixWriteBuffer", "can not write data with davix: %s (%d)",
814  davixErr->getErrMsg().c_str(), davixErr->getStatus());
815  DavixError::clearError(&davixErr);
816  } else {
817  fOffset += ret;
818  eventStop(start_time, ret, false);
819  }
820 
821  return ret;
822 }
823 
824 ////////////////////////////////////////////////////////////////////////////////
825 
826 Long64_t TDavixFile::DavixPReadBuffer(Davix_fd *fd, char *buf, Long64_t pos, Int_t len)
827 {
828  DavixError *davixErr = NULL;
829  Double_t start_time = eventStart();
830 
831  Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, pos, &davixErr);
832  if (ret < 0) {
833  Error("DavixPReadBuffer", "can not read data with davix: %s (%d)",
834  davixErr->getErrMsg().c_str(), davixErr->getStatus());
835  DavixError::clearError(&davixErr);
836  } else {
837  eventStop(start_time, ret);
838  }
839 
840 
841  return ret;
842 }
843 
844 ////////////////////////////////////////////////////////////////////////////////
845 
846 Long64_t TDavixFile::DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
847 {
848  DavixError *davixErr = NULL;
849  Double_t start_time = eventStart();
850  DavIOVecInput in[nbuf];
851  DavIOVecOuput out[nbuf];
852 
853  int lastPos = 0;
854  for (Int_t i = 0; i < nbuf; ++i) {
855  in[i].diov_buffer = &buf[lastPos];
856  in[i].diov_offset = pos[i];
857  in[i].diov_size = len[i];
858  lastPos += len[i];
859  }
860 
861  Long64_t ret = d_ptr->davixPosix->preadVec(fd, in, out, nbuf, &davixErr);
862  if (ret < 0) {
863  Error("DavixReadBuffers", "can not read data with davix: %s (%d)",
864  davixErr->getErrMsg().c_str(), davixErr->getStatus());
865  DavixError::clearError(&davixErr);
866  } else {
867  eventStop(start_time, ret);
868  }
869 
870  return ret;
871 }
static void ConfigureDavixLogLevel()
Definition: TDavixFile.cxx:123
Definition: TMutex.h:30
double read(const std::string &file_name)
reading
virtual Long64_t GetSize() const
Returns the current file size.
Definition: TDavixFile.cxx:743
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:847
static const std::string gUserAgent
Definition: TDavixFile.cxx:58
const char * s3_seckey_opt
Definition: TDavixFile.cxx:70
long long Long64_t
Definition: RtypesCore.h:69
Long64_t fBytesWrite
Number of bytes written to this file.
Definition: TFile.h:68
const char * open_mode_read
Definition: TDavixFile.cxx:75
const char * open_mode_update
Definition: TDavixFile.cxx:78
void Init(Bool_t init)
Initialize a TFile object.
Definition: TDavixFile.cxx:552
const char Option_t
Definition: RtypesCore.h:62
const char * s3_alternate_opt
Definition: TDavixFile.cxx:74
static void TDavixFile_http_get_ucert(std::string &ucert, std::string &ukey)
Definition: TDavixFile.cxx:151
void setS3Auth(const std::string &secret, const std::string &access, const std::string &region, const std::string &token)
Definition: TDavixFile.cxx:385
static auto awsToken(TRequestParams *parameters, const char *token) -> decltype(parameters->setAwsToken(token), void())
Definition: TDavixFile.cxx:342
Davix::RequestParams * davixParam
Small helper to keep current directory context.
Davix_fd * Open()
Definition: TDavixFile.cxx:252
static void SetFileReadCalls(Int_t readcalls=0)
Definition: TFile.cxx:4417
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read specified byte range from remote file via HTTP.
Definition: TDavixFile.cxx:612
#define gROOT
Definition: TROOT.h:375
void addDird(void *fd)
Definition: TDavixFile.cxx:725
bool strToBool(const char *str, bool defvalue)
Definition: TDavixFile.cxx:96
Double_t eventStart()
Definition: TDavixFile.cxx:757
int Int_t
Definition: RtypesCore.h:41
bool Bool_t
Definition: RtypesCore.h:59
TDavixFileInternal * d_ptr
Definition: TDavixFile.h:68
Long64_t DavixReadBuffer(Davix_fd *fd, char *buf, Int_t len)
Definition: TDavixFile.cxx:786
ERelativeTo
Definition: TFile.h:171
#define NULL
Definition: RtypesCore.h:88
Int_t fReadCalls
Number of read calls ( not counting the cache calls )
Definition: TFile.h:82
virtual Bool_t WriteBuffer(const char *buffer, Int_t bufferLength)
Write a buffer to the file.
Definition: TDavixFile.cxx:684
void setAwsAlternate(const bool &option)
Definition: TDavixFile.cxx:380
Davix_fd * getDavixFileInstance()
void setAwsRegion(const std::string &region)
Definition: TDavixFile.cxx:368
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition: TUrl.cxx:387
const char * open_mode_create
Definition: TDavixFile.cxx:76
Long64_t DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Definition: TDavixFile.cxx:846
static void SetFileBytesWritten(Long64_t bytes=0)
Definition: TFile.cxx:4414
TUrl fUrl
!URL of file
Definition: TFile.h:103
Int_t fD
File descriptor.
Definition: TFile.h:75
static auto awsAlternate(TRequestParams *parameters, bool option) -> decltype(parameters->setAwsAlternate(option), void())
Definition: TDavixFile.cxx:356
Davix::DavPosix * davixPosix
virtual void Seek(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
Definition: TDavixFile.cxx:585
static Long64_t GetFileBytesRead()
Static function returning the total number of bytes read from all files.
Definition: TFile.cxx:4377
static Long64_t GetFileBytesWritten()
Static function returning the total number of bytes written to all files.
Definition: TFile.cxx:4386
void Info(const char *location, const char *msgfmt,...)
static TMutex createLock
Definition: TDavixFile.cxx:80
void Error(const char *location, const char *msgfmt,...)
Int_t DavixStat(const char *url, struct stat *st)
Definition: TDavixFile.cxx:516
static Context * davix_context_s
Definition: TDavixFile.cxx:81
Long64_t DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len)
Definition: TDavixFile.cxx:806
Long64_t fEND
Last used byte in file.
Definition: TFile.h:72
virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len)
Definition: TDavixFile.cxx:649
const char * s3_token_opt
Definition: TDavixFile.cxx:73
virtual TString GetNewUrl()
Definition: TDavixFile.cxx:568
static Davix::Context * getDavixInstance()
Definition: TDavixFile.cxx:239
std::vector< std::string > getReplicas()
virtual Int_t GetValue(const char *name, Int_t dflt)
Returns the integer value for a resource.
Definition: TEnv.cxx:482
void removeDird(void *fd)
Definition: TDavixFile.cxx:733
static auto awsRegion(TRequestParams *parameters, const char *region) -> decltype(parameters->setAwsRegion(region), void())
Definition: TDavixFile.cxx:328
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:873
bool isMyDird(void *fd)
Definition: TDavixFile.cxx:716
TLine * l
Definition: textangle.C:4
static Int_t GetFileReadCalls()
Static function returning the total number of read calls from all files.
Definition: TFile.cxx:4394
void eventStop(Double_t t, Long64_t len, bool read=true)
set TFile state info
Definition: TDavixFile.cxx:767
void Warning(const char *location, const char *msgfmt,...)
bool isno(const char *str)
Definition: TDavixFile.cxx:86
virtual void Init(Bool_t create)
Initialize a TFile object.
Definition: TFile.cxx:584
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
Definition: TDavixFile.cxx:665
#define gPerfStats
const Bool_t kFALSE
Definition: RtypesCore.h:92
const char * grid_mode_opt
Definition: TDavixFile.cxx:68
static const std::string VERSION
Definition: TDavixFile.cxx:56
#define ClassImp(name)
Definition: Rtypes.h:336
double f(double x)
static Int_t init()
double Double_t
Definition: RtypesCore.h:55
R__EXTERN TEnv * gEnv
Definition: TEnv.h:170
The TTimeStamp encapsulates seconds and ns since EPOCH.
Definition: TTimeStamp.h:71
const char * s3_acckey_opt
Definition: TDavixFile.cxx:71
int configure_open_flag(const std::string &str, int old_flag)
Definition: TDavixFile.cxx:107
const char * s3_region_opt
Definition: TDavixFile.cxx:72
void enableGridMode()
Enable the grid mode The grid Mode configure automatically all grid-CA path, VOMS authentication and ...
Definition: TDavixFile.cxx:709
typedef void((*Func_t)())
Definition: file.py:1
void setAwsToken(const std::string &token)
Definition: TDavixFile.cxx:374
const char * ca_check_opt
Definition: TDavixFile.cxx:69
void MakeZombie()
Definition: TObject.h:49
#define snprintf
Definition: civetweb.c:822
static int TDavixFile_http_authn_cert_X509(void *userdata, const Davix::SessionInfo &info, Davix::X509Credential *cert, Davix::DavixError **err)
Definition: TDavixFile.cxx:211
R__EXTERN Int_t gDebug
Definition: Rtypes.h:83
Long64_t fArchiveOffset
!Offset at which file starts in archive
Definition: TFile.h:94
#define gDirectory
Definition: TDirectory.h:211
TDavixFile(const char *url, Option_t *option="", const char *ftitle="", Int_t compress=1)
Open function for TDavixFile.
Definition: TDavixFile.cxx:534
Long64_t fOffset
!Seek offset cache
Definition: TFile.h:89
static void SetFileBytesRead(Long64_t bytes=0)
Definition: TFile.cxx:4411
Long64_t DavixPReadBuffer(Davix_fd *fd, char *buf, Long64_t pos, Int_t len)
Definition: TDavixFile.cxx:826
void setCACheck(Bool_t check)
Enable or disable certificate authority check.
Definition: TDavixFile.cxx:702
const Bool_t kTRUE
Definition: RtypesCore.h:91
void parseParams(Option_t *option)
intput params
Definition: TDavixFile.cxx:448
Long64_t fBytesRead
Number of bytes read from this file.
Definition: TFile.h:69
const char * open_mode_new
Definition: TDavixFile.cxx:77