Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
TDavixFile.cxx
Go to the documentation of this file.
1// @(#)root/net:$Id$
2// Author: Adrien Devresse and Tigran Mkrtchyan
3
4/*************************************************************************
5 * Copyright (C) 1995-2013, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12//////////////////////////////////////////////////////////////////////////
13// //
14// TDavixFile //
15// //
16// A TDavixFile is like a normal TFile except that it uses //
17// libdavix to read/write remote files. //
18// It supports HTTP and HTTPS in a number of dialects and options //
19// e.g. S3 is one of them //
20// Other caracteristics come from the full support of Davix, //
21// e.g. full redirection support in any circumstance //
22// //
23// Authors: Adrien Devresse (CERN IT/SDC) //
24// Tigran Mkrtchyan (DESY) //
25// //
26// Checks and ROOT5 porting: //
27// Fabrizio Furano (CERN IT/SDC) //
28// //
29// September 2013 //
30// //
31//////////////////////////////////////////////////////////////////////////
32
33#include "ROOT/RLogger.hxx"
34#include "TDavixFile.h"
35#include "TROOT.h"
36#include "TSocket.h"
37#include "Bytes.h"
38#include "TError.h"
39#include "TEnv.h"
40#include "TBase64.h"
41#include "TVirtualPerfStats.h"
42#include "TDavixFileInternal.h"
43#include "snprintf.h"
44
45#include <cerrno>
46#include <cstdlib>
47#include <unistd.h>
48#include <fcntl.h>
49#include <davix.hpp>
50#include <sstream>
51#include <string>
52#include <cstring>
53
54
55static const std::string VERSION = "0.2.0";
56
57static const std::string gUserAgent = "ROOT/" + std::string(gROOT->GetVersion()) +
58" TDavixFile/" + VERSION + " davix/" + Davix::version();
59
60// The prefix that is used to find the variables in the gEnv
61#define ENVPFX "Davix."
62
64
65using namespace Davix;
66
67const char* grid_mode_opt = "grid_mode=yes";
68const char* ca_check_opt = "ca_check=no";
69const char* s3_seckey_opt = "s3seckey=";
70const char* s3_acckey_opt = "s3acckey=";
71const char* s3_region_opt = "s3region=";
72const char* s3_token_opt = "s3token=";
73const char* s3_alternate_opt = "s3alternate=";
74const char* open_mode_read = "READ";
75const char* open_mode_create = "CREATE";
76const char* open_mode_new = "NEW";
77const char* open_mode_update = "UPDATE";
78
80static Context* davix_context_s = NULL;
81
82////////////////////////////////////////////////////////////////////////////////
83
85{
86 static ROOT::Experimental::RLogChannel sLog("ROOT.TDavix");
87 return sLog;
88}
89
90////////////////////////////////////////////////////////////////////////////////
91
92bool isno(const char *str)
93{
94 if (!str) return false;
95
96 if (!strcmp(str, "n") || !strcmp(str, "no") || !strcmp(str, "0") || !strcmp(str, "false")) return true;
97
98 return false;
99
100}
101
102bool strToBool(const char *str, bool defvalue) {
103 if(!str) return defvalue;
104
105 if(strcmp(str, "n") == 0 || strcmp(str, "no") == 0 || strcmp(str, "0") == 0 || strcmp(str, "false") == 0) return false;
106 if(strcmp(str, "y") == 0 || strcmp(str, "yes") == 0 || strcmp(str, "1") == 0 || strcmp(str, "true") == 0) return true;
107
108 return defvalue;
109}
110
111////////////////////////////////////////////////////////////////////////////////
112
113int configure_open_flag(const std::string &str, int old_flag)
114{
115 if (strcasecmp(str.c_str(), open_mode_read) == 0)
116 old_flag |= O_RDONLY;
117 if ((strcasecmp(str.c_str(), open_mode_create) == 0)
118 || (strcasecmp(str.c_str(), open_mode_new) == 0)) {
119 old_flag |= (O_CREAT | O_WRONLY | O_TRUNC);
120 }
121 if ((strcasecmp(str.c_str(), open_mode_update) == 0)) {
122 old_flag |= (O_RDWR);
123 }
124 return old_flag;
125}
126
127////////////////////////////////////////////////////////////////////////////////
128
130{
131 Int_t log_level = (gEnv) ? gEnv->GetValue("Davix.Debug", 0) : 0;
132
133 switch (log_level) {
134 case 0:
135 davix_set_log_level(0);
136 break;
137 case 1:
138 davix_set_log_level(DAVIX_LOG_WARNING);
139 break;
140 case 2:
141 davix_set_log_level(DAVIX_LOG_VERBOSE);
142 break;
143 case 3:
144 davix_set_log_level(DAVIX_LOG_DEBUG);
145 break;
146 default:
147 davix_set_log_level(DAVIX_LOG_ALL);
148 break;
149 }
150}
151
152////////////////////////////////////////////////////////////////////////////////
153
154bool normalizeToken(const std::string &input_token, std::string &output_token)
155{
156 static const std::string whitespace = " \t\f\n\v\r";
157 // Tokens are meant for use in an HTTP header; these are forbidden
158 // characters.
159 static const std::string nonheader_whitespace = "\r\n";
160 auto begin = input_token.find_first_not_of(whitespace);
161 // No token here, but not an error.
162 if (begin == std::string::npos) {
163 output_token = "";
164 return true;
165 }
166
167 std::string token = input_token.substr(begin);
168 auto end = token.find_last_not_of(whitespace);
169 token = token.substr(0, end + 1);
170
171 // If non-permitted header characters are present ("\r\n"),
172 // then this token is not permitted.
173 if (token.find(nonheader_whitespace) != std::string::npos) {
174 output_token = "";
175 R__LOG_ERROR(TDavixLogChannel()) << "Token discovery failure: token contains non-permitted character sequence (\\r\\n)";
176 return false;
177 }
178 output_token = token;
179 return true;
180}
181
182bool findTokenInFile(const std::string &token_file, std::string &output_token)
183{
184 R__LOG_INFO(TDavixLogChannel()) << "Looking for token in file " << token_file.c_str();
185 int fd = open(token_file.c_str(), O_RDONLY);
186 if (fd == -1) {
187 output_token = "";
188 if (errno == ENOENT) {
189 return true;
190 }
191 R__LOG_ERROR(TDavixLogChannel()) << "Cannot open '" << token_file << "', error: " << strerror(errno);
192 return false;
193 }
194
195 // As a pragmatic matter, we limit tokens to 16KB; this is often the limit
196 // in size of HTTP headers for many web servers.
197 // This also avoids unbounded memory use in case if the user points us to
198 // a large file.
199 static const size_t max_size = 16384;
200 std::vector<char> input_buffer;
201 input_buffer.resize(max_size);
202 ssize_t retval = read(fd, &input_buffer[0], max_size);
203 close(fd);
204
205 if (retval == -1) {
206 output_token = "";
207 R__LOG_ERROR(TDavixLogChannel()) << "Token discovery failure: failed to read file " << token_file.c_str() << "', error: " << strerror(errno);
208 return false;
209 }
210 if (retval == 16384) {
211 // Token may have been truncated! Erring on the side of caution.
212 R__LOG_ERROR(TDavixLogChannel()) << "Token discovery failure: token was larger than 16KB limit.";
213 return false;
214 }
215
216 std::string token(&input_buffer[0], retval);
217
218 return normalizeToken(token, output_token);
219}
220
221// Token discovery process is based on WLCG specification:
222// https://github.com/WLCG-AuthZ-WG/bearer-token-discovery/blob/master/specification.md
223std::string DiscoverToken()
224{
225 const char *bearer_token = getenv("BEARER_TOKEN");
226 std::string token;
227 if (bearer_token && *bearer_token){
228 if (!normalizeToken(bearer_token, token)) {return "";}
229 if (!token.empty()) {return token;}
230 }
231
232 const char *bearer_token_file = getenv("BEARER_TOKEN_FILE");
233 if (bearer_token_file) {
234 if (!findTokenInFile(bearer_token_file, token)) {return "";}
235 if (!token.empty()) {return token;}
236 }
237
238#ifndef WIN32
239 uid_t euid = geteuid();
240 std::string fname = "/bt_u";
241 fname += std::to_string(euid);
242
243 const char *xdg_runtime_dir = getenv("XDG_RUNTIME_DIR");
244 if (xdg_runtime_dir) {
245 std::string xdg_token_file = std::string(xdg_runtime_dir) + fname;
246 if (!findTokenInFile(xdg_token_file, token)) {return "";}
247 if (!token.empty()) {return token;}
248 }
249
250 if (!findTokenInFile("/tmp" + fname, token)) {return "";}
251 return token;
252#else
253 // WLCG profile doesn't define search paths on Windows;
254 // skip this for now.
255 return "";
256#endif
257}
258
259///////////////////////////////////////////////////////////////////
260// Authn implementation, Locate and get VOMS cred if exist
261
262////////////////////////////////////////////////////////////////////////////////
263
264static void TDavixFile_http_get_ucert(std::string &ucert, std::string &ukey)
265{
266 char default_proxy[64];
267 const char *genvvar = 0, *genvvar1 = 0;
268 // The gEnv has higher priority, let's look for a proxy cert
269 genvvar = gEnv->GetValue("Davix.GSI.UserProxy", (const char *) NULL);
270 if (genvvar) {
271 ucert = ukey = genvvar;
272 if (gDebug > 0)
273 Info("TDavixFile_http_get_ucert", "Found proxy in gEnv");
274 return;
275 }
276
277 // Try explicit environment for proxy
278 if (getenv("X509_USER_PROXY")) {
279 if (gDebug > 0)
280 Info("TDavixFile_http_get_ucert", "Found proxy in X509_USER_PROXY");
281 ucert = ukey = getenv("X509_USER_PROXY");
282 return;
283 }
284
285 // Try with default location
286 snprintf(default_proxy, sizeof(default_proxy), "/tmp/x509up_u%d",
287 geteuid());
288
289 if (access(default_proxy, R_OK) == 0) {
290 if (gDebug > 0)
291 Info("TDavixFile_http_get_ucert", "Found proxy in /tmp");
292 ucert = ukey = default_proxy;
293 return;
294 }
295
296 // It seems we got no proxy, let's try to gather the keys
297 genvvar = gEnv->GetValue("Davix.GSI.UserCert", (const char *) NULL);
298 genvvar1 = gEnv->GetValue("Davix.GSI.UserKey", (const char *) NULL);
299 if (genvvar || genvvar1) {
300 if (gDebug > 0)
301 Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv");
302
303 ucert = genvvar;
304 ukey = genvvar1;
305 return;
306 }
307
308 // try with X509_* environment
309 if (getenv("X509_USER_CERT"))
310 ucert = getenv("X509_USER_CERT");
311 if (getenv("X509_USER_KEY"))
312 ukey = getenv("X509_USER_KEY");
313
314 if ((ucert.size() > 0) || (ukey.size() > 0)) {
315 if (gDebug > 0)
316 Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv");
317 }
318 return;
319
320}
321
322////////////////////////////////////////////////////////////////////////////////
323
324static int TDavixFile_http_authn_cert_X509(void *userdata, const Davix::SessionInfo &info,
325 Davix::X509Credential *cert, Davix::DavixError **err)
326{
327 (void) userdata; // keep quiete compilation warnings
328 (void) info;
329 std::string ucert, ukey;
330 TDavixFile_http_get_ucert(ucert, ukey);
331
332 if (ucert.empty() || ukey.empty()) {
333 Davix::DavixError::setupError(err, "TDavixFile",
334 Davix::StatusCode::AuthentificationError,
335 "Could not set the user's proxy or certificate");
336 return -1;
337 }
338 return cert->loadFromFilePEM(ukey, ucert, "", err);
339}
340/////////////////////////////////////////////////////////////////////////////////////////////
341
342////////////////////////////////////////////////////////////////////////////////
343
345{
346 delete davixPosix;
347 delete davixParam;
348}
349
350////////////////////////////////////////////////////////////////////////////////
351
353{
354 if (davix_context_s == NULL) {
355 TLockGuard guard(&createLock);
356 if (davix_context_s == NULL) {
357 davix_context_s = new Context();
358 }
359 }
360 return davix_context_s;
361}
362
363////////////////////////////////////////////////////////////////////////////////
364
366{
367 DavixError *davixErr = NULL;
368 Davix_fd *fd = davixPosix->open(davixParam, fUrl.GetUrl(), oflags, &davixErr);
369 if (fd == NULL) {
370 // An error has occurred.. We might be able to recover with metalinks.
371 // Try to populate the replicas vector. If successful, TFile will try
372 // the replicas one by one
373
374 replicas.clear();
375 DavixError *davixErr2 = NULL;
376 try {
377 DavFile file(*davixContext, Davix::Uri(fUrl.GetUrl()));
378 std::vector<DavFile> replicasLocal = file.getReplicas(NULL, &davixErr2);
379 for(size_t i = 0; i < replicasLocal.size(); i++) {
380 replicas.push_back(replicasLocal[i].getUri().getString());
381 }
382 }
383 catch(...) {}
384 DavixError::clearError(&davixErr2);
385
386 if(replicas.empty()) {
387 // I was unable to retrieve a list of replicas: propagate the original
388 // error.
389 Error("DavixOpen", "can not open file \"%s\" with davix: %s (%d)",
390 fUrl.GetUrl(),
391 davixErr->getErrMsg().c_str(), davixErr->getStatus());
392 }
393 DavixError::clearError(&davixErr);
394 } else {
395 // setup ROOT style read
396 davixPosix->fadvise(fd, 0, 300, Davix::AdviseRandom);
397 }
398
399 return fd;
400}
401
402////////////////////////////////////////////////////////////////////////////////
403
405{
406 DavixError *davixErr = NULL;
407 if (davixFd != NULL && davixPosix->close(davixFd, &davixErr)) {
408 Error("DavixClose", "can not to close file with davix: %s (%d)",
409 davixErr->getErrMsg().c_str(), davixErr->getStatus());
410 DavixError::clearError(&davixErr);
411 }
412}
413
414////////////////////////////////////////////////////////////////////////////////
415
417{
418 const char *env_var = NULL;
419
420 if (gDebug > 1)
421 Info("enableGridMode", " grid mode enabled !");
422
423 if( ( env_var = getenv("X509_CERT_DIR")) == NULL){
424 env_var= "/etc/grid-security/certificates/";
425 }
426 davixParam->addCertificateAuthorityPath(env_var);
427 if (gDebug > 0)
428 Info("enableGridMode", "Adding CAdir %s", env_var);
429}
430
431////////////////////////////////////////////////////////////////////////////////
432
433// Only newer versions of davix support setting the S3 region and STS tokens.
434// But it's only possible to check the davix version through a #define starting from
435// 0.6.4.
436// I have no way to check if setAwsRegion is available, so let's use SFINAE. :-)
437// The first overload will always take priority - if "substitution" fails, meaning
438// setAwsRegion is not there, the compiler will pick the second overload with
439// the ellipses. (...)
440
441template<typename TRequestParams = Davix::RequestParams>
442static auto awsRegion(TRequestParams *parameters, const char *region)
443 -> decltype(parameters->setAwsRegion(region), void())
444{
445 if (gDebug > 1) Info("awsRegion", "Setting S3 Region to '%s' - v4 signature will be used", region);
446 parameters->setAwsRegion(region);
447}
448
449template<typename TRequestParams = Davix::RequestParams>
450static void awsRegion(...) {
451 Warning("setAwsRegion", "Unable to set AWS region, not supported by this version of davix");
452}
453
454// Identical SFINAE trick as above for setAwsToken
455template<typename TRequestParams = Davix::RequestParams>
456static auto awsToken(TRequestParams *parameters, const char *token)
457 -> decltype(parameters->setAwsToken(token), void())
458{
459 if (gDebug > 1) Info("awsToken", "Setting S3 STS temporary credentials");
460 parameters->setAwsToken(token);
461}
462
463template<typename TRequestParams = Davix::RequestParams>
464static void awsToken(...) {
465 Warning("awsToken", "Unable to set AWS token, not supported by this version of davix");
466}
467
468// Identical SFINAE trick as above for setAwsAlternate
469template<typename TRequestParams = Davix::RequestParams>
470static auto awsAlternate(TRequestParams *parameters, bool option)
471 -> decltype(parameters->setAwsAlternate(option), void())
472{
473 if (gDebug > 1) Info("awsAlternate", "Setting S3 path-based bucket option (s3alternate)");
474 parameters->setAwsAlternate(option);
475}
476
477template<typename TRequestParams = Davix::RequestParams>
478static void awsAlternate(...) {
479 Warning("awsAlternate", "Unable to set AWS path-based bucket option (s3alternate), not supported by this version of davix");
480}
481
482void TDavixFileInternal::setAwsRegion(const std::string & region) {
483 if(!region.empty()) {
484 awsRegion(davixParam, region.c_str());
485 }
486}
487
488void TDavixFileInternal::setAwsToken(const std::string & token) {
489 if(!token.empty()) {
490 awsToken(davixParam, token.c_str());
491 }
492}
493
496}
497
498
499void TDavixFileInternal::setS3Auth(const std::string &secret, const std::string &access,
500 const std::string &region, const std::string &token)
501{
502 if (gDebug > 1) {
503 Info("setS3Auth", " Aws S3 tokens configured");
504 }
505 davixParam->setAwsAuthorizationKeys(secret, access);
506 davixParam->setProtocol(RequestProtocol::AwsS3);
507
508 setAwsRegion(region);
509 setAwsToken(token);
510}
511
512////////////////////////////////////////////////////////////////////////////////
513
515{
516 const char *env_var = NULL, *env_var2 = NULL;
517 // default opts
518 davixParam->setTransparentRedirectionSupport(true);
519 davixParam->setClientCertCallbackX509(&TDavixFile_http_authn_cert_X509, NULL);
520
521 // setup CADIR
522 env_var = gEnv->GetValue("Davix.GSI.CAdir", (const char *) NULL);
523 if (env_var) {
524 davixParam->addCertificateAuthorityPath(env_var);
525 if (gDebug > 0)
526 Info("parseConfig", "Add CAdir: %s", env_var);
527 }
528
529 // CA Check
530 bool ca_check_local = !isno(gEnv->GetValue("Davix.GSI.CACheck", (const char *)"y"));
531 davixParam->setSSLCAcheck(ca_check_local);
532 if (gDebug > 0)
533 Info("parseConfig", "Setting CAcheck to %s", ((ca_check_local) ? ("true") : ("false")));
534
535 // WLCG Bearer tokens check
536 std::string prefix = "Bearer ";
537 auto token = DiscoverToken();
538 if (!token.empty()) {
539 // header: "Authorization: Bearer mytoken"
540 R__LOG_INFO(TDavixLogChannel()) << "Using Bearer token starting with: " << token.substr(0, 3);
541 davixParam->addHeader("Authorization", prefix + token);
542 }
543
544 // S3 Auth
545 if (((env_var = gEnv->GetValue("Davix.S3.SecretKey", getenv("S3_SECRET_KEY"))) != NULL)
546 && ((env_var2 = gEnv->GetValue("Davix.S3.AccessKey", getenv("S3_ACCESS_KEY"))) != NULL)) {
547 Info("parseConfig", "Setting S3 SecretKey and AccessKey. Access Key : %s ", env_var2);
548 davixParam->setAwsAuthorizationKeys(env_var, env_var2);
549
550 // need to set region?
551 if ( (env_var = gEnv->GetValue("Davix.S3.Region", getenv("S3_REGION"))) != NULL) {
552 setAwsRegion(env_var);
553 }
554 // need to set STS token?
555 if( (env_var = gEnv->GetValue("Davix.S3.Token", getenv("S3_TOKEN"))) != NULL) {
556 setAwsToken(env_var);
557 }
558 // need to set aws alternate?
559 if( (env_var = gEnv->GetValue("Davix.S3.Alternate", getenv("S3_ALTERNATE"))) != NULL) {
560 setAwsAlternate(strToBool(env_var, false));
561 }
562 }
563
564 env_var = gEnv->GetValue("Davix.GSI.GridMode", (const char *)"y");
565 if (!isno(env_var))
567}
568
569////////////////////////////////////////////////////////////////////////////////
570/// intput params
571
573{
574 std::stringstream ss(option);
575 std::string item;
576 std::vector<std::string> parsed_options;
577 // parameters
578 std::string s3seckey, s3acckey, s3region, s3token;
579
580 while (std::getline(ss, item, ' ')) {
581 parsed_options.push_back(item);
582 }
583
584 for (std::vector<std::string>::iterator it = parsed_options.begin(); it < parsed_options.end(); ++it) {
585 // grid mode option
586 if ((strcasecmp(it->c_str(), grid_mode_opt)) == 0) {
588 }
589 // ca check option
590 if ((strcasecmp(it->c_str(), ca_check_opt)) == 0) {
591 davixParam->setSSLCAcheck(false);
592 }
593 // s3 sec key
594 if (strncasecmp(it->c_str(), s3_seckey_opt, strlen(s3_seckey_opt)) == 0) {
595 s3seckey = std::string(it->c_str() + strlen(s3_seckey_opt));
596 }
597 // s3 access key
598 if (strncasecmp(it->c_str(), s3_acckey_opt, strlen(s3_acckey_opt)) == 0) {
599 s3acckey = std::string(it->c_str() + strlen(s3_acckey_opt));
600 }
601 // s3 region
602 if (strncasecmp(it->c_str(), s3_region_opt, strlen(s3_region_opt)) == 0) {
603 s3region = std::string(it->c_str() + strlen(s3_region_opt));
604 }
605 // s3 sts token
606 if (strncasecmp(it->c_str(), s3_token_opt, strlen(s3_token_opt)) == 0) {
607 s3token = std::string(it->c_str() + strlen(s3_token_opt));
608 }
609 // s3 alternate option
610 if (strncasecmp(it->c_str(), s3_alternate_opt, strlen(s3_alternate_opt)) == 0) {
611 setAwsAlternate(strToBool(it->c_str() + strlen(s3_alternate_opt), false));
612 }
613 // open mods
615 }
616
617 if (s3seckey.size() > 0) {
618 setS3Auth(s3seckey, s3acckey, s3region, s3token);
619 }
620
621 if (oflags == 0) // default open mode
622 oflags = O_RDONLY;
623}
624
625////////////////////////////////////////////////////////////////////////////////
626
628{
629 davixPosix = new DavPosix(davixContext);
630 davixParam = new RequestParams();
631 davixParam->setUserAgent(gUserAgent);
632 davixParam->setMetalinkMode(Davix::MetalinkMode::Disable);
634 parseConfig();
636}
637
638////////////////////////////////////////////////////////////////////////////////
639
640Int_t TDavixFileInternal::DavixStat(const char *url, struct stat *st)
641{
642 DavixError *davixErr = NULL;
643
644 if (davixPosix->stat(davixParam, url, st, &davixErr) < 0) {
645
646 Error("DavixStat", "can not stat the file with davix: %s (%d)",
647 davixErr->getErrMsg().c_str(), davixErr->getStatus());
648 DavixError::clearError(&davixErr);
649 return 0;
650 }
651 return 1;
652}
653
654/////////////////////////////////////////////////////////////////////////////////////////////
655
656////////////////////////////////////////////////////////////////////////////////
657
658TDavixFile::TDavixFile(const char *url, Option_t *opt, const char *ftitle, Int_t compress)
659 : TFile(url, strstr(opt, "_WITHOUT_GLOBALREGISTRATION") != nullptr ? "WEB_WITHOUT_GLOBALREGISTRATION" : "WEB"),
660 d_ptr(new TDavixFileInternal(fUrl, opt))
661{
662 (void) ftitle;
663 (void) compress;
664 Init(kFALSE);
665}
666
667////////////////////////////////////////////////////////////////////////////////
668
670{
671 d_ptr->Close();
672 delete d_ptr;
673}
674
675////////////////////////////////////////////////////////////////////////////////
676
678{
679 (void) init;
680 //initialize davix
681 d_ptr->init();
682 // pre-open file
683 if ((d_ptr->getDavixFileInstance()) == NULL){
684 MakeZombie();
686 return;
687 }
689 fOffset = 0;
690 fD = -2; // so TFile::IsOpen() will return true when in TFile::~TFi */
691}
692
694 std::vector<std::string> replicas = d_ptr->getReplicas();
695 TString newUrl;
696 if(!replicas.empty()) {
697 std::stringstream ss;
698 for(size_t i = 0; i < replicas.size(); i++) {
699 ss << replicas[i];
700 if(i != replicas.size()-1) ss << "|";
701 }
702 newUrl = ss.str();
703 }
704 return newUrl;
705}
706
707////////////////////////////////////////////////////////////////////////////////
708/// Set position from where to start reading.
709
711{
712 TLockGuard guard(&(d_ptr->positionLock));
713 switch (pos) {
714 case kBeg:
716 break;
717 case kCur:
718 fOffset += offset;
719 break;
720 case kEnd:
721 // this option is not used currently in the ROOT code
722 if (fArchiveOffset)
723 Error("Seek", "seeking from end in archive is not (yet) supported");
724 fOffset = fEND - offset; // is fEND really EOF or logical EOF?
725 break;
726 }
727
728 if (gDebug > 1)
729 Info("Seek", " move cursor to %lld"
730 , fOffset);
731}
732
733////////////////////////////////////////////////////////////////////////////////
734/// Read specified byte range from remote file via HTTP.
735/// Returns kTRUE in case of error.
736
738{
739 TLockGuard guard(&(d_ptr->positionLock));
740 Davix_fd *fd;
741 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
742 return kTRUE;
743 Long64_t ret = DavixReadBuffer(fd, buf, len);
744 if (ret < 0)
745 return kTRUE;
746
747 if (gDebug > 1)
748 Info("ReadBuffer", "%lld bytes of data read sequentially"
749 " (%d requested)", ret, len);
750
751 return kFALSE;
752}
753
754////////////////////////////////////////////////////////////////////////////////
755
757{
758 Davix_fd *fd;
759 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
760 return kTRUE;
761
762 Long64_t ret = DavixPReadBuffer(fd, buf, pos, len);
763 if (ret < 0)
764 return kTRUE;
765
766 if (gDebug > 1)
767 Info("ReadBuffer", "%lld bytes of data read from offset"
768 " %lld (%d requested)", ret, pos, len);
769 return kFALSE;
770}
771
772////////////////////////////////////////////////////////////////////////////////
773
775{
776 Davix_fd *fd;
777 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
778 return kFALSE;
779
780 d_ptr->davixPosix->fadvise(fd, static_cast<dav_off_t>(offs), static_cast<dav_size_t>(len), Davix::AdviseRandom);
781
782 if (gDebug > 1)
783 Info("ReadBufferAsync", "%d bytes of data prefected from offset"
784 " %lld ", len, offs);
785 return kFALSE;
786}
787
788////////////////////////////////////////////////////////////////////////////////
789
791{
792 Davix_fd *fd;
793 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
794 return kTRUE;
795
796 Long64_t ret = DavixReadBuffers(fd, buf, pos, len, nbuf);
797 if (ret < 0)
798 return kTRUE;
799
800 if (gDebug > 1)
801 Info("ReadBuffers", "%lld bytes of data read from a list of %d buffers",
802 ret, nbuf);
803
804 return kFALSE;
805}
806
807////////////////////////////////////////////////////////////////////////////////
808
810{
811 Davix_fd *fd;
812 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
813 return kTRUE;
814
815 Long64_t ret = DavixWriteBuffer(fd, buf, len);
816 if (ret < 0)
817 return kTRUE;
818
819 if (gDebug > 1)
820 Info("WriteBuffer", "%lld bytes of data write"
821 " %d requested", ret, len);
822 return kFALSE;
823}
824
825////////////////////////////////////////////////////////////////////////////////
826
828{
829 d_ptr->davixParam->setSSLCAcheck((bool)check);
830}
831
832////////////////////////////////////////////////////////////////////////////////
833
835{
837}
838
839////////////////////////////////////////////////////////////////////////////////
840
842{
844 std::vector<void *>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd);
845 return (f != dirdVec.end());
846}
847
848////////////////////////////////////////////////////////////////////////////////
849
851{
853 dirdVec.push_back(fd);
854}
855
856////////////////////////////////////////////////////////////////////////////////
857
859{
861 std::vector<void *>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd);
862 if (f != dirdVec.end())
863 dirdVec.erase(f);
864}
865
866////////////////////////////////////////////////////////////////////////////////
867
869{
870 struct stat st;
871 Int_t ret = d_ptr->DavixStat(fUrl.GetUrl(), &st);
872 if (ret) {
873 if (gDebug > 1)
874 Info("GetSize", "file size requested: %lld", (Long64_t)st.st_size);
875 return st.st_size;
876 }
877 return -1;
878}
879
880////////////////////////////////////////////////////////////////////////////////
881
883{
884 if (gPerfStats)
885 return TTimeStamp();
886 return 0;
887}
888
889////////////////////////////////////////////////////////////////////////////////
890/// set TFile state info
891
893{
894 if(read) {
895 fBytesRead += len;
896 fReadCalls += 1;
897
900
901 if (gPerfStats)
902 gPerfStats->FileReadEvent(this, (Int_t) len, t_start);
903 } else {
904 fBytesWrite += len;
906 }
907}
908
909////////////////////////////////////////////////////////////////////////////////
910
912{
913 DavixError *davixErr = NULL;
914 Double_t start_time = eventStart();
915
916 Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, fOffset, &davixErr);
917 if (ret < 0) {
918 Error("DavixReadBuffer", "can not read data with davix: %s (%d)",
919 davixErr->getErrMsg().c_str(), davixErr->getStatus());
920 DavixError::clearError(&davixErr);
921 } else {
922 fOffset += ret;
923 eventStop(start_time, ret);
924 }
925
926 return ret;
927}
928
929////////////////////////////////////////////////////////////////////////////////
930
931Long64_t TDavixFile::DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len)
932{
933 DavixError *davixErr = NULL;
934 Double_t start_time = eventStart();
935
936 Long64_t ret = d_ptr->davixPosix->pwrite(fd, buf, len, fOffset, &davixErr);
937 if (ret < 0) {
938 Error("DavixWriteBuffer", "can not write data with davix: %s (%d)",
939 davixErr->getErrMsg().c_str(), davixErr->getStatus());
940 DavixError::clearError(&davixErr);
941 } else {
942 fOffset += ret;
943 eventStop(start_time, ret, false);
944 }
945
946 return ret;
947}
948
949////////////////////////////////////////////////////////////////////////////////
950
952{
953 DavixError *davixErr = NULL;
954 Double_t start_time = eventStart();
955
956 Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, pos, &davixErr);
957 if (ret < 0) {
958 Error("DavixPReadBuffer", "can not read data with davix: %s (%d)",
959 davixErr->getErrMsg().c_str(), davixErr->getStatus());
960 DavixError::clearError(&davixErr);
961 } else {
962 eventStop(start_time, ret);
963 }
964
965
966 return ret;
967}
968
969////////////////////////////////////////////////////////////////////////////////
970
971Long64_t TDavixFile::DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
972{
973 DavixError *davixErr = NULL;
974 Double_t start_time = eventStart();
975 DavIOVecInput in[nbuf];
976 DavIOVecOuput out[nbuf];
977
978 int lastPos = 0;
979 for (Int_t i = 0; i < nbuf; ++i) {
980 in[i].diov_buffer = &buf[lastPos];
981 in[i].diov_offset = pos[i];
982 in[i].diov_size = len[i];
983 lastPos += len[i];
984 }
985
986 Long64_t ret = d_ptr->davixPosix->preadVec(fd, in, out, nbuf, &davixErr);
987 if (ret < 0) {
988 Error("DavixReadBuffers", "can not read data with davix: %s (%d)",
989 davixErr->getErrMsg().c_str(), davixErr->getStatus());
990 DavixError::clearError(&davixErr);
991 } else {
992 eventStop(start_time, ret);
993 }
994
995 return ret;
996}
#define R__LOG_ERROR(...)
Definition RLogger.hxx:362
#define R__LOG_INFO(...)
Definition RLogger.hxx:364
#define f(i)
Definition RSha256.hxx:104
int Int_t
Definition RtypesCore.h:45
constexpr Bool_t kFALSE
Definition RtypesCore.h:101
long long Long64_t
Definition RtypesCore.h:80
constexpr Bool_t kTRUE
Definition RtypesCore.h:100
const char Option_t
Definition RtypesCore.h:66
#define ClassImp(name)
Definition Rtypes.h:377
const char * s3_seckey_opt
bool normalizeToken(const std::string &input_token, std::string &output_token)
static const std::string VERSION
static void ConfigureDavixLogLevel()
static auto awsRegion(TRequestParams *parameters, const char *region) -> decltype(parameters->setAwsRegion(region), void())
bool findTokenInFile(const std::string &token_file, std::string &output_token)
static TMutex createLock
static auto awsAlternate(TRequestParams *parameters, bool option) -> decltype(parameters->setAwsAlternate(option), void())
std::string DiscoverToken()
static int TDavixFile_http_authn_cert_X509(void *userdata, const Davix::SessionInfo &info, Davix::X509Credential *cert, Davix::DavixError **err)
const char * s3_alternate_opt
static auto awsToken(TRequestParams *parameters, const char *token) -> decltype(parameters->setAwsToken(token), void())
const char * s3_token_opt
bool isno(const char *str)
const char * ca_check_opt
const char * open_mode_new
const char * open_mode_update
const char * open_mode_create
const char * s3_acckey_opt
static Context * davix_context_s
const char * grid_mode_opt
ROOT::Experimental::RLogChannel & TDavixLogChannel()
int configure_open_flag(const std::string &str, int old_flag)
static void TDavixFile_http_get_ucert(std::string &ucert, std::string &ukey)
static const std::string gUserAgent
bool strToBool(const char *str, bool defvalue)
const char * s3_region_opt
const char * open_mode_read
ROOT::Experimental::RLogChannel & TDavixLogChannel()
#define gDirectory
Definition TDirectory.h:384
R__EXTERN TEnv * gEnv
Definition TEnv.h:170
void Info(const char *location, const char *msgfmt,...)
Use this function for informational messages.
Definition TError.cxx:218
void Error(const char *location, const char *msgfmt,...)
Use this function in case an error occurred.
Definition TError.cxx:185
void Warning(const char *location, const char *msgfmt,...)
Use this function in warning situations.
Definition TError.cxx:229
Option_t Option_t option
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
Int_t gDebug
Definition TROOT.cxx:595
#define gROOT
Definition TROOT.h:406
#define gPerfStats
#define snprintf
Definition civetweb.c:1540
A log configuration for a channel, e.g.
Definition RLogger.hxx:101
void addDird(void *fd)
Davix::RequestParams * davixParam
bool isMyDird(void *fd)
std::vector< void * > dirdVec
void setAwsToken(const std::string &token)
void setS3Auth(const std::string &secret, const std::string &access, const std::string &region, const std::string &token)
void setAwsAlternate(const bool &option)
static Davix::Context * getDavixInstance()
Davix::DavPosix * davixPosix
void parseParams(Option_t *option)
intput params
std::vector< std::string > getReplicas()
Int_t DavixStat(const char *url, struct stat *st)
Davix_fd * getDavixFileInstance()
Davix_fd * Open()
Davix::Context * davixContext
void setAwsRegion(const std::string &region)
std::vector< std::string > replicas
void removeDird(void *fd)
virtual Long64_t GetSize() const
Returns the current file size.
void Init(Bool_t init)
Initialize a TFile object.
Long64_t DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Long64_t DavixReadBuffer(Davix_fd *fd, char *buf, Int_t len)
Long64_t DavixPReadBuffer(Davix_fd *fd, char *buf, Long64_t pos, Int_t len)
virtual Bool_t WriteBuffer(const char *buffer, Int_t bufferLength)
Write a buffer to the file.
virtual TString GetNewUrl()
Long64_t DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len)
virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len)
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read specified byte range from remote file via HTTP.
TDavixFile(const char *url, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault)
Open function for TDavixFile.
void enableGridMode()
Enable the grid mode The grid Mode configure automatically all grid-CA path, VOMS authentication and ...
virtual void Seek(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
TDavixFileInternal * d_ptr
Definition TDavixFile.h:72
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
void eventStop(Double_t t, Long64_t len, bool read=true)
set TFile state info
void setCACheck(Bool_t check)
Enable or disable certificate authority check.
Double_t eventStart()
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
Definition TEnv.cxx:491
A ROOT file is composed of a header, followed by consecutive data records (TKey instances) with a wel...
Definition TFile.h:53
Int_t fReadCalls
Number of read calls ( not counting the cache calls )
Definition TFile.h:89
static void SetFileBytesWritten(Long64_t bytes=0)
Definition TFile.cxx:4588
Long64_t fBytesRead
Number of bytes read from this file.
Definition TFile.h:76
static Long64_t GetFileBytesWritten()
Static function returning the total number of bytes written to all files.
Definition TFile.cxx:4560
static void SetFileBytesRead(Long64_t bytes=0)
Definition TFile.cxx:4585
static void SetFileReadCalls(Int_t readcalls=0)
Definition TFile.cxx:4591
TUrl fUrl
!URL of file
Definition TFile.h:110
static Long64_t GetFileBytesRead()
Static function returning the total number of bytes read from all files.
Definition TFile.cxx:4551
Long64_t fArchiveOffset
!Offset at which file starts in archive
Definition TFile.h:101
virtual void Init(Bool_t create)
Initialize a TFile object.
Definition TFile.cxx:603
ERelativeTo
Definition TFile.h:192
@ kCur
Definition TFile.h:192
@ kBeg
Definition TFile.h:192
@ kEnd
Definition TFile.h:192
Int_t fD
File descriptor.
Definition TFile.h:82
Long64_t fBytesWrite
Number of bytes written to this file.
Definition TFile.h:75
Long64_t fOffset
!Seek offset cache
Definition TFile.h:96
Long64_t fEND
Last used byte in file.
Definition TFile.h:79
static Int_t GetFileReadCalls()
Static function returning the total number of read calls from all files.
Definition TFile.cxx:4568
void MakeZombie()
Definition TObject.h:53
Basic string class.
Definition TString.h:139
The TTimeStamp encapsulates seconds and ns since EPOCH.
Definition TTimeStamp.h:45
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition TUrl.cxx:390
TLine l
Definition textangle.C:4