Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
TDavixFile.cxx
Go to the documentation of this file.
1// @(#)root/net:$Id$
2// Author: Adrien Devresse and Tigran Mkrtchyan
3
4/*************************************************************************
5 * Copyright (C) 1995-2013, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12//////////////////////////////////////////////////////////////////////////
13// //
14// TDavixFile //
15// //
16// A TDavixFile is like a normal TFile except that it uses //
17// libdavix to read/write remote files. //
18// It supports HTTP and HTTPS in a number of dialects and options //
19// e.g. S3 is one of them //
20// Other caracteristics come from the full support of Davix, //
21// e.g. full redirection support in any circumstance //
22// //
23// Authors: Adrien Devresse (CERN IT/SDC) //
24// Tigran Mkrtchyan (DESY) //
25// //
26// Checks and ROOT5 porting: //
27// Fabrizio Furano (CERN IT/SDC) //
28// //
29// September 2013 //
30// //
31//////////////////////////////////////////////////////////////////////////
32
33#include "ROOT/RLogger.hxx"
34#include "TDavixFile.h"
35#include "TROOT.h"
36#include "TSocket.h"
37#include "Bytes.h"
38#include "TError.h"
39#include "TEnv.h"
40#include "TBase64.h"
41#include "TVirtualPerfStats.h"
42#include "TDavixFileInternal.h"
43#include "utils.h"
44#include "snprintf.h"
45
46#include <cerrno>
47#include <cstdlib>
48#include <unistd.h>
49#include <fcntl.h>
50#include <davix.hpp>
51#include <sstream>
52#include <string>
53#include <cstring>
54#include <memory>
55
56static const std::string VERSION = "0.2.0";
57
58static const std::string gUserAgent = "ROOT/" + std::string(gROOT->GetVersion()) +
59" TDavixFile/" + VERSION + " davix/" + Davix::version();
60
61// The prefix that is used to find the variables in the gEnv
62#define ENVPFX "Davix."
63
64
65using namespace Davix;
66
67const char* grid_mode_opt = "grid_mode=yes";
68const char* ca_check_opt = "ca_check=no";
69const char* s3_seckey_opt = "s3seckey=";
70const char* s3_acckey_opt = "s3acckey=";
71const char* s3_region_opt = "s3region=";
72const char* s3_token_opt = "s3token=";
73const char* s3_alternate_opt = "s3alternate=";
74const char* open_mode_read = "READ";
75const char* open_mode_create = "CREATE";
76const char* open_mode_new = "NEW";
77const char* open_mode_update = "UPDATE";
78
80static Context* davix_context_s = NULL;
81
82////////////////////////////////////////////////////////////////////////////////
83
85{
86 static ROOT::RLogChannel sLog("ROOT.TDavix");
87 return sLog;
88}
89
90////////////////////////////////////////////////////////////////////////////////
91
92bool strToBool(const char *str, bool defvalue) {
93 if(!str) return defvalue;
94
95 if(strcmp(str, "n") == 0 || strcmp(str, "no") == 0 || strcmp(str, "0") == 0 || strcmp(str, "false") == 0) return false;
96 if(strcmp(str, "y") == 0 || strcmp(str, "yes") == 0 || strcmp(str, "1") == 0 || strcmp(str, "true") == 0) return true;
97
98 return defvalue;
99}
100
101////////////////////////////////////////////////////////////////////////////////
102
103int configure_open_flag(const std::string &str, int old_flag)
104{
105 if (strcasecmp(str.c_str(), open_mode_read) == 0)
107 if ((strcasecmp(str.c_str(), open_mode_create) == 0)
108 || (strcasecmp(str.c_str(), open_mode_new) == 0)) {
110 }
111 if ((strcasecmp(str.c_str(), open_mode_update) == 0)) {
112 old_flag |= (O_RDWR);
113 }
114 return old_flag;
115}
116
117////////////////////////////////////////////////////////////////////////////////
118
120{
121 Int_t log_level = (gEnv) ? gEnv->GetValue("Davix.Debug", 0) : 0;
122
123 switch (log_level) {
124 case 0:
126 break;
127 case 1:
129 break;
130 case 2:
132 break;
133 case 3:
135 break;
136 default:
138 break;
139 }
140}
141
142////////////////////////////////////////////////////////////////////////////////
143
144bool normalizeToken(const std::string &input_token, std::string &output_token)
145{
146 static const std::string whitespace = " \t\f\n\v\r";
147 // Tokens are meant for use in an HTTP header; these are forbidden
148 // characters.
149 static const std::string nonheader_whitespace = "\r\n";
150 auto begin = input_token.find_first_not_of(whitespace);
151 // No token here, but not an error.
152 if (begin == std::string::npos) {
153 output_token = "";
154 return true;
155 }
156
157 std::string token = input_token.substr(begin);
158 auto end = token.find_last_not_of(whitespace);
159 token = token.substr(0, end + 1);
160
161 // If non-permitted header characters are present ("\r\n"),
162 // then this token is not permitted.
163 if (token.find(nonheader_whitespace) != std::string::npos) {
164 output_token = "";
165 R__LOG_ERROR(TDavixLogChannel()) << "Token discovery failure: token contains non-permitted character sequence (\\r\\n)";
166 return false;
167 }
168 output_token = token;
169 return true;
170}
171
172bool findTokenInFile(const std::string &token_file, std::string &output_token)
173{
174 R__LOG_INFO(TDavixLogChannel()) << "Looking for token in file " << token_file.c_str();
175 int fd = open(token_file.c_str(), O_RDONLY);
176 if (fd == -1) {
177 output_token = "";
178 if (errno == ENOENT) {
179 return true;
180 }
181 R__LOG_ERROR(TDavixLogChannel()) << "Cannot open '" << token_file << "', error: " << strerror(errno);
182 return false;
183 }
184
185 // As a pragmatic matter, we limit tokens to 16KB; this is often the limit
186 // in size of HTTP headers for many web servers.
187 // This also avoids unbounded memory use in case if the user points us to
188 // a large file.
189 static const size_t max_size = 16384;
190 std::vector<char> input_buffer;
191 input_buffer.resize(max_size);
192 ssize_t retval = read(fd, &input_buffer[0], max_size);
193 close(fd);
194
195 if (retval == -1) {
196 output_token = "";
197 R__LOG_ERROR(TDavixLogChannel()) << "Token discovery failure: failed to read file " << token_file.c_str() << "', error: " << strerror(errno);
198 return false;
199 }
200 if (retval == 16384) {
201 // Token may have been truncated! Erring on the side of caution.
202 R__LOG_ERROR(TDavixLogChannel()) << "Token discovery failure: token was larger than 16KB limit.";
203 return false;
204 }
205
206 std::string token(&input_buffer[0], retval);
207
208 return normalizeToken(token, output_token);
209}
210
211// Token discovery process is based on WLCG specification:
212// https://github.com/WLCG-AuthZ-WG/bearer-token-discovery/blob/master/specification.md
213std::string DiscoverToken()
214{
215 const char *bearer_token = std::getenv("BEARER_TOKEN");
216 std::string token;
217 if (bearer_token && *bearer_token){
218 if (!normalizeToken(bearer_token, token)) {return "";}
219 if (!token.empty()) {return token;}
220 }
221
222 const char *bearer_token_file = std::getenv("BEARER_TOKEN_FILE");
223 if (bearer_token_file) {
224 if (!findTokenInFile(bearer_token_file, token)) {return "";}
225 if (!token.empty()) {return token;}
226 }
227
228#ifndef WIN32
229 uid_t euid = geteuid();
230 std::string fname = "/bt_u";
231 fname += std::to_string(euid);
232
233 const char *xdg_runtime_dir = std::getenv("XDG_RUNTIME_DIR");
234 if (xdg_runtime_dir) {
235 std::string xdg_token_file = std::string(xdg_runtime_dir) + fname;
236 if (!findTokenInFile(xdg_token_file, token)) {return "";}
237 if (!token.empty()) {return token;}
238 }
239
240 if (!findTokenInFile("/tmp" + fname, token)) {return "";}
241 return token;
242#else
243 // WLCG profile doesn't define search paths on Windows;
244 // skip this for now.
245 return "";
246#endif
247}
248
249///////////////////////////////////////////////////////////////////
250// Authn implementation, Locate and get VOMS cred if exist
251
252////////////////////////////////////////////////////////////////////////////////
253
254static void TDavixFile_http_get_ucert(std::string &ucert, std::string &ukey)
255{
256 char default_proxy[64];
257 const char *genvvar = 0, *genvvar1 = 0;
258 // The gEnv has higher priority, let's look for a proxy cert
259 genvvar = gEnv->GetValue("Davix.GSI.UserProxy", (const char *) NULL);
260 if (genvvar) {
261 ucert = ukey = genvvar;
262 if (gDebug > 0)
263 Info("TDavixFile_http_get_ucert", "Found proxy in gEnv");
264 return;
265 }
266
267 // Try explicit environment for proxy
268 if (std::getenv("X509_USER_PROXY")) {
269 if (gDebug > 0)
270 Info("TDavixFile_http_get_ucert", "Found proxy in X509_USER_PROXY");
271 ucert = ukey = std::getenv("X509_USER_PROXY");
272 return;
273 }
274
275 // Try with default location
276 snprintf(default_proxy, sizeof(default_proxy), "/tmp/x509up_u%d",
277 geteuid());
278
279 if (access(default_proxy, R_OK) == 0) {
280 if (gDebug > 0)
281 Info("TDavixFile_http_get_ucert", "Found proxy in /tmp");
283 return;
284 }
285
286 // It seems we got no proxy, let's try to gather the keys
287 genvvar = gEnv->GetValue("Davix.GSI.UserCert", (const char *) NULL);
288 genvvar1 = gEnv->GetValue("Davix.GSI.UserKey", (const char *) NULL);
289 if (genvvar || genvvar1) {
290 if (gDebug > 0)
291 Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv");
292
293 ucert = genvvar;
294 ukey = genvvar1;
295 return;
296 }
297
298 // try with X509_* environment
299 if (std::getenv("X509_USER_CERT"))
300 ucert = std::getenv("X509_USER_CERT");
301 if (std::getenv("X509_USER_KEY"))
302 ukey = std::getenv("X509_USER_KEY");
303
304 if ((ucert.size() > 0) || (ukey.size() > 0)) {
305 if (gDebug > 0)
306 Info("TDavixFile_http_get_ucert", "Found cert and key in gEnv");
307 }
308 return;
309
310}
311
312////////////////////////////////////////////////////////////////////////////////
313
314static int TDavixFile_http_authn_cert_X509(void *userdata, const Davix::SessionInfo &info,
315 Davix::X509Credential *cert, Davix::DavixError **err)
316{
317 (void) userdata; // keep quiete compilation warnings
318 (void) info;
319 std::string ucert, ukey;
321
322 if (ucert.empty() || ukey.empty()) {
323 Davix::DavixError::setupError(err, "TDavixFile",
324 Davix::StatusCode::AuthentificationError,
325 "Could not set the user's proxy or certificate");
326 return -1;
327 }
328 return cert->loadFromFilePEM(ukey, ucert, "", err);
329}
330/////////////////////////////////////////////////////////////////////////////////////////////
331
332////////////////////////////////////////////////////////////////////////////////
333
339
340////////////////////////////////////////////////////////////////////////////////
341
343{
344 if (davix_context_s == NULL) {
345 TLockGuard guard(&createLock);
346 if (davix_context_s == NULL) {
347 davix_context_s = new Context();
348 }
349 }
350 return davix_context_s;
351}
352
353////////////////////////////////////////////////////////////////////////////////
354
356{
359 if (fd == NULL) {
360 // An error has occurred.. We might be able to recover with metalinks.
361 // Try to populate the replicas vector. If successful, TFile will try
362 // the replicas one by one
363
364 replicas.clear();
366 try {
367 DavFile file(*davixContext, Davix::Uri(fUrl.GetUrl()));
368 std::vector<DavFile> replicasLocal = file.getReplicas(NULL, &davixErr2);
369 for(size_t i = 0; i < replicasLocal.size(); i++) {
370 replicas.push_back(replicasLocal[i].getUri().getString());
371 }
372 }
373 catch(...) {}
374 DavixError::clearError(&davixErr2);
375
376 if(replicas.empty()) {
377 // I was unable to retrieve a list of replicas: propagate the original
378 // error.
379 Error("DavixOpen", "can not open file \"%s\" with davix: %s (%d)",
380 fUrl.GetUrl(),
381 davixErr->getErrMsg().c_str(), davixErr->getStatus());
382 }
383 DavixError::clearError(&davixErr);
384 } else {
385 // setup ROOT style read
386 davixPosix->fadvise(fd, 0, 300, Davix::AdviseRandom);
387 }
388
389 return fd;
390}
391
392////////////////////////////////////////////////////////////////////////////////
393
395{
397 if (davixFd != NULL && davixPosix->close(davixFd, &davixErr)) {
398 Error("DavixClose", "can not to close file with davix: %s (%d)",
399 davixErr->getErrMsg().c_str(), davixErr->getStatus());
400 DavixError::clearError(&davixErr);
401 }
402}
403
404////////////////////////////////////////////////////////////////////////////////
405
407{
408 const char *env_var = NULL;
409
410 if (gDebug > 1)
411 Info("enableGridMode", " grid mode enabled !");
412
413 if( ( env_var = std::getenv("X509_CERT_DIR")) == NULL){
414 env_var= "/etc/grid-security/certificates/";
415 }
416 davixParam->addCertificateAuthorityPath(env_var);
417 if (gDebug > 0)
418 Info("enableGridMode", "Adding CAdir %s", env_var);
419}
420
421////////////////////////////////////////////////////////////////////////////////
422
423// Only newer versions of davix support setting the S3 region and STS tokens.
424// But it's only possible to check the davix version through a #define starting from
425// 0.6.4.
426// I have no way to check if setAwsRegion is available, so let's use SFINAE. :-)
427// The first overload will always take priority - if "substitution" fails, meaning
428// setAwsRegion is not there, the compiler will pick the second overload with
429// the ellipses. (...)
430
431template<typename TRequestParams = Davix::RequestParams>
432static auto awsRegion(TRequestParams *parameters, const char *region)
433 -> decltype(parameters->setAwsRegion(region), void())
434{
435 if (gDebug > 1) Info("awsRegion", "Setting S3 Region to '%s' - v4 signature will be used", region);
436 parameters->setAwsRegion(region);
437}
438
439template<typename TRequestParams = Davix::RequestParams>
440static void awsRegion(...) {
441 Warning("setAwsRegion", "Unable to set AWS region, not supported by this version of davix");
442}
443
444// Identical SFINAE trick as above for setAwsToken
445template<typename TRequestParams = Davix::RequestParams>
446static auto awsToken(TRequestParams *parameters, const char *token)
447 -> decltype(parameters->setAwsToken(token), void())
448{
449 if (gDebug > 1) Info("awsToken", "Setting S3 STS temporary credentials");
450 parameters->setAwsToken(token);
451}
452
453template<typename TRequestParams = Davix::RequestParams>
454static void awsToken(...) {
455 Warning("awsToken", "Unable to set AWS token, not supported by this version of davix");
456}
457
458// Identical SFINAE trick as above for setAwsAlternate
459template<typename TRequestParams = Davix::RequestParams>
460static auto awsAlternate(TRequestParams *parameters, bool option)
461 -> decltype(parameters->setAwsAlternate(option), void())
462{
463 if (gDebug > 1) Info("awsAlternate", "Setting S3 path-based bucket option (s3alternate)");
464 parameters->setAwsAlternate(option);
465}
466
467template<typename TRequestParams = Davix::RequestParams>
468static void awsAlternate(...) {
469 Warning("awsAlternate", "Unable to set AWS path-based bucket option (s3alternate), not supported by this version of davix");
470}
471
472void TDavixFileInternal::setAwsRegion(const std::string & region) {
473 if(!region.empty()) {
474 awsRegion(davixParam, region.c_str());
475 }
476}
477
478void TDavixFileInternal::setAwsToken(const std::string & token) {
479 if(!token.empty()) {
480 awsToken(davixParam, token.c_str());
481 }
482}
483
487
488
489void TDavixFileInternal::setS3Auth(const std::string &secret, const std::string &access,
490 const std::string &region, const std::string &token)
491{
492 if (gDebug > 1) {
493 Info("setS3Auth", " Aws S3 tokens configured");
494 }
495 davixParam->setAwsAuthorizationKeys(secret, access);
496 davixParam->setProtocol(RequestProtocol::AwsS3);
497
499 setAwsToken(token);
500}
501
502////////////////////////////////////////////////////////////////////////////////
503
505{
506 const char *env_var = NULL, *env_var2 = NULL;
507 // default opts
508 davixParam->setTransparentRedirectionSupport(true);
509 davixParam->setClientCertCallbackX509(&TDavixFile_http_authn_cert_X509, NULL);
510
511 // setup CADIR
512 env_var = gEnv->GetValue("Davix.GSI.CAdir", (const char *) NULL);
513 if (env_var) {
514 davixParam->addCertificateAuthorityPath(env_var);
515 if (gDebug > 0)
516 Info("parseConfig", "Add CAdir: %s", env_var);
517 }
518
519 // CA Check
520 bool ca_check_local = !isno(gEnv->GetValue("Davix.GSI.CACheck", (const char *)"y"));
521 davixParam->setSSLCAcheck(ca_check_local);
522 if (gDebug > 0)
523 Info("parseConfig", "Setting CAcheck to %s", ((ca_check_local) ? ("true") : ("false")));
524
525 // WLCG Bearer tokens check
526 std::string prefix = "Bearer ";
527 auto token = DiscoverToken();
528 if (!token.empty()) {
529 // header: "Authorization: Bearer mytoken"
530 R__LOG_INFO(TDavixLogChannel()) << "Using Bearer token starting with: " << token.substr(0, 3);
531 davixParam->addHeader("Authorization", prefix + token);
532 }
533
534 // S3 Auth
535 if (((env_var = gEnv->GetValue("Davix.S3.SecretKey", std::getenv("S3_SECRET_KEY"))) != NULL)
536 && ((env_var2 = gEnv->GetValue("Davix.S3.AccessKey", std::getenv("S3_ACCESS_KEY"))) != NULL)) {
537 Info("parseConfig", "Setting S3 SecretKey and AccessKey. Access Key : %s ", env_var2);
538 davixParam->setAwsAuthorizationKeys(env_var, env_var2);
539
540 // need to set region?
541 if ( (env_var = gEnv->GetValue("Davix.S3.Region", std::getenv("S3_REGION"))) != NULL) {
543 }
544 // need to set STS token?
545 if( (env_var = gEnv->GetValue("Davix.S3.Token", std::getenv("S3_TOKEN"))) != NULL) {
547 }
548 // need to set aws alternate?
549 if( (env_var = gEnv->GetValue("Davix.S3.Alternate", std::getenv("S3_ALTERNATE"))) != NULL) {
551 }
552 }
553
554 env_var = gEnv->GetValue("Davix.GSI.GridMode", (const char *)"y");
555 if (!isno(env_var))
557}
558
559////////////////////////////////////////////////////////////////////////////////
560/// intput params
561
563{
564 std::stringstream ss(option);
565 std::string item;
566 std::vector<std::string> parsed_options;
567 // parameters
568 std::string s3seckey, s3acckey, s3region, s3token;
569
570 while (std::getline(ss, item, ' ')) {
571 parsed_options.push_back(item);
572 }
573
574 for (std::vector<std::string>::iterator it = parsed_options.begin(); it < parsed_options.end(); ++it) {
575 // grid mode option
576 if ((strcasecmp(it->c_str(), grid_mode_opt)) == 0) {
578 }
579 // ca check option
580 if ((strcasecmp(it->c_str(), ca_check_opt)) == 0) {
581 davixParam->setSSLCAcheck(false);
582 }
583 // s3 sec key
584 if (strncasecmp(it->c_str(), s3_seckey_opt, strlen(s3_seckey_opt)) == 0) {
585 s3seckey = std::string(it->c_str() + strlen(s3_seckey_opt));
586 }
587 // s3 access key
588 if (strncasecmp(it->c_str(), s3_acckey_opt, strlen(s3_acckey_opt)) == 0) {
589 s3acckey = std::string(it->c_str() + strlen(s3_acckey_opt));
590 }
591 // s3 region
592 if (strncasecmp(it->c_str(), s3_region_opt, strlen(s3_region_opt)) == 0) {
593 s3region = std::string(it->c_str() + strlen(s3_region_opt));
594 }
595 // s3 sts token
596 if (strncasecmp(it->c_str(), s3_token_opt, strlen(s3_token_opt)) == 0) {
597 s3token = std::string(it->c_str() + strlen(s3_token_opt));
598 }
599 // s3 alternate option
600 if (strncasecmp(it->c_str(), s3_alternate_opt, strlen(s3_alternate_opt)) == 0) {
601 setAwsAlternate(strToBool(it->c_str() + strlen(s3_alternate_opt), false));
602 }
603 // open mods
605 }
606
607 if (s3seckey.size() > 0) {
609 }
610
611 if (oflags == 0) // default open mode
613}
614
615////////////////////////////////////////////////////////////////////////////////
616
618{
619 davixPosix = new DavPosix(davixContext);
620 davixParam = new RequestParams();
621 davixParam->setUserAgent(gUserAgent);
622 davixParam->setMetalinkMode(Davix::MetalinkMode::Disable);
624 parseConfig();
626}
627
628////////////////////////////////////////////////////////////////////////////////
629
630Int_t TDavixFileInternal::DavixStat(const char *url, struct stat *st)
631{
633
634 if (davixPosix->stat(davixParam, url, st, &davixErr) < 0) {
635
636 Error("DavixStat", "can not stat the file with davix: %s (%d)",
637 davixErr->getErrMsg().c_str(), davixErr->getStatus());
638 DavixError::clearError(&davixErr);
639 return 0;
640 }
641 return 1;
642}
643
644/////////////////////////////////////////////////////////////////////////////////////////////
645
646////////////////////////////////////////////////////////////////////////////////
647
648TDavixFile::TDavixFile(const char *url, Option_t *opt, const char *ftitle, Int_t compress)
649 : TFile(url, strstr(opt, "_WITHOUT_GLOBALREGISTRATION") != nullptr ? "WEB_WITHOUT_GLOBALREGISTRATION" : "WEB"),
650 d_ptr(new TDavixFileInternal(fUrl, opt))
651{
652 (void) ftitle;
653 (void) compress;
654 Init(kFALSE);
655}
656
657////////////////////////////////////////////////////////////////////////////////
658
660{
661 d_ptr->Close();
662 delete d_ptr;
663}
664
665////////////////////////////////////////////////////////////////////////////////
666
668{
669 (void) init;
670 //initialize davix
671 d_ptr->init();
672 // pre-open file
673 if ((d_ptr->getDavixFileInstance()) == NULL){
674 MakeZombie();
676 return;
677 }
679 fOffset = 0;
680 fD = -2; // so TFile::IsOpen() will return true when in TFile::~TFi */
681}
682
684 std::vector<std::string> replicas = d_ptr->getReplicas();
686 if(!replicas.empty()) {
687 std::stringstream ss;
688 for(size_t i = 0; i < replicas.size(); i++) {
689 ss << replicas[i];
690 if(i != replicas.size()-1) ss << "|";
691 }
692 newUrl = ss.str();
693 }
694 return newUrl;
695}
696
697////////////////////////////////////////////////////////////////////////////////
698/// Set position from where to start reading.
699
701{
702 TLockGuard guard(&(d_ptr->positionLock));
703 switch (pos) {
704 case kBeg:
706 break;
707 case kCur:
708 fOffset += offset;
709 break;
710 case kEnd:
711 // this option is not used currently in the ROOT code
712 if (fArchiveOffset)
713 Error("Seek", "seeking from end in archive is not (yet) supported");
714 fOffset = fEND - offset; // is fEND really EOF or logical EOF?
715 break;
716 }
717
718 if (gDebug > 1)
719 Info("Seek", " move cursor to %lld"
720 , fOffset);
721}
722
723////////////////////////////////////////////////////////////////////////////////
724/// Read specified byte range from remote file via HTTP.
725/// Returns kTRUE in case of error.
726
728{
729 TLockGuard guard(&(d_ptr->positionLock));
730 Davix_fd *fd;
731 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
732 return kTRUE;
733 Long64_t ret = DavixReadBuffer(fd, buf, len);
734 if (ret < 0)
735 return kTRUE;
736
737 if (gDebug > 1)
738 Info("ReadBuffer", "%lld bytes of data read sequentially"
739 " (%d requested)", ret, len);
740
741 return kFALSE;
742}
743
744////////////////////////////////////////////////////////////////////////////////
745
747{
748 Davix_fd *fd;
749 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
750 return kTRUE;
751
752 Long64_t ret = DavixPReadBuffer(fd, buf, pos, len);
753 if (ret < 0)
754 return kTRUE;
755
756 if (gDebug > 1)
757 Info("ReadBuffer", "%lld bytes of data read from offset"
758 " %lld (%d requested)", ret, pos, len);
759 return kFALSE;
760}
761
762////////////////////////////////////////////////////////////////////////////////
763
765{
766 Davix_fd *fd;
767 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
768 return kFALSE;
769
770 d_ptr->davixPosix->fadvise(fd, static_cast<dav_off_t>(offs), static_cast<dav_size_t>(len), Davix::AdviseRandom);
771
772 if (gDebug > 1)
773 Info("ReadBufferAsync", "%d bytes of data prefected from offset"
774 " %lld ", len, offs);
775 return kFALSE;
776}
777
778////////////////////////////////////////////////////////////////////////////////
779
781{
782 Davix_fd *fd;
783 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
784 return kTRUE;
785
786 Long64_t ret = DavixReadBuffers(fd, buf, pos, len, nbuf);
787 if (ret < 0)
788 return kTRUE;
789
790 if (gDebug > 1)
791 Info("ReadBuffers", "%lld bytes of data read from a list of %d buffers",
792 ret, nbuf);
793
794 return kFALSE;
795}
796
797////////////////////////////////////////////////////////////////////////////////
798
800{
801 Davix_fd *fd;
802 if ((fd = d_ptr->getDavixFileInstance()) == NULL)
803 return kTRUE;
804
805 Long64_t ret = DavixWriteBuffer(fd, buf, len);
806 if (ret < 0)
807 return kTRUE;
808
809 if (gDebug > 1)
810 Info("WriteBuffer", "%lld bytes of data write"
811 " %d requested", ret, len);
812 return kFALSE;
813}
814
815////////////////////////////////////////////////////////////////////////////////
816
818{
819 d_ptr->davixParam->setSSLCAcheck((bool)check);
820}
821
822////////////////////////////////////////////////////////////////////////////////
823
828
829////////////////////////////////////////////////////////////////////////////////
830
832{
834 std::vector<void *>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd);
835 return (f != dirdVec.end());
836}
837
838////////////////////////////////////////////////////////////////////////////////
839
841{
843 dirdVec.push_back(fd);
844}
845
846////////////////////////////////////////////////////////////////////////////////
847
849{
851 std::vector<void *>::iterator f = std::find(dirdVec.begin(), dirdVec.end(), fd);
852 if (f != dirdVec.end())
853 dirdVec.erase(f);
854}
855
856////////////////////////////////////////////////////////////////////////////////
857
859{
860 struct stat st;
862 if (ret) {
863 if (gDebug > 1)
864 Info("GetSize", "file size requested: %lld", (Long64_t)st.st_size);
865 return st.st_size;
866 }
867 return -1;
868}
869
870////////////////////////////////////////////////////////////////////////////////
871
873{
874 if (gPerfStats)
875 return TTimeStamp();
876 return 0;
877}
878
879////////////////////////////////////////////////////////////////////////////////
880/// set TFile state info
881
883{
884 if(read) {
885 fBytesRead += len;
886 fReadCalls += 1;
887
890
891 if (gPerfStats)
892 gPerfStats->FileReadEvent(this, (Int_t) len, t_start);
893 } else {
894 fBytesWrite += len;
896 }
897}
898
899////////////////////////////////////////////////////////////////////////////////
900
902{
904 Double_t start_time = eventStart();
905
906 Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, fOffset, &davixErr);
907 if (ret < 0) {
908 Error("DavixReadBuffer", "can not read data with davix: %s (%d)",
909 davixErr->getErrMsg().c_str(), davixErr->getStatus());
910 DavixError::clearError(&davixErr);
911 } else {
912 fOffset += ret;
913 eventStop(start_time, ret);
914 }
915
916 return ret;
917}
918
919////////////////////////////////////////////////////////////////////////////////
920
922{
924 Double_t start_time = eventStart();
925
926 Long64_t ret = d_ptr->davixPosix->pwrite(fd, buf, len, fOffset, &davixErr);
927 if (ret < 0) {
928 Error("DavixWriteBuffer", "can not write data with davix: %s (%d)",
929 davixErr->getErrMsg().c_str(), davixErr->getStatus());
930 DavixError::clearError(&davixErr);
931 } else {
932 fOffset += ret;
933 eventStop(start_time, ret, false);
934 }
935
936 return ret;
937}
938
939////////////////////////////////////////////////////////////////////////////////
940
942{
944 Double_t start_time = eventStart();
945
946 Long64_t ret = d_ptr->davixPosix->pread(fd, buf, len, pos, &davixErr);
947 if (ret < 0) {
948 Error("DavixPReadBuffer", "can not read data with davix: %s (%d)",
949 davixErr->getErrMsg().c_str(), davixErr->getStatus());
950 DavixError::clearError(&davixErr);
951 } else {
952 eventStop(start_time, ret);
953 }
954
955
956 return ret;
957}
958
959////////////////////////////////////////////////////////////////////////////////
960
962{
964 Double_t start_time = eventStart();
965 auto in = std::make_unique<DavIOVecInput[]>(nbuf);
966 auto out = std::make_unique<DavIOVecOuput[]>(nbuf);
967
968 int lastPos = 0;
969 for (Int_t i = 0; i < nbuf; ++i) {
970 in[i].diov_buffer = &buf[lastPos];
971 in[i].diov_offset = pos[i];
972 in[i].diov_size = len[i];
973 lastPos += len[i];
974 }
975
976 Long64_t ret = d_ptr->davixPosix->preadVec(fd, in.get(), out.get(), nbuf, &davixErr);
977 if (ret < 0) {
978 Error("DavixReadBuffers", "can not read data with davix: %s (%d)",
979 davixErr->getErrMsg().c_str(), davixErr->getStatus());
980 DavixError::clearError(&davixErr);
981 } else {
982 eventStop(start_time, ret);
983 }
984
985 return ret;
986}
#define R__LOG_ERROR(...)
Definition RLogger.hxx:357
#define R__LOG_INFO(...)
Definition RLogger.hxx:359
#define f(i)
Definition RSha256.hxx:104
int Int_t
Signed integer 4 bytes (int)
Definition RtypesCore.h:59
constexpr Bool_t kFALSE
Definition RtypesCore.h:108
long long Long64_t
Portable signed long integer 8 bytes.
Definition RtypesCore.h:83
constexpr Bool_t kTRUE
Definition RtypesCore.h:107
const char Option_t
Option string (const char)
Definition RtypesCore.h:80
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
const char * s3_seckey_opt
bool normalizeToken(const std::string &input_token, std::string &output_token)
static const std::string VERSION
static void ConfigureDavixLogLevel()
static auto awsRegion(TRequestParams *parameters, const char *region) -> decltype(parameters->setAwsRegion(region), void())
bool findTokenInFile(const std::string &token_file, std::string &output_token)
static TMutex createLock
static auto awsAlternate(TRequestParams *parameters, bool option) -> decltype(parameters->setAwsAlternate(option), void())
std::string DiscoverToken()
static int TDavixFile_http_authn_cert_X509(void *userdata, const Davix::SessionInfo &info, Davix::X509Credential *cert, Davix::DavixError **err)
const char * s3_alternate_opt
static auto awsToken(TRequestParams *parameters, const char *token) -> decltype(parameters->setAwsToken(token), void())
ROOT::RLogChannel & TDavixLogChannel()
const char * s3_token_opt
const char * ca_check_opt
const char * open_mode_new
const char * open_mode_update
const char * open_mode_create
const char * s3_acckey_opt
static Context * davix_context_s
const char * grid_mode_opt
int configure_open_flag(const std::string &str, int old_flag)
static void TDavixFile_http_get_ucert(std::string &ucert, std::string &ukey)
static const std::string gUserAgent
bool strToBool(const char *str, bool defvalue)
const char * s3_region_opt
const char * open_mode_read
#define gDirectory
Definition TDirectory.h:385
R__EXTERN TEnv * gEnv
Definition TEnv.h:170
void Info(const char *location, const char *msgfmt,...)
Use this function for informational messages.
Definition TError.cxx:241
void Error(const char *location, const char *msgfmt,...)
Use this function in case an error occurred.
Definition TError.cxx:208
void Warning(const char *location, const char *msgfmt,...)
Use this function in warning situations.
Definition TError.cxx:252
Option_t Option_t option
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
Int_t gDebug
Global variable setting the debug level. Set to 0 to disable, increase it in steps of 1 to increase t...
Definition TROOT.cxx:627
#define gROOT
Definition TROOT.h:411
#define gPerfStats
#define snprintf
Definition civetweb.c:1579
A log configuration for a channel, e.g.
Definition RLogger.hxx:98
const_iterator begin() const
const_iterator end() const
void addDird(void *fd)
Davix::RequestParams * davixParam
bool isMyDird(void *fd)
std::vector< void * > dirdVec
void setAwsToken(const std::string &token)
void setS3Auth(const std::string &secret, const std::string &access, const std::string &region, const std::string &token)
void setAwsAlternate(const bool &option)
static Davix::Context * getDavixInstance()
Davix::DavPosix * davixPosix
void parseParams(Option_t *option)
intput params
std::vector< std::string > getReplicas()
Int_t DavixStat(const char *url, struct stat *st)
Davix_fd * getDavixFileInstance()
Davix_fd * Open()
Davix::Context * davixContext
void setAwsRegion(const std::string &region)
std::vector< std::string > replicas
void removeDird(void *fd)
Bool_t WriteBuffer(const char *buffer, Int_t bufferLength) override
Write a buffer to the file.
TString GetNewUrl() override
Long64_t GetSize() const override
Returns the current file size.
Long64_t DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Long64_t DavixReadBuffer(Davix_fd *fd, char *buf, Int_t len)
Long64_t DavixPReadBuffer(Davix_fd *fd, char *buf, Long64_t pos, Int_t len)
Long64_t DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len)
void Seek(Long64_t offset, ERelativeTo pos=kBeg) override
Set position from where to start reading.
TDavixFile(const char *url, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault)
Open function for TDavixFile.
void enableGridMode()
Enable the grid mode The grid Mode configure automatically all grid-CA path, VOMS authentication and ...
Bool_t ReadBuffer(char *buf, Int_t len) override
Read specified byte range from remote file via HTTP.
Bool_t ReadBufferAsync(Long64_t offs, Int_t len) override
TDavixFileInternal * d_ptr
Definition TDavixFile.h:69
void Init(Bool_t init) override
Initialize a TFile object.
void eventStop(Double_t t, Long64_t len, bool read=true)
set TFile state info
void setCACheck(Bool_t check)
Enable or disable certificate authority check.
Double_t eventStart()
Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf) override
Read the nbuf blocks described in arrays pos and len.
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
Definition TEnv.cxx:490
A ROOT file is an on-disk file, usually with extension .root, that stores objects in a file-system-li...
Definition TFile.h:131
Int_t fReadCalls
Number of read calls ( not counting the cache calls )
Definition TFile.h:168
static void SetFileBytesWritten(Long64_t bytes=0)
Definition TFile.cxx:4293
Long64_t fBytesRead
Number of bytes read from this file.
Definition TFile.h:155
static Long64_t GetFileBytesWritten()
Static function returning the total number of bytes written to all files.
Definition TFile.cxx:4265
static void SetFileBytesRead(Long64_t bytes=0)
Definition TFile.cxx:4290
static void SetFileReadCalls(Int_t readcalls=0)
Definition TFile.cxx:4296
TUrl fUrl
!URL of file
Definition TFile.h:189
static Long64_t GetFileBytesRead()
Static function returning the total number of bytes read from all files.
Definition TFile.cxx:4256
Long64_t fArchiveOffset
!Offset at which file starts in archive
Definition TFile.h:180
virtual void Init(Bool_t create)
Initialize a TFile object.
Definition TFile.cxx:616
ERelativeTo
Definition TFile.h:278
@ kCur
Definition TFile.h:278
@ kBeg
Definition TFile.h:278
@ kEnd
Definition TFile.h:278
Int_t fD
File descriptor.
Definition TFile.h:161
Long64_t fBytesWrite
Number of bytes written to this file.
Definition TFile.h:154
Long64_t fOffset
!Seek offset cache
Definition TFile.h:175
Long64_t fEND
Last used byte in file.
Definition TFile.h:158
static Int_t GetFileReadCalls()
Static function returning the total number of read calls from all files.
Definition TFile.cxx:4273
void MakeZombie()
Definition TObject.h:53
Basic string class.
Definition TString.h:138
The TTimeStamp encapsulates seconds and ns since EPOCH.
Definition TTimeStamp.h:45
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition TUrl.cxx:395
bool isno(const char *str)
Definition utils.h:11
TLine l
Definition textangle.C:4