58#include <XrdClient/XrdClient.hh>
62#include <XProtocol/XProtocol.hh>
115 const char *logicalurl) :
116 TNetFile((logicalurl ? logicalurl : url), ftitle, compress,
kFALSE)
118 TUrl urlnoanchor(url);
129 Info(
"TXNetFile",
"(eXtended TNetFile) %s",
130 gROOT->GetVersion());
176 if (strlen(uu.
GetUser()) > 0) {
182 if (strlen(uu.
GetHost()) > 0) {
203 static const char *keys[6] = {
"cachesz=",
"readaheadsz=",
"rmpolicy=",
204 "mxredir=",
"readaheadstrategy=",
"readtrimblksz=" };
209 for (i = 0; i < (
sizeof(keys)/
sizeof(keys[0])); i++) {
210 Int_t j =
s.Index(keys[i]);
212 TString val(
s(j+strlen(keys[i]),
s.Length()));
216 if (!
TString(val(k++)).IsDigit())
223 cachesz = val.
Atoi();
225 readaheadsz = val.
Atoi();
227 rmpolicy = val.
Atoi();
229 mxredir = val.
Atoi();
231 rastrategy = val.
Atoi();
233 readtrimblksz = val.
Atoi();
240 Info(
"ParseCacheOptions",
"found: cachesz = %d, readaheadsz = %d, "
241 "rmpolicy = %d, mxredir = %d, rastrategy = %d, readtrimblksz = %d",
242 cachesz, readaheadsz, rmpolicy, mxredir, rastrategy, readtrimblksz);
254 Int_t cachesz = -1, readaheadsz = -1, rmpolicy = -1, mxredir = -1, np = 0;
255 Int_t readaheadstrategy = -1, readtrimblksz = -1;
271 ::Warning(
"TXNetFile",
"<%s> is not staged - StageOnly flag is set!",url);
282 parallelopen =
kTRUE;
292 Error(
"CreateXClient",
"fatal error: new object creation failed -"
293 " out of system resources.");
300 cachesz, readaheadsz, rmpolicy, mxredir,
301 readaheadstrategy, readtrimblksz);
305 if (
fClient->GetClientConn()) {
307 Info(
"CreateXClient",
"setting maxredir = %d", mxredir);
308 fClient->GetClientConn()->SetMaxRedirCnt(mxredir);
315 Info(
"CreateXClient",
"setting cachesz = %d, readaheadsz = %d, "
317 cachesz, readaheadsz, rmpolicy);
318 fClient->SetCacheParameters(cachesz, readaheadsz, rmpolicy);
320 if (readaheadstrategy >= 0) {
322 Info(
"CreateXClient",
"setting readaheadstrategy = %d", readaheadstrategy);
323 fClient->SetReadAheadStrategy(readaheadstrategy);
326 if (readtrimblksz >= 0) {
328 Info(
"CreateXClient",
"setting readtrimblksz = %d", readtrimblksz);
329 fClient->SetBlockReadTrimming(readtrimblksz);
336 if (!
Open(option, parallelopen)) {
339 Info(
"CreateXClient",
"remote file could not be open");
358 Error(
"CreateXClient",
"getting rootd server protocol");
363 s->SetRemoteProtocol(rproto);
365 ->GetCurrentUrl()).GetUrl().c_str());
370 Info(
"CreateXClient",
" url: %s",uu.
Data());
371 s->SetUrl(uu.
Data());
372 s->SetService(
"rootd");
393 Error(
"CreateXClient",
"rootd: underlying socket undefined");
398 Info(
"CreateXClient",
"rootd: fall back not enabled - closing");
426 Int_t len =
sizeof(cproto);
427 memcpy((
char *)&cproto,
429 Int_t ns =
s->SendRaw(&cproto, len);
431 ::Error(
"TXNetFile::GetRootdProtocol",
432 "sending %d bytes to rootd server [%s:%d]",
433 len, (
s->GetInetAddress()).GetHostName(),
s->GetPort());
440 Int_t nr =
s->RecvRaw(ibuf, len);
442 ::Error(
"TXNetFile::GetRootdProtocol",
443 "reading %d bytes from rootd server [%s:%d]",
444 len, (
s->GetInetAddress()).GetHostName(),
s->GetPort());
453 len =
sizeof(rproto);
454 nr =
s->RecvRaw(&rproto, len);
456 ::Error(
"TXNetFile::GetRootdProtocol",
457 "reading %d bytes from rootd server [%s:%d]",
458 len, (
s->GetInetAddress()).GetHostName(),
s->GetPort());
465 ::Info(
"TXNetFile::GetRootdProtocol",
466 "remote rootd: buf1: %d, buf2: %d rproto: %d",
480 kXR_unt16 openOpt = 0;
481 memset(&openOpt, 0,
sizeof(openOpt));
489 openOpt |= kXR_force;
493 Bool_t read = (opt ==
"READ");
496 Bool_t create = (opt ==
"CREATE" || opt ==
"NEW");
499 Bool_t recreate = (opt ==
"RECREATE");
505 if (!create && !recreate && !
update && !read) {
512 if (create ||
update || recreate)
524 Error(
"Open",
"no write permission, could not open file %s",
529 openOpt |= kXR_open_updt;
538 openOpt |= kXR_delete;
545 mkpath = (*(p + strlen(
"mkpath=")) ==
'1') ?
kTRUE :
kFALSE;
547 openOpt |= kXR_mkpath;
550 openOpt |= kXR_open_read;
554 kXR_unt16 openMode = kXR_or | kXR_gr | kXR_ur | kXR_uw;
558 if (!
fClient->Open(openMode, openOpt, doitparallel)) {
560 Info(
"Open",
"remote file could not be open");
594 Error(
"ReadBuffer",
"ReadBuffer is not possible because object"
595 " is in 'zombie' state");
601 Info(
"ReadBuffer",
"Calling TNetFile::ReadBuffer");
606 Error(
"ReadBuffer",
"The remote file is not open");
650 if (nr != bufferLength) {
651 Error(
"ReadBuffer",
"error reading all requested bytes, got %d of %d",
657 Info(
"ReadBuffer",
"%d bytes of data read from offset"
658 " %lld (%d requested)", nr,
fOffset, bufferLength);
673 gPerfStats->FileReadEvent(
this, bufferLength, start);
699 Error(
"ReadBuffer",
"ReadBuffer is not possible because object"
700 " is in 'zombie' state");
706 Error(
"ReadBufferAsync",
"Not supported for rootd");
711 Error(
"ReadBuffer",
"The remote file is not open");
745 gPerfStats->FileReadEvent(
this, bufferLength, start);
748 Info(
"ReadBufferAsync",
"%d bytes of data read request from offset"
749 " %lld", bufferLength, offs);
766 Error(
"ReadBuffers",
"ReadBuffers is not possible because object"
767 " is in 'zombie' state");
773 Info(
"ReadBuffers",
"Calling TNetFile::ReadBuffers");
778 Error(
"ReadBuffers",
"The remote file is not open");
786 for (
Int_t i = 0; i < nbuf; i++)
791 for (
Int_t i = 0; i < nbuf; i++) {
792 expected_nr += len[i];
808 Info(
"ReadBuffers",
"response from ReadV(%d) nr: %lld", nbuf, nr);
810 if (nr == expected_nr) {
813 Info(
"ReadBuffers",
"%lld bytes of data read from a list of %d buffers",
817 Info(
"ReadBuffers",
"%lld bytes of data read with a smaller (%d) TFileCacheRead buffer size?",
844 Info(
"ReadBuffers",
"XrdClient->ReadV failed, executing TFile::ReadBuffers");
863 Error(
"WriteBuffer",
"WriteBuffer is not possible because object"
864 " is in 'zombie' state");
870 Info(
"WriteBuffer",
"file not writable");
876 Info(
"WriteBuffer",
"Calling TNetFile::WriteBuffer");
881 Error(
"WriteBuffer",
"The remote file is not open");
896 "error writing %d bytes of data wrote to offset %lld",
902 Info(
"WriteBuffer",
" %d bytes of data wrote to offset"
903 " %lld", bufferLength ,
fOffset);
925 Info(
"Init",
"TFile::Init already called once");
931 Info(
"Init",
"rootd: calling directly TFile::Init");
948 bool usecachesave =
fClient->UseCache(0);
954 fClient->UseCache(usecachesave);
962 if (
fClient->GetClientConn() &&
fClient->GetClientConn()->fRedirOpaque.length() <= 0) {
972 Info(
"Init",
"open request failed!");
987 Info(
"IsOpen",
"Calling TNetFile::IsOpen");
1004 if (
fClient->IsOpen_inprogress()) {
1026 Info(
"ReOpen",
"Calling TNetFile::ReOpen");
1041 Info(
"Close",
"Calling TNetFile::Close");
1064 Error(
"Flush",
"Flush is not possible because object is"
1065 " in 'zombie' state");
1071 Info(
"Flush",
"file not writable - do nothing");
1077 Info(
"Flush",
"Calling TNetFile::Flush");
1083 Error(
"Flush",
"The remote file is not open");
1093 Info(
"Flush",
"XrdClient::Sync called.");
1103 Error(
"SysStat",
"SysStat is not possible because object is"
1104 " in 'zombie' state");
1111 Info(
"SysStat",
"calling TNetFile::SysStat");
1117 struct XrdClientStatInfo stinfo;
1119 *
id = (
Long_t)(stinfo.id);
1121 *flags = (
Long_t)(stinfo.flags);
1122 *modtime = (
Long_t)(stinfo.modtime);
1124 Info(
"SysStat",
"got stats = %ld %lld %ld %ld",
1125 *
id, *size, *flags, *modtime);
1129 if (!
IsOpen())
Info(
"SysStat",
"could not stat remote file. Not opened.");
1131 Info(
"SysStat",
"could not stat remote file");
1149 Error(
"SysClose",
"SysClose is not possible because object is"
1150 " in 'zombie' state");
1156 Info(
"SysClose",
"Calling TNetFile::SysClose");
1174 Info(
"SysOpen",
"Calling TNetFile::SysOpen");
1205 if (allowRE.
Length() > 0)
1215 if (allowCO.
Length() > 0)
1289 if (socks4Port > 0) {
1292 socks4Host =
"127.0.0.1";
1297 const char *cenv = 0;
1301 if (autolog.
Length() > 0 &&
1302 (!(cenv =
gSystem->
Getenv(
"XrdSecPWDAUTOLOG")) || strlen(cenv) <= 0))
1311 if (alogfile.
Length() > 0)
1315 if (verisrv.
Length() > 0 &&
1316 (!(cenv =
gSystem->
Getenv(
"XrdSecPWDVERIFYSRV")) || strlen(cenv) <= 0))
1353 if (deplen.
Length() > 0 &&
1354 (!(cenv =
gSystem->
Getenv(
"XrdSecGSIPROXYDEPLEN")) || strlen(cenv) <= 0))
1358 if (pxybits.
Length() > 0)
1362 if (crlcheck.
Length() > 0 &&
1363 (!(cenv =
gSystem->
Getenv(
"XrdSecGSICRLCHECK")) || strlen(cenv) <= 0))
1367 if (delegpxy.
Length() > 0 &&
1368 (!(cenv =
gSystem->
Getenv(
"XrdSecGSIDELEGPROXY")) || strlen(cenv) <= 0))
1372 if (signpxy.
Length() > 0 &&
1373 (!(cenv =
gSystem->
Getenv(
"XrdSecGSISIGNPROXY")) || strlen(cenv) <= 0))
1390 Long64_t bytessubmitted, byteshit, misscount, readreqcnt;
1391 Float_t missrate, bytesusefulness;
1393 if (
fClient->GetCacheInfo(size, bytessubmitted,
1394 byteshit, misscount,
1395 missrate, readreqcnt,
1396 bytesusefulness) ) {
1402 newbsz = std::max(newbsz, size);
1419 fClient->RemoveAllDataFromCache();
1428 Long64_t bytessubmitted, byteshit, misscount, readreqcnt;
1429 Float_t missrate, bytesusefulness;
1432 byteshit, misscount,
1433 missrate, readreqcnt,
1436 return ((bytes < 0) ? 0 : bytes);
1444 Printf(
"TXNetFile caching information:");
1447 Long64_t bytessubmitted, byteshit, misscount, readreqcnt;
1448 Float_t missrate, bytesusefulness;
1451 byteshit, misscount,
1452 missrate, readreqcnt,
1454 Printf(
" Max size: %d", size);
1455 Printf(
" Bytes submitted: %lld", bytessubmitted);
1456 Printf(
" Bytes hit (estimation): %lld", byteshit);
1457 Printf(
" Miss count: %lld", misscount);
1458 Printf(
" Miss rate: %f", missrate);
1459 Printf(
" Read requests count: %lld", readreqcnt);
1460 Printf(
" Bytes usefulness: %f\n", bytesusefulness);
1462 Printf(
" -- No Xrd client instance allocated --\n");
UShort_t net2host(UShort_t x)
static void update(gsl_integration_workspace *workspace, double a1, double b1, double area1, double error1, double a2, double b2, double area2, double error2)
char * Form(const char *fmt,...)
void Printf(const char *fmt,...)
R__EXTERN TSystem * gSystem
R__EXTERN TVirtualMonitoringWriter * gMonitoringWriter
#define NAME_TRANSACTIONTIMEOUT
#define NAME_CONNECTTIMEOUT
#define DFLT_REQUESTTIMEOUT
#define NAME_READCACHESIZE
#define NAME_REQUESTTIMEOUT
#define DFLT_DFLTTCPWINDOWSIZE
#define DFLT_FIRSTCONNECTMAXCNT
#define DFLT_MAXREDIRECTCOUNT
#define NAME_REDIRDOMAINDENY_RE
#define NAME_MAXREDIRECTCOUNT
#define NAME_FIRSTCONNECTMAXCNT
#define DFLT_TRANSACTIONTIMEOUT
#define NAME_RECONNECTWAIT
#define NAME_REDIRDOMAINALLOW_RE
#define DFLT_READAHEADSIZE
#define NAME_DFLTTCPWINDOWSIZE
#define NAME_CONNECTDOMAINDENY_RE
#define NAME_READAHEADSIZE
#define NAME_MULTISTREAMCNT
#define NAME_CONNECTDOMAINALLOW_RE
#define DFLT_READCACHESIZE
#define DFLT_CONNECTTIMEOUT
#define NAME_KEEPSOCKOPENIFNOTXRD
#define DFLT_MULTISTREAMCNT
#define DFLT_RECONNECTWAIT
#define EnvPutInt(name, val)
#define EnvPutString(name, val)
#define XrdSysMutexHelper
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
Bool_t fWritable
True if directory is writable.
Int_t GetBufferSize() const override
Return the buffer size to create new TKeys.
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
A cache when reading files over the network.
virtual Bool_t Matches(const char *s)
virtual Bool_t IsStaged(const char *)
Just check if the file exists locally.
static TFileStager * Open(const char *stager)
Open a stager, after having loaded the relevant plug-in.
static std::atomic< Long64_t > fgBytesRead
Number of bytes read by all TFile objects.
Int_t fReadCalls
Number of read calls ( not counting the cache calls )
static void SetFileBytesWritten(Long64_t bytes=0)
Long64_t fBytesRead
Number of bytes read from this file.
static Bool_t GetOnlyStaged()
Returns staged only flag.
virtual Int_t ReOpen(Option_t *mode)
Reopen a file with a different access mode.
static Long64_t GetFileBytesWritten()
Static function returning the total number of bytes written to all files.
static void SetFileBytesRead(Long64_t bytes=0)
TList * fOpenPhases
!Time info about open phases
static void SetFileReadCalls(Int_t readcalls=0)
void Print(Option_t *option="") const override
Print all objects in the file.
static UInt_t GetOpenTimeout()
Returns open timeout (in ms).
Int_t WriteBufferViaCache(const char *buf, Int_t len)
Write buffer via cache.
static Long64_t GetFileBytesRead()
Static function returning the total number of bytes read from all files.
Int_t ReadBufferViaCache(char *buf, Int_t len)
Read buffer via cache.
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
Long64_t fArchiveOffset
!Offset at which file starts in archive
virtual void Init(Bool_t create)
Initialize a TFile object.
TString fOption
File options.
EAsyncOpenStatus
Asynchronous open request status.
Bool_t FlushWriteCache()
Flush the write cache if active.
Long64_t fBytesWrite
Number of bytes written to this file.
Bool_t fInitDone
!True if the file has been initialized
Long64_t fOffset
!Seek offset cache
static std::atomic< Long64_t > fgBytesWrite
Number of bytes written by all TFile objects.
EAsyncOpenStatus fAsyncOpenStatus
!Status of an asynchronous open request
void Close(Option_t *option="") override
Close a file.
static std::atomic< Int_t > fgReadCalls
Number of bytes read from all TFile objects.
static Int_t GetFileReadCalls()
Static function returning the total number of read calls from all files.
TFileCacheRead * GetCacheRead(const TObject *tree=nullptr) const
Return a pointer to the current read cache.
Int_t SysOpen(const char *pathname, Int_t flags, UInt_t mode)
Open a remote file. Requires fOption to be set correctly.
Int_t SysStat(Int_t fd, Long_t *id, Long64_t *size, Long_t *flags, Long_t *modtime)
Return file stat information.
virtual void Create(const char *url, Option_t *option, Int_t netopt)
Create a NetFile object.
Bool_t IsOpen() const
Retruns kTRUE if file is open, kFALSE otherwise.
Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read a list of buffers given in pos[] and len[] and return it in a single buffer.
Int_t SysClose(Int_t fd)
Close currently open file.
Bool_t ReadBuffer(char *buf, Int_t len)
Read specified byte range from remote file via rootd daemon.
void Init(Bool_t create)
Initialize a TNetFile object.
Bool_t WriteBuffer(const char *buf, Int_t len)
Write specified byte range to remote file via rootd daemon.
void Close(Option_t *option="")
Close remote file.
void Flush()
Flush file to disk.
Int_t ReOpen(Option_t *mode)
Reopen a file with a different access mode, like from READ to UPDATE or from NEW, CREATE,...
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
R__ALWAYS_INLINE Bool_t IsZombie() const
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
static Int_t GetClientProtocol()
Static method returning supported client protocol.
Int_t Atoi() const
Return integer value of string.
const char * Data() const
Bool_t IsDigit() const
Returns true if all characters in string are digits (0-9) or white spaces, i.e.
void ToUpper()
Change string to upper case.
Bool_t BeginsWith(const char *s, ECaseCompare cmp=kExact) const
TString & Remove(Ssiz_t pos)
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
virtual const char * Getenv(const char *env)
Get environment variable.
virtual Bool_t AccessPathName(const char *path, EAccessMode mode=kFileExists)
Returns FALSE if one can access a file using the specified access mode.
virtual void IgnoreSignal(ESignals sig, Bool_t ignore=kTRUE)
If ignore is true ignore the specified signal, else restore previous behaviour.
virtual void Setenv(const char *name, const char *value)
Set environment variable.
virtual const char * HomeDirectory(const char *userName=nullptr)
Return the user's home directory.
virtual void Abort(int code=0)
Abort the application.
The TTimeStamp encapsulates seconds and ns since EPOCH.
This class represents a WWW compatible URL.
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
void SetAnchor(const char *anchor)
const char * GetUser() const
const char * GetHost() const
const char * GetHostFQDN() const
Return fully qualified domain name of url host.
const char * GetOptions() const
virtual Bool_t SendFileOpenProgress(TFile *, TList *, const char *, Bool_t=kFALSE)
virtual Bool_t SendFileReadProgress(TFile *)
Int_t ParseOptions(const char *opts, Int_t &cachesz, Int_t &readaheadsz, Int_t &rmpolicy, Int_t &mxredir, Int_t &rastrategy, Int_t &readtrimblksz)
Parse input options for cache parameters.
virtual void Print(Option_t *option="") const
Print the local statistics.
void SynchronizeCacheSize()
Synchronize the cache size Alternative purging policy.
static TFileStager * fgFileStager
Int_t SysStat(Int_t fd, Long_t *id, Long64_t *size, Long_t *flags, Long_t *modtime)
Override TNetFile::SysStat (see parent's method for more details).
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len, where pos[i] is the seek position of block i of...
virtual Int_t ReOpen(const Option_t *mode)
Re-open the file (see TNetFile::ReOpen() or TFile::ReOpen() for more details).
virtual ~TXNetFile()
Destructor.
void Init(Bool_t create)
Initialize the file.
static void SetEnv()
Set the relevant environment variables.
virtual void ResetCache()
Reset the cache.
static void FormUrl(TUrl uut, TString &uu)
Form url for rootd socket.
Int_t SysOpen(const char *pathname, Int_t flags, UInt_t mode)
Override TNetFile::SysOpen (see parent's method for more details).
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Override TNetFile::ReadBuffer to deal with the xrootd server.
virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len)
Implementation dealing with the xrootd server.
virtual Bool_t WriteBuffer(const char *buffer, Int_t BufferLength)
Override TNetFile::WriteBuffer to deal with the xrootd server.
virtual void Close(const Option_t *opt="")
Close the file (see TNetFile::Close() or TFile::Close() for more details).
virtual Int_t GetBytesToPrefetch() const
Max number of bytes to prefetch.
void CreateXClient(const char *url, Option_t *option, Int_t netopt, Bool_t parallelopen)
The real creation work is done here.
virtual void Flush()
Flushes un-written data.
Int_t SysClose(Int_t fd)
Override TNetFile::SysClose (see parent's method for more details).
Bool_t Open(Option_t *option, Bool_t parallelopen)
The real creation work is done here.
virtual Bool_t IsOpen() const
Return kTRUE if the file is open, kFALSE otherwise.
static Int_t GetRootdProtocol(TSocket *s)
Find out the remote rootd protocol version.
virtual TFile::EAsyncOpenStatus GetAsyncOpenStatus()
Return status of asynchronous request.
static constexpr double s
static constexpr double ns