57constexpr char const *gSQliteVfsName =
"ROOT-Davix-readonly";
68 VfsRootFile() : pos(&
c) {}
81int VfsRdOnlyClose(sqlite3_file *pFile)
83 Davix::DavixError *err =
nullptr;
84 VfsRootFile *p =
reinterpret_cast<VfsRootFile *
>(pFile);
85 auto retval = p->pos.close(p->fd, &err);
88 return (retval == -1) ? SQLITE_IOERR_CLOSE : SQLITE_OK;
93int VfsRdOnlyRead(sqlite3_file *pFile,
void *zBuf,
int count, sqlite_int64 offset)
95 Davix::DavixError *err =
nullptr;
96 VfsRootFile *p =
reinterpret_cast<VfsRootFile *
>(pFile);
97 auto retval = p->pos.pread(p->fd, zBuf, count, offset, &err);
98 return (retval == -1) ? SQLITE_IOERR : SQLITE_OK;
103int VfsRdOnlyWrite(sqlite3_file * ,
const void * ,
int , sqlite_int64 )
105 return SQLITE_OPEN_READONLY;
110int VfsRdOnlyTruncate(sqlite3_file * , sqlite_int64 )
112 return SQLITE_OPEN_READONLY;
117int VfsRdOnlySync(sqlite3_file * ,
int )
124int VfsRdOnlyFileSize(sqlite3_file *pFile, sqlite_int64 *pSize)
126 VfsRootFile *p =
reinterpret_cast<VfsRootFile *
>(pFile);
133int VfsRdOnlyLock(sqlite3_file * ,
int )
140int VfsRdOnlyUnlock(sqlite3_file * ,
int )
147int VfsRdOnlyCheckReservedLock(sqlite3_file * ,
int *pResOut)
155int VfsRdOnlyFileControl(sqlite3_file * ,
int ,
void * )
157 return SQLITE_NOTFOUND;
162int VfsRdOnlySectorSize(sqlite3_file * )
164 return SQLITE_OPEN_READONLY;
169int VfsRdOnlyDeviceCharacteristics(sqlite3_file * )
171 return SQLITE_OPEN_READONLY;
176int VfsRdOnlyOpen(sqlite3_vfs * ,
const char *zName, sqlite3_file *pFile,
int flags,
int * )
179 VfsRootFile *p =
new (pFile) VfsRootFile();
180 p->pFile.pMethods =
nullptr;
184 static const sqlite3_io_methods io_methods = {
194 VfsRdOnlyCheckReservedLock,
195 VfsRdOnlyFileControl,
197 VfsRdOnlyDeviceCharacteristics,
207 if (flags & (SQLITE_OPEN_READWRITE | SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_EXCLUSIVE))
210 Davix::DavixError *err =
nullptr;
211 p->fd = p->pos.open(
nullptr, zName, O_RDONLY, &err);
214 ::Error(
"VfsRdOnlyOpen",
"%s\n", err->getErrMsg().c_str());
219 if (p->pos.stat(
nullptr, zName, &buf,
nullptr) == -1) {
222 p->size = buf.st_size;
224 p->pFile.pMethods = &io_methods;
234int VfsRdOnlyDelete(sqlite3_vfs * ,
const char * ,
int )
236 return SQLITE_IOERR_DELETE;
241int VfsRdOnlyAccess(sqlite3_vfs * ,
const char * ,
int flags,
int *pResOut)
244 if (flags == SQLITE_ACCESS_READWRITE) {
245 return SQLITE_OPEN_READONLY;
252int VfsRdOnlyFullPathname(sqlite3_vfs * ,
const char *zPath,
int nOut,
char *zOut)
254 zOut[nOut - 1] =
'\0';
255 sqlite3_snprintf(nOut, zOut,
"%s", zPath);
261int VfsRdOnlyRandomness(sqlite3_vfs * ,
int nBuf,
char *zBuf)
263 for (
int i = 0; i < nBuf; ++i) {
271int VfsRdOnlySleep(sqlite3_vfs * ,
int microseconds)
280int VfsRdOnlyGetLastError(sqlite3_vfs * ,
int ,
char * )
287int VfsRdOnlyCurrentTimeInt64(sqlite3_vfs * , sqlite3_int64 *piNow)
289 static constexpr sqlite3_int64 unixEpoch = 24405875 * (sqlite3_int64)8640000;
292 *piNow = ((sqlite3_int64)t) * 1000 + unixEpoch;
298int VfsRdOnlyCurrentTime(sqlite3_vfs *vfs,
double *prNow)
301 int rc = VfsRdOnlyCurrentTimeInt64(vfs, &i);
302 *prNow = i / 86400000.0;
308static struct sqlite3_vfs kSqlite3Vfs = {
318 VfsRdOnlyFullPathname,
325 VfsRdOnlyCurrentTime,
326 VfsRdOnlyGetLastError,
327 VfsRdOnlyCurrentTimeInt64,
336bool RegisterDavixVfs()
340 retval = sqlite3_vfs_register(&kSqlite3Vfs,
false);
341 return (retval == SQLITE_OK);
347bool IsURL(
const std::string &fileName)
349 if (fileName.compare(0, 7,
"http://") == 0)
351 if (fileName.compare(0, 8,
"https://") == 0)
365struct RSqliteDSDataSet {
366 sqlite3 *fDb =
nullptr;
367 sqlite3_stmt *fQuery =
nullptr;
372 :
fType(
type), fIsActive(false), fInteger(0), fReal(0.0), fText(), fBlob(), fNull(nullptr)
380 default:
throw std::runtime_error(
"Internal error");
395 static bool isDavixAvailable = RegisterDavixVfs();
399 if (IsURL(fileName)) {
400 if (!isDavixAvailable)
401 throw std::runtime_error(
"Processing remote files is not available. "
402 "Please compile ROOT with Davix support to read from HTTP(S) locations.");
404 sqlite3_open_v2(fileName.c_str(), &
fDataSet->fDb, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, gSQliteVfsName);
406 retval = sqlite3_open_v2(fileName.c_str(), &
fDataSet->fDb, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX,
nullptr);
408 if (retval != SQLITE_OK)
411 retval = sqlite3_prepare_v2(
fDataSet->fDb, query.c_str(), -1, &
fDataSet->fQuery,
nullptr);
412 if (retval != SQLITE_OK)
415 int colCount = sqlite3_column_count(
fDataSet->fQuery);
416 retval = sqlite3_step(
fDataSet->fQuery);
417 if ((retval != SQLITE_ROW) && (retval != SQLITE_DONE))
421 for (
int i = 0; i < colCount; ++i) {
423 int type = SQLITE_NULL;
426 const char *declTypeCstr = sqlite3_column_decltype(
fDataSet->fQuery, i);
427 if (declTypeCstr ==
nullptr) {
428 if (retval == SQLITE_ROW)
431 std::string declType(declTypeCstr);
432 std::transform(declType.begin(), declType.end(), declType.begin(), ::toupper);
433 if (declType ==
"INTEGER")
434 type = SQLITE_INTEGER;
435 else if (declType ==
"FLOAT")
437 else if (declType ==
"TEXT")
439 else if (declType ==
"BLOB")
442 throw std::runtime_error(
"Unexpected column decl type");
467 default:
throw std::runtime_error(
"Unhandled data type");
503 std::string errmsg =
"The type selected for column \"";
505 errmsg +=
"\" does not correspond to column type, which is ";
507 throw std::runtime_error(errmsg);
510 fValues[index].fIsActive =
true;
519 std::vector<std::pair<ULong64_t, ULong64_t>> entryRanges;
520 int retval = sqlite3_step(
fDataSet->fQuery);
522 case SQLITE_DONE:
return entryRanges;
540 for (
unsigned i = 0; i <
N; ++i) {
545 throw std::runtime_error(
"Unknown column: " + std::string(colName));
560 int retval = sqlite3_reset(
fDataSet->fQuery);
561 if (retval != SQLITE_OK)
562 throw std::runtime_error(
"SQlite error, reset");
576 ROOT::RDataFrame rdf(std::make_unique<RSqliteDS>(std::string(fileName), std::string(query)));
586 for (
unsigned i = 0; i <
N; ++i) {
595 nbytes = sqlite3_column_bytes(
fDataSet->fQuery, i);
599 fValues[i].fText =
reinterpret_cast<const char *
>(sqlite3_column_text(
fDataSet->fQuery, i));
603 nbytes = sqlite3_column_bytes(
fDataSet->fQuery, i);
604 fValues[i].fBlob.resize(nbytes);
606 std::memcpy(
fValues[i].fBlob.data(), sqlite3_column_blob(
fDataSet->fQuery, i), nbytes);
610 default:
throw std::runtime_error(
"Unhandled column type");
621 ::Warning(
"SetNSlots",
"Currently the SQlite data source faces performance degradation in multi-threaded mode. "
622 "Consider turning off IMT.");
631 std::string errmsg =
"SQlite error: ";
632 errmsg += sqlite3_errstr(errcode);
633 throw std::runtime_error(errmsg);
unsigned long long ULong64_t
void Error(const char *location, const char *msgfmt,...)
void Warning(const char *location, const char *msgfmt,...)
R__EXTERN TRandom * gRandom
R__EXTERN TSystem * gSystem
std::vector< void * > Record_t
void SetNSlots(unsigned int nSlots) final
Almost a no-op, many slots can in fact reduce the performance due to thread synchronization.
static constexpr char const * fgTypeNames[]
Corresponds to the types defined in ETypes.
std::string GetLabel() final
Return a string representation of the datasource type.
void Initialise() final
Resets the SQlite query engine at the beginning of the event loop.
std::vector< std::string > fColumnNames
~RSqliteDS()
Frees the sqlite resources and closes the file.
bool HasColumn(std::string_view colName) const final
A linear search through the columns for the given name.
std::vector< ETypes > fColumnTypes
std::string GetTypeName(std::string_view colName) const final
Returns the C++ type for a given column name, implemented as a linear search through all the columns.
ETypes
All the types known to SQlite. Changes require changing fgTypeNames, too.
Record_t GetColumnReadersImpl(std::string_view name, const std::type_info &) final
Activates the given column's result value.
RSqliteDS(const std::string &fileName, const std::string &query)
Build the dataframe.
std::unique_ptr< Internal::RSqliteDSDataSet > fDataSet
std::vector< std::pair< ULong64_t, ULong64_t > > GetEntryRanges() final
Returns a range of size 1 as long as more rows are available in the SQL result set.
const std::vector< std::string > & GetColumnNames() const final
Returns the SELECT queries names.
bool SetEntry(unsigned int slot, ULong64_t entry) final
Stores the result of the current active sqlite query row as a C++ value.
void SqliteError(int errcode)
Helper function to throw an exception if there is a fatal sqlite error, e.g. an I/O error.
std::vector< Value_t > fValues
The data source is inherently single-threaded and returns only one row at a time. This vector holds t...
ROOT's RDataFrame offers a high level interface for analyses of data stored in TTrees,...
virtual UInt_t Integer(UInt_t imax)
Returns a random integer uniformly distributed on the interval [ 0, imax-1 ].
virtual void Sleep(UInt_t milliSec)
Sleep milliSec milli seconds.
basic_string_view< char > string_view
RDataFrame MakeSqliteDataFrame(std::string_view fileName, std::string_view query)
Factory method to create a SQlite RDataFrame.
Namespace for new ROOT classes and functions.
void * fPtr
Points to one of the values; an address to this pointer is returned by GetColumnReadersImpl.
std::vector< unsigned char > fBlob