Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RCsvDS.hxx
Go to the documentation of this file.
1// Author: Enric Tejedor CERN 10/2017
2
3/*************************************************************************
4 * Copyright (C) 1995-2022, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#ifndef ROOT_RCSVTDS
12#define ROOT_RCSVTDS
13
14#include "ROOT/RDataFrame.hxx"
15#include "ROOT/RDataSource.hxx"
16
17#include <cstdint>
18#include <deque>
19#include <list>
20#include <unordered_map>
21#include <set>
22#include <memory>
23#include <vector>
24
25#include <TRegexp.h>
26
27namespace ROOT {
28
29namespace Internal {
30class RRawFile;
31}
32
33namespace RDF {
34
35class RCsvDS final : public ROOT::RDF::RDataSource {
36
37private:
38 // Possible values are D, O, L, T. This is possible only because we treat double, bool, Long64_t and string
39 using ColType_t = char;
40 static const std::unordered_map<ColType_t, std::string> fgColTypeMap;
41
42 // Regular expressions for type inference
44
45 std::uint64_t fDataPos = 0;
46 bool fReadHeaders = false;
47 unsigned int fNSlots = 0U;
48 std::unique_ptr<ROOT::Internal::RRawFile> fCsvFile;
49 const char fDelimiter;
52 ULong64_t fProcessedLines = 0ULL; // marks the progress of the consumption of the csv lines
53 std::vector<std::string> fHeaders; // the column names
54 std::unordered_map<std::string, ColType_t> fColTypes;
55 std::set<std::string> fColContainingEmpty; // store columns which had empty entry
56 std::list<ColType_t> fColTypesList; // column types, order is the same as fHeaders, values the same as fColTypes
57 std::vector<std::vector<void *>> fColAddresses; // fColAddresses[column][slot] (same ordering as fHeaders)
58 std::vector<Record_t> fRecords; // fRecords[entry][column] (same ordering as fHeaders)
59 std::vector<std::vector<double>> fDoubleEvtValues; // one per column per slot
60 std::vector<std::vector<Long64_t>> fLong64EvtValues; // one per column per slot
61 std::vector<std::vector<std::string>> fStringEvtValues; // one per column per slot
62 // This must be a deque to avoid the specialisation vector<bool>. This would not
63 // work given that the pointer to the boolean in that case cannot be taken
64 std::vector<std::deque<bool>> fBoolEvtValues; // one per column per slot
65
66 void FillHeaders(const std::string &);
67 void FillRecord(const std::string &, Record_t &);
68 void GenerateHeaders(size_t);
69 std::vector<void *> GetColumnReadersImpl(std::string_view, const std::type_info &) final;
70 void ValidateColTypes(std::vector<std::string> &) const;
71 void InferColTypes(std::vector<std::string> &);
72 void InferType(const std::string &, unsigned int);
73 std::vector<std::string> ParseColumns(const std::string &);
74 size_t ParseValue(const std::string &, std::vector<std::string> &, size_t);
75 ColType_t GetType(std::string_view colName) const;
76 void FreeRecords();
77
78protected:
79 std::string AsString() final;
80
81public:
82 RCsvDS(std::string_view fileName, bool readHeaders = true, char delimiter = ',', Long64_t linesChunkSize = -1LL,
83 std::unordered_map<std::string, char> &&colTypes = {});
84 void Finalize() final;
85 ~RCsvDS();
86 std::size_t GetNFiles() const final { return 1; }
87 const std::vector<std::string> &GetColumnNames() const final;
88 std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() final;
89 std::string GetTypeName(std::string_view colName) const final;
90 bool HasColumn(std::string_view colName) const final;
91 bool SetEntry(unsigned int slot, ULong64_t entry) final;
92 void SetNSlots(unsigned int nSlots) final;
93 std::string GetLabel() final;
94};
95
96////////////////////////////////////////////////////////////////////////////////////////////////
97/// \brief Factory method to create a CSV RDataFrame.
98/// \param[in] fileName Path of the CSV file.
99/// \param[in] readHeaders `true` if the CSV file contains headers as first row, `false` otherwise
100/// (default `true`).
101/// \param[in] delimiter Delimiter character (default ',').
102/// \param[in] linesChunkSize bunch of lines to read, use -1 to read all
103/// \param[in] colTypes Allow user to specify custom column types, accepts an unordered map with keys being
104/// column type, values being type alias ('O' for boolean, 'D' for double, 'L' for
105/// Long64_t, 'T' for std::string)
106RDataFrame FromCSV(std::string_view fileName, bool readHeaders = true, char delimiter = ',',
107 Long64_t linesChunkSize = -1LL, std::unordered_map<std::string, char> &&colTypes = {});
108
109} // ns RDF
110
111} // ns ROOT
112
113#endif
long long Long64_t
Definition RtypesCore.h:69
unsigned long long ULong64_t
Definition RtypesCore.h:70
RDataFrame data source class for reading CSV files.
Definition RCsvDS.hxx:35
std::string GetTypeName(std::string_view colName) const final
Type of a column as a string, e.g.
Definition RCsvDS.cxx:495
void FillRecord(const std::string &, Record_t &)
Definition RCsvDS.cxx:120
void Finalize() final
Convenience method called after concluding an event-loop.
Definition RCsvDS.cxx:413
std::size_t GetNFiles() const final
Returns the number of files from which the dataset is constructed.
Definition RCsvDS.hxx:86
ColType_t GetType(std::string_view colName) const
Definition RCsvDS.cxx:484
std::vector< std::vector< double > > fDoubleEvtValues
Definition RCsvDS.hxx:59
void InferType(const std::string &, unsigned int)
Definition RCsvDS.cxx:255
std::uint64_t fDataPos
Definition RCsvDS.hxx:45
void SetNSlots(unsigned int nSlots) final
Inform RDataSource of the number of processing slots (i.e.
Definition RCsvDS.cxx:536
static const std::unordered_map< ColType_t, std::string > fgColTypeMap
Definition RCsvDS.hxx:40
size_t ParseValue(const std::string &, std::vector< std::string > &, size_t)
Definition RCsvDS.cxx:287
static const TRegexp fgTrueRegex
Definition RCsvDS.hxx:43
void GenerateHeaders(size_t)
Definition RCsvDS.cxx:163
std::vector< std::vector< void * > > fColAddresses
Definition RCsvDS.hxx:57
unsigned int fNSlots
Definition RCsvDS.hxx:47
const std::vector< std::string > & GetColumnNames() const final
Returns a reference to the collection of the dataset's column names.
Definition RCsvDS.cxx:421
const Long64_t fLinesChunkSize
Definition RCsvDS.hxx:50
std::string AsString() final
Definition RCsvDS.cxx:95
std::vector< std::string > fHeaders
Definition RCsvDS.hxx:53
ULong64_t fEntryRangesRequested
Definition RCsvDS.hxx:51
ULong64_t fProcessedLines
Definition RCsvDS.hxx:52
bool HasColumn(std::string_view colName) const final
Checks if the dataset has a certain column.
Definition RCsvDS.cxx:500
void InferColTypes(std::vector< std::string > &)
Definition RCsvDS.cxx:222
std::unordered_map< std::string, ColType_t > fColTypes
Definition RCsvDS.hxx:54
std::vector< std::vector< Long64_t > > fLong64EvtValues
Definition RCsvDS.hxx:60
const char fDelimiter
Definition RCsvDS.hxx:49
static const TRegexp fgDoubleRegex2
Definition RCsvDS.hxx:43
std::vector< Record_t > fRecords
Definition RCsvDS.hxx:58
std::set< std::string > fColContainingEmpty
Definition RCsvDS.hxx:55
static const TRegexp fgFalseRegex
Definition RCsvDS.hxx:43
static const TRegexp fgDoubleRegex3
Definition RCsvDS.hxx:43
void ValidateColTypes(std::vector< std::string > &) const
Definition RCsvDS.cxx:203
static const TRegexp fgIntRegex
Definition RCsvDS.hxx:43
std::vector< std::string > ParseColumns(const std::string &)
Definition RCsvDS.cxx:276
void FillHeaders(const std::string &)
Definition RCsvDS.cxx:111
std::unique_ptr< ROOT::Internal::RRawFile > fCsvFile
Definition RCsvDS.hxx:48
std::vector< std::pair< ULong64_t, ULong64_t > > GetEntryRanges() final
Return ranges of entries to distribute to tasks.
Definition RCsvDS.cxx:426
std::string GetLabel() final
Return a string representation of the datasource type.
Definition RCsvDS.cxx:553
std::vector< void * > GetColumnReadersImpl(std::string_view, const std::type_info &) final
type-erased vector of pointers to pointers to column values - one per slot
Definition RCsvDS.cxx:171
static const TRegexp fgDoubleRegex1
Definition RCsvDS.hxx:43
std::vector< std::vector< std::string > > fStringEvtValues
Definition RCsvDS.hxx:61
std::vector< std::deque< bool > > fBoolEvtValues
Definition RCsvDS.hxx:64
bool SetEntry(unsigned int slot, ULong64_t entry) final
Advance the "cursors" returned by GetColumnReaders to the selected entry for a particular slot.
Definition RCsvDS.cxx:505
std::list< ColType_t > fColTypesList
Definition RCsvDS.hxx:56
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
std::vector< void * > Record_t
ROOT's RDataFrame offers a modern, high-level interface for analysis of data stored in TTree ,...
Regular expression class.
Definition TRegexp.h:31
RDataFrame FromCSV(std::string_view fileName, bool readHeaders=true, char delimiter=',', Long64_t linesChunkSize=-1LL, std::unordered_map< std::string, char > &&colTypes={})
Factory method to create a CSV RDataFrame.
Definition RCsvDS.cxx:558
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...