Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RDataSource.hxx
Go to the documentation of this file.
1// Author: Enrico Guiraud, Danilo Piparo CERN 09/2017
2
3/*************************************************************************
4 * Copyright (C) 1995-2018, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#ifndef ROOT_RDATASOURCE
12#define ROOT_RDATASOURCE
13
15#include <string_view>
16#include "RtypesCore.h" // ULong64_t
17#include "TString.h"
18
19#include <algorithm> // std::transform
20#include <string>
21#include <typeinfo>
22#include <vector>
23
24namespace ROOT {
25namespace RDF {
26class RDataSource;
27}
28}
29
30/// Print a RDataSource at the prompt
31namespace cling {
32std::string printValue(ROOT::RDF::RDataSource *ds);
33} // namespace cling
34
35namespace ROOT {
36
37namespace Internal {
38namespace TDS {
39
40/// Mother class of TTypedPointerHolder. The instances
41/// of this class can be put in a container. Upon destruction,
42/// the correct deletion of the pointer is performed in the
43/// derived class.
45protected:
46 void *fPointer{nullptr};
47
48public:
49 TPointerHolder(void *ptr) : fPointer(ptr) {}
50 void *GetPointer() { return fPointer; }
51 void *GetPointerAddr() { return &fPointer; }
53 virtual ~TPointerHolder(){};
54};
55
56/// Class to wrap a pointer and delete the memory associated to it
57/// correctly
58template <typename T>
59class TTypedPointerHolder final : public TPointerHolder {
60public:
61 TTypedPointerHolder(T *ptr) : TPointerHolder((void *)ptr) {}
62
64 {
65 const auto typedPtr = static_cast<T *>(fPointer);
66 return new TTypedPointerHolder(new T(*typedPtr));
67 }
68
69 ~TTypedPointerHolder() { delete static_cast<T *>(fPointer); }
70};
71
72} // ns TDS
73} // ns Internal
74
75namespace RDF {
76
77// clang-format off
78/**
79\class ROOT::RDF::RDataSource
80\ingroup dataframe
81\brief RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
82
83A concrete RDataSource implementation (i.e. a class that inherits from RDataSource and implements all of its pure
84methods) provides an adaptor that RDataFrame can leverage to read any kind of tabular data formats.
85RDataFrame calls into RDataSource to retrieve information about the data, retrieve (thread-local) readers or "cursors"
86for selected columns and to advance the readers to the desired data entry.
87
88The sequence of calls that RDataFrame (or any other client of a RDataSource) performs is the following:
89
90 - SetNSlots() : inform RDataSource of the desired level of parallelism
91 - GetColumnReaders() : retrieve from RDataSource per-thread readers for the desired columns
92 - Initialize() : inform RDataSource that an event-loop is about to start
93 - GetEntryRanges() : retrieve from RDataSource a set of ranges of entries that can be processed concurrently
94 - InitSlot() : inform RDataSource that a certain thread is about to start working on a certain range of entries
95 - SetEntry() : inform RDataSource that a certain thread is about to start working on a certain entry
96 - FinalizeSlot() : inform RDataSource that a certain thread finished working on a certain range of entries
97 - Finalize() : inform RDataSource that an event-loop finished
98
99RDataSource implementations must support running multiple event-loops consecutively (although sequentially) on the same dataset.
100 - \b SetNSlots() is called once per RDataSource object, typically when it is associated to a RDataFrame.
101 - \b GetColumnReaders() can be called several times, potentially with the same arguments, also in-between event-loops, but not during an event-loop.
102 - \b GetEntryRanges() will be called several times, including during an event loop, as additional ranges are needed. It will not be called concurrently.
103 - \b Initialize() and \b Finalize() are called once per event-loop, right before starting and right after finishing.
104 - \b InitSlot(), \b SetEntry(), and \b FinalizeSlot() can be called concurrently from multiple threads, multiple times per event-loop.
105
106 Advanced users that plan to implement a custom RDataSource can check out existing implementations, e.g. RCsvDS or RNTupleDS.
107 See the inheritance diagram below for the full list of existing concrete implementations.
108*/
110 // clang-format on
111protected:
112 using Record_t = std::vector<void *>;
113 friend std::string cling::printValue(::ROOT::RDF::RDataSource *);
114
115 virtual std::string AsString() { return "generic data source"; };
116
117public:
118 virtual ~RDataSource() = default;
119
120 // clang-format off
121 /// \brief Inform RDataSource of the number of processing slots (i.e. worker threads) used by the associated RDataFrame.
122 /// Slots numbers are used to simplify parallel execution: RDataFrame guarantees that different threads will always
123 /// pass different slot values when calling methods concurrently.
124 // clang-format on
125 virtual void SetNSlots(unsigned int nSlots) = 0;
126
127 /// \brief Returns the number of files from which the dataset is constructed
128 virtual std::size_t GetNFiles() const { return 0; }
129
130 // clang-format off
131 /// \brief Returns a reference to the collection of the dataset's column names
132 // clang-format on
133 virtual const std::vector<std::string> &GetColumnNames() const = 0;
134
135 /// \brief Checks if the dataset has a certain column
136 /// \param[in] colName The name of the column
137 virtual bool HasColumn(std::string_view colName) const = 0;
138
139 // clang-format off
140 /// \brief Type of a column as a string, e.g. `GetTypeName("x") == "double"`. Required for jitting e.g. `df.Filter("x>0")`.
141 /// \param[in] colName The name of the column
142 // clang-format on
143 virtual std::string GetTypeName(std::string_view colName) const = 0;
144
145 // clang-format off
146 /// Called at most once per column by RDF. Return vector of pointers to pointers to column values - one per slot.
147 /// \tparam T The type of the data stored in the column
148 /// \param[in] columnName The name of the column
149 ///
150 /// These pointers are veritable cursors: it's a responsibility of the RDataSource implementation that they point to
151 /// the "right" memory region.
152 // clang-format on
153 template <typename T>
154 std::vector<T **> GetColumnReaders(std::string_view columnName)
155 {
156 auto typeErasedVec = GetColumnReadersImpl(columnName, typeid(T));
157 std::vector<T **> typedVec(typeErasedVec.size());
158 std::transform(typeErasedVec.begin(), typeErasedVec.end(), typedVec.begin(),
159 [](void *p) { return static_cast<T **>(p); });
160 return typedVec;
161 }
162
163 /// If the other GetColumnReaders overload returns an empty vector, this overload will be called instead.
164 /// \param[in] slot The data processing slot that needs to be considered
165 /// \param[in] name The name of the column for which a column reader needs to be returned
166 /// \param[in] tid A type_info
167 /// At least one of the two must return a non-empty/non-null value.
168 virtual std::unique_ptr<ROOT::Detail::RDF::RColumnReaderBase>
169 GetColumnReaders(unsigned int /*slot*/, std::string_view /*name*/, const std::type_info &)
170 {
171 return {};
172 }
173
174 // clang-format off
175 /// \brief Return ranges of entries to distribute to tasks.
176 /// They are required to be contiguous intervals with no entries skipped. Supposing a dataset with nEntries, the
177 /// intervals must start at 0 and end at nEntries, e.g. [0-5],[5-10] for 10 entries.
178 /// This function will be invoked repeatedly by RDataFrame as it needs additional entries to process.
179 /// The same entry range should not be returned more than once.
180 /// Returning an empty collection of ranges signals to RDataFrame that the processing can stop.
181 // clang-format on
182 virtual std::vector<std::pair<ULong64_t, ULong64_t>> GetEntryRanges() = 0;
183
184 // clang-format off
185 /// \brief Advance the "cursors" returned by GetColumnReaders to the selected entry for a particular slot.
186 /// \param[in] slot The data processing slot that needs to be considered
187 /// \param[in] entry The entry which needs to be pointed to by the reader pointers
188 /// Slots are adopted to accommodate parallel data processing.
189 /// Different workers will loop over different ranges and
190 /// will be labelled by different "slot" values.
191 /// Returns *true* if the entry has to be processed, *false* otherwise.
192 // clang-format on
193 virtual bool SetEntry(unsigned int slot, ULong64_t entry) = 0;
194
195 // clang-format off
196 /// \brief Convenience method called before starting an event-loop.
197 /// This method might be called multiple times over the lifetime of a RDataSource, since
198 /// users can run multiple event-loops with the same RDataFrame.
199 /// Ideally, `Initialize` should set the state of the RDataSource so that multiple identical event-loops
200 /// will produce identical results.
201 // clang-format on
202 virtual void Initialize() {}
203
204 // clang-format off
205 /// \brief Convenience method called at the start of the data processing associated to a slot.
206 /// \param[in] slot The data processing slot wihch needs to be initialized
207 /// \param[in] firstEntry The first entry of the range that the task will process.
208 /// This method might be called multiple times per thread per event-loop.
209 // clang-format on
210 virtual void InitSlot(unsigned int /*slot*/, ULong64_t /*firstEntry*/) {}
211
212 // clang-format off
213 /// \brief Convenience method called at the end of the data processing associated to a slot.
214 /// \param[in] slot The data processing slot wihch needs to be finalized
215 /// This method might be called multiple times per thread per event-loop.
216 // clang-format on
217 virtual void FinalizeSlot(unsigned int /*slot*/) {}
218
219 // clang-format off
220 /// \brief Convenience method called after concluding an event-loop.
221 /// See Initialize for more details.
222 // clang-format on
223 virtual void Finalize() {}
224
225 /// \brief Return a string representation of the datasource type.
226 /// The returned string will be used by ROOT::RDF::SaveGraph() to represent
227 /// the datasource in the visualization of the computation graph.
228 /// Concrete datasources can override the default implementation.
229 virtual std::string GetLabel() { return "Custom Datasource"; }
230
231protected:
232 /// type-erased vector of pointers to pointers to column values - one per slot
233 virtual Record_t GetColumnReadersImpl(std::string_view name, const std::type_info &) = 0;
234};
235
236} // ns RDF
237
238} // ns ROOT
239
240/// Print a RDataSource at the prompt
241namespace cling {
242inline std::string printValue(ROOT::RDF::RDataSource *ds)
243{
244 return ds->AsString();
245}
246} // namespace cling
247
248#endif // ROOT_TDATASOURCE
unsigned long long ULong64_t
Definition RtypesCore.h:81
winID h TVirtualViewer3D TVirtualGLPainter p
char name[80]
Definition TGX11.cxx:110
Mother class of TTypedPointerHolder.
virtual TPointerHolder * GetDeepCopy()=0
Class to wrap a pointer and delete the memory associated to it correctly.
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual bool HasColumn(std::string_view colName) const =0
Checks if the dataset has a certain column.
virtual void Finalize()
Convenience method called after concluding an event-loop.
virtual void InitSlot(unsigned int, ULong64_t)
Convenience method called at the start of the data processing associated to a slot.
virtual void FinalizeSlot(unsigned int)
Convenience method called at the end of the data processing associated to a slot.
virtual ~RDataSource()=default
virtual std::string AsString()
virtual bool SetEntry(unsigned int slot, ULong64_t entry)=0
Advance the "cursors" returned by GetColumnReaders to the selected entry for a particular slot.
std::vector< void * > Record_t
virtual std::string GetLabel()
Return a string representation of the datasource type.
virtual void SetNSlots(unsigned int nSlots)=0
Inform RDataSource of the number of processing slots (i.e.
virtual std::size_t GetNFiles() const
Returns the number of files from which the dataset is constructed.
virtual const std::vector< std::string > & GetColumnNames() const =0
Returns a reference to the collection of the dataset's column names.
virtual std::vector< std::pair< ULong64_t, ULong64_t > > GetEntryRanges()=0
Return ranges of entries to distribute to tasks.
virtual Record_t GetColumnReadersImpl(std::string_view name, const std::type_info &)=0
type-erased vector of pointers to pointers to column values - one per slot
virtual std::string GetTypeName(std::string_view colName) const =0
Type of a column as a string, e.g.
std::vector< T ** > GetColumnReaders(std::string_view columnName)
Called at most once per column by RDF.
virtual std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase > GetColumnReaders(unsigned int, std::string_view, const std::type_info &)
If the other GetColumnReaders overload returns an empty vector, this overload will be called instead.
virtual void Initialize()
Convenience method called before starting an event-loop.
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...