Logo ROOT  

Reference Guide
 
Loading...
Searching...
No Matches
List of all members | Classes | Public Member Functions | Private Member Functions | Private Attributes | List of all members
ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args > Class Template Reference

template<typename... Args>
class ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >

In this class, the processes of loading clusters (see RClusterLoader) and creating batches from those clusters (see RBatchLoader) are combined, allowing batches from the training and validation sets to be loaded directly from a dataset in an RDataFrame.

Definition at line 52 of file RDataLoaderEngine.hxx.

Classes

struct  REpochGuard
 Opens a training or validation epoch and closes it again when done. More...
 

Public Member Functions

 RDataLoaderEngine (const std::vector< ROOT::RDF::RNode > &rdfs, const std::size_t batchSize, const std::size_t batchesInMemory, const std::vector< std::string > &cols, const std::vector< std::size_t > &vecSizes={}, const float vecPadding=0.0, const float testSize=0.0, bool shuffle=true, bool dropRemainder=true, const std::size_t setSeed=0, bool loadEager=false, std::string sampleType="", float sampleRatio=1.0, bool replacement=false)
 
 ~RDataLoaderEngine ()
 
void Activate ()
 Activate the loading process by spawning the loading thread.
 
void ActivateTrainingEpoch ()
 Activate the training epoch by starting the batchloader.
 
void ActivateValidationEpoch ()
 
void CreateTrainBatches ()
 Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)
 
void CreateValidationBatches ()
 Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)
 
void DeActivate ()
 
void DeActivateTrainingEpoch ()
 
void DeActivateValidationEpoch ()
 
RFlat2DMatrix GetTrainBatch ()
 Loads a training batch from the queue.
 
RFlat2DMatrix GetValidationBatch ()
 Loads a validation batch from the queue.
 
bool IsActive ()
 
bool IsTrainingActive ()
 
bool IsValidationActive ()
 
void LoadData ()
 Main loop for loading clusters and creating batches.
 
std::size_t NumberOfTrainingBatches ()
 
std::size_t NumberOfValidationBatches ()
 
void Save (std::string_view dataset_name, std::string_view filename, bool isTraining, std::string_view outputFormat)
 Materialize one train/test split to disk by draining a full epoch through the normal batch pipeline and Fill() each batch into filename instead of yielding it.
 
std::size_t TrainRemainderRows ()
 
std::size_t ValidationRemainderRows ()
 

Private Member Functions

std::vector< RColumnLayout > MakeColumnLayout () const
 Describe how the loader's columns map onto a batch-tensor row.
 

Private Attributes

std::size_t fBatchesInMemory
 
std::size_t fBatchSize
 
std::size_t fBufferCapacity
 
std::unique_ptr< RClusterLoader< Args... > > fClusterLoader
 
std::vector< std::string > fCols
 
std::unique_ptr< RDatasetLoader< Args... > > fDatasetLoader
 
bool fDropRemainder
 
bool fEpochActive {false}
 
std::size_t fHighWatermark
 
bool fIsActive {false}
 
bool fLoadEager
 
std::condition_variable fLoadingCondition
 
std::mutex fLoadingMutex
 
std::unique_ptr< std::thread > fLoadingThread
 
std::size_t fLowWatermark
 
std::size_t fNumTrainingEntries
 
std::size_t fNumValidationEntries
 
std::vector< ROOT::RDF::RNode > fRdfs
 
bool fReplacement
 
RFlat2DMatrix fSampledTrainingDataset
 
RFlat2DMatrix fSampledValidationDataset
 
float fSampleRatio
 
std::string fSampleType
 
std::size_t fSetSeed
 
bool fShuffle
 
std::unique_ptr< RFlat2DMatrixOperators > fTensorOperators
 
float fTestSize
 
std::unique_ptr< RBatchLoader > fTrainingBatchLoader
 
std::size_t fTrainingClusterIdx {0}
 
RFlat2DMatrix fTrainingDataset
 
std::vector< RFlat2DMatrix > fTrainingDatasets
 
bool fTrainingEpochActive {false}
 
std::size_t fTrainingEpochCount {0}
 
std::unique_ptr< RSampler > fTrainingSampler
 
std::unique_ptr< RBatchLoader > fValidationBatchLoader
 
std::size_t fValidationClusterIdx {0}
 
RFlat2DMatrix fValidationDataset
 
std::vector< RFlat2DMatrix > fValidationDatasets
 
bool fValidationEpochActive {false}
 
std::size_t fValidationEpochCount {0}
 
std::unique_ptr< RSampler > fValidationSampler
 
std::vector< std::size_t > fVecSizes
 

#include <ROOT/ML/RDataLoaderEngine.hxx>

Constructor & Destructor Documentation

◆ RDataLoaderEngine()

template<typename... Args>
ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::RDataLoaderEngine ( const std::vector< ROOT::RDF::RNode > & rdfs,
const std::size_t batchSize,
const std::size_t batchesInMemory,
const std::vector< std::string > & cols,
const std::vector< std::size_t > & vecSizes = {},
const float vecPadding = 0.0,
const float testSize = 0.0,
bool shuffle = true,
bool dropRemainder = true,
const std::size_t setSeed = 0,
bool loadEager = false,
std::string sampleType = "",
float sampleRatio = 1.0,
bool replacement = false )
inline

Definition at line 163 of file RDataLoaderEngine.hxx.

◆ ~RDataLoaderEngine()

template<typename... Args>
ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::~RDataLoaderEngine ( )
inline

Definition at line 237 of file RDataLoaderEngine.hxx.

Member Function Documentation

◆ Activate()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::Activate ( )
inline

Activate the loading process by spawning the loading thread.

Definition at line 260 of file RDataLoaderEngine.hxx.

◆ ActivateTrainingEpoch()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::ActivateTrainingEpoch ( )
inline

Activate the training epoch by starting the batchloader.

Definition at line 304 of file RDataLoaderEngine.hxx.

◆ ActivateValidationEpoch()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::ActivateValidationEpoch ( )
inline

Definition at line 332 of file RDataLoaderEngine.hxx.

◆ CreateTrainBatches()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::CreateTrainBatches ( )
inline

Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)

Definition at line 533 of file RDataLoaderEngine.hxx.

◆ CreateValidationBatches()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::CreateValidationBatches ( )
inline

Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)

Definition at line 553 of file RDataLoaderEngine.hxx.

◆ DeActivate()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::DeActivate ( )
inline

Definition at line 239 of file RDataLoaderEngine.hxx.

◆ DeActivateTrainingEpoch()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::DeActivateTrainingEpoch ( )
inline

Definition at line 320 of file RDataLoaderEngine.hxx.

◆ DeActivateValidationEpoch()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::DeActivateValidationEpoch ( )
inline

Definition at line 347 of file RDataLoaderEngine.hxx.

◆ GetTrainBatch()

template<typename... Args>
RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::GetTrainBatch ( )
inline

Loads a training batch from the queue.

Definition at line 572 of file RDataLoaderEngine.hxx.

◆ GetValidationBatch()

template<typename... Args>
RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::GetValidationBatch ( )
inline

Loads a validation batch from the queue.

Definition at line 579 of file RDataLoaderEngine.hxx.

◆ IsActive()

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::IsActive ( )
inline

Definition at line 591 of file RDataLoaderEngine.hxx.

◆ IsTrainingActive()

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::IsTrainingActive ( )
inline

Definition at line 597 of file RDataLoaderEngine.hxx.

◆ IsValidationActive()

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::IsValidationActive ( )
inline

Definition at line 603 of file RDataLoaderEngine.hxx.

◆ LoadData()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::LoadData ( )
inline

Main loop for loading clusters and creating batches.

The producer (loading thread) will keep loading clusters and creating batches until the end of the epoch is reached, or the generator is deactivated.

Definition at line 362 of file RDataLoaderEngine.hxx.

◆ MakeColumnLayout()

template<typename... Args>
std::vector< RColumnLayout > ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::MakeColumnLayout ( ) const
inlineprivate

Describe how the loader's columns map onto a batch-tensor row.

Definition at line 115 of file RDataLoaderEngine.hxx.

◆ NumberOfTrainingBatches()

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::NumberOfTrainingBatches ( )
inline

Definition at line 585 of file RDataLoaderEngine.hxx.

◆ NumberOfValidationBatches()

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::NumberOfValidationBatches ( )
inline

Definition at line 586 of file RDataLoaderEngine.hxx.

◆ Save()

template<typename... Args>
void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::Save ( std::string_view dataset_name,
std::string_view filename,
bool isTraining,
std::string_view outputFormat )
inline

Materialize one train/test split to disk by draining a full epoch through the normal batch pipeline and Fill() each batch into filename instead of yielding it.

Filters, shuffling, the train/validation split and the batch_size/drop_remainder settings are all inherited from the loader's configuration.

Parameters
outputFormatEither "ttree" or "rntuple".

Definition at line 283 of file RDataLoaderEngine.hxx.

◆ TrainRemainderRows()

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::TrainRemainderRows ( )
inline

Definition at line 588 of file RDataLoaderEngine.hxx.

◆ ValidationRemainderRows()

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::ValidationRemainderRows ( )
inline

Definition at line 589 of file RDataLoaderEngine.hxx.

Member Data Documentation

◆ fBatchesInMemory

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fBatchesInMemory
private

Definition at line 60 of file RDataLoaderEngine.hxx.

◆ fBatchSize

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fBatchSize
private

Definition at line 56 of file RDataLoaderEngine.hxx.

◆ fBufferCapacity

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fBufferCapacity
private

Definition at line 61 of file RDataLoaderEngine.hxx.

◆ fClusterLoader

template<typename... Args>
std::unique_ptr<RClusterLoader<Args...> > ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fClusterLoader
private

Definition at line 71 of file RDataLoaderEngine.hxx.

◆ fCols

template<typename... Args>
std::vector<std::string> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fCols
private

Definition at line 54 of file RDataLoaderEngine.hxx.

◆ fDatasetLoader

template<typename... Args>
std::unique_ptr<RDatasetLoader<Args...> > ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fDatasetLoader
private

Definition at line 70 of file RDataLoaderEngine.hxx.

◆ fDropRemainder

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fDropRemainder
private

Definition at line 85 of file RDataLoaderEngine.hxx.

◆ fEpochActive

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fEpochActive {false}
private

Definition at line 94 of file RDataLoaderEngine.hxx.

◆ fHighWatermark

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fHighWatermark
private

Definition at line 63 of file RDataLoaderEngine.hxx.

◆ fIsActive

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fIsActive {false}
private

Definition at line 92 of file RDataLoaderEngine.hxx.

◆ fLoadEager

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadEager
private

Definition at line 87 of file RDataLoaderEngine.hxx.

◆ fLoadingCondition

template<typename... Args>
std::condition_variable ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadingCondition
private

Definition at line 82 of file RDataLoaderEngine.hxx.

◆ fLoadingMutex

template<typename... Args>
std::mutex ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadingMutex
private

Definition at line 83 of file RDataLoaderEngine.hxx.

◆ fLoadingThread

template<typename... Args>
std::unique_ptr<std::thread> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadingThread
private

Definition at line 81 of file RDataLoaderEngine.hxx.

◆ fLowWatermark

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLowWatermark
private

Definition at line 62 of file RDataLoaderEngine.hxx.

◆ fNumTrainingEntries

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fNumTrainingEntries
private

Definition at line 98 of file RDataLoaderEngine.hxx.

◆ fNumValidationEntries

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fNumValidationEntries
private

Definition at line 99 of file RDataLoaderEngine.hxx.

◆ fRdfs

template<typename... Args>
std::vector<ROOT::RDF::RNode> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fRdfs
private

Definition at line 79 of file RDataLoaderEngine.hxx.

◆ fReplacement

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fReplacement
private

Definition at line 90 of file RDataLoaderEngine.hxx.

◆ fSampledTrainingDataset

template<typename... Args>
RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampledTrainingDataset
private

Definition at line 108 of file RDataLoaderEngine.hxx.

◆ fSampledValidationDataset

template<typename... Args>
RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampledValidationDataset
private

Definition at line 109 of file RDataLoaderEngine.hxx.

◆ fSampleRatio

template<typename... Args>
float ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampleRatio
private

Definition at line 89 of file RDataLoaderEngine.hxx.

◆ fSampleType

template<typename... Args>
std::string ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampleType
private

Definition at line 88 of file RDataLoaderEngine.hxx.

◆ fSetSeed

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSetSeed
private

Definition at line 57 of file RDataLoaderEngine.hxx.

◆ fShuffle

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fShuffle
private

Definition at line 86 of file RDataLoaderEngine.hxx.

◆ fTensorOperators

template<typename... Args>
std::unique_ptr<RFlat2DMatrixOperators> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTensorOperators
private

Definition at line 77 of file RDataLoaderEngine.hxx.

◆ fTestSize

template<typename... Args>
float ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTestSize
private

Definition at line 68 of file RDataLoaderEngine.hxx.

◆ fTrainingBatchLoader

template<typename... Args>
std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingBatchLoader
private

Definition at line 72 of file RDataLoaderEngine.hxx.

◆ fTrainingClusterIdx

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingClusterIdx {0}
private

Definition at line 65 of file RDataLoaderEngine.hxx.

◆ fTrainingDataset

template<typename... Args>
RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingDataset
private

Definition at line 105 of file RDataLoaderEngine.hxx.

◆ fTrainingDatasets

template<typename... Args>
std::vector<RFlat2DMatrix> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingDatasets
private

Definition at line 102 of file RDataLoaderEngine.hxx.

◆ fTrainingEpochActive

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingEpochActive {false}
private

Definition at line 95 of file RDataLoaderEngine.hxx.

◆ fTrainingEpochCount

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingEpochCount {0}
private

Definition at line 111 of file RDataLoaderEngine.hxx.

◆ fTrainingSampler

template<typename... Args>
std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingSampler
private

Definition at line 74 of file RDataLoaderEngine.hxx.

◆ fValidationBatchLoader

template<typename... Args>
std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationBatchLoader
private

Definition at line 73 of file RDataLoaderEngine.hxx.

◆ fValidationClusterIdx

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationClusterIdx {0}
private

Definition at line 66 of file RDataLoaderEngine.hxx.

◆ fValidationDataset

template<typename... Args>
RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationDataset
private

Definition at line 106 of file RDataLoaderEngine.hxx.

◆ fValidationDatasets

template<typename... Args>
std::vector<RFlat2DMatrix> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationDatasets
private

Definition at line 103 of file RDataLoaderEngine.hxx.

◆ fValidationEpochActive

template<typename... Args>
bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationEpochActive {false}
private

Definition at line 96 of file RDataLoaderEngine.hxx.

◆ fValidationEpochCount

template<typename... Args>
std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationEpochCount {0}
private

Definition at line 112 of file RDataLoaderEngine.hxx.

◆ fValidationSampler

template<typename... Args>
std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationSampler
private

Definition at line 75 of file RDataLoaderEngine.hxx.

◆ fVecSizes

template<typename... Args>
std::vector<std::size_t> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fVecSizes
private

Definition at line 55 of file RDataLoaderEngine.hxx.

  • tree/ml/inc/ROOT/ML/RDataLoaderEngine.hxx