ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args > Class Template Reference

template<typename... Args>
class ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >

In this class, the processes of loading clusters (see RClusterLoader) and creating batches from those clusters (see RBatchLoader) are combined, allowing batches from the training and validation sets to be loaded directly from a dataset in an RDataFrame.

Definition at line 49 of file RDataLoaderEngine.hxx.

Public Member Functions
	RDataLoaderEngine (const std::vector< ROOT::RDF::RNode > &rdfs, const std::size_t batchSize, const std::size_t batchesInMemory, const std::vector< std::string > &cols, const std::vector< std::size_t > &vecSizes={}, const float vecPadding=0.0, const float testSize=0.0, bool shuffle=true, bool dropRemainder=true, const std::size_t setSeed=0, bool loadEager=false, std::string sampleType="", float sampleRatio=1.0, bool replacement=false)
	~RDataLoaderEngine ()
void	Activate ()
	Activate the loading process by spawning the loading thread.
void	ActivateTrainingEpoch ()
	Activate the training epoch by starting the batchloader.
void	ActivateValidationEpoch ()
void	CreateTrainBatches ()
	Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)
void	CreateValidationBatches ()
	Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)
void	DeActivate ()
void	DeActivateTrainingEpoch ()
void	DeActivateValidationEpoch ()
RFlat2DMatrix	GetTrainBatch ()
	Loads a training batch from the queue.
RFlat2DMatrix	GetValidationBatch ()
	Loads a validation batch from the queue.
bool	IsActive ()
bool	IsTrainingActive ()
bool	IsValidationActive ()
void	LoadData ()
	Main loop for loading clusters and creating batches.
std::size_t	NumberOfTrainingBatches ()
std::size_t	NumberOfValidationBatches ()
std::size_t	TrainRemainderRows ()
std::size_t	ValidationRemainderRows ()

Private Attributes
std::size_t	fBatchesInMemory
std::size_t	fBatchSize
std::size_t	fBufferCapacity
std::unique_ptr< RClusterLoader< Args... > >	fClusterLoader
std::vector< std::string >	fCols
std::unique_ptr< RDatasetLoader< Args... > >	fDatasetLoader
bool	fDropRemainder
bool	fEpochActive {false}
std::size_t	fHighWatermark
bool	fIsActive {false}
bool	fLoadEager
std::condition_variable	fLoadingCondition
std::mutex	fLoadingMutex
std::unique_ptr< std::thread >	fLoadingThread
std::size_t	fLowWatermark
std::size_t	fNumTrainingEntries
std::size_t	fNumValidationEntries
std::vector< ROOT::RDF::RNode >	fRdfs
bool	fReplacement
RFlat2DMatrix	fSampledTrainingDataset
RFlat2DMatrix	fSampledValidationDataset
float	fSampleRatio
std::string	fSampleType
std::size_t	fSetSeed
bool	fShuffle
std::unique_ptr< RFlat2DMatrixOperators >	fTensorOperators
float	fTestSize
std::unique_ptr< RBatchLoader >	fTrainingBatchLoader
std::size_t	fTrainingClusterIdx {0}
RFlat2DMatrix	fTrainingDataset
std::vector< RFlat2DMatrix >	fTrainingDatasets
bool	fTrainingEpochActive {false}
std::size_t	fTrainingEpochCount {0}
std::unique_ptr< RSampler >	fTrainingSampler
std::unique_ptr< RBatchLoader >	fValidationBatchLoader
std::size_t	fValidationClusterIdx {0}
RFlat2DMatrix	fValidationDataset
std::vector< RFlat2DMatrix >	fValidationDatasets
bool	fValidationEpochActive {false}
std::size_t	fValidationEpochCount {0}
std::unique_ptr< RSampler >	fValidationSampler
std::vector< std::size_t >	fVecSizes

#include <ROOT/ML/RDataLoaderEngine.hxx>

Constructor & Destructor Documentation

RDataLoaderEngine()

template<typename... Args>

ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::RDataLoaderEngine ( const std::vector< ROOT::RDF::RNode > & rdfs, const std::size_t batchSize, const std::size_t batchesInMemory, const std::vector< std::string > & cols, const std::vector< std::size_t > & vecSizes = {}, const float vecPadding = 0.0, const float testSize = 0.0, bool shuffle = true, bool dropRemainder = true, const std::size_t setSeed = 0, bool loadEager = false, std::string sampleType = "", float sampleRatio = 1.0, bool replacement = false )

inline

Definition at line 112 of file RDataLoaderEngine.hxx.

~RDataLoaderEngine()

template<typename... Args>

ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::~RDataLoaderEngine ( )

inline

Definition at line 185 of file RDataLoaderEngine.hxx.

Member Function Documentation

Activate()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::Activate ( )

inline

Activate the loading process by spawning the loading thread.

Definition at line 208 of file RDataLoaderEngine.hxx.

ActivateTrainingEpoch()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::ActivateTrainingEpoch ( )

inline

Activate the training epoch by starting the batchloader.

Definition at line 226 of file RDataLoaderEngine.hxx.

ActivateValidationEpoch()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::ActivateValidationEpoch ( )

inline

Definition at line 254 of file RDataLoaderEngine.hxx.

CreateTrainBatches()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::CreateTrainBatches ( )

inline

Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)

Definition at line 454 of file RDataLoaderEngine.hxx.

CreateValidationBatches()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::CreateValidationBatches ( )

inline

Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)

Definition at line 474 of file RDataLoaderEngine.hxx.

DeActivate()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::DeActivate ( )

inline

Definition at line 187 of file RDataLoaderEngine.hxx.

DeActivateTrainingEpoch()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::DeActivateTrainingEpoch ( )

inline

Definition at line 242 of file RDataLoaderEngine.hxx.

DeActivateValidationEpoch()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::DeActivateValidationEpoch ( )

inline

Definition at line 269 of file RDataLoaderEngine.hxx.

GetTrainBatch()

template<typename... Args>

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::GetTrainBatch ( )

inline

Loads a training batch from the queue.

Definition at line 493 of file RDataLoaderEngine.hxx.

GetValidationBatch()

template<typename... Args>

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::GetValidationBatch ( )

inline

Loads a validation batch from the queue.

Definition at line 500 of file RDataLoaderEngine.hxx.

IsActive()

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::IsActive ( )

inline

Definition at line 512 of file RDataLoaderEngine.hxx.

IsTrainingActive()

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::IsTrainingActive ( )

inline

Definition at line 518 of file RDataLoaderEngine.hxx.

IsValidationActive()

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::IsValidationActive ( )

inline

Definition at line 524 of file RDataLoaderEngine.hxx.

LoadData()

template<typename... Args>

void ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::LoadData ( )

inline

Main loop for loading clusters and creating batches.

The producer (loading thread) will keep loading clusters and creating batches until the end of the epoch is reached, or the generator is deactivated.

Definition at line 284 of file RDataLoaderEngine.hxx.

NumberOfTrainingBatches()

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::NumberOfTrainingBatches ( )

inline

Definition at line 506 of file RDataLoaderEngine.hxx.

NumberOfValidationBatches()

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::NumberOfValidationBatches ( )

inline

Definition at line 507 of file RDataLoaderEngine.hxx.

TrainRemainderRows()

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::TrainRemainderRows ( )

inline

Definition at line 509 of file RDataLoaderEngine.hxx.

ValidationRemainderRows()

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::ValidationRemainderRows ( )

inline

Definition at line 510 of file RDataLoaderEngine.hxx.

Member Data Documentation

fBatchesInMemory

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fBatchesInMemory

private

Definition at line 57 of file RDataLoaderEngine.hxx.

fBatchSize

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fBatchSize

private

Definition at line 53 of file RDataLoaderEngine.hxx.

fBufferCapacity

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fBufferCapacity

private

Definition at line 58 of file RDataLoaderEngine.hxx.

fClusterLoader

template<typename... Args>

std::unique_ptr<RClusterLoader<Args...> > ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fClusterLoader

private

Definition at line 68 of file RDataLoaderEngine.hxx.

fCols

template<typename... Args>

std::vector<std::string> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fCols

private

Definition at line 51 of file RDataLoaderEngine.hxx.

fDatasetLoader

template<typename... Args>

std::unique_ptr<RDatasetLoader<Args...> > ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fDatasetLoader

private

Definition at line 67 of file RDataLoaderEngine.hxx.

fDropRemainder

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fDropRemainder

private

Definition at line 82 of file RDataLoaderEngine.hxx.

fEpochActive

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fEpochActive {false}

private

Definition at line 91 of file RDataLoaderEngine.hxx.

fHighWatermark

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fHighWatermark

private

Definition at line 60 of file RDataLoaderEngine.hxx.

fIsActive

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fIsActive {false}

private

Definition at line 89 of file RDataLoaderEngine.hxx.

fLoadEager

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadEager

private

Definition at line 84 of file RDataLoaderEngine.hxx.

fLoadingCondition

template<typename... Args>

std::condition_variable ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadingCondition

private

Definition at line 79 of file RDataLoaderEngine.hxx.

fLoadingMutex

template<typename... Args>

std::mutex ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadingMutex

private

Definition at line 80 of file RDataLoaderEngine.hxx.

fLoadingThread

template<typename... Args>

std::unique_ptr<std::thread> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLoadingThread

private

Definition at line 78 of file RDataLoaderEngine.hxx.

fLowWatermark

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fLowWatermark

private

Definition at line 59 of file RDataLoaderEngine.hxx.

fNumTrainingEntries

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fNumTrainingEntries

private

Definition at line 95 of file RDataLoaderEngine.hxx.

fNumValidationEntries

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fNumValidationEntries

private

Definition at line 96 of file RDataLoaderEngine.hxx.

fRdfs

template<typename... Args>

std::vector<ROOT::RDF::RNode> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fRdfs

private

Definition at line 76 of file RDataLoaderEngine.hxx.

fReplacement

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fReplacement

private

Definition at line 87 of file RDataLoaderEngine.hxx.

fSampledTrainingDataset

template<typename... Args>

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampledTrainingDataset

private

Definition at line 105 of file RDataLoaderEngine.hxx.

fSampledValidationDataset

template<typename... Args>

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampledValidationDataset

private

Definition at line 106 of file RDataLoaderEngine.hxx.

fSampleRatio

template<typename... Args>

float ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampleRatio

private

Definition at line 86 of file RDataLoaderEngine.hxx.

fSampleType

template<typename... Args>

std::string ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSampleType

private

Definition at line 85 of file RDataLoaderEngine.hxx.

fSetSeed

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fSetSeed

private

Definition at line 54 of file RDataLoaderEngine.hxx.

fShuffle

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fShuffle

private

Definition at line 83 of file RDataLoaderEngine.hxx.

fTensorOperators

template<typename... Args>

std::unique_ptr<RFlat2DMatrixOperators> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTensorOperators

private

Definition at line 74 of file RDataLoaderEngine.hxx.

fTestSize

template<typename... Args>

float ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTestSize

private

Definition at line 65 of file RDataLoaderEngine.hxx.

fTrainingBatchLoader

template<typename... Args>

std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingBatchLoader

private

Definition at line 69 of file RDataLoaderEngine.hxx.

fTrainingClusterIdx

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingClusterIdx {0}

private

Definition at line 62 of file RDataLoaderEngine.hxx.

fTrainingDataset

template<typename... Args>

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingDataset

private

Definition at line 102 of file RDataLoaderEngine.hxx.

fTrainingDatasets

template<typename... Args>

std::vector<RFlat2DMatrix> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingDatasets

private

Definition at line 99 of file RDataLoaderEngine.hxx.

fTrainingEpochActive

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingEpochActive {false}

private

Definition at line 92 of file RDataLoaderEngine.hxx.

fTrainingEpochCount

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingEpochCount {0}

private

Definition at line 108 of file RDataLoaderEngine.hxx.

fTrainingSampler

template<typename... Args>

std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fTrainingSampler

private

Definition at line 71 of file RDataLoaderEngine.hxx.

fValidationBatchLoader

template<typename... Args>

std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationBatchLoader

private

Definition at line 70 of file RDataLoaderEngine.hxx.

fValidationClusterIdx

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationClusterIdx {0}

private

Definition at line 63 of file RDataLoaderEngine.hxx.

fValidationDataset

template<typename... Args>

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationDataset

private

Definition at line 103 of file RDataLoaderEngine.hxx.

fValidationDatasets

template<typename... Args>

std::vector<RFlat2DMatrix> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationDatasets

private

Definition at line 100 of file RDataLoaderEngine.hxx.

fValidationEpochActive

template<typename... Args>

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationEpochActive {false}

private

Definition at line 93 of file RDataLoaderEngine.hxx.

fValidationEpochCount

template<typename... Args>

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationEpochCount {0}

private

Definition at line 109 of file RDataLoaderEngine.hxx.

fValidationSampler

template<typename... Args>

std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fValidationSampler

private

Definition at line 72 of file RDataLoaderEngine.hxx.

fVecSizes

template<typename... Args>

std::vector<std::size_t> ROOT::Experimental::Internal::ML::RDataLoaderEngine< Args >::fVecSizes

private

Definition at line 52 of file RDataLoaderEngine.hxx.