In this class, the processes of loading clusters (see RClusterLoader) and creating batches from those clusters (see RBatchLoader) are combined, allowing batches from the training and validation sets to be loaded directly from a dataset in an RDataFrame.

Definition at line 64 of file RDataLoaderEngine.hxx.

Classes
struct	REpochGuard
	Opens a training or validation epoch and closes it again when done. More...

Public Member Functions
	RDataLoaderEngine (const std::vector< ROOT::RDF::RNode > &rdfs, const std::size_t batchSize, const std::size_t batchesInMemory, const std::vector< std::string > &cols, const std::vector< std::size_t > &vecSizes={}, const float vecPadding=0.0, const float testSize=0.0, bool shuffle=true, bool dropRemainder=true, const std::size_t setSeed=0, bool loadEager=false, std::string sampleType="", float sampleRatio=1.0, bool replacement=false)

	~RDataLoaderEngine ()

void	Activate ()
	Activate the loading process by spawning the loading thread.

void	ActivateTrainingEpoch ()
	Activate the training epoch by starting the batchloader.

void	ActivateValidationEpoch ()

void	CreateTrainBatches ()
	Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)

void	CreateValidationBatches ()
	Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)

void	DeActivate ()

void	DeActivateTrainingEpoch ()

void	DeActivateValidationEpoch ()

RFlat2DMatrix	GetTrainBatch ()
	Loads a training batch from the queue.

RFlat2DMatrix	GetValidationBatch ()
	Loads a validation batch from the queue.

bool	IsActive ()

bool	IsTrainingActive ()

bool	IsValidationActive ()

void	LoadData ()
	Main loop for loading clusters and creating batches.

std::size_t	NumberOfTrainingBatches ()

std::size_t	NumberOfValidationBatches ()

void	Save (std::string_view dataset_name, std::string_view filename, bool isTraining, std::string_view outputFormat)
	Materialize one train/test split to disk by draining a full epoch through the normal batch pipeline and Fill() each batch into `filename` instead of yielding it.

std::size_t	TrainRemainderRows ()

std::size_t	ValidationRemainderRows ()

Private Member Functions
std::vector< RColumnLayout >	MakeColumnLayout ()
	Describe how the loader's columns map onto a batch-tensor row.

Private Attributes
std::size_t	fBatchSize

std::size_t	fBufferCapacity

std::unique_ptr< RClusterLoader >	fClusterLoader

std::vector< std::string >	fCols

std::vector< RColumnLayout >	fColumnLayout

std::unique_ptr< RDatasetLoader >	fDatasetLoader

bool	fDropRemainder

bool	fIsActive {false}

bool	fLoadEager

std::condition_variable	fLoadingCondition

std::mutex	fLoadingMutex

std::unique_ptr< std::thread >	fLoadingThread

std::size_t	fLowWatermark

std::size_t	fNumTrainingEntries

std::size_t	fNumValidationEntries

std::vector< ROOT::RDF::RNode >	fRdfs

bool	fReplacement

RFlat2DMatrix	fSampledTrainingDataset

RFlat2DMatrix	fSampledValidationDataset

float	fSampleRatio

std::string	fSampleType

std::size_t	fSetSeed

bool	fShuffle

std::unique_ptr< RFlat2DMatrixOperators >	fTensorOperators

float	fTestSize

std::unique_ptr< RBatchLoader >	fTrainingBatchLoader

std::size_t	fTrainingClusterIdx {0}

RFlat2DMatrix	fTrainingDataset

std::vector< RFlat2DMatrix >	fTrainingDatasets

bool	fTrainingEpochActive {false}

std::size_t	fTrainingEpochCount {0}

std::unique_ptr< RSampler >	fTrainingSampler

std::unique_ptr< RBatchLoader >	fValidationBatchLoader

std::size_t	fValidationClusterIdx {0}

RFlat2DMatrix	fValidationDataset

std::vector< RFlat2DMatrix >	fValidationDatasets

bool	fValidationEpochActive {false}

std::size_t	fValidationEpochCount {0}

std::unique_ptr< RSampler >	fValidationSampler

std::vector< std::size_t >	fVecSizes

#include <ROOT/ML/RDataLoaderEngine.hxx>

Constructor & Destructor Documentation

◆ RDataLoaderEngine()

ROOT::Experimental::Internal::ML::RDataLoaderEngine::RDataLoaderEngine	(	const std::vector< ROOT::RDF::RNode > &	rdfs,
		const std::size_t	batchSize,
		const std::size_t	batchesInMemory,
		const std::vector< std::string > &	cols,
		const std::vector< std::size_t > &	vecSizes = {},
		const float	vecPadding = 0.0,
		const float	testSize = 0.0,
		bool	shuffle = true,
		bool	dropRemainder = true,
		const std::size_t	setSeed = 0,
		bool	loadEager = false,
		std::string	sampleType = "",
		float	sampleRatio = 1.0,
		bool	replacement = false )

Definition at line 54 of file RDataLoaderEngine.cxx.

◆ ~RDataLoaderEngine()

ROOT::Experimental::Internal::ML::RDataLoaderEngine::~RDataLoaderEngine ( )

Definition at line 132 of file RDataLoaderEngine.cxx.

Member Function Documentation

◆ Activate()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::Activate ( )

Activate the loading process by spawning the loading thread.

Definition at line 158 of file RDataLoaderEngine.cxx.

◆ ActivateTrainingEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::ActivateTrainingEpoch ( )

Activate the training epoch by starting the batchloader.

Definition at line 203 of file RDataLoaderEngine.cxx.

◆ ActivateValidationEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::ActivateValidationEpoch ( )

Definition at line 231 of file RDataLoaderEngine.cxx.

◆ CreateTrainBatches()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::CreateTrainBatches ( )

Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)

Definition at line 432 of file RDataLoaderEngine.cxx.

◆ CreateValidationBatches()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::CreateValidationBatches ( )

Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)

Definition at line 453 of file RDataLoaderEngine.cxx.

◆ DeActivate()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::DeActivate ( )

Definition at line 137 of file RDataLoaderEngine.cxx.

◆ DeActivateTrainingEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::DeActivateTrainingEpoch ( )

Definition at line 219 of file RDataLoaderEngine.cxx.

◆ DeActivateValidationEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::DeActivateValidationEpoch ( )

Definition at line 246 of file RDataLoaderEngine.cxx.

◆ GetTrainBatch()

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::GetTrainBatch ( )

Loads a training batch from the queue.

Definition at line 473 of file RDataLoaderEngine.cxx.

◆ GetValidationBatch()

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::GetValidationBatch ( )

Loads a validation batch from the queue.

Definition at line 480 of file RDataLoaderEngine.cxx.

◆ IsActive()

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::IsActive ( )

Definition at line 506 of file RDataLoaderEngine.cxx.

◆ IsTrainingActive()

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::IsTrainingActive ( )

Definition at line 512 of file RDataLoaderEngine.cxx.

◆ IsValidationActive()

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::IsValidationActive ( )

Definition at line 518 of file RDataLoaderEngine.cxx.

◆ LoadData()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::LoadData ( )

Main loop for loading clusters and creating batches.

The producer (loading thread) will keep loading clusters and creating batches until the end of the epoch is reached, or the generator is deactivated.

Definition at line 261 of file RDataLoaderEngine.cxx.

◆ MakeColumnLayout()

std::vector< RColumnLayout > ROOT::Experimental::Internal::ML::RDataLoaderEngine::MakeColumnLayout ( )

private

Describe how the loader's columns map onto a batch-tensor row.

Definition at line 15 of file RDataLoaderEngine.cxx.

◆ NumberOfTrainingBatches()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::NumberOfTrainingBatches ( )

Definition at line 486 of file RDataLoaderEngine.cxx.

◆ NumberOfValidationBatches()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::NumberOfValidationBatches ( )

Definition at line 491 of file RDataLoaderEngine.cxx.

◆ Save()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::Save	(	std::string_view	dataset_name,
		std::string_view	filename,
		bool	isTraining,
		std::string_view	outputFormat )

Materialize one train/test split to disk by draining a full epoch through the normal batch pipeline and Fill() each batch into filename instead of yielding it.

Filters, shuffling, the train/validation split and the batch_size/drop_remainder settings are all inherited from the loader's configuration.

Parameters

outputFormat Either "ttree" or "rntuple".

Definition at line 181 of file RDataLoaderEngine.cxx.

◆ TrainRemainderRows()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::TrainRemainderRows ( )

Definition at line 496 of file RDataLoaderEngine.cxx.

◆ ValidationRemainderRows()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::ValidationRemainderRows ( )

Definition at line 501 of file RDataLoaderEngine.cxx.

Member Data Documentation

◆ fBatchSize

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fBatchSize

private

Definition at line 69 of file RDataLoaderEngine.hxx.

◆ fBufferCapacity

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fBufferCapacity

private

Definition at line 73 of file RDataLoaderEngine.hxx.

◆ fClusterLoader

std::unique_ptr<RClusterLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fClusterLoader

private

Definition at line 82 of file RDataLoaderEngine.hxx.

◆ fCols

std::vector<std::string> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fCols

private

Definition at line 66 of file RDataLoaderEngine.hxx.

◆ fColumnLayout

std::vector<RColumnLayout> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fColumnLayout

private

Definition at line 68 of file RDataLoaderEngine.hxx.

◆ fDatasetLoader

std::unique_ptr<RDatasetLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fDatasetLoader

private

Definition at line 81 of file RDataLoaderEngine.hxx.

◆ fDropRemainder

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fDropRemainder

private

Definition at line 96 of file RDataLoaderEngine.hxx.

◆ fIsActive

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fIsActive {false}

private

Definition at line 103 of file RDataLoaderEngine.hxx.

◆ fLoadEager

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadEager

private

Definition at line 98 of file RDataLoaderEngine.hxx.

◆ fLoadingCondition

std::condition_variable ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadingCondition

private

Definition at line 93 of file RDataLoaderEngine.hxx.

◆ fLoadingMutex

std::mutex ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadingMutex

private

Definition at line 94 of file RDataLoaderEngine.hxx.

◆ fLoadingThread

std::unique_ptr<std::thread> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadingThread

private

Definition at line 92 of file RDataLoaderEngine.hxx.

◆ fLowWatermark

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLowWatermark

private

Definition at line 74 of file RDataLoaderEngine.hxx.

◆ fNumTrainingEntries

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fNumTrainingEntries

private

Definition at line 108 of file RDataLoaderEngine.hxx.

◆ fNumValidationEntries

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fNumValidationEntries

private

Definition at line 109 of file RDataLoaderEngine.hxx.

◆ fRdfs

std::vector<ROOT::RDF::RNode> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fRdfs

private

Definition at line 90 of file RDataLoaderEngine.hxx.

◆ fReplacement

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fReplacement

private

Definition at line 101 of file RDataLoaderEngine.hxx.

◆ fSampledTrainingDataset

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampledTrainingDataset

private

Definition at line 118 of file RDataLoaderEngine.hxx.

◆ fSampledValidationDataset

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampledValidationDataset

private

Definition at line 119 of file RDataLoaderEngine.hxx.

◆ fSampleRatio

float ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampleRatio

private

Definition at line 100 of file RDataLoaderEngine.hxx.

◆ fSampleType

std::string ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampleType

private

Definition at line 99 of file RDataLoaderEngine.hxx.

◆ fSetSeed

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSetSeed

private

Definition at line 70 of file RDataLoaderEngine.hxx.

◆ fShuffle

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fShuffle

private

Definition at line 97 of file RDataLoaderEngine.hxx.

◆ fTensorOperators

std::unique_ptr<RFlat2DMatrixOperators> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTensorOperators

private

Definition at line 88 of file RDataLoaderEngine.hxx.

◆ fTestSize

float ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTestSize

private

Definition at line 79 of file RDataLoaderEngine.hxx.

◆ fTrainingBatchLoader

std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingBatchLoader

private

Definition at line 83 of file RDataLoaderEngine.hxx.

◆ fTrainingClusterIdx

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingClusterIdx {0}

private

Definition at line 76 of file RDataLoaderEngine.hxx.

◆ fTrainingDataset

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingDataset

private

Definition at line 115 of file RDataLoaderEngine.hxx.

◆ fTrainingDatasets

std::vector<RFlat2DMatrix> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingDatasets

private

Definition at line 112 of file RDataLoaderEngine.hxx.

◆ fTrainingEpochActive

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingEpochActive {false}

private

Definition at line 105 of file RDataLoaderEngine.hxx.

◆ fTrainingEpochCount

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingEpochCount {0}

private

Definition at line 121 of file RDataLoaderEngine.hxx.

◆ fTrainingSampler

std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingSampler

private

Definition at line 85 of file RDataLoaderEngine.hxx.

◆ fValidationBatchLoader

std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationBatchLoader

private

Definition at line 84 of file RDataLoaderEngine.hxx.

◆ fValidationClusterIdx

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationClusterIdx {0}

private

Definition at line 77 of file RDataLoaderEngine.hxx.

◆ fValidationDataset

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationDataset

private

Definition at line 116 of file RDataLoaderEngine.hxx.

◆ fValidationDatasets

std::vector<RFlat2DMatrix> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationDatasets

private

Definition at line 113 of file RDataLoaderEngine.hxx.

◆ fValidationEpochActive

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationEpochActive {false}

private

Definition at line 106 of file RDataLoaderEngine.hxx.

◆ fValidationEpochCount

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationEpochCount {0}

private

Definition at line 122 of file RDataLoaderEngine.hxx.

◆ fValidationSampler

std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationSampler

private

Definition at line 86 of file RDataLoaderEngine.hxx.

◆ fVecSizes

std::vector<std::size_t> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fVecSizes

private

Definition at line 67 of file RDataLoaderEngine.hxx.

Libraries for ROOT::Experimental::Internal::ML::RDataLoaderEngine:

[legend]

The documentation for this class was generated from the following files:

tree/ml/inc/ROOT/ML/RDataLoaderEngine.hxx
tree/ml/src/RDataLoaderEngine.cxx

Classes

Public Member Functions

Private Member Functions

Private Attributes

Constructor & Destructor Documentation

◆ RDataLoaderEngine()

◆ ~RDataLoaderEngine()

Member Function Documentation

◆ Activate()

◆ ActivateTrainingEpoch()

◆ ActivateValidationEpoch()

◆ CreateTrainBatches()

◆ CreateValidationBatches()

◆ DeActivate()

◆ DeActivateTrainingEpoch()

◆ DeActivateValidationEpoch()

◆ GetTrainBatch()

◆ GetValidationBatch()

◆ IsActive()

◆ IsTrainingActive()

◆ IsValidationActive()

◆ LoadData()

◆ MakeColumnLayout()

◆ NumberOfTrainingBatches()

◆ NumberOfValidationBatches()

◆ Save()

◆ TrainRemainderRows()

◆ ValidationRemainderRows()

Member Data Documentation

◆ fBatchSize

◆ fBufferCapacity

◆ fClusterLoader

◆ fCols

◆ fColumnLayout

◆ fDatasetLoader

◆ fDropRemainder

◆ fIsActive

◆ fLoadEager

◆ fLoadingCondition

◆ fLoadingMutex

◆ fLoadingThread

◆ fLowWatermark

◆ fNumTrainingEntries

◆ fNumValidationEntries

◆ fRdfs

◆ fReplacement

◆ fSampledTrainingDataset

◆ fSampledValidationDataset

◆ fSampleRatio

◆ fSampleType

◆ fSetSeed

◆ fShuffle

◆ fTensorOperators

◆ fTestSize

◆ fTrainingBatchLoader

◆ fTrainingClusterIdx

◆ fTrainingDataset

◆ fTrainingDatasets

◆ fTrainingEpochActive

◆ fTrainingEpochCount

◆ fTrainingSampler

◆ fValidationBatchLoader

◆ fValidationClusterIdx

◆ fValidationDataset

◆ fValidationDatasets

◆ fValidationEpochActive

◆ fValidationEpochCount

◆ fValidationSampler

◆ fVecSizes