Logo ROOT  

Reference Guide
 
Loading...
Searching...
No Matches
List of all members | Classes | Public Member Functions | Private Member Functions | Private Attributes | List of all members
ROOT::Experimental::Internal::ML::RDataLoaderEngine Class Reference

In this class, the processes of loading clusters (see RClusterLoader) and creating batches from those clusters (see RBatchLoader) are combined, allowing batches from the training and validation sets to be loaded directly from a dataset in an RDataFrame.

Definition at line 64 of file RDataLoaderEngine.hxx.

Classes

struct  REagerEpoch
 Rows of an eager epoch, turned into batches as they are requested. More...
 
struct  REpochGuard
 Opens a training or validation epoch and closes it again when done. More...
 

Public Member Functions

 RDataLoaderEngine (const std::vector< ROOT::RDF::RNode > &rdfs, const std::size_t batchSize, const std::size_t batchesInMemory, const std::vector< std::string > &cols, const std::vector< std::size_t > &vecSizes={}, const float vecPadding=0.0, const float testSize=0.0, bool shuffle=true, bool dropRemainder=true, const std::size_t setSeed=0, bool loadEager=false, std::string sampleType="", float sampleRatio=1.0, bool replacement=false)
 
 ~RDataLoaderEngine ()
 
void Activate ()
 Activate the loading process by spawning the loading thread.
 
void ActivateTrainingEpoch ()
 Activate the training epoch by starting the batchloader.
 
void ActivateValidationEpoch ()
 
void CreateTrainBatches ()
 Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)
 
void CreateValidationBatches ()
 Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)
 
void DeActivate ()
 
void DeActivateTrainingEpoch ()
 
void DeActivateValidationEpoch ()
 
RFlat2DMatrix GetTrainBatch ()
 Loads a training batch from the queue.
 
RFlat2DMatrix GetValidationBatch ()
 Loads a validation batch from the queue.
 
bool IsActive ()
 
bool IsTrainingActive ()
 
bool IsValidationActive ()
 
void LoadData ()
 Main loop for loading clusters and creating batches.
 
std::size_t NumberOfTrainingBatches ()
 
std::size_t NumberOfValidationBatches ()
 
void Save (std::string_view dataset_name, std::string_view filename, bool isTraining, std::string_view outputFormat)
 Materialize one train/test split to disk by draining a full epoch through the normal batch pipeline and Fill() each batch into filename instead of yielding it.
 
std::size_t TrainRemainderRows ()
 
std::size_t ValidationRemainderRows ()
 

Private Member Functions

void CreateNextEagerBatch (RBatchLoader &batchLoader, REagerEpoch &epoch)
 Create the next batch of an eager epoch, unless one is already queued or the epoch is done.
 
std::vector< RColumnLayout > MakeColumnLayout ()
 Describe how the loader's columns map onto a batch-tensor row.
 
REagerEpoch MakeEagerEpoch (const std::vector< std::size_t > &rows)
 The eager epoch reading the given rows of the dataset, in shuffled order when shuffling.
 

Private Attributes

std::size_t fBatchSize
 
std::size_t fBufferCapacity
 
std::unique_ptr< RClusterLoader > fClusterLoader
 
std::vector< std::string > fCols
 
std::vector< RColumnLayout > fColumnLayout
 
RFlat2DMatrix fDataset
 
std::unique_ptr< RDatasetLoader > fDatasetLoader
 
bool fDropRemainder
 
bool fIsActive {false}
 
bool fLoadEager
 
std::condition_variable fLoadingCondition
 
std::mutex fLoadingMutex
 
std::unique_ptr< std::thread > fLoadingThread
 
std::size_t fLowWatermark
 
std::size_t fNumTrainingEntries
 
std::size_t fNumValidationEntries
 
std::vector< ROOT::RDF::RNode > fRdfs
 
bool fReplacement
 
RFlat2DMatrix fSampledTrainingDataset
 
RFlat2DMatrix fSampledValidationDataset
 
float fSampleRatio
 
std::string fSampleType
 
std::size_t fSetSeed
 
bool fShuffle
 
std::unique_ptr< RFlat2DMatrixOperators > fTensorOperators
 
float fTestSize
 
std::unique_ptr< RBatchLoader > fTrainingBatchLoader
 
std::size_t fTrainingClusterIdx {0}
 
REagerEpoch fTrainingEagerEpoch
 
bool fTrainingEpochActive {false}
 
std::size_t fTrainingEpochCount {0}
 
std::vector< std::vector< std::size_t > > fTrainingRows
 
std::unique_ptr< RSampler > fTrainingSampler
 
std::unique_ptr< RBatchLoader > fValidationBatchLoader
 
std::size_t fValidationClusterIdx {0}
 
REagerEpoch fValidationEagerEpoch
 
bool fValidationEpochActive {false}
 
std::size_t fValidationEpochCount {0}
 
std::vector< std::vector< std::size_t > > fValidationRows
 
std::unique_ptr< RSampler > fValidationSampler
 
std::vector< std::size_t > fVecSizes
 

#include <ROOT/ML/RDataLoaderEngine.hxx>

Constructor & Destructor Documentation

◆ RDataLoaderEngine()

ROOT::Experimental::Internal::ML::RDataLoaderEngine::RDataLoaderEngine ( const std::vector< ROOT::RDF::RNode > & rdfs,
const std::size_t batchSize,
const std::size_t batchesInMemory,
const std::vector< std::string > & cols,
const std::vector< std::size_t > & vecSizes = {},
const float vecPadding = 0.0,
const float testSize = 0.0,
bool shuffle = true,
bool dropRemainder = true,
const std::size_t setSeed = 0,
bool loadEager = false,
std::string sampleType = "",
float sampleRatio = 1.0,
bool replacement = false )

Definition at line 71 of file RDataLoaderEngine.cxx.

◆ ~RDataLoaderEngine()

ROOT::Experimental::Internal::ML::RDataLoaderEngine::~RDataLoaderEngine ( )

Definition at line 144 of file RDataLoaderEngine.cxx.

Member Function Documentation

◆ Activate()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::Activate ( )

Activate the loading process by spawning the loading thread.

Definition at line 170 of file RDataLoaderEngine.cxx.

◆ ActivateTrainingEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::ActivateTrainingEpoch ( )

Activate the training epoch by starting the batchloader.

Definition at line 215 of file RDataLoaderEngine.cxx.

◆ ActivateValidationEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::ActivateValidationEpoch ( )

Definition at line 243 of file RDataLoaderEngine.cxx.

◆ CreateNextEagerBatch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::CreateNextEagerBatch ( RBatchLoader & batchLoader,
REagerEpoch & epoch )
private

Create the next batch of an eager epoch, unless one is already queued or the epoch is done.

Definition at line 506 of file RDataLoaderEngine.cxx.

◆ CreateTrainBatches()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::CreateTrainBatches ( )

Create training batches by first loading a chunk (see RClusterLoader) and split it into batches (see RBatchLoader)

Definition at line 452 of file RDataLoaderEngine.cxx.

◆ CreateValidationBatches()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::CreateValidationBatches ( )

Creates validation batches by first loading a chunk (see RClusterLoader), and then split it into batches (see RBatchLoader)

Definition at line 472 of file RDataLoaderEngine.cxx.

◆ DeActivate()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::DeActivate ( )

Definition at line 149 of file RDataLoaderEngine.cxx.

◆ DeActivateTrainingEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::DeActivateTrainingEpoch ( )

Definition at line 231 of file RDataLoaderEngine.cxx.

◆ DeActivateValidationEpoch()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::DeActivateValidationEpoch ( )

Definition at line 258 of file RDataLoaderEngine.cxx.

◆ GetTrainBatch()

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::GetTrainBatch ( )

Loads a training batch from the queue.

Definition at line 525 of file RDataLoaderEngine.cxx.

◆ GetValidationBatch()

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::GetValidationBatch ( )

Loads a validation batch from the queue.

Definition at line 536 of file RDataLoaderEngine.cxx.

◆ IsActive()

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::IsActive ( )

Definition at line 566 of file RDataLoaderEngine.cxx.

◆ IsTrainingActive()

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::IsTrainingActive ( )

Definition at line 572 of file RDataLoaderEngine.cxx.

◆ IsValidationActive()

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::IsValidationActive ( )

Definition at line 578 of file RDataLoaderEngine.cxx.

◆ LoadData()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::LoadData ( )

Main loop for loading clusters and creating batches.

The producer (loading thread) will keep loading clusters and creating batches until the end of the epoch is reached, or the generator is deactivated.

Definition at line 273 of file RDataLoaderEngine.cxx.

◆ MakeColumnLayout()

std::vector< RColumnLayout > ROOT::Experimental::Internal::ML::RDataLoaderEngine::MakeColumnLayout ( )
private

Describe how the loader's columns map onto a batch-tensor row.

Definition at line 32 of file RDataLoaderEngine.cxx.

◆ MakeEagerEpoch()

RDataLoaderEngine::REagerEpoch ROOT::Experimental::Internal::ML::RDataLoaderEngine::MakeEagerEpoch ( const std::vector< std::size_t > & rows)
private

The eager epoch reading the given rows of the dataset, in shuffled order when shuffling.

Definition at line 490 of file RDataLoaderEngine.cxx.

◆ NumberOfTrainingBatches()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::NumberOfTrainingBatches ( )

Definition at line 546 of file RDataLoaderEngine.cxx.

◆ NumberOfValidationBatches()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::NumberOfValidationBatches ( )

Definition at line 551 of file RDataLoaderEngine.cxx.

◆ Save()

void ROOT::Experimental::Internal::ML::RDataLoaderEngine::Save ( std::string_view dataset_name,
std::string_view filename,
bool isTraining,
std::string_view outputFormat )

Materialize one train/test split to disk by draining a full epoch through the normal batch pipeline and Fill() each batch into filename instead of yielding it.

Filters, shuffling, the train/validation split and the batch_size/drop_remainder settings are all inherited from the loader's configuration.

Parameters
outputFormatEither "ttree" or "rntuple".

Definition at line 193 of file RDataLoaderEngine.cxx.

◆ TrainRemainderRows()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::TrainRemainderRows ( )

Definition at line 556 of file RDataLoaderEngine.cxx.

◆ ValidationRemainderRows()

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::ValidationRemainderRows ( )

Definition at line 561 of file RDataLoaderEngine.cxx.

Member Data Documentation

◆ fBatchSize

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fBatchSize
private

Definition at line 69 of file RDataLoaderEngine.hxx.

◆ fBufferCapacity

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fBufferCapacity
private

Definition at line 73 of file RDataLoaderEngine.hxx.

◆ fClusterLoader

std::unique_ptr<RClusterLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fClusterLoader
private

Definition at line 82 of file RDataLoaderEngine.hxx.

◆ fCols

std::vector<std::string> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fCols
private

Definition at line 66 of file RDataLoaderEngine.hxx.

◆ fColumnLayout

std::vector<RColumnLayout> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fColumnLayout
private

Definition at line 68 of file RDataLoaderEngine.hxx.

◆ fDataset

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::fDataset
private

Definition at line 112 of file RDataLoaderEngine.hxx.

◆ fDatasetLoader

std::unique_ptr<RDatasetLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fDatasetLoader
private

Definition at line 81 of file RDataLoaderEngine.hxx.

◆ fDropRemainder

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fDropRemainder
private

Definition at line 96 of file RDataLoaderEngine.hxx.

◆ fIsActive

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fIsActive {false}
private

Definition at line 103 of file RDataLoaderEngine.hxx.

◆ fLoadEager

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadEager
private

Definition at line 98 of file RDataLoaderEngine.hxx.

◆ fLoadingCondition

std::condition_variable ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadingCondition
private

Definition at line 93 of file RDataLoaderEngine.hxx.

◆ fLoadingMutex

std::mutex ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadingMutex
private

Definition at line 94 of file RDataLoaderEngine.hxx.

◆ fLoadingThread

std::unique_ptr<std::thread> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLoadingThread
private

Definition at line 92 of file RDataLoaderEngine.hxx.

◆ fLowWatermark

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fLowWatermark
private

Definition at line 74 of file RDataLoaderEngine.hxx.

◆ fNumTrainingEntries

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fNumTrainingEntries
private

Definition at line 108 of file RDataLoaderEngine.hxx.

◆ fNumValidationEntries

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fNumValidationEntries
private

Definition at line 109 of file RDataLoaderEngine.hxx.

◆ fRdfs

std::vector<ROOT::RDF::RNode> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fRdfs
private

Definition at line 90 of file RDataLoaderEngine.hxx.

◆ fReplacement

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fReplacement
private

Definition at line 101 of file RDataLoaderEngine.hxx.

◆ fSampledTrainingDataset

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampledTrainingDataset
private

Definition at line 118 of file RDataLoaderEngine.hxx.

◆ fSampledValidationDataset

RFlat2DMatrix ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampledValidationDataset
private

Definition at line 119 of file RDataLoaderEngine.hxx.

◆ fSampleRatio

float ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampleRatio
private

Definition at line 100 of file RDataLoaderEngine.hxx.

◆ fSampleType

std::string ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSampleType
private

Definition at line 99 of file RDataLoaderEngine.hxx.

◆ fSetSeed

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fSetSeed
private

Definition at line 70 of file RDataLoaderEngine.hxx.

◆ fShuffle

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fShuffle
private

Definition at line 97 of file RDataLoaderEngine.hxx.

◆ fTensorOperators

std::unique_ptr<RFlat2DMatrixOperators> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTensorOperators
private

Definition at line 88 of file RDataLoaderEngine.hxx.

◆ fTestSize

float ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTestSize
private

Definition at line 79 of file RDataLoaderEngine.hxx.

◆ fTrainingBatchLoader

std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingBatchLoader
private

Definition at line 83 of file RDataLoaderEngine.hxx.

◆ fTrainingClusterIdx

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingClusterIdx {0}
private

Definition at line 76 of file RDataLoaderEngine.hxx.

◆ fTrainingEagerEpoch

REagerEpoch ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingEagerEpoch
private

Definition at line 134 of file RDataLoaderEngine.hxx.

◆ fTrainingEpochActive

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingEpochActive {false}
private

Definition at line 105 of file RDataLoaderEngine.hxx.

◆ fTrainingEpochCount

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingEpochCount {0}
private

Definition at line 121 of file RDataLoaderEngine.hxx.

◆ fTrainingRows

std::vector<std::vector<std::size_t> > ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingRows
private

Definition at line 115 of file RDataLoaderEngine.hxx.

◆ fTrainingSampler

std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fTrainingSampler
private

Definition at line 85 of file RDataLoaderEngine.hxx.

◆ fValidationBatchLoader

std::unique_ptr<RBatchLoader> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationBatchLoader
private

Definition at line 84 of file RDataLoaderEngine.hxx.

◆ fValidationClusterIdx

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationClusterIdx {0}
private

Definition at line 77 of file RDataLoaderEngine.hxx.

◆ fValidationEagerEpoch

REagerEpoch ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationEagerEpoch
private

Definition at line 135 of file RDataLoaderEngine.hxx.

◆ fValidationEpochActive

bool ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationEpochActive {false}
private

Definition at line 106 of file RDataLoaderEngine.hxx.

◆ fValidationEpochCount

std::size_t ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationEpochCount {0}
private

Definition at line 122 of file RDataLoaderEngine.hxx.

◆ fValidationRows

std::vector<std::vector<std::size_t> > ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationRows
private

Definition at line 116 of file RDataLoaderEngine.hxx.

◆ fValidationSampler

std::unique_ptr<RSampler> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fValidationSampler
private

Definition at line 86 of file RDataLoaderEngine.hxx.

◆ fVecSizes

std::vector<std::size_t> ROOT::Experimental::Internal::ML::RDataLoaderEngine::fVecSizes
private

Definition at line 67 of file RDataLoaderEngine.hxx.

Libraries for ROOT::Experimental::Internal::ML::RDataLoaderEngine:
[legend]
The documentation for this class was generated from the following files: