Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
_ml_dataloader.py
Go to the documentation of this file.
1# Author: Dante Niewenhuis, VU Amsterdam 07/2023
2# Author: Kristupas Pranckietis, Vilnius University 05/2024
3# Author: Nopphakorn Subsa-Ard, King Mongkut's University of Technology Thonburi (KMUTT) (TH) 08/2024
4# Author: Vincenzo Eduardo Padulano, CERN 10/2024
5# Author: Martin Føll, University of Oslo (UiO) & CERN 01/2026
6# Author: Silia Taider, CERN 02/2026
7
8################################################################################
9# Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. #
10# All rights reserved. #
11# #
12# For the licensing terms see $ROOTSYS/LICENSE. #
13# For the list of contributors see $ROOTSYS/README/CREDITS. #
14################################################################################
15
16from __future__ import annotations
17
18import atexit
19import os
20from typing import TYPE_CHECKING, Any, Callable, Tuple
21
22if TYPE_CHECKING:
23 from collections.abc import Iterable
24
25 import numpy as np
26 import tensorflow as tf
27 import torch
28
29 import ROOT
30
31 # a dataset read from files: the name of its TTree or RNTuple, and one or more file paths
32 Spec = Tuple[str, str | os.PathLike | Iterable[str | os.PathLike]]
33
34
35def _normalize_target(target: str | list[str] | None) -> list[str]:
36 """`target` can be a string, list of strings, or None. Normalize to a list of strings
37 to be consistent with `columns`."""
38 if target is None or target == "":
39 return []
40 if isinstance(target, str):
41 return [target]
42 return list(target)
43
44
45def _exclude_target_and_weights(columns: list[str], target_columns: list[str], weights_column: str) -> list[str]:
46 """Drop any target/weights column names that also appear in `columns`."""
47 excluded = set(target_columns) | ({weights_column} if weights_column else set())
48 return [str(f) for f in columns if str(f) not in excluded]
49
50
51def _raise_if_implicit_mt(rdfs: list) -> None:
52 """Raise if ImplicitMT is enabled, or was when one of the RDataFrames was created."""
53 import ROOT
54
55 if ROOT.IsImplicitMTEnabled() or any(rdf.GetNSlots() > 1 for rdf in rdfs):
56 raise RuntimeError(
57 "Cannot use RDataLoader with implicit multi-threading enabled, call ROOT.DisableImplicitMT() before creating the data loader and its inputs."
58 )
59
60
61def _is_spec(item: Any) -> bool:
62 """Whether `item` is a `(dataset_name, files)` tuple."""
63 return isinstance(item, tuple) and len(item) == 2 and isinstance(item[0], str)
64
65
66def _open_files(inputs: Any) -> list:
67 """Return the inputs as a list of RDataFrames, each `(dataset_name, files)` tuple being read into one RDataFrame."""
68 import ROOT
69
70 if _is_spec(inputs) or not hasattr(inputs, "__iter__"):
71 inputs = [inputs]
72
73 rdataframes = []
74 for item in inputs:
75 if isinstance(item, (str, os.PathLike)):
76 raise TypeError("File paths are given together with the dataset to read, as (dataset_name, files) tuples")
77 if _is_spec(item):
78 dataset_name, files = item
79 files = [files] if isinstance(files, (str, os.PathLike)) else list(files)
80 item = ROOT.RDataFrame(dataset_name, [os.fspath(f) for f in files])
81 rdataframes.append(item)
82 return rdataframes
83
84
87 self,
88 x_rdf: ROOT.RDF.RNode,
89 columns: list[str] | None,
90 target_columns: list[str],
91 weights_column: str,
92 max_vec_sizes: dict[str, int] | None = None,
93 ) -> list[int]:
94 """
95 Resolve the columns to load based on the given RDataFrame and columns.
96
97 Args:
98 x_rdf (RNode): RDataFrame or RNode object.
99 columns (list[str], optional): Feature columns to load.
100 Defaults to all columns in the RDataFrame,
101 except for target_columns and weights_column.
102 target_columns (list[str]): Name(s) of target column(s).
103 weights_column (str): Name of the weight column, or "" for none.
104 max_vec_sizes (dict[str, int], optional):
105 Mapping from vector column name
106 to the maximum size of the vector.
107 Required when using vector based columns.
108
109 Returns:
110 list[int]: List of max vector sizes
111 """
112 if max_vec_sizes is None:
113 max_vec_sizes = {}
114
115 if columns:
116 columns = _exclude_target_and_weights(columns, target_columns, weights_column)
117 else:
118 rdf_column_names = [str(c) for c in x_rdf.GetColumnNames()]
119 columns = _exclude_target_and_weights(rdf_column_names, target_columns, weights_column)
120
121 ordered_columns = columns + target_columns + ([weights_column] if weights_column else [])
122
124 self.all_columns = []
125
126 max_vec_sizes_list = []
127 num_feature_expanded = 0
128 num_target_expanded = 0
129
130 for i, name in enumerate(ordered_columns):
131 name_str = str(name)
132 self.given_columns.append(name_str)
133 column_type = x_rdf.GetColumnType(name_str)
134
135 if "RVec" in column_type:
136 # Add column for each element if column is a vector
137 if name_str in max_vec_sizes:
138 max_vec_sizes_list.append(max_vec_sizes[name_str])
139 for j in range(max_vec_sizes[name_str]):
140 self.all_columns.append(f"{name_str}_{j}")
141 n_added = max_vec_sizes[name_str]
142
143 else:
144 raise ValueError(
145 f"No max size given for feature {name_str}. \
146 Given max sizes: {max_vec_sizes}"
147 )
148
149 else:
150 self.all_columns.append(name_str)
151 n_added = 1
152
153 if i < len(columns):
154 num_feature_expanded += n_added
155 elif i < len(columns) + len(target_columns):
156 num_target_expanded += n_added
157
158 self.train_slice = slice(0, num_feature_expanded)
159 self.target_slice = slice(num_feature_expanded, num_feature_expanded + num_target_expanded)
161
162 return max_vec_sizes_list
163
165 self,
166 inputs: ROOT.RDF.RNode | Spec | Iterable[ROOT.RDF.RNode | Spec],
167 *,
168 batch_size: int = 0,
169 batches_in_memory: int = 1,
170 columns: list[str] | None = None,
171 max_vec_sizes: dict[str, int] | None = None,
172 vec_padding: int = 0,
173 target: str | list[str] | None = None,
174 weights: str = "",
175 test_size: float = 0.0,
176 shuffle: bool = True,
177 drop_remainder: bool = True,
178 set_seed: int = 0,
179 load_eager: bool = False,
180 sampling_type: str = "",
181 sampling_ratio: float = 1.0,
182 replacement: bool = False,
183 ) -> None:
184 """Wrapper around the C++ DataLoader
185
186 Args:
187 inputs (ROOT.RDF.RNode | Spec | Iterable[ROOT.RDF.RNode | Spec]):
188 RDataFrame or ``(dataset_name, files)`` tuple, or a list of them, to load from.
189 A tuple is used to construct a single RDataFrame to read ``dataset_name`` from one or more files.
190 batch_size (int):
191 Number of entries per batch returned by the generator.
192 batches_in_memory (int):
193 Approximate number of batches that should be kept in memory at
194 the same time. Higher value results in faster loading, but
195 also higher memory usage. Defaults to 1.
196 columns (list[str] | None):
197 Names of feature columns to load. If not given, all columns are used.
198 max_vec_sizes (dict[str, int] | None):
199 Mapping from vector column name to the maximum size of the vector.
200 Required when using vector based columns.
201 vec_padding (int):
202 Value used to pad vectors with if the vector is smaller
203 than the given max vector length. Defaults to 0.
204 target (str | list[str] | None):
205 Name or list of names of target column(s).
206 weights (str):
207 Column used to weight events.
208 Can only be used when a target is given.
209 test_size (float):
210 The ratio of batches being kept for validation.
211 Value has to be between 0 and 1. Defaults to 0.0.
212 shuffle (bool):
213 Batches consist of random events and are shuffled every epoch.
214 Defaults to True.
215 drop_remainder (bool):
216 Drop the remainder of data that is too small to compose full batch.
217 Defaults to True.
218 set_seed (int):
219 For reproducibility: Set the seed for the random number generator used
220 to split the dataset into training and validation as well as shuffing of the entries.
221 Defaults to 0 which means that the seed is set to the random device.
222 load_eager (bool):
223 If True, load the full dataset(s) into memory.
224 If False, load data lazily in clusters. Defaults to False.
225 sampling_type (str):
226 Describes the mode of sampling from the minority and majority dataframes.
227 Supported values are ``"undersampling"`` and ``"oversampling"``. Requires ``load_eager=True``.
228 Defaults to ``""``.
229 For 'undersampling' and 'oversampling' it requires a list of exactly two dataframes as input,
230 where the dataframe with the most entries is the majority dataframe
231 and the dataframe with the fewest entries is the minority dataframe.
232 sampling_ratio (float):
233 Ratio of minority and majority entries in the resampled dataset.
234 Requires ``load_eager=True`` and ``sampling_type="undersampling"`` or ``"oversampling"``. Defaults to 1.0.
235 replacement (bool):
236 Whether the sampling is with (True) or without (False) replacement.
237 Requires ``load_eager=True`` and ``sampling_type="undersampling"``. Defaults to False.
238 """
239
240 from ROOT import RDF
241
242 rdataframes = [] if inputs is None else _open_files(inputs)
243 if not rdataframes:
244 raise ValueError("Detected empty input, at least one valid input is required.")
245
246 self.noded_rdfs = [RDF.AsRNode(rdf) for rdf in rdataframes]
248
250 self.weights_column = weights
253
254 if self.weights_given and not self.target_given:
255 raise ValueError("Weights can only be used when a target is provided")
256
257 max_vec_sizes_list = self._resolve_columns(
258 self.noded_rdfs[0], columns, self.target_columns, self.weights_column, max_vec_sizes
259 )
260
262 self.batch_size = batch_size
263
264 if self.weights_given:
265 # The weight column is always the single column right after the target(s)
267
268 import ROOT
269
270 # The DataLoader will create a separate C++ thread for I/O.
271 # Enable thread safety in ROOT from here, to make sure there is no
272 # interference between the main Python thread (which might call into
273 # cling via cppyy) and the I/O thread.
275
277 self.noded_rdfs,
278 batch_size,
279 batches_in_memory,
280 self.given_columns,
281 max_vec_sizes_list,
282 vec_padding,
283 test_size,
284 shuffle,
285 drop_remainder,
286 set_seed,
287 load_eager,
288 sampling_type,
289 sampling_ratio,
290 replacement,
291 )
292
293 atexit.register(self.DeActivateDeActivate)
294
295 @property
296 def isActive(self):
297 return self.engine.IsActive()
298
300 return self.engine.IsTrainingActive()
301
303 return self.engine.IsValidationActive()
304
305 def Activate(self):
306 """Initialize the generator to be used for a loop, this spawns the loading thread"""
307 self.engine.Activate()
308
309 def DeActivate(self):
310 """Deactivate the generator"""
311 self.engine.DeActivate()
312
313 def ActivateTrainingEpoch(self):
314 """Activate the training epoch of the generator"""
315 _raise_if_implicit_mt(self.noded_rdfs)
316 self.engine.ActivateTrainingEpoch()
317
318 def ActivateValidationEpoch(self):
319 """Activate the validation epoch of the generator"""
320 _raise_if_implicit_mt(self.noded_rdfs)
321 self.engine.ActivateValidationEpoch()
322
323 def DeActivateTrainingEpoch(self):
324 """Deactivate the training epoch of the generator"""
325 self.engine.DeActivateTrainingEpoch()
326
327 def DeActivateValidationEpoch(self):
328 """Deactivate the validation epoch of the generator"""
329 self.engine.DeActivateValidationEpoch()
330
331 def CreateTrainBatches(self):
332 """Create the first training batches from the first cluster"""
333 self.engine.CreateTrainBatches()
334
335 def CreateValidationBatches(self):
336 """Create the first validation batches from the first cluster"""
337 self.engine.CreateValidationBatches()
338
339 @property
340 def num_training_batches(self) -> int:
341 return self.engine.NumberOfTrainingBatches()
342
343 @property
344 def num_validation_batches(self) -> int:
345 return self.engine.NumberOfValidationBatches()
346
347 @property
348 def train_remainder_rows(self) -> int:
349 return self.engine.TrainRemainderRows()
350
351 @property
352 def val_remainder_rows(self) -> int:
353 return self.engine.ValidationRemainderRows()
354
355 def GetSample(self):
356 """
357 Return a sample of data that has the same size and types as the actual
358 result. This sample can be used to define the shape and size of the
359 output
360
361 Returns:
362 np.ndarray: data sample
363 """
364 try:
365 import numpy as np
366 except ImportError:
367 raise ImportError("Failed to import numpy needed for the ML dataloader")
368
369 # Split the target and weight
370 if not self.target_given:
371 return np.zeros((self.batch_size, self.num_columns))
372
373 num_feature_columns = self.train_slice.stop - self.train_slice.start
374 num_target_columns = self.target_slice.stop - self.target_slice.start
375
376 if not self.weights_given:
377 if num_target_columns == 1:
378 return np.zeros((self.batch_size, num_feature_columns)), np.zeros((self.batch_size)).reshape(-1, 1)
379
380 return np.zeros((self.batch_size, num_feature_columns)), np.zeros((self.batch_size, num_target_columns))
381
382 if num_target_columns == 1:
383 return (
384 np.zeros((self.batch_size, num_feature_columns)),
385 np.zeros((self.batch_size)).reshape(-1, 1),
386 np.zeros((self.batch_size)).reshape(-1, 1),
387 )
388
389 return (
390 np.zeros((self.batch_size, num_feature_columns)),
391 np.zeros((self.batch_size, num_target_columns)),
392 np.zeros((self.batch_size)).reshape(-1, 1),
393 )
394
395 def _get_raw_array(self, batch) -> np.ndarray:
396 try:
397 import numpy as np
398 except ImportError:
399 raise ImportError("Failed to import numpy needed for the ML dataloader")
400
401 batch_size, num_columns = tuple(batch.GetShape())
402 rvec = batch.ReleaseData()
403 return np.asarray(rvec).reshape(batch_size, num_columns)
404
405 def _split_target_and_weights(
406 self, data: np.ndarray
407 ) -> np.ndarray | Tuple[np.ndarray, np.ndarray] | Tuple[np.ndarray, np.ndarray, np.ndarray]:
408 # Splice target column from the data if target is given
409 if self.target_given:
410 train_data = data[:, self.train_slice]
411 target_data = data[:, self.target_slice]
412
413 # Splice weight column from the data if weight is given
414 if self.weights_given:
415 weights_data = data[:, self.weights_index]
416
417 if self.target_slice.stop - self.target_slice.start == 1:
418 return train_data, target_data.reshape(-1, 1), weights_data.reshape(-1, 1)
419
420 return train_data, target_data, weights_data.reshape(-1, 1)
421
422 if self.target_slice.stop - self.target_slice.start == 1:
423 return train_data, target_data.reshape(-1, 1)
424
425 return train_data, target_data
426
427 return data
428
429 def ConvertBatchToNumpy(self, batch) -> np.ndarray:
430 """Convert the batch into a NumPy array
431
432 Args:
433 batch: Batch returned from the DataLoader
434
435 Returns:
436 np.ndarray: converted batch
437 """
438 return self._split_target_and_weights(self._get_raw_array(batch))
439
440 def ConvertBatchToPyTorch(self, batch: Any, device=None) -> torch.Tensor:
441 """Convert the batch into a PyTorch tensor
442
443 Args:
444 batch: Batch returned from the DataLoader
445
446 Returns:
447 torch.Tensor: converted batch
448 """
449 import torch
450
451 split = self._split_target_and_weights(self._get_raw_array(batch))
452 return (
453 tuple(torch.as_tensor(arr, device=device) for arr in split)
454 if isinstance(split, tuple)
455 else torch.as_tensor(split, device=device)
456 )
457
458 def ConvertBatchToTF(self, batch: Any) -> Any:
459 """
460 Convert the batch into a TensorFlow tensor
461
462 Args:
463 batch: Batch returned from the DataLoader
464
465 Returns:
466 tensorflow.Tensor: converted batch
467 """
468 import tensorflow as tf
469
470 arr = self._get_raw_array(batch)
471 batch_size = arr.shape[0]
472 return_data = tf.constant(arr)
473
474 if batch_size != self.batch_size:
475 return_data = tf.pad(return_data, tf.constant([[0, self.batch_size - batch_size], [0, 0]]))
476
477 # Splice target column from the data if target is given
478 if self.target_given:
479 train_data = return_data[:, self.train_slice]
480 target_data = return_data[:, self.target_slice]
481
482 # Splice weight column from the data if weight is given
483 if self.weights_given:
484 weights_data = return_data[:, self.weights_index : self.weights_index + 1]
485
486 return train_data, target_data, weights_data
487
488 return train_data, target_data
489
490 return return_data
491
492 def ConvertBatchToJAX(self, batch: Any, device=None) -> Any:
493 """
494 Convert the batch into a JAX array
495
496 Args:
497 batch: Batch returned from the DataLoader
498
499 Returns:
500 jax.Array: converted batch
501 """
502 import jax
503 import jax.numpy as jnp
504
505 split = self._split_target_and_weights(jnp.asarray(self._get_raw_array(batch)))
506
507 if isinstance(device, str):
508 device = jax.devices(device)[0]
509
510 return (
511 tuple(jax.device_put(arr, device=device) for arr in split)
512 if isinstance(split, tuple)
513 else jax.device_put(split, device=device)
514 )
515
516 # Return a batch when available
517 def GetTrainBatch(self) -> Any:
518 """Return the next training batch of data from the given RDataFrame
519
520 Returns:
521 (np.ndarray): Batch of data of size.
522 """
523
524 batch = self.engine.GetTrainBatch()
525 return batch if (batch and batch.GetSize() > 0) else None
526
527 def GetValidationBatch(self) -> Any:
528 """Return the next training batch of data from the given RDataFrame
529
530 Returns:
531 (np.ndarray): Batch of data of size.
532 """
533
534 batch = self.engine.GetValidationBatch()
535 return batch if (batch and batch.GetSize() > 0) else None
536
537 def Save(self, dataset_name: str, filename: str, is_training: bool, output_format: str) -> None:
538 _raise_if_implicit_mt(self.noded_rdfs)
539 self.engine.Save(dataset_name, filename, is_training, output_format)
540
541
542# context managers for the loading thread
543class _TrainingEpochContext:
544 def __init__(self, internal: _RDataLoader):
545 self._internal = internal
546 # init loading thread
547 internal.Activate()
548 internal.CreateTrainBatches()
549
550 def __enter__(self):
551 self._internal.ActivateTrainingEpoch()
552 return self
553
554 def __exit__(self, type, value, traceback):
555 self._internal.DeActivateTrainingEpoch()
556
557
558class _ValidationEpochContext:
559 def __init__(self, internal: _RDataLoader):
560 self._internal = internal
561 internal.Activate()
562 internal.CreateValidationBatches()
563
564 def __enter__(self):
565 self._internal.ActivateValidationEpoch()
566 return self
567
568 def __exit__(self, type, value, traceback):
569 self._internal.DeActivateValidationEpoch()
570
571
572# formatted iterator (returned by as_torch / as_numpy / as_tensorflow)
573class FormattedLoader:
574 r"""
575 \ingroup Py_ML
576 Iterable that converts each batch to the requested format.
577 Returned by the as_torch / as_numpy / as_tensorflow methods on RDataLoader.
578 """
579
580 def __init__(
581 self,
582 internal: _RDataLoader,
583 conversion_fn: Callable,
584 is_training: bool,
585 ):
586 self._internal = internal
587 self._conversion_fn = conversion_fn
588 self._is_training = is_training
589
590 def _make_gen(self):
591 ctx_cls = _TrainingEpochContext if self._is_training else _ValidationEpochContext
592 get_batch = self._internal.GetTrainBatch if self._is_training else self._internal.GetValidationBatch
593
594 with ctx_cls(self._internal):
595 while True:
596 batch = get_batch()
597 if batch is None:
598 break
599 yield self._conversion_fn(batch)
600
601 def __iter__(self):
602 return self._make_gen()
603
604
605class RDataLoader:
606 r"""
607 \ingroup Py_ML
608 Entry point for ML batch loading from a ROOT RDataFrame.
609
610 Usage without a validation split::
611
612 train = ROOT.Experimental.ML.RDataLoader(df, batch_size=1000, ...)
613 for x, y in train.as_torch():
614 ...
615
616 Usage with a validation split::
617
618 dl = ROOT.Experimental.ML.RDataLoader(df, batch_size=1000, ...)
619 train, val = dl.train_test_split(test_size=0.2)
620 for x, y in train.as_torch():
621 ...
622 for x, y in val.as_numpy():
623 ...
624 """
625
626 def __init__(
627 self,
628 inputs: ROOT.RDF.RNode | Spec | Iterable[ROOT.RDF.RNode | Spec],
629 *,
630 batch_size: int = 64,
631 batches_in_memory: int = 10,
632 columns: list[str] | None = None,
633 max_vec_sizes: dict[str, int] | None = None,
634 vec_padding: float = 0.0,
635 target: str | list[str] | None = None,
636 weights: str = "",
637 shuffle: bool = True,
638 drop_remainder: bool = True,
639 set_seed: int = 0,
640 load_eager: bool = False,
641 sampling_type: str = "",
642 sampling_ratio: float = 1.0,
643 replacement: bool = False,
644 ) -> None:
645 r"""
646 \ingroup Py_ML
647
648 Args:
649 inputs:
650 RDataFrame or (dataset_name, files) tuple, or a list of them, to load from.
651 A tuple is used to construct a single RDataFrame to read dataset_name from one or more files.
652 batch_size:
653 Number of entries per batch.
654 batches_in_memory:
655 Approximate number of batches held in the shuffle buffer at any
656 time. Larger values improve shuffle quality across cluster
657 boundaries at the cost of higher memory usage. Acts as a soft
658 cap: the buffer may temporarily exceed this. Defaults to 10.
659 columns:
660 Names of feature columns to load. If not given, all columns are used.
661 max_vec_sizes:
662 Maximum size per vector column. Required for RVec columns.
663 vec_padding:
664 Padding value for vectors shorter than their max size. Defaults to 0.
665 target:
666 Name or list of names of target column(s).
667 weights:
668 Column to use for event weighting. Requires a target.
669 shuffle:
670 Whether to shuffle data across cluster boundaries every epoch.
671 Defaults to True.
672 drop_remainder:
673 Drop the last batch if smaller than batch_size. Defaults to True.
674 set_seed:
675 Seed for the random number generator. 0 means a random seed is
676 drawn from the system. Defaults to 0.
677 load_eager:
678 If True, load the full dataset into memory before training.
679 If False (default), load lazily in chunks.
680 sampling_type:
681 Resampling strategy: "undersampling" or "oversampling".
682 Requires load_eager=True and exactly two input dataframes.
683 sampling_ratio:
684 Ratio of minority to majority entries in the resampled dataset.
685 Requires load_eager=True and sampling_type set.
686 replacement:
687 Whether undersampling is with replacement. Requires load_eager=True
688 and sampling_type="undersampling".
689 """
690 # Store all constructor parameters. The C++ backend (_RDataLoader) is
691 # created lazily on the first call to as_torch/as_numpy/as_tensorflow or
692 # train_test_split.
693 self._params = dict(
694 inputs=inputs,
695 batch_size=batch_size,
696 batches_in_memory=batches_in_memory,
697 columns=columns,
698 max_vec_sizes=max_vec_sizes,
699 vec_padding=vec_padding,
700 target=target,
701 weights=weights,
702 shuffle=shuffle,
703 drop_remainder=drop_remainder,
704 set_seed=set_seed,
705 load_eager=load_eager,
706 sampling_type=sampling_type, # TODO(staider) consider turning into an enum
707 sampling_ratio=sampling_ratio,
708 replacement=replacement,
709 )
710 self._internal: _RDataLoader | None = None
711 self._test_size: float | None = None
712 self._is_training: bool = True # default: full dataset treated as training
713
714 @classmethod
715 def _from_internal(cls, internal: _RDataLoader, is_training: bool) -> RDataLoader:
716 """
717 Internal factory that creates a split bound RDataLoader sharing an
718 already-constructed C++ backend, train_test_split uses this to return two
719 RDataLoader instances that both point at the same _RDataLoader object.
720 """
721 obj = cls.__new__(cls)
722 obj._params = None
723 obj._internal = internal
724 obj._test_size = None
725 obj._is_training = is_training
726 return obj
727
728 def _ensure_created(self, test_size: float = 0.0) -> None:
729 """
730 Construct the C++ backend if not already done.
731 """
732 if self._internal is not None:
733 # Already constructed, guard against accidentally calling with a different split
734 if self._params is not None and test_size != self._test_size:
735 raise RuntimeError(
736 f"RDataLoader was already initialised with test_size="
737 f"{self._test_size}. Create a new RDataLoader to use a different split."
738 )
739 return
740
741 self._internal = _RDataLoader(**self._params, test_size=test_size)
742 self._test_size = test_size
743
744 def train_test_split(self, test_size: float = 0.2) -> Tuple[RDataLoader, RDataLoader]:
745 r"""
746 \ingroup Py_ML
747 Partition the dataset into training and validation splits.
748 Returns two RDataLoader instances that share the same underlying C++
749 backend and can each be iterated independently.
750 """
751 if not (0.0 < test_size < 1.0):
752 raise ValueError(f"test_size must be in (0.0, 1.0), got {test_size}")
753
754 self._ensure_created(test_size)
755 return (
756 RDataLoader._from_internal(self._internal, is_training=True),
757 RDataLoader._from_internal(self._internal, is_training=False),
758 )
759
760 def save(self, dataset_name: str, filename: str | os.PathLike, *, output_format: str = "ttree") -> None:
761 r"""
762 \ingroup Py_ML
763 Write this split to disk as it would be loaded for training. This function preserves the
764 input column names when writing the output dataset. Input columns of collection types keep
765 the same type, but their values will be fixed-sized and padded according to the
766 configuration during loading (e.g. via `max_vec_sizes` and `vec_padding`).
767
768 Args:
769 dataset_name: Name of the output TTree (or RNTuple).
770 filename: Output file path.
771 output_format: `"ttree"` (default) or `"rntuple"`.
772 """
773 output_formats = ("ttree", "rntuple")
774 if output_format not in output_formats:
775 raise ValueError(f"output_format must be one of {list(output_formats)}, got {output_format!r}")
776
777 self._ensure_created()
778 self._internal.Save(dataset_name, str(filename), self._is_training, output_format)
779
780 def as_numpy(self) -> FormattedLoader:
781 r"""
782 \ingroup Py_ML
783 Return an iterable that yields batches as NumPy arrays.
784 """
785 self._ensure_created()
786 return FormattedLoader(self._internal, self._internal.ConvertBatchToNumpy, self._is_training)
787
788 def as_torch(self, device: str | torch.device | None = None) -> FormattedLoader:
789 r"""
790 \ingroup Py_ML
791 Return an iterable that yields batches as PyTorch tensors.
792
793 Args:
794 device: If given, the returned tensors are moved to the specified device.
795 """
796 try:
797 import torch # noqa F401
798 except ImportError:
799 raise ImportError("Failed to import torch needed for the ML dataloader")
800
801 self._ensure_created()
802 conversion_fn = lambda batch: self._internal.ConvertBatchToPyTorch(batch, device) # noqa: E731
803 return FormattedLoader(self._internal, conversion_fn, self._is_training)
804
805 def as_tensorflow(self) -> tf.data.Dataset:
806 r"""
807 \ingroup Py_ML
808 Return a tf.data.Dataset over batches as TensorFlow tensors.
809 """
810 try:
811 import tensorflow as tf
812 except ImportError:
813 raise ImportError("Failed to import tensorflow needed for the ML dataloader")
814
815 self._ensure_created()
816
817 batch_size = self._internal.batch_size
818 num_feature_columns = len(self._internal.feature_columns)
819 num_target_columns = len(self._internal.target_columns)
820
821 # No target and weights given
822 if not self._internal.target_given:
823 batch_signature = tf.TensorSpec(shape=(batch_size, num_feature_columns), dtype=tf.float32)
824
825 # Target given, no weights given
826 elif not self._internal.weights_given:
827 batch_signature = (
828 tf.TensorSpec(shape=(batch_size, num_feature_columns), dtype=tf.float32),
829 tf.TensorSpec(shape=(batch_size, num_target_columns), dtype=tf.float32),
830 )
831
832 # Target and weights given
833 else:
834 batch_signature = (
835 tf.TensorSpec(shape=(batch_size, num_feature_columns), dtype=tf.float32),
836 tf.TensorSpec(shape=(batch_size, num_target_columns), dtype=tf.float32),
837 tf.TensorSpec(shape=(batch_size, 1), dtype=tf.float32),
838 )
839
840 loader = FormattedLoader(self._internal, self._internal.ConvertBatchToTF, self._is_training)
841 return tf.data.Dataset.from_generator(lambda: loader, output_signature=batch_signature)
842
843 def as_jax(self, device: str | Any = None) -> FormattedLoader:
844 r"""
845 \ingroup Py_ML
846 Return an iterable that yields batches as JAX arrays.
847
848 Args:
849 device: If given, the returned arrays are moved to the specified device.
850 Can be a string (e.g. "cpu", "gpu", "tpu") or any of JAX's device objects.
851 """
852 try:
853 import jax # noqa F401
854 except ImportError:
855 raise ImportError("Failed to import jax needed for the ML dataloader")
856
857 self._ensure_created()
858 conversion_fn = lambda batch: self._internal.ConvertBatchToJAX(batch, device) # noqa: E731
859 return FormattedLoader(self._internal, conversion_fn, self._is_training)
860
861 @property
862 def columns(self) -> list[str]:
863 r"""
864 \ingroup Py_ML
865 All column names as they appear in each batch tensor.
866 """
867 if self._internal is None:
868 columns = _exclude_target_and_weights(
869 self._params["columns"] or [], self.target_columns, self.weights_column
870 )
871 weights = self.weights_column
872 return columns + self.target_columns + ([weights] if weights else [])
873 return self._internal.all_columns
874
875 @property
876 def feature_columns(self) -> list[str]:
877 r"""
878 \ingroup Py_ML
879 Feature column names (columns minus target and weights).
880 """
881 if self._internal is None:
882 return _exclude_target_and_weights(self._params["columns"] or [], self.target_columns, self.weights_column)
883 return self._internal.feature_columns
884
885 @property
886 def target_columns(self) -> list[str]:
887 r"""
888 \ingroup Py_ML
889 Target column names.
890 """
891 if self._internal is None:
892 return _normalize_target(self._params["target"])
893 return self._internal.target_columns
894
895 @property
896 def weights_column(self) -> str:
897 r"""
898 \ingroup Py_ML
899 Weights column name, or empty string if not set.
900 """
901 if self._internal is None:
902 return self._params["weights"] if self._params["weights"] is not None else ""
903 return self._internal.weights_column
904
905 @property
906 def num_batches(self) -> int:
907 r"""
908 \ingroup Py_ML
909 Total number of batches in this split for one epoch.
910 """
911 if self._internal is None:
912 raise RuntimeError(
913 "num_batches is available after the first call to "
914 "as_torch / as_numpy / as_tensorflow / train_test_split."
915 )
916 if self._is_training:
917 return self._internal.num_training_batches
918 return self._internal.num_validation_batches
919
920 @property
921 def last_batch_no_of_rows(self) -> int:
922 r"""
923 \ingroup Py_ML
924 Number of rows in the last (remainder) batch, 0 if no remainder.
925 """
926 if self._internal is None:
927 raise RuntimeError(
928 "last_batch_no_of_rows is available after the first call to "
929 "as_torch / as_numpy / as_tensorflow / train_test_split."
930 )
931 if self._is_training:
932 return self._internal.train_remainder_rows
933 return self._internal.val_remainder_rows
934
935
936def _inject_dataloader_api(parentmodule):
937 """
938 Inject the public Python API into the ROOT.Experimental.ML namespace.
939 Only RDataLoader is part of the public surface.
940 """
941 for cls in [RDataLoader, FormattedLoader]:
942 setattr(parentmodule, cls.__name__, cls)
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
In this class, the processes of loading clusters (see RClusterLoader) and creating batches from those...
The public interface to the RDataFrame federation of classes.
ROOT's RDataFrame offers a modern, high-level interface for analysis of data stored in TTree ,...
Activate(self)
Initialize the generator to be used for a loop, this spawns the loading thread.
None __init__(self, ROOT.RDF.RNode|Spec|Iterable[ROOT.RDF.RNode|Spec] inputs, *int batch_size=0, int batches_in_memory=1, list[str]|None columns=None, dict[str, int]|None max_vec_sizes=None, int vec_padding=0, str|list[str]|None target=None, str weights="", float test_size=0.0, bool shuffle=True, bool drop_remainder=True, int set_seed=0, bool load_eager=False, str sampling_type="", float sampling_ratio=1.0, bool replacement=False)
Wrapper around the C++ DataLoader.
list[int] _resolve_columns(self, ROOT.RDF.RNode x_rdf, list[str]|None columns, list[str] target_columns, str weights_column, dict[str, int]|None max_vec_sizes=None)
Resolve the columns to load based on the given RDataFrame and columns.
list _open_files(Any inputs)
Return the inputs as a list of RDataFrames, each (dataset_name, files) tuple being read into one RDat...
list[str] _exclude_target_and_weights(list[str] columns, list[str] target_columns, str weights_column)
Drop any target/weights column names that also appear in columns.
None _raise_if_implicit_mt(list rdfs)
Raise if ImplicitMT is enabled, or was when one of the RDataFrames was created.
list[str] _normalize_target(str|list[str]|None target)
target can be a string, list of strings, or None.
bool _is_spec(Any item)
Whether item is a (dataset_name, files) tuple.
Bool_t IsImplicitMTEnabled()
Returns true if the implicit multi-threading in ROOT is enabled.
Definition TROOT.cxx:673
void EnableThreadSafety()
Enable support for multi-threading within the ROOT code in particular, enables the global mutex to ma...
Definition TROOT.cxx:579