7#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
8#include <numpy/arrayobject.h>
24 PyGILState_STATE m_GILState;
27 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
28 ~PyGILRAII() { PyGILState_Release(m_GILState); }
85 "Specify as 0.2 or 20% to use a fifth of the data set as validation set."
86 "Specify as 100 to use exactly 100 events. (Default: 20%)");
87 DeclareOptionRef(
fUserCodeName =
"",
"UserCode",
"Necessary python code provided by the user to be executed before loading and training the PyTorch Model");
102 Int_t nValidationSamples = 0;
107 if (fNumValidationString.EndsWith(
"%")) {
112 Double_t valSizeAsDouble = fNumValidationString.Atof() / 100.0;
113 nValidationSamples = GetEventCollection(
Types::kTraining).size() * valSizeAsDouble;
115 Log() << kFATAL <<
"Cannot parse number \"" << fNumValidationString
116 <<
"\". Expected string like \"20%\" or \"20.0%\"." <<
Endl;
118 }
else if (fNumValidationString.IsFloat()) {
119 Double_t valSizeAsDouble = fNumValidationString.Atof();
121 if (valSizeAsDouble < 1.0) {
123 nValidationSamples = GetEventCollection(
Types::kTraining).size() * valSizeAsDouble;
126 nValidationSamples = valSizeAsDouble;
129 Log() << kFATAL <<
"Cannot parse number \"" << fNumValidationString <<
"\". Expected string like \"0.2\" or \"100\"."
135 if (nValidationSamples < 0) {
136 Log() << kFATAL <<
"Validation size \"" << fNumValidationString <<
"\" is negative." <<
Endl;
139 if (nValidationSamples == 0) {
140 Log() << kFATAL <<
"Validation size \"" << fNumValidationString <<
"\" is zero." <<
Endl;
143 if (nValidationSamples >= (
Int_t)trainingSetSize) {
144 Log() << kFATAL <<
"Validation size \"" << fNumValidationString
145 <<
"\" is larger than or equal in size to training set (size=\"" << trainingSetSize <<
"\")." <<
Endl;
148 return nValidationSamples;
162 Log() << kINFO <<
"Using PyTorch - setting special configuration options " <<
Endl;
163 PyRunString(
"import torch",
"Error importing pytorch");
169 PyRunString(
"torch_major_version = int(torch.__version__.split('.')[0])");
170 PyObject *pyTorchVersion = PyDict_GetItemString(
fLocalNS,
"torch_major_version");
171 int torchVersion = PyLong_AsLong(pyTorchVersion);
172 Log() << kINFO <<
"Using PyTorch version " << torchVersion <<
Endl;
176 if (num_threads > 0) {
177 Log() << kINFO <<
"Setting the CPU number of threads = " << num_threads <<
Endl;
195 Log() << kINFO <<
" Setup PyTorch Model " <<
Endl;
207 PyRunString(
"print('custom objects for loading model : ',load_model_custom_objects)");
210 PyRunString(
"fit = load_model_custom_objects[\"train_func\"]",
211 "Failed to load train function from file. Please use key: 'train_func' and pass training loop function as the value.");
212 Log() << kINFO <<
"Loaded pytorch train function: " <<
Endl;
216 PyRunString(
"if 'optimizer' in load_model_custom_objects:\n"
217 " optimizer = load_model_custom_objects['optimizer']\n"
219 " optimizer = torch.optim.SGD\n",
220 "Please use key: 'optimizer' and pass a pytorch optimizer as the value for a custom optimizer.");
221 Log() << kINFO <<
"Loaded pytorch optimizer: " <<
Endl;
225 PyRunString(
"criterion = load_model_custom_objects[\"criterion\"]",
226 "Failed to load loss function from file. Using MSE Loss as default. Please use key: 'criterion' and pass a pytorch loss function as the value.");
227 Log() << kINFO <<
"Loaded pytorch loss function: " <<
Endl;
231 PyRunString(
"predict = load_model_custom_objects[\"predict_func\"]",
232 "Can't find user predict function object from file. Please use key: 'predict' and pass a predict function for evaluating the model as the value.");
233 Log() << kINFO <<
"Loaded pytorch predict function: " <<
Endl;
238 if (loadTrainedModel) {
244 PyRunString(
"model = torch.jit.load('"+filenameLoadModel+
"')",
245 "Failed to load PyTorch model from file: "+filenameLoadModel);
246 Log() << kINFO <<
"Load model from file: " << filenameLoadModel <<
Endl;
257 else Log() << kFATAL <<
"Selected analysis type is not implemented" <<
Endl;
261 npy_intp dimsVals[2] = {(npy_intp)1, (npy_intp)
fNVars};
262 PyArrayObject* pVals = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsVals, NPY_FLOAT, (
void*)
fVals);
266 npy_intp dimsOutput[2] = {(npy_intp)1, (npy_intp)
fNOutputs};
267 PyArrayObject* pOutput = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsOutput, NPY_FLOAT, (
void*)&
fOutput[0]);
277 TMVA::Internal::PyGILRAII raii;
280 Log() << kFATAL <<
"Python is not initialized" <<
Endl;
285 PyRunString(
"import sys; sys.argv = ['']",
"Set sys.argv failed");
286 PyRunString(
"import torch",
"import PyTorch failed");
290 Log() << kFATAL <<
"import torch in global namespace failed!" <<
Endl;
307 UInt_t nTrainingEvents = nAllEvents - nValEvents;
309 Log() << kINFO <<
"Split TMVA training data in " << nTrainingEvents <<
" training events and "
310 << nValEvents <<
" validation events" <<
Endl;
312 float* trainDataX =
new float[nTrainingEvents*
fNVars];
313 float* trainDataY =
new float[nTrainingEvents*
fNOutputs];
314 float* trainDataWeights =
new float[nTrainingEvents];
315 for (
UInt_t i=0; i<nTrainingEvents; i++) {
319 trainDataX[j + i*
fNVars] =
e->GetValue(j);
332 trainDataY[j + i*
fNOutputs] =
e->GetTarget(j);
335 else Log() << kFATAL <<
"Can not fill target vector because analysis type is not known" <<
Endl;
338 trainDataWeights[i] =
e->GetWeight();
341 npy_intp dimsTrainX[2] = {(npy_intp)nTrainingEvents, (npy_intp)
fNVars};
342 npy_intp dimsTrainY[2] = {(npy_intp)nTrainingEvents, (npy_intp)
fNOutputs};
343 npy_intp dimsTrainWeights[1] = {(npy_intp)nTrainingEvents};
344 PyArrayObject* pTrainDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainX, NPY_FLOAT, (
void*)trainDataX);
345 PyArrayObject* pTrainDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainY, NPY_FLOAT, (
void*)trainDataY);
346 PyArrayObject* pTrainDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsTrainWeights, NPY_FLOAT, (
void*)trainDataWeights);
349 PyDict_SetItemString(
fLocalNS,
"trainWeights", (
PyObject*)pTrainDataWeights);
359 float* valDataX =
new float[nValEvents*
fNVars];
360 float* valDataY =
new float[nValEvents*
fNOutputs];
361 float* valDataWeights =
new float[nValEvents];
363 for (
UInt_t i=0; i< nValEvents ; i++) {
364 UInt_t ievt = nTrainingEvents + i;
368 valDataX[j + i*
fNVars] =
e->GetValue(j);
382 else Log() << kFATAL <<
"Can not fill target vector because analysis type is not known" <<
Endl;
384 valDataWeights[i] =
e->GetWeight();
387 npy_intp dimsValX[2] = {(npy_intp)nValEvents, (npy_intp)
fNVars};
388 npy_intp dimsValY[2] = {(npy_intp)nValEvents, (npy_intp)
fNOutputs};
389 npy_intp dimsValWeights[1] = {(npy_intp)nValEvents};
390 PyArrayObject* pValDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValX, NPY_FLOAT, (
void*)valDataX);
391 PyArrayObject* pValDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValY, NPY_FLOAT, (
void*)valDataY);
392 PyArrayObject* pValDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsValWeights, NPY_FLOAT, (
void*)valDataWeights);
400 Log() << kINFO <<
"Print Training Model Architecture" <<
Endl;
407 PyDict_SetItemString(
fLocalNS,
"batchSize", pBatchSize);
408 PyDict_SetItemString(
fLocalNS,
"numEpochs", pNumEpochs);
411 PyRunString(
"train_dataset = torch.utils.data.TensorDataset(torch.Tensor(trainX), torch.Tensor(trainY))",
412 "Failed to create pytorch train Dataset.");
414 PyRunString(
"train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batchSize, shuffle=False)",
415 "Failed to create pytorch train Dataloader.");
419 PyRunString(
"val_dataset = torch.utils.data.TensorDataset(torch.Tensor(valX), torch.Tensor(valY))",
420 "Failed to create pytorch validation Dataset.");
422 PyRunString(
"val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=batchSize, shuffle=False)",
423 "Failed to create pytorch validation Dataloader.");
430 "schedulerSteps = {}\n"
431 "for c in strScheduleSteps.split(';'):\n"
432 " x = c.split(',')\n"
433 " schedulerSteps[int(x[0])] = float(x[1])\n",
437 PyRunString(
"def schedule(optimizer, epoch, schedulerSteps=schedulerSteps):\n"
438 " if epoch in schedulerSteps:\n"
439 " for param_group in optimizer.param_groups:\n"
440 " param_group['lr'] = float(schedulerSteps[epoch])\n",
447 PyRunString(
"schedule = None; schedulerSteps = None",
"Failed to set scheduler to None.");
454 " if curr_val<=best_val:\n"
455 " best_val = curr_val\n"
456 " best_model_jitted = torch.jit.script(model)\n"
457 " torch.jit.save(best_model_jitted, save_path)\n"
459 "Failed to setup training with option: SaveBestOnly");
460 Log() << kINFO <<
"Option SaveBestOnly: Only model weights with smallest validation loss will be stored" <<
Endl;
463 PyRunString(
"save_best = None",
"Failed to set save_best to None.");
470 PyRunString(
"trained_model = fit(model, train_loader, val_loader, num_epochs=numEpochs, batch_size=batchSize,"
471 "optimizer=optimizer, criterion=criterion, save_best=save_best, scheduler=(schedule, schedulerSteps))",
472 "Failed to train model");
483 PyRunString(
"trained_model_jitted = torch.jit.script(trained_model)",
484 "Model not scriptable. Failed to convert to torch script.");
496 delete[] trainDataWeights;
499 delete[] valDataWeights;
521 PyRunString(
"for i,p in enumerate(predict(model, vals)): output[i]=p\n",
522 "Failed to get predictions");
539 if (firstEvt > lastEvt || lastEvt > nEvents) lastEvt = nEvents;
540 if (firstEvt < 0) firstEvt = 0;
541 nEvents = lastEvt-firstEvt;
550 <<
" sample (" << nEvents <<
" events)" <<
Endl;
552 float* data =
new float[nEvents*
fNVars];
553 for (
UInt_t i=0; i<nEvents; i++) {
557 data[j + i*
fNVars] =
e->GetValue(j);
561 npy_intp dimsData[2] = {(npy_intp)nEvents, (npy_intp)
fNVars};
562 PyArrayObject* pDataMvaValues = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsData, NPY_FLOAT, (
void*)data);
563 if (pDataMvaValues==0)
Log() <<
"Failed to load data to Python array" <<
Endl;
568 if (pModel==0)
Log() << kFATAL <<
"Failed to get model Python object" <<
Endl;
571 if (pPredict==0)
Log() << kFATAL <<
"Failed to get Python predict function" <<
Endl;
575 PyArrayObject* pPredictions = (PyArrayObject*) PyObject_CallFunctionObjArgs(pPredict, pModel, pDataMvaValues, NULL);
576 if (pPredictions==0)
Log() << kFATAL <<
"Failed to get predictions" <<
Endl;
581 std::vector<double> mvaValues(nEvents);
582 float* predictionsData = (
float*) PyArray_DATA(pPredictions);
583 for (
UInt_t i=0; i<nEvents; i++) {
589 <<
"Elapsed time for evaluation of " << nEvents <<
" events: "
608 PyRunString(
"for i,p in enumerate(predict(model, vals)): output[i]=p\n",
609 "Failed to get predictions");
637 PyRunString(
"for i,p in enumerate(predict(model, vals)): output[i]=p\n",
638 "Failed to get predictions");
650 Log() <<
"PyTorch is a scientific computing package supporting" <<
Endl;
651 Log() <<
"automatic differentiation. This method wraps the training" <<
Endl;
652 Log() <<
"and predictions steps of the PyTorch Python package for" <<
Endl;
653 Log() <<
"TMVA, so that dataloading, preprocessing and evaluation" <<
Endl;
654 Log() <<
"can be done within the TMVA system. To use this PyTorch" <<
Endl;
655 Log() <<
"interface, you need to generatea model with PyTorch first." <<
Endl;
656 Log() <<
"Then, this model can be loaded and trained in TMVA." <<
Endl;
#define REGISTER_METHOD(CLASS)
for example
char * Form(const char *fmt,...)
OptionBase * DeclareOptionRef(T &ref, const TString &name, const TString &desc="")
Class that contains all the data information.
UInt_t GetNClasses() const
UInt_t GetNTargets() const
Types::ETreeType GetCurrentType() const
Long64_t GetNEvents(Types::ETreeType type=Types::kMaxTreeType) const
Long64_t GetNTrainingEvents() const
void SetCurrentEvent(Long64_t ievt) const
void SetTarget(UInt_t itgt, Float_t value)
set the target value (dimension itgt) to value
Float_t GetTarget(UInt_t itgt) const
const char * GetName() const
Types::EAnalysisType GetAnalysisType() const
const TString & GetWeightFileDir() const
const TString & GetMethodName() const
const Event * GetEvent() const
DataSetInfo & DataInfo() const
virtual void TestClassification()
initialization
UInt_t GetNVariables() const
TransformationHandler & GetTransformationHandler(Bool_t takeReroutedIfAvailable=true)
void NoErrorCalc(Double_t *const err, Double_t *const errUpper)
const Event * GetTrainingEvent(Long64_t ievt) const
Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t)
std::vector< Float_t > & GetMulticlassValues()
std::vector< float > fOutput
MethodPyTorch(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
virtual void TestClassification()
initialization
std::vector< Double_t > GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t logProgress)
get all the MVA values for the events of the current Data type
TString fNumValidationString
UInt_t GetNumValidationSamples()
Validation of the ValidationSize option.
void GetHelpMessage() const
TString fLearningRateSchedule
std::vector< Float_t > & GetRegressionValues()
TString fFilenameTrainedModel
void SetupPyTorchModel(Bool_t loadTrainedModel)
Double_t GetMvaValue(Double_t *errLower, Double_t *errUpper)
static int PyIsInitialized()
Check Python interpreter initialization status.
static PyObject * fGlobalNS
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=Py_single_input)
Execute Python code from string.
Timing information for training and evaluation of MVA methods.
TString GetElapsedTime(Bool_t Scientific=kTRUE)
returns pretty string with elapsed time
Singleton class for Global types used by TMVA.
Bool_t IsFloat() const
Returns kTRUE if string contains a floating point or integer number.
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString.
create variable transformations
MsgLogger & Endl(MsgLogger &ml)