Logo ROOT  
Reference Guide
MethodPyKeras.cxx
Go to the documentation of this file.
1// @(#)root/tmva/pymva $Id$
2// Author: Stefan Wunsch, 2016
3
4#include <Python.h>
6
7#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
8#include <numpy/arrayobject.h>
9
10#include "TMVA/Types.h"
11#include "TMVA/Config.h"
13#include "TMVA/Results.h"
16#include "TMVA/Tools.h"
17#include "TMVA/Timer.h"
18#include "TSystem.h"
19
20using namespace TMVA;
21
22namespace TMVA {
23namespace Internal {
24class PyGILRAII {
25 PyGILState_STATE m_GILState;
26
27public:
28 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
29 ~PyGILRAII() { PyGILState_Release(m_GILState); }
30};
31} // namespace Internal
32} // namespace TMVA
33
34REGISTER_METHOD(PyKeras)
35
37
38MethodPyKeras::MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption)
39 : PyMethodBase(jobName, Types::kPyKeras, methodTitle, dsi, theOption) {
40 fNumEpochs = 10;
41 fBatchSize = 100;
42 fVerbose = 1;
43 fContinueTraining = false;
44 fSaveBestOnly = true;
46 fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
47 fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
48 fTensorBoard = ""; // empty string deactivates TensorBoard callback
49}
50
51MethodPyKeras::MethodPyKeras(DataSetInfo &theData, const TString &theWeightFile)
52 : PyMethodBase(Types::kPyKeras, theData, theWeightFile) {
53 fNumEpochs = 10;
54 fNumThreads = 0;
55 fBatchSize = 100;
56 fVerbose = 1;
57 fContinueTraining = false;
58 fSaveBestOnly = true;
60 fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
61 fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
62 fTensorBoard = ""; // empty string deactivates TensorBoard callback
63}
64
66}
67
69 if (type == Types::kRegression) return kTRUE;
70 if (type == Types::kClassification && numberClasses == 2) return kTRUE;
71 if (type == Types::kMulticlass && numberClasses >= 2) return kTRUE;
72 return kFALSE;
73}
74
75///////////////////////////////////////////////////////////////////////////////
76
78 DeclareOptionRef(fFilenameModel, "FilenameModel", "Filename of the initial Keras model");
79 DeclareOptionRef(fFilenameTrainedModel, "FilenameTrainedModel", "Filename of the trained output Keras model");
80 DeclareOptionRef(fBatchSize, "BatchSize", "Training batch size");
81 DeclareOptionRef(fNumEpochs, "NumEpochs", "Number of training epochs");
82 DeclareOptionRef(fNumThreads, "NumThreads", "Number of CPU threads (only for Tensorflow backend)");
83 DeclareOptionRef(fGpuOptions, "GpuOptions", "GPU options for tensorflow, such as allow_growth");
84 DeclareOptionRef(fUseTFKeras, "tf.keras", "Use tensorflow from Keras");
85 DeclareOptionRef(fVerbose, "Verbose", "Keras verbosity during training");
86 DeclareOptionRef(fContinueTraining, "ContinueTraining", "Load weights from previous training");
87 DeclareOptionRef(fSaveBestOnly, "SaveBestOnly", "Store only weights with smallest validation loss");
88 DeclareOptionRef(fTriesEarlyStopping, "TriesEarlyStopping", "Number of epochs with no improvement in validation loss after which training will be stopped. The default or a negative number deactivates this option.");
89 DeclareOptionRef(fLearningRateSchedule, "LearningRateSchedule", "Set new learning rate during training at specific epochs, e.g., \"50,0.01;70,0.005\"");
90 DeclareOptionRef(fTensorBoard, "TensorBoard",
91 "Write a log during training to visualize and monitor the training performance with TensorBoard");
92
93 DeclareOptionRef(fNumValidationString = "20%", "ValidationSize", "Part of the training data to use for validation. "
94 "Specify as 0.2 or 20% to use a fifth of the data set as validation set. "
95 "Specify as 100 to use exactly 100 events. (Default: 20%)");
96 DeclareOptionRef(fUserCodeName = "", "UserCode",
97 "Optional python code provided by the user to be executed before loading the Keras model");
98}
99
100////////////////////////////////////////////////////////////////////////////////
101/// Validation of the ValidationSize option. Allowed formats are 20%, 0.2 and
102/// 100 etc.
103/// - 20% and 0.2 selects 20% of the training set as validation data.
104/// - 100 selects 100 events as the validation data.
105///
106/// @return number of samples in validation set
107///
109{
110 Int_t nValidationSamples = 0;
111 UInt_t trainingSetSize = GetEventCollection(Types::kTraining).size();
112
113 // Parsing + Validation
114 // --------------------
115 if (fNumValidationString.EndsWith("%")) {
116 // Relative spec. format 20%
117 TString intValStr = TString(fNumValidationString.Strip(TString::kTrailing, '%'));
118
119 if (intValStr.IsFloat()) {
120 Double_t valSizeAsDouble = fNumValidationString.Atof() / 100.0;
121 nValidationSamples = GetEventCollection(Types::kTraining).size() * valSizeAsDouble;
122 } else {
123 Log() << kFATAL << "Cannot parse number \"" << fNumValidationString
124 << "\". Expected string like \"20%\" or \"20.0%\"." << Endl;
125 }
126 } else if (fNumValidationString.IsFloat()) {
127 Double_t valSizeAsDouble = fNumValidationString.Atof();
128
129 if (valSizeAsDouble < 1.0) {
130 // Relative spec. format 0.2
131 nValidationSamples = GetEventCollection(Types::kTraining).size() * valSizeAsDouble;
132 } else {
133 // Absolute spec format 100 or 100.0
134 nValidationSamples = valSizeAsDouble;
135 }
136 } else {
137 Log() << kFATAL << "Cannot parse number \"" << fNumValidationString << "\". Expected string like \"0.2\" or \"100\"."
138 << Endl;
139 }
140
141 // Value validation
142 // ----------------
143 if (nValidationSamples < 0) {
144 Log() << kFATAL << "Validation size \"" << fNumValidationString << "\" is negative." << Endl;
145 }
146
147 if (nValidationSamples == 0) {
148 Log() << kFATAL << "Validation size \"" << fNumValidationString << "\" is zero." << Endl;
149 }
150
151 if (nValidationSamples >= (Int_t)trainingSetSize) {
152 Log() << kFATAL << "Validation size \"" << fNumValidationString
153 << "\" is larger than or equal in size to training set (size=\"" << trainingSetSize << "\")." << Endl;
154 }
155
156 return nValidationSamples;
157}
158
159/// Function processing the options
160/// This is called only when creating the method before training not when
161/// readinf from XML file. Called from MethodBase::ProcessSetup
162/// that is called from Factory::BookMethod
164
165 // Set default filename for trained model if option is not used
167 fFilenameTrainedModel = GetWeightFileDir() + "/TrainedModel_" + GetName() + ".h5";
168 }
169
170 // Setup model, either the initial model from `fFilenameModel` or
171 // the trained model from `fFilenameTrainedModel`
172 if (fContinueTraining) Log() << kINFO << "Continue training with trained model" << Endl;
174}
175
176void MethodPyKeras::SetupKerasModel(bool loadTrainedModel) {
177
178 // initialize first Keras. This is done only here when class has
179 // all state variable set from options or read from XML file
180 // Import Keras
181
182 if (fUseTFKeras)
183 Log() << kINFO << "Setting up tf.keras" << Endl;
184 else
185 Log() << kINFO << "Setting up keras with " << gSystem->Getenv("KERAS_BACKEND") << " backend" << Endl;
186
187 bool useTFBackend = kFALSE;
188 bool kerasIsCompatible = kTRUE;
189 bool kerasIsPresent = kFALSE;
190
191 if (!fUseTFKeras) {
192 auto ret = PyRun_String("import keras", Py_single_input, fGlobalNS, fLocalNS);
193 // need importing also in global namespace
194 if (ret != nullptr) ret = PyRun_String("import keras", Py_single_input, fGlobalNS, fGlobalNS);
195 if (ret != nullptr)
196 kerasIsPresent = kTRUE;
197 if (kerasIsPresent) {
198 // check compatibility with tensorflow
199 if (GetKerasBackend() == kTensorFlow ) {
200 useTFBackend = kTRUE;
201
202 PyRunString("keras_major_version = int(keras.__version__.split('.')[0])");
203 PyRunString("keras_minor_version = int(keras.__version__.split('.')[1])");
204 PyObject *pyKerasMajorVersion = PyDict_GetItemString(fLocalNS, "keras_major_version");
205 PyObject *pyKerasMinorVersion = PyDict_GetItemString(fLocalNS, "keras_minor_version");
206 int kerasMajorVersion = PyLong_AsLong(pyKerasMajorVersion);
207 int kerasMinorVersion = PyLong_AsLong(pyKerasMinorVersion);
208 Log() << kINFO << "Using Keras version " << kerasMajorVersion << "." << kerasMinorVersion << Endl;
209 // only version 2.3 is latest multi-backend version.
210 // version 2.4 is just tf.keras and should not be used in standalone and will not work in this workflow
211 // see https://github.com/keras-team/keras/releases/tag/2.4.0
212 // for example variable keras.backend.tensorflow_backend will not exist anymore in keras 2.4
213 kerasIsCompatible = (kerasMajorVersion >= 2 && kerasMinorVersion == 3);
214
215 }
216 } else {
217 // Keras is not found. try tyo use tf.keras
218 Log() << kINFO << "Keras is not found. Trying using tf.keras" << Endl;
219 fUseTFKeras = 1;
220 }
221 }
222
223 // import Tensoprflow (if requested or because is keras backend)
224 if (fUseTFKeras || useTFBackend) {
225 auto ret = PyRun_String("import tensorflow as tf", Py_single_input, fGlobalNS, fLocalNS);
226 if (ret != nullptr) ret = PyRun_String("import tensorflow as tf", Py_single_input, fGlobalNS, fGlobalNS);
227 if (ret == nullptr) {
228 Log() << kFATAL << "Importing TensorFlow failed" << Endl;
229 }
230 // check tensorflow version
231 PyRunString("tf_major_version = int(tf.__version__.split('.')[0])");
232 PyObject *pyTfVersion = PyDict_GetItemString(fLocalNS, "tf_major_version");
233 int tfVersion = PyLong_AsLong(pyTfVersion);
234 Log() << kINFO << "Using TensorFlow version " << tfVersion << Endl;
235
236 if (tfVersion < 2) {
237 if (fUseTFKeras == 1) {
238 Log() << kWARNING << "Using TensorFlow version 1.x which does not contain tf.keras - use then TensorFlow as Keras backend" << Endl;
240 // case when Keras was not found
241 if (!kerasIsPresent) {
242 Log() << kFATAL << "Keras is not present and not a suitable TensorFlow version is found " << Endl;
243 return;
244 }
245 }
246 }
247 else {
248 // using version larger than 2.0 - can use tf.keras
249 if (!kerasIsCompatible) {
250 Log() << kWARNING << "The Keras version is not compatible with TensorFlow 2. Use instead tf.keras" << Endl;
251 fUseTFKeras = 1;
252 }
253 }
254
255 // if keras 2.3 and tensorflow 2 are found. Use tf.keras or keras ?
256 // at the moment default is tf.keras=false to keep compatibility
257 // but this might change in future releases
258 if (fUseTFKeras) {
259 Log() << kINFO << "Use Keras version from TensorFlow : tf.keras" << Endl;
260 fKerasString = "tf.keras";
261 PyRunString("K = tf.keras.backend");
262 PyRun_String("K = tf.keras.backend", Py_single_input, fGlobalNS, fGlobalNS);
263 }
264 else {
265 Log() << kINFO << "Use TensorFlow as Keras backend" << Endl;
266 fKerasString = "keras";
267 PyRunString("from keras.backend import tensorflow_backend as K");
268 PyRun_String("from keras.backend import tensorflow_backend as K", Py_single_input, fGlobalNS, fGlobalNS);
269 }
270
271 // extra options for tensorflow
272 // use different naming in tf2 for ConfigProto and Session
273 TString configProto = (tfVersion >= 2) ? "tf.compat.v1.ConfigProto" : "tf.ConfigProto";
274 TString session = (tfVersion >= 2) ? "tf.compat.v1.Session" : "tf.Session";
275
276 // in case specify number of threads
277 int num_threads = fNumThreads;
278 if (num_threads > 0) {
279 Log() << kINFO << "Setting the CPU number of threads = " << num_threads << Endl;
280
282 TString::Format("session_conf = %s(intra_op_parallelism_threads=%d,inter_op_parallelism_threads=%d)",
283 configProto.Data(), num_threads, num_threads));
284 } else
285 PyRunString(TString::Format("session_conf = %s()", configProto.Data()));
286
287 // applying GPU options such as allow_growth=True to avoid allocating all memory on GPU
288 // that prevents running later TMVA-GPU
289 // Also new Nvidia RTX cards (e.g. RTX 2070) require this option
290 if (!fGpuOptions.IsNull()) {
291 TObjArray *optlist = fGpuOptions.Tokenize(",");
292 for (int item = 0; item < optlist->GetEntries(); ++item) {
293 Log() << kINFO << "Applying GPU option: gpu_options." << optlist->At(item)->GetName() << Endl;
294 PyRunString(TString::Format("session_conf.gpu_options.%s", optlist->At(item)->GetName()));
295 }
296 }
297 PyRunString(TString::Format("sess = %s(config=session_conf)", session.Data()));
298
299 if (tfVersion < 2) {
300 PyRunString("K.set_session(sess)");
301 } else {
302 PyRunString("tf.compat.v1.keras.backend.set_session(sess)");
303 }
304 }
305 // case not using a Tensorflow backend
306 else {
307 fKerasString = "keras";
308 if (fNumThreads > 0)
309 Log() << kWARNING << "Cannot set the given " << fNumThreads << " threads when not using tensorflow as backend"
310 << Endl;
311 if (!fGpuOptions.IsNull()) {
312 Log() << kWARNING << "Cannot set the given GPU option " << fGpuOptions
313 << " when not using tensorflow as backend" << Endl;
314 }
315 }
316
317 /*
318 * Load Keras model from file
319 */
320
321 Log() << kINFO << " Loading Keras Model " << Endl;
322
323 PyRunString("load_model_custom_objects=None");
324
325
326
327 if (!fUserCodeName.IsNull()) {
328 Log() << kINFO << " Executing user initialization code from " << fUserCodeName << Endl;
329
330
331 // run some python code provided by user for model initialization if needed
332 TString cmd = "exec(open('" + fUserCodeName + "').read())";
333 TString errmsg = "Error executing the provided user code";
334 PyRunString(cmd, errmsg);
335
336 PyRunString("print('custom objects for loading model : ',load_model_custom_objects)");
337 }
338
339 // Load initial model or already trained model
340 TString filenameLoadModel;
341 if (loadTrainedModel) {
342 filenameLoadModel = fFilenameTrainedModel;
343 }
344 else {
345 filenameLoadModel = fFilenameModel;
346 }
347
348 PyRunString("model = " + fKerasString + ".models.load_model('" + filenameLoadModel +
349 "', custom_objects=load_model_custom_objects)", "Failed to load Keras model from file: " + filenameLoadModel);
350
351 Log() << kINFO << "Loaded model from file: " << filenameLoadModel << Endl;
352
353
354 /*
355 * Init variables and weights
356 */
357
358 // Get variables, classes and target numbers
362 else Log() << kFATAL << "Selected analysis type is not implemented" << Endl;
363
364 // Init evaluation (needed for getMvaValue)
365 fVals = new float[fNVars]; // holds values used for classification and regression
366 npy_intp dimsVals[2] = {(npy_intp)1, (npy_intp)fNVars};
367 PyArrayObject* pVals = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsVals, NPY_FLOAT, (void*)fVals);
368 PyDict_SetItemString(fLocalNS, "vals", (PyObject*)pVals);
369
370 fOutput.resize(fNOutputs); // holds classification probabilities or regression output
371 npy_intp dimsOutput[2] = {(npy_intp)1, (npy_intp)fNOutputs};
372 PyArrayObject* pOutput = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsOutput, NPY_FLOAT, (void*)&fOutput[0]);
373 PyDict_SetItemString(fLocalNS, "output", (PyObject*)pOutput);
374
375 // Mark the model as setup
376 fModelIsSetup = true;
377}
378
379/// Initialization function called from MethodBase::SetupMethod()
380/// Note that option string are not yet filled with their values.
381/// This is done before ProcessOption method or after reading from XML file
383
385
386 if (!PyIsInitialized()) {
387 Log() << kFATAL << "Python is not initialized" << Endl;
388 }
389 _import_array(); // required to use numpy arrays
390
391 // NOTE: sys.argv has to be cleared because otherwise TensorFlow breaks
392 PyRunString("import sys; sys.argv = ['']", "Set sys.argv failed");
393
394 // Set flag that model is not setup
395 fModelIsSetup = false;
396}
397
399
400 if(!fModelIsSetup) Log() << kFATAL << "Model is not setup for training" << Endl;
401
402 /*
403 * Load training data to numpy array
404 */
405
406 UInt_t nAllEvents = Data()->GetNTrainingEvents();
407 UInt_t nValEvents = GetNumValidationSamples();
408 UInt_t nTrainingEvents = nAllEvents - nValEvents;
409
410 Log() << kINFO << "Split TMVA training data in " << nTrainingEvents << " training events and "
411 << nValEvents << " validation events" << Endl;
412
413 float* trainDataX = new float[nTrainingEvents*fNVars];
414 float* trainDataY = new float[nTrainingEvents*fNOutputs];
415 float* trainDataWeights = new float[nTrainingEvents];
416 for (UInt_t i=0; i<nTrainingEvents; i++) {
417 const TMVA::Event* e = GetTrainingEvent(i);
418 // Fill variables
419 for (UInt_t j=0; j<fNVars; j++) {
420 trainDataX[j + i*fNVars] = e->GetValue(j);
421 }
422 // Fill targets
423 // NOTE: For classification, convert class number in one-hot vector,
424 // e.g., 1 -> [0, 1] or 0 -> [1, 0] for binary classification
426 for (UInt_t j=0; j<fNOutputs; j++) {
427 trainDataY[j + i*fNOutputs] = 0;
428 }
429 trainDataY[e->GetClass() + i*fNOutputs] = 1;
430 }
431 else if (GetAnalysisType() == Types::kRegression) {
432 for (UInt_t j=0; j<fNOutputs; j++) {
433 trainDataY[j + i*fNOutputs] = e->GetTarget(j);
434 }
435 }
436 else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
437 // Fill weights
438 // NOTE: If no weight branch is given, this defaults to ones for all events
439 trainDataWeights[i] = e->GetWeight();
440 }
441
442 npy_intp dimsTrainX[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNVars};
443 npy_intp dimsTrainY[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNOutputs};
444 npy_intp dimsTrainWeights[1] = {(npy_intp)nTrainingEvents};
445 PyArrayObject* pTrainDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainX, NPY_FLOAT, (void*)trainDataX);
446 PyArrayObject* pTrainDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainY, NPY_FLOAT, (void*)trainDataY);
447 PyArrayObject* pTrainDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsTrainWeights, NPY_FLOAT, (void*)trainDataWeights);
448 PyDict_SetItemString(fLocalNS, "trainX", (PyObject*)pTrainDataX);
449 PyDict_SetItemString(fLocalNS, "trainY", (PyObject*)pTrainDataY);
450 PyDict_SetItemString(fLocalNS, "trainWeights", (PyObject*)pTrainDataWeights);
451
452 /*
453 * Load validation data to numpy array
454 */
455
456 // NOTE: from TMVA, we get the validation data as a subset of all the training data
457 // we will not use test data for validation. They will be used for the real testing
458
459
460 float* valDataX = new float[nValEvents*fNVars];
461 float* valDataY = new float[nValEvents*fNOutputs];
462 float* valDataWeights = new float[nValEvents];
463 //validation events follows the trainig one in the TMVA training vector
464 for (UInt_t i=0; i< nValEvents ; i++) {
465 UInt_t ievt = nTrainingEvents + i; // TMVA event index
466 const TMVA::Event* e = GetTrainingEvent(ievt);
467 // Fill variables
468 for (UInt_t j=0; j<fNVars; j++) {
469 valDataX[j + i*fNVars] = e->GetValue(j);
470 }
471 // Fill targets
473 for (UInt_t j=0; j<fNOutputs; j++) {
474 valDataY[j + i*fNOutputs] = 0;
475 }
476 valDataY[e->GetClass() + i*fNOutputs] = 1;
477 }
478 else if (GetAnalysisType() == Types::kRegression) {
479 for (UInt_t j=0; j<fNOutputs; j++) {
480 valDataY[j + i*fNOutputs] = e->GetTarget(j);
481 }
482 }
483 else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
484 // Fill weights
485 valDataWeights[i] = e->GetWeight();
486 }
487
488 npy_intp dimsValX[2] = {(npy_intp)nValEvents, (npy_intp)fNVars};
489 npy_intp dimsValY[2] = {(npy_intp)nValEvents, (npy_intp)fNOutputs};
490 npy_intp dimsValWeights[1] = {(npy_intp)nValEvents};
491 PyArrayObject* pValDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValX, NPY_FLOAT, (void*)valDataX);
492 PyArrayObject* pValDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValY, NPY_FLOAT, (void*)valDataY);
493 PyArrayObject* pValDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsValWeights, NPY_FLOAT, (void*)valDataWeights);
494 PyDict_SetItemString(fLocalNS, "valX", (PyObject*)pValDataX);
495 PyDict_SetItemString(fLocalNS, "valY", (PyObject*)pValDataY);
496 PyDict_SetItemString(fLocalNS, "valWeights", (PyObject*)pValDataWeights);
497
498 /*
499 * Train Keras model
500 */
501 Log() << kINFO << "Training Model Summary" << Endl;
502 PyRunString("model.summary()");
503
504 // Setup parameters
505
506 PyObject* pBatchSize = PyLong_FromLong(fBatchSize);
507 PyObject* pNumEpochs = PyLong_FromLong(fNumEpochs);
508 PyObject* pVerbose = PyLong_FromLong(fVerbose);
509 PyDict_SetItemString(fLocalNS, "batchSize", pBatchSize);
510 PyDict_SetItemString(fLocalNS, "numEpochs", pNumEpochs);
511 PyDict_SetItemString(fLocalNS, "verbose", pVerbose);
512
513 // Setup training callbacks
514 PyRunString("callbacks = []");
515
516 // Callback: Save only weights with smallest validation loss
517 if (fSaveBestOnly) {
518 PyRunString("callbacks.append(" + fKerasString +".callbacks.ModelCheckpoint('"+fFilenameTrainedModel+"', monitor='val_loss', verbose=verbose, save_best_only=True, mode='auto'))", "Failed to setup training callback: SaveBestOnly");
519 Log() << kINFO << "Option SaveBestOnly: Only model weights with smallest validation loss will be stored" << Endl;
520 }
521
522 // Callback: Stop training early if no improvement in validation loss is observed
523 if (fTriesEarlyStopping>=0) {
524 TString tries;
525 tries.Form("%i", fTriesEarlyStopping);
526 PyRunString("callbacks.append(" + fKerasString + ".callbacks.EarlyStopping(monitor='val_loss', patience="+tries+", verbose=verbose, mode='auto'))", "Failed to setup training callback: TriesEarlyStopping");
527 Log() << kINFO << "Option TriesEarlyStopping: Training will stop after " << tries << " number of epochs with no improvement of validation loss" << Endl;
528 }
529
530 // Callback: Learning rate scheduler
531 if (fLearningRateSchedule!="") {
532 // Setup a python dictionary with the desired learning rate steps
533 PyRunString("strScheduleSteps = '"+fLearningRateSchedule+"'\n"
534 "schedulerSteps = {}\n"
535 "for c in strScheduleSteps.split(';'):\n"
536 " x = c.split(',')\n"
537 " schedulerSteps[int(x[0])] = float(x[1])\n",
538 "Failed to setup steps for scheduler function from string: "+fLearningRateSchedule,
539 Py_file_input);
540 // Set scheduler function as piecewise function with given steps
541 PyRunString("def schedule(epoch, model=model, schedulerSteps=schedulerSteps):\n"
542 " if epoch in schedulerSteps: return float(schedulerSteps[epoch])\n"
543 " else: return float(model.optimizer.lr.get_value())\n",
544 "Failed to setup scheduler function with string: "+fLearningRateSchedule,
545 Py_file_input);
546 // Setup callback
547 PyRunString("callbacks.append(" + fKerasString + ".callbacks.LearningRateScheduler(schedule))",
548 "Failed to setup training callback: LearningRateSchedule");
549 Log() << kINFO << "Option LearningRateSchedule: Set learning rate during training: " << fLearningRateSchedule << Endl;
550 }
551
552 // Callback: TensorBoard
553 if (fTensorBoard != "") {
554 TString logdir = TString("'") + fTensorBoard + TString("'");
556 "callbacks.append(" + fKerasString + ".callbacks.TensorBoard(log_dir=" + logdir +
557 ", histogram_freq=0, batch_size=batchSize, write_graph=True, write_grads=False, write_images=False))",
558 "Failed to setup training callback: TensorBoard");
559 Log() << kINFO << "Option TensorBoard: Log files for training monitoring are stored in: " << logdir << Endl;
560 }
561
562 // Train model
563 PyRunString("history = model.fit(trainX, trainY, sample_weight=trainWeights, batch_size=batchSize, epochs=numEpochs, verbose=verbose, validation_data=(valX, valY, valWeights), callbacks=callbacks)",
564 "Failed to train model");
565
566
567 std::vector<float> fHistory; // Hold training history (val_acc or loss etc)
568 fHistory.resize(fNumEpochs); // holds training loss or accuracy output
569 npy_intp dimsHistory[1] = { (npy_intp)fNumEpochs};
570 PyArrayObject* pHistory = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsHistory, NPY_FLOAT, (void*)&fHistory[0]);
571 PyDict_SetItemString(fLocalNS, "HistoryOutput", (PyObject*)pHistory);
572
573 // Store training history data
574 Int_t iHis=0;
575 PyRunString("number_of_keys=len(history.history.keys())");
576 PyObject* PyNkeys=PyDict_GetItemString(fLocalNS, "number_of_keys");
577 int nkeys=PyLong_AsLong(PyNkeys);
578 for (iHis=0; iHis<nkeys; iHis++) {
579
580 PyRunString(TString::Format("copy_string=str(list(history.history.keys())[%d])",iHis));
581 PyObject* stra=PyDict_GetItemString(fLocalNS, "copy_string");
582 if(!stra) break;
583#if PY_MAJOR_VERSION < 3 // for Python2
584 const char *stra_name = PyBytes_AsString(stra);
585 // need to add string delimiter for Python2
586 TString sname = TString::Format("'%s'",stra_name);
587 const char * name = sname.Data();
588#else // for Python3
589 PyObject* repr = PyObject_Repr(stra);
590 PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~");
591 const char *name = PyBytes_AsString(str);
592#endif
593
594 Log() << kINFO << "Getting training history for item:" << iHis << " name = " << name << Endl;
595 PyRunString(TString::Format("for i,p in enumerate(history.history[%s]):\n HistoryOutput[i]=p\n",name),
596 TString::Format("Failed to get %s from training history",name));
597 for (size_t i=0; i<fHistory.size(); i++)
598 fTrainHistory.AddValue(name,i+1,fHistory[i]);
599
600 }
601//#endif
602
603 /*
604 * Store trained model to file (only if option 'SaveBestOnly' is NOT activated,
605 * because we do not want to override the best model checkpoint)
606 */
607
608 if (!fSaveBestOnly) {
609 PyRunString("model.save('"+fFilenameTrainedModel+"', overwrite=True)",
610 "Failed to save trained model: "+fFilenameTrainedModel);
611 Log() << kINFO << "Trained model written to file: " << fFilenameTrainedModel << Endl;
612 }
613
614 /*
615 * Clean-up
616 */
617
618 delete[] trainDataX;
619 delete[] trainDataY;
620 delete[] trainDataWeights;
621 delete[] valDataX;
622 delete[] valDataY;
623 delete[] valDataWeights;
624}
625
628}
629
631 // Cannot determine error
632 NoErrorCalc(errLower, errUpper);
633
634 // Check whether the model is setup
635 // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
636 if (!fModelIsSetup) {
637 // Setup the trained model
638 SetupKerasModel(true);
639 }
640
641 // Get signal probability (called mvaValue here)
642 const TMVA::Event* e = GetEvent();
643 for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
644 PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
645 "Failed to get predictions");
646
648}
649
650std::vector<Double_t> MethodPyKeras::GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t logProgress) {
651 // Check whether the model is setup
652 // NOTE: Unfortunately this is needed because during evaluation ProcessOptions is not called again
653 if (!fModelIsSetup) {
654 // Setup the trained model
655 SetupKerasModel(true);
656 }
657
658 // Load data to numpy array
659 Long64_t nEvents = Data()->GetNEvents();
660 if (firstEvt > lastEvt || lastEvt > nEvents) lastEvt = nEvents;
661 if (firstEvt < 0) firstEvt = 0;
662 nEvents = lastEvt-firstEvt;
663
664 // use timer
665 Timer timer( nEvents, GetName(), kTRUE );
666
667 if (logProgress)
668 Log() << kHEADER << Form("[%s] : ",DataInfo().GetName())
669 << "Evaluation of " << GetMethodName() << " on "
670 << (Data()->GetCurrentType() == Types::kTraining ? "training" : "testing")
671 << " sample (" << nEvents << " events)" << Endl;
672
673 float* data = new float[nEvents*fNVars];
674 for (UInt_t i=0; i<nEvents; i++) {
675 Data()->SetCurrentEvent(i);
676 const TMVA::Event *e = GetEvent();
677 for (UInt_t j=0; j<fNVars; j++) {
678 data[j + i*fNVars] = e->GetValue(j);
679 }
680 }
681
682 npy_intp dimsData[2] = {(npy_intp)nEvents, (npy_intp)fNVars};
683 PyArrayObject* pDataMvaValues = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsData, NPY_FLOAT, (void*)data);
684 if (pDataMvaValues==0) Log() << "Failed to load data to Python array" << Endl;
685
686 // Get prediction for all events
687 PyObject* pModel = PyDict_GetItemString(fLocalNS, "model");
688 if (pModel==0) Log() << kFATAL << "Failed to get model Python object" << Endl;
689 PyArrayObject* pPredictions = (PyArrayObject*) PyObject_CallMethod(pModel, (char*)"predict", (char*)"O", pDataMvaValues);
690 if (pPredictions==0) Log() << kFATAL << "Failed to get predictions" << Endl;
691 delete[] data;
692
693 // Load predictions to double vector
694 // NOTE: The signal probability is given at the output
695 std::vector<double> mvaValues(nEvents);
696 float* predictionsData = (float*) PyArray_DATA(pPredictions);
697 for (UInt_t i=0; i<nEvents; i++) {
698 mvaValues[i] = (double) predictionsData[i*fNOutputs + TMVA::Types::kSignal];
699 }
700
701 if (logProgress) {
702 Log() << kINFO
703 << "Elapsed time for evaluation of " << nEvents << " events: "
704 << timer.GetElapsedTime() << " " << Endl;
705 }
706
707
708 return mvaValues;
709}
710
711std::vector<Float_t>& MethodPyKeras::GetRegressionValues() {
712 // Check whether the model is setup
713 // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
714 if (!fModelIsSetup){
715 // Setup the model and load weights
716 SetupKerasModel(true);
717 }
718
719 // Get regression values
720 const TMVA::Event* e = GetEvent();
721 for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
722 PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
723 "Failed to get predictions");
724
725 // Use inverse transformation of targets to get final regression values
726 Event * eTrans = new Event(*e);
727 for (UInt_t i=0; i<fNOutputs; ++i) {
728 eTrans->SetTarget(i,fOutput[i]);
729 }
730
731 const Event* eTrans2 = GetTransformationHandler().InverseTransform(eTrans);
732 for (UInt_t i=0; i<fNOutputs; ++i) {
733 fOutput[i] = eTrans2->GetTarget(i);
734 }
735
736 return fOutput;
737}
738
739std::vector<Float_t>& MethodPyKeras::GetMulticlassValues() {
740 // Check whether the model is setup
741 // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
742 if (!fModelIsSetup){
743 // Setup the model and load weights
744 SetupKerasModel(true);
745 }
746
747 // Get class probabilites
748 const TMVA::Event* e = GetEvent();
749 for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
750 PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
751 "Failed to get predictions");
752
753 return fOutput;
754}
755
757}
758
760// typical length of text line:
761// "|--------------------------------------------------------------|"
762 Log() << Endl;
763 Log() << "Keras is a high-level API for the Theano and Tensorflow packages." << Endl;
764 Log() << "This method wraps the training and predictions steps of the Keras" << Endl;
765 Log() << "Python package for TMVA, so that dataloading, preprocessing and" << Endl;
766 Log() << "evaluation can be done within the TMVA system. To use this Keras" << Endl;
767 Log() << "interface, you have to generate a model with Keras first. Then," << Endl;
768 Log() << "this model can be loaded and trained in TMVA." << Endl;
769 Log() << Endl;
770}
771
773 // get the keras backend
774
775 // in case we use tf.keras backend is tensorflow
776 if (UseTFKeras()) return kTensorFlow;
777
778 // check first if using tensorflow backend
779 PyRunString("keras_backend_is_set = keras.backend.backend() == \"tensorflow\"");
780 PyObject * keras_backend = PyDict_GetItemString(fLocalNS,"keras_backend_is_set");
781 if (keras_backend != nullptr && keras_backend == Py_True)
782 return kTensorFlow;
783
784 PyRunString("keras_backend_is_set = keras.backend.backend() == \"theano\"");
785 keras_backend = PyDict_GetItemString(fLocalNS,"keras_backend_is_set");
786 if (keras_backend != nullptr && keras_backend == Py_True)
787 return kTheano;
788
789 PyRunString("keras_backend_is_set = keras.backend.backend() == \"cntk\"");
790 keras_backend = PyDict_GetItemString(fLocalNS,"keras_backend_is_set");
791 if (keras_backend != nullptr && keras_backend == Py_True)
792 return kCNTK;
793
794 return kUndefined;
795}
796
798 // get the keras backend name
800 if (type == kTensorFlow) return "TensorFlow";
801 if (type == kTheano) return "Theano";
802 if (type == kCNTK) return "CNTK";
803 return "Undefined";
804}
#define PyBytes_AsString
Definition: CPyCppyy.h:86
#define REGISTER_METHOD(CLASS)
for example
double
Definition: Converters.cxx:939
_object PyObject
Definition: PyMethodBase.h:42
#define Py_single_input
Definition: PyMethodBase.h:44
#define e(i)
Definition: RSha256.hxx:103
int Int_t
Definition: RtypesCore.h:45
unsigned int UInt_t
Definition: RtypesCore.h:46
const Bool_t kFALSE
Definition: RtypesCore.h:101
bool Bool_t
Definition: RtypesCore.h:63
double Double_t
Definition: RtypesCore.h:59
long long Long64_t
Definition: RtypesCore.h:80
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define ClassImp(name)
Definition: Rtypes.h:364
char name[80]
Definition: TGX11.cxx:110
int type
Definition: TGX11.cxx:121
char * Form(const char *fmt,...)
R__EXTERN TSystem * gSystem
Definition: TSystem.h:559
OptionBase * DeclareOptionRef(T &ref, const TString &name, const TString &desc="")
MsgLogger & Log() const
Definition: Configurable.h:122
Class that contains all the data information.
Definition: DataSetInfo.h:62
UInt_t GetNClasses() const
Definition: DataSetInfo.h:155
UInt_t GetNTargets() const
Definition: DataSetInfo.h:128
Types::ETreeType GetCurrentType() const
Definition: DataSet.h:194
Long64_t GetNEvents(Types::ETreeType type=Types::kMaxTreeType) const
Definition: DataSet.h:206
Long64_t GetNTrainingEvents() const
Definition: DataSet.h:68
void SetCurrentEvent(Long64_t ievt) const
Definition: DataSet.h:88
void SetTarget(UInt_t itgt, Float_t value)
set the target value (dimension itgt) to value
Definition: Event.cxx:359
Float_t GetTarget(UInt_t itgt) const
Definition: Event.h:102
const char * GetName() const
Definition: MethodBase.h:334
Types::EAnalysisType GetAnalysisType() const
Definition: MethodBase.h:437
const TString & GetWeightFileDir() const
Definition: MethodBase.h:492
const TString & GetMethodName() const
Definition: MethodBase.h:331
const Event * GetEvent() const
Definition: MethodBase.h:751
DataSetInfo & DataInfo() const
Definition: MethodBase.h:410
virtual void TestClassification()
initialization
UInt_t GetNVariables() const
Definition: MethodBase.h:345
TransformationHandler & GetTransformationHandler(Bool_t takeReroutedIfAvailable=true)
Definition: MethodBase.h:394
void NoErrorCalc(Double_t *const err, Double_t *const errUpper)
Definition: MethodBase.cxx:836
TrainingHistory fTrainHistory
Definition: MethodBase.h:425
DataSet * Data() const
Definition: MethodBase.h:409
const Event * GetTrainingEvent(Long64_t ievt) const
Definition: MethodBase.h:771
void GetHelpMessage() const
void Init()
Initialization function called from MethodBase::SetupMethod() Note that option string are not yet fil...
std::vector< float > fOutput
virtual void TestClassification()
initialization
void ProcessOptions()
Function processing the options This is called only when creating the method before training not when...
Bool_t UseTFKeras() const
Definition: MethodPyKeras.h:80
EBackendType
enumeration defining the used Keras backend
Definition: MethodPyKeras.h:74
void SetupKerasModel(Bool_t loadTrainedModel)
std::vector< Float_t > & GetMulticlassValues()
UInt_t GetNumValidationSamples()
Validation of the ValidationSize option.
Double_t GetMvaValue(Double_t *errLower, Double_t *errUpper)
std::vector< Float_t > & GetRegressionValues()
TString fNumValidationString
Definition: MethodPyKeras.h:95
Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t)
TString GetKerasBackendName()
MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
TString fLearningRateSchedule
Definition: MethodPyKeras.h:93
EBackendType GetKerasBackend()
Get the Keras backend (can be: TensorFlow, Theano or CNTK)
TString fFilenameTrainedModel
std::vector< Double_t > GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t logProgress)
get all the MVA values for the events of the current Data type
static int PyIsInitialized()
Check Python interpreter initialization status.
static PyObject * fGlobalNS
Definition: PyMethodBase.h:130
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=Py_single_input)
Execute Python code from string.
PyObject * fLocalNS
Definition: PyMethodBase.h:131
Timing information for training and evaluation of MVA methods.
Definition: Timer.h:58
TString GetElapsedTime(Bool_t Scientific=kTRUE)
returns pretty string with elapsed time
Definition: Timer.cxx:146
void AddValue(TString Property, Int_t stage, Double_t value)
const Event * InverseTransform(const Event *, Bool_t suppressIfNoTargets=true) const
Singleton class for Global types used by TMVA.
Definition: Types.h:73
@ kSignal
Definition: Types.h:137
EAnalysisType
Definition: Types.h:128
@ kMulticlass
Definition: Types.h:131
@ kClassification
Definition: Types.h:129
@ kRegression
Definition: Types.h:130
@ kTraining
Definition: Types.h:145
@ kHEADER
Definition: Types.h:65
@ kINFO
Definition: Types.h:60
@ kWARNING
Definition: Types.h:61
@ kFATAL
Definition: Types.h:63
An array of TObjects.
Definition: TObjArray.h:37
Int_t GetEntries() const
Return the number of objects in array (i.e.
Definition: TObjArray.cxx:523
TObject * At(Int_t idx) const
Definition: TObjArray.h:166
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:359
Basic string class.
Definition: TString.h:136
Bool_t IsFloat() const
Returns kTRUE if string contains a floating point or integer number.
Definition: TString.cxx:1816
const char * Data() const
Definition: TString.h:369
@ kTrailing
Definition: TString.h:267
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition: TString.cxx:2222
Bool_t IsNull() const
Definition: TString.h:407
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString.
Definition: TString.cxx:2336
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2314
virtual const char * Getenv(const char *env)
Get environment variable.
Definition: TSystem.cxx:1663
create variable transformations
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:158
Double_t Log(Double_t x)
Definition: TMath.h:760