Logo ROOT   6.12/07
Reference Guide
MethodPyKeras.cxx
Go to the documentation of this file.
1 // @(#)root/tmva/pymva $Id$
2 // Author: Stefan Wunsch, 2016
3 
4 #include <Python.h>
5 #include "TMVA/MethodPyKeras.h"
6 
7 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
8 #include <numpy/arrayobject.h>
9 
10 #include "TMVA/Types.h"
11 #include "TMVA/Config.h"
12 #include "TMVA/ClassifierFactory.h"
13 #include "TMVA/Results.h"
16 
17 using namespace TMVA;
18 
19 REGISTER_METHOD(PyKeras)
20 
22 
23 MethodPyKeras::MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption)
24  : PyMethodBase(jobName, Types::kPyKeras, methodTitle, dsi, theOption) {
25  fNumEpochs = 10;
26  fBatchSize = 100;
27  fVerbose = 1;
28  fContinueTraining = false;
29  fSaveBestOnly = true;
31  fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
32  fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
33 }
34 
35 MethodPyKeras::MethodPyKeras(DataSetInfo &theData, const TString &theWeightFile)
36  : PyMethodBase(Types::kPyKeras, theData, theWeightFile) {
37  fNumEpochs = 10;
38  fBatchSize = 100;
39  fVerbose = 1;
40  fContinueTraining = false;
41  fSaveBestOnly = true;
43  fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
44  fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
45 }
46 
48 }
49 
51  if (type == Types::kRegression) return kTRUE;
52  if (type == Types::kClassification && numberClasses == 2) return kTRUE;
53  if (type == Types::kMulticlass && numberClasses >= 2) return kTRUE;
54  return kFALSE;
55 }
56 
57 ///////////////////////////////////////////////////////////////////////////////
58 
60  DeclareOptionRef(fFilenameModel, "FilenameModel", "Filename of the initial Keras model");
61  DeclareOptionRef(fFilenameTrainedModel, "FilenameTrainedModel", "Filename of the trained output Keras model");
62  DeclareOptionRef(fBatchSize, "BatchSize", "Training batch size");
63  DeclareOptionRef(fNumEpochs, "NumEpochs", "Number of training epochs");
64  DeclareOptionRef(fVerbose, "Verbose", "Keras verbosity during training");
65  DeclareOptionRef(fContinueTraining, "ContinueTraining", "Load weights from previous training");
66  DeclareOptionRef(fSaveBestOnly, "SaveBestOnly", "Store only weights with smallest validation loss");
67  DeclareOptionRef(fTriesEarlyStopping, "TriesEarlyStopping", "Number of epochs with no improvement in validation loss after which training will be stopped. The default or a negative number deactivates this option.");
68  DeclareOptionRef(fLearningRateSchedule, "LearningRateSchedule", "Set new learning rate during training at specific epochs, e.g., \"50,0.01;70,0.005\"");
69 }
70 
72  // Set default filename for trained model if option is not used
73  if (fFilenameTrainedModel.IsNull()) {
74  fFilenameTrainedModel = GetWeightFileDir() + "/TrainedModel_" + GetName() + ".h5";
75  }
76  // Setup model, either the initial model from `fFilenameModel` or
77  // the trained model from `fFilenameTrainedModel`
78  if (fContinueTraining) Log() << kINFO << "Continue training with trained model" << Endl;
80 }
81 
82 void MethodPyKeras::SetupKerasModel(bool loadTrainedModel) {
83  /*
84  * Load Keras model from file
85  */
86 
87  // Load initial model or already trained model
88  TString filenameLoadModel;
89  if (loadTrainedModel) {
90  filenameLoadModel = fFilenameTrainedModel;
91  }
92  else {
93  filenameLoadModel = fFilenameModel;
94  }
95  PyRunString("model = keras.models.load_model('"+filenameLoadModel+"')",
96  "Failed to load Keras model from file: "+filenameLoadModel);
97  Log() << kINFO << "Load model from file: " << filenameLoadModel << Endl;
98 
99  /*
100  * Init variables and weights
101  */
102 
103  // Get variables, classes and target numbers
104  fNVars = GetNVariables();
107  else Log() << kFATAL << "Selected analysis type is not implemented" << Endl;
108 
109  // Init evaluation (needed for getMvaValue)
110  fVals = new float[fNVars]; // holds values used for classification and regression
111  npy_intp dimsVals[2] = {(npy_intp)1, (npy_intp)fNVars};
112  PyArrayObject* pVals = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsVals, NPY_FLOAT, (void*)fVals);
113  PyDict_SetItemString(fLocalNS, "vals", (PyObject*)pVals);
114 
115  fOutput.resize(fNOutputs); // holds classification probabilities or regression output
116  npy_intp dimsOutput[2] = {(npy_intp)1, (npy_intp)fNOutputs};
117  PyArrayObject* pOutput = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsOutput, NPY_FLOAT, (void*)&fOutput[0]);
118  PyDict_SetItemString(fLocalNS, "output", (PyObject*)pOutput);
119 
120  // Mark the model as setup
121  fModelIsSetup = true;
122 }
123 
125  if (!PyIsInitialized()) {
126  Log() << kFATAL << "Python is not initialized" << Endl;
127  }
128  _import_array(); // required to use numpy arrays
129 
130  // Import Keras
131  // NOTE: sys.argv has to be cleared because otherwise TensorFlow breaks
132  PyRunString("import sys; sys.argv = ['']", "Set sys.argv failed");
133  PyRunString("import keras", "Import Keras failed");
134 
135  // Set flag that model is not setup
136  fModelIsSetup = false;
137 }
138 
140  if(!fModelIsSetup) Log() << kFATAL << "Model is not setup for training" << Endl;
141 
142  /*
143  * Load training data to numpy array
144  */
145 
146  UInt_t nTrainingEvents = Data()->GetNTrainingEvents();
147 
148  float* trainDataX = new float[nTrainingEvents*fNVars];
149  float* trainDataY = new float[nTrainingEvents*fNOutputs];
150  float* trainDataWeights = new float[nTrainingEvents];
151  for (UInt_t i=0; i<nTrainingEvents; i++) {
152  const TMVA::Event* e = GetTrainingEvent(i);
153  // Fill variables
154  for (UInt_t j=0; j<fNVars; j++) {
155  trainDataX[j + i*fNVars] = e->GetValue(j);
156  }
157  // Fill targets
158  // NOTE: For classification, convert class number in one-hot vector,
159  // e.g., 1 -> [0, 1] or 0 -> [1, 0] for binary classification
161  for (UInt_t j=0; j<fNOutputs; j++) {
162  trainDataY[j + i*fNOutputs] = 0;
163  }
164  trainDataY[e->GetClass() + i*fNOutputs] = 1;
165  }
166  else if (GetAnalysisType() == Types::kRegression) {
167  for (UInt_t j=0; j<fNOutputs; j++) {
168  trainDataY[j + i*fNOutputs] = e->GetTarget(j);
169  }
170  }
171  else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
172  // Fill weights
173  // NOTE: If no weight branch is given, this defaults to ones for all events
174  trainDataWeights[i] = e->GetWeight();
175  }
176 
177  npy_intp dimsTrainX[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNVars};
178  npy_intp dimsTrainY[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNOutputs};
179  npy_intp dimsTrainWeights[1] = {(npy_intp)nTrainingEvents};
180  PyArrayObject* pTrainDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainX, NPY_FLOAT, (void*)trainDataX);
181  PyArrayObject* pTrainDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainY, NPY_FLOAT, (void*)trainDataY);
182  PyArrayObject* pTrainDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsTrainWeights, NPY_FLOAT, (void*)trainDataWeights);
183  PyDict_SetItemString(fLocalNS, "trainX", (PyObject*)pTrainDataX);
184  PyDict_SetItemString(fLocalNS, "trainY", (PyObject*)pTrainDataY);
185  PyDict_SetItemString(fLocalNS, "trainWeights", (PyObject*)pTrainDataWeights);
186 
187  /*
188  * Load validation data to numpy array
189  */
190 
191  // NOTE: In TMVA, test data is actually validation data
192 
193  UInt_t nValEvents = Data()->GetNTestEvents();
194 
195  float* valDataX = new float[nValEvents*fNVars];
196  float* valDataY = new float[nValEvents*fNOutputs];
197  float* valDataWeights = new float[nValEvents];
198  for (UInt_t i=0; i<nValEvents; i++) {
199  const TMVA::Event* e = GetTestingEvent(i);
200  // Fill variables
201  for (UInt_t j=0; j<fNVars; j++) {
202  valDataX[j + i*fNVars] = e->GetValue(j);
203  }
204  // Fill targets
206  for (UInt_t j=0; j<fNOutputs; j++) {
207  valDataY[j + i*fNOutputs] = 0;
208  }
209  valDataY[e->GetClass() + i*fNOutputs] = 1;
210  }
211  else if (GetAnalysisType() == Types::kRegression) {
212  for (UInt_t j=0; j<fNOutputs; j++) {
213  valDataY[j + i*fNOutputs] = e->GetTarget(j);
214  }
215  }
216  else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
217  // Fill weights
218  valDataWeights[i] = e->GetWeight();
219  }
220 
221  npy_intp dimsValX[2] = {(npy_intp)nValEvents, (npy_intp)fNVars};
222  npy_intp dimsValY[2] = {(npy_intp)nValEvents, (npy_intp)fNOutputs};
223  npy_intp dimsValWeights[1] = {(npy_intp)nValEvents};
224  PyArrayObject* pValDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValX, NPY_FLOAT, (void*)valDataX);
225  PyArrayObject* pValDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValY, NPY_FLOAT, (void*)valDataY);
226  PyArrayObject* pValDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsValWeights, NPY_FLOAT, (void*)valDataWeights);
227  PyDict_SetItemString(fLocalNS, "valX", (PyObject*)pValDataX);
228  PyDict_SetItemString(fLocalNS, "valY", (PyObject*)pValDataY);
229  PyDict_SetItemString(fLocalNS, "valWeights", (PyObject*)pValDataWeights);
230 
231  /*
232  * Train Keras model
233  */
234 
235  // Setup parameters
236 
237  PyObject* pBatchSize = PyLong_FromLong(fBatchSize);
238  PyObject* pNumEpochs = PyLong_FromLong(fNumEpochs);
239  PyObject* pVerbose = PyLong_FromLong(fVerbose);
240  PyDict_SetItemString(fLocalNS, "batchSize", pBatchSize);
241  PyDict_SetItemString(fLocalNS, "numEpochs", pNumEpochs);
242  PyDict_SetItemString(fLocalNS, "verbose", pVerbose);
243 
244  // Setup training callbacks
245  PyRunString("callbacks = []");
246 
247  // Callback: Save only weights with smallest validation loss
248  if (fSaveBestOnly) {
249  PyRunString("callbacks.append(keras.callbacks.ModelCheckpoint('"+fFilenameTrainedModel+"', monitor='val_loss', verbose=verbose, save_best_only=True, mode='auto'))", "Failed to setup training callback: SaveBestOnly");
250  Log() << kINFO << "Option SaveBestOnly: Only model weights with smallest validation loss will be stored" << Endl;
251  }
252 
253  // Callback: Stop training early if no improvement in validation loss is observed
254  if (fTriesEarlyStopping>=0) {
255  TString tries;
256  tries.Form("%i", fTriesEarlyStopping);
257  PyRunString("callbacks.append(keras.callbacks.EarlyStopping(monitor='val_loss', patience="+tries+", verbose=verbose, mode='auto'))", "Failed to setup training callback: TriesEarlyStopping");
258  Log() << kINFO << "Option TriesEarlyStopping: Training will stop after " << tries << " number of epochs with no improvement of validation loss" << Endl;
259  }
260 
261  // Callback: Learning rate scheduler
262  if (fLearningRateSchedule!="") {
263  // Setup a python dictionary with the desired learning rate steps
264  PyRunString("strScheduleSteps = '"+fLearningRateSchedule+"'\n"
265  "schedulerSteps = {}\n"
266  "for c in strScheduleSteps.split(';'):\n"
267  " x = c.split(',')\n"
268  " schedulerSteps[int(x[0])] = float(x[1])\n",
269  "Failed to setup steps for scheduler function from string: "+fLearningRateSchedule,
270  Py_file_input);
271  // Set scheduler function as piecewise function with given steps
272  PyRunString("def schedule(epoch, model=model, schedulerSteps=schedulerSteps):\n"
273  " if epoch in schedulerSteps: return float(schedulerSteps[epoch])\n"
274  " else: return float(model.optimizer.lr.get_value())\n",
275  "Failed to setup scheduler function with string: "+fLearningRateSchedule,
276  Py_file_input);
277  // Setup callback
278  PyRunString("callbacks.append(keras.callbacks.LearningRateScheduler(schedule))",
279  "Failed to setup training callback: LearningRateSchedule");
280  Log() << kINFO << "Option LearningRateSchedule: Set learning rate during training: " << fLearningRateSchedule << Endl;
281  }
282 
283  // Train model
284  PyRunString("history = model.fit(trainX, trainY, sample_weight=trainWeights, batch_size=batchSize, nb_epoch=numEpochs, verbose=verbose, validation_data=(valX, valY, valWeights), callbacks=callbacks)",
285  "Failed to train model");
286 
287  /*
288  * Store trained model to file (only if option 'SaveBestOnly' is NOT activated,
289  * because we do not want to override the best model checkpoint)
290  */
291 
292  if (!fSaveBestOnly) {
293  PyRunString("model.save('"+fFilenameTrainedModel+"', overwrite=True)",
294  "Failed to save trained model: "+fFilenameTrainedModel);
295  Log() << kINFO << "Trained model written to file: " << fFilenameTrainedModel << Endl;
296  }
297 
298  /*
299  * Clean-up
300  */
301 
302  delete[] trainDataX;
303  delete[] trainDataY;
304  delete[] trainDataWeights;
305  delete[] valDataX;
306  delete[] valDataY;
307  delete[] valDataWeights;
308 }
309 
312 }
313 
315  // Cannot determine error
316  NoErrorCalc(errLower, errUpper);
317 
318  // Check whether the model is setup
319  // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
320  if (!fModelIsSetup) {
321  // Setup the trained model
322  SetupKerasModel(true);
323  }
324 
325  // Get signal probability (called mvaValue here)
326  const TMVA::Event* e = GetEvent();
327  for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
328  PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
329  "Failed to get predictions");
330 
332 }
333 
334 std::vector<Double_t> MethodPyKeras::GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t) {
335  // Check whether the model is setup
336  // NOTE: Unfortunately this is needed because during evaluation ProcessOptions is not called again
337  if (!fModelIsSetup) {
338  // Setup the trained model
339  SetupKerasModel(true);
340  }
341 
342  // Load data to numpy array
343  Long64_t nEvents = Data()->GetNEvents();
344  if (firstEvt > lastEvt || lastEvt > nEvents) lastEvt = nEvents;
345  if (firstEvt < 0) firstEvt = 0;
346  nEvents = lastEvt-firstEvt;
347 
348  float* data = new float[nEvents*fNVars];
349  for (UInt_t i=0; i<nEvents; i++) {
350  Data()->SetCurrentEvent(i);
351  const TMVA::Event *e = GetEvent();
352  for (UInt_t j=0; j<fNVars; j++) {
353  data[j + i*fNVars] = e->GetValue(j);
354  }
355  }
356 
357  npy_intp dimsData[2] = {(npy_intp)nEvents, (npy_intp)fNVars};
358  PyArrayObject* pDataMvaValues = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsData, NPY_FLOAT, (void*)data);
359  if (pDataMvaValues==0) Log() << "Failed to load data to Python array" << Endl;
360 
361  // Get prediction for all events
362  PyObject* pModel = PyDict_GetItemString(fLocalNS, "model");
363  if (pModel==0) Log() << kFATAL << "Failed to get model Python object" << Endl;
364  PyArrayObject* pPredictions = (PyArrayObject*) PyObject_CallMethod(pModel, (char*)"predict", (char*)"O", pDataMvaValues);
365  if (pPredictions==0) Log() << kFATAL << "Failed to get predictions" << Endl;
366  delete[] data;
367 
368  // Load predictions to double vector
369  // NOTE: The signal probability is given at the output
370  std::vector<double> mvaValues(nEvents);
371  float* predictionsData = (float*) PyArray_DATA(pPredictions);
372  for (UInt_t i=0; i<nEvents; i++) {
373  mvaValues[i] = (double) predictionsData[i*fNOutputs + TMVA::Types::kSignal];
374  }
375 
376  return mvaValues;
377 }
378 
379 std::vector<Float_t>& MethodPyKeras::GetRegressionValues() {
380  // Check whether the model is setup
381  // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
382  if (!fModelIsSetup){
383  // Setup the model and load weights
384  SetupKerasModel(true);
385  }
386 
387  // Get regression values
388  const TMVA::Event* e = GetEvent();
389  for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
390  PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
391  "Failed to get predictions");
392 
393  // Use inverse transformation of targets to get final regression values
394  Event * eTrans = new Event(*e);
395  for (UInt_t i=0; i<fNOutputs; ++i) {
396  eTrans->SetTarget(i,fOutput[i]);
397  }
398 
399  const Event* eTrans2 = GetTransformationHandler().InverseTransform(eTrans);
400  for (UInt_t i=0; i<fNOutputs; ++i) {
401  fOutput[i] = eTrans2->GetTarget(i);
402  }
403 
404  return fOutput;
405 }
406 
407 std::vector<Float_t>& MethodPyKeras::GetMulticlassValues() {
408  // Check whether the model is setup
409  // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
410  if (!fModelIsSetup){
411  // Setup the model and load weights
412  SetupKerasModel(true);
413  }
414 
415  // Get class probabilites
416  const TMVA::Event* e = GetEvent();
417  for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
418  PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
419  "Failed to get predictions");
420 
421  return fOutput;
422 }
423 
425 }
426 
428 // typical length of text line:
429 // "|--------------------------------------------------------------|"
430  Log() << Endl;
431  Log() << "Keras is a high-level API for the Theano and Tensorflow packages." << Endl;
432  Log() << "This method wraps the training and predictions steps of the Keras" << Endl;
433  Log() << "Python package for TMVA, so that dataloading, preprocessing and" << Endl;
434  Log() << "evaluation can be done within the TMVA system. To use this Keras" << Endl;
435  Log() << "interface, you have to generate a model with Keras first. Then," << Endl;
436  Log() << "this model can be loaded and trained in TMVA." << Endl;
437  Log() << Endl;
438 }
Double_t GetMvaValue(Double_t *errLower, Double_t *errUpper)
void SetCurrentEvent(Long64_t ievt) const
Definition: DataSet.h:99
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:158
Singleton class for Global types used by TMVA.
Definition: Types.h:73
long long Long64_t
Definition: RtypesCore.h:69
MsgLogger & Log() const
Definition: Configurable.h:122
OptionBase * DeclareOptionRef(T &ref, const TString &name, const TString &desc="")
EAnalysisType
Definition: Types.h:125
TransformationHandler & GetTransformationHandler(Bool_t takeReroutedIfAvailable=true)
Definition: MethodBase.h:383
bool Bool_t
Definition: RtypesCore.h:59
UInt_t GetNClasses() const
Definition: DataSetInfo.h:136
static int PyIsInitialized()
Check Python interpreter initialization status.
const TString & GetWeightFileDir() const
Definition: MethodBase.h:479
std::vector< Float_t > & GetRegressionValues()
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=Py_single_input)
Execute Python code from string.
void GetHelpMessage() const
const Event * GetEvent() const
Definition: MethodBase.h:738
DataSet * Data() const
Definition: MethodBase.h:398
TString fFilenameTrainedModel
Definition: MethodPyKeras.h:88
UInt_t GetClass() const
Definition: Event.h:81
DataSetInfo & DataInfo() const
Definition: MethodBase.h:399
Class that contains all the data information.
Definition: DataSetInfo.h:60
Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t)
Double_t GetWeight() const
return the event weight - depending on whether the flag IgnoreNegWeightsInTraining is or not...
Definition: Event.cxx:382
Long64_t GetNTrainingEvents() const
Definition: DataSet.h:79
const Event * GetTrainingEvent(Long64_t ievt) const
Definition: MethodBase.h:758
TString fLearningRateSchedule
Definition: MethodPyKeras.h:81
const Event * GetTestingEvent(Long64_t ievt) const
Definition: MethodBase.h:764
Float_t GetTarget(UInt_t itgt) const
Definition: Event.h:97
UInt_t GetNTargets() const
Definition: DataSetInfo.h:111
const char * GetName() const
Definition: MethodBase.h:323
unsigned int UInt_t
Definition: RtypesCore.h:42
const Event * InverseTransform(const Event *, Bool_t suppressIfNoTargets=true) const
void SetTarget(UInt_t itgt, Float_t value)
set the target value (dimension itgt) to value
Definition: Event.cxx:360
std::vector< Float_t > & GetMulticlassValues()
Long64_t GetNTestEvents() const
Definition: DataSet.h:80
UInt_t GetNVariables() const
Definition: MethodBase.h:334
const Bool_t kFALSE
Definition: RtypesCore.h:88
Float_t GetValue(UInt_t ivar) const
return value of i&#39;th variable
Definition: Event.cxx:237
void SetupKerasModel(Bool_t loadTrainedModel)
#define ClassImp(name)
Definition: Rtypes.h:359
double Double_t
Definition: RtypesCore.h:55
std::vector< Double_t > GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t logProgress)
get all the MVA values for the events of the current Data type
int type
Definition: TGX11.cxx:120
virtual void TestClassification()
initialization
you should not use this method at all Int_t Int_t Double_t Double_t Double_t e
Definition: TRolke.cxx:630
#define REGISTER_METHOD(CLASS)
for example
Abstract ClassifierFactory template that handles arbitrary types.
PyObject * fLocalNS
Definition: PyMethodBase.h:143
Long64_t GetNEvents(Types::ETreeType type=Types::kMaxTreeType) const
Definition: DataSet.h:215
Types::EAnalysisType GetAnalysisType() const
Definition: MethodBase.h:426
MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
const Bool_t kTRUE
Definition: RtypesCore.h:87
virtual void TestClassification()
initialization
std::vector< float > fOutput
Definition: MethodPyKeras.h:85
_object PyObject
Definition: TPyArg.h:20
void NoErrorCalc(Double_t *const err, Double_t *const errUpper)
Definition: MethodBase.cxx:829