Logo ROOT  
Reference Guide
PyMethodBase.cxx
Go to the documentation of this file.
1 // @(#)root/tmva/pymva $Id$
2 // Authors: Omar Zapata, Lorenzo Moneta, Sergei Gleyzer 2015, Stefan Wunsch 2017
3 
4 /**********************************************************************************
5  * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6  * Package: TMVA *
7  * Class : PyMethodBase *
8  * *
9  * Description: *
10  * Virtual base class for all MVA method based on python *
11  * *
12  **********************************************************************************/
13 
14 #include <Python.h> // Needs to be included first to avoid redefinition of _POSIX_C_SOURCE
15 #include <TMVA/PyMethodBase.h>
16 
17 #include "TMVA/DataSet.h"
18 #include "TMVA/DataSetInfo.h"
19 #include "TMVA/MsgLogger.h"
20 #include "TMVA/Results.h"
21 #include "TMVA/Timer.h"
22 
23 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
24 #include <numpy/arrayobject.h>
25 
26 #include <cwchar>
27 
28 using namespace TMVA;
29 
30 namespace TMVA {
31 namespace Internal {
32 class PyGILRAII {
33  PyGILState_STATE m_GILState;
34 
35 public:
36  PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
37  ~PyGILRAII() { PyGILState_Release(m_GILState); }
38 };
39 } // namespace Internal
40 } // namespace TMVA
41 
43 
44 // NOTE: Introduce here nothing that breaks if multiple instances
45 // of the same method share these objects, e.g., the local namespace.
49 
53 
56 
57 ///////////////////////////////////////////////////////////////////////////////
58 
59 PyMethodBase::PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi,
60  const TString &theOption)
61  : MethodBase(jobName, methodType, methodTitle, dsi, theOption),
62  fClassifier(NULL)
63 {
64  if (!PyIsInitialized()) {
65  PyInitialize();
66  }
67 
68  // Set up private local namespace for each method instance
69  fLocalNS = PyDict_New();
70  if (!fLocalNS) {
71  Log() << kFATAL << "Can't init local namespace" << Endl;
72  }
73 }
74 
75 ///////////////////////////////////////////////////////////////////////////////
76 
78  DataSetInfo &dsi,
79  const TString &weightFile): MethodBase(methodType, dsi, weightFile),
80  fClassifier(NULL)
81 {
82  if (!PyIsInitialized()) {
83  PyInitialize();
84  }
85 
86  // Set up private local namespace for each method instance
87  fLocalNS = PyDict_New();
88  if (!fLocalNS) {
89  Log() << kFATAL << "Can't init local namespace" << Endl;
90  }
91 }
92 
93 ///////////////////////////////////////////////////////////////////////////////
94 
96 {
97 }
98 
99 ///////////////////////////////////////////////////////////////////////////////
100 /// Evaluate Python code
101 ///
102 /// \param[in] code Python code as string
103 /// \return Python object from evaluation of code line
104 ///
105 /// Take a Python code as input and evaluate it in the local namespace. Then,
106 /// return the result as Python object.
107 
109 {
110  if(!PyIsInitialized()) PyInitialize();
111  PyObject *pycode = Py_BuildValue("(sOO)", code.Data(), fGlobalNS, fLocalNS);
112  PyObject *result = PyObject_CallObject(fEval, pycode);
113  Py_DECREF(pycode);
114  return result;
115 }
116 
117 ///////////////////////////////////////////////////////////////////////////////
118 /// Initialize Python interpreter
119 ///
120 /// NOTE: We introduce a shared global namespace `fGlobalNS`, but using
121 /// a private local namespace `fLocalNS`. This prohibits the interference
122 /// of instances of the same method with the same factory, e.g., by overriding
123 /// variables in the same local namespace.
124 
126 {
128 
129  bool pyIsInitialized = PyIsInitialized();
130  if (!pyIsInitialized) {
131  Py_Initialize();
132  }
133 
135  if (!pyIsInitialized) {
136  _import_array();
137  }
138 
139  fMain = PyImport_AddModule("__main__");
140  if (!fMain) {
141  Log << kFATAL << "Can't import __main__" << Endl;
142  Log << Endl;
143  }
144 
145  fGlobalNS = PyModule_GetDict(fMain);
146  if (!fGlobalNS) {
147  Log << kFATAL << "Can't init global namespace" << Endl;
148  Log << Endl;
149  }
150 
151  #if PY_MAJOR_VERSION < 3
152  //preparing objects for eval
153  PyObject *bName = PyUnicode_FromString("__builtin__");
154  // Import the file as a Python module.
155  fModuleBuiltin = PyImport_Import(bName);
156  if (!fModuleBuiltin) {
157  Log << kFATAL << "Can't import __builtin__" << Endl;
158  Log << Endl;
159  }
160  #else
161  //preparing objects for eval
162  PyObject *bName = PyUnicode_FromString("builtins");
163  // Import the file as a Python module.
164  fModuleBuiltin = PyImport_Import(bName);
165  if (!fModuleBuiltin) {
166  Log << kFATAL << "Can't import builtins" << Endl;
167  Log << Endl;
168  }
169  #endif
170 
171  PyObject *mDict = PyModule_GetDict(fModuleBuiltin);
172  fEval = PyDict_GetItemString(mDict, "eval");
173  fOpen = PyDict_GetItemString(mDict, "open");
174 
175  Py_DECREF(bName);
176  Py_DECREF(mDict);
177  //preparing objects for pickle
178  PyObject *pName = PyUnicode_FromString("pickle");
179  // Import the file as a Python module.
180  fModulePickle = PyImport_Import(pName);
181  if (!fModulePickle) {
182  Log << kFATAL << "Can't import pickle" << Endl;
183  Log << Endl;
184  }
185  PyObject *pDict = PyModule_GetDict(fModulePickle);
186  fPickleDumps = PyDict_GetItemString(pDict, "dump");
187  fPickleLoads = PyDict_GetItemString(pDict, "load");
188 
189  Py_DECREF(pName);
190  Py_DECREF(pDict);
191 }
192 
193 ///////////////////////////////////////////////////////////////////////////////
194 // Finalize Python interpreter
195 
197 {
198  Py_Finalize();
199  if (fEval) Py_DECREF(fEval);
200  if (fModuleBuiltin) Py_DECREF(fModuleBuiltin);
201  if (fPickleDumps) Py_DECREF(fPickleDumps);
202  if (fPickleLoads) Py_DECREF(fPickleLoads);
203  if(fMain) Py_DECREF(fMain);//objects fGlobalNS and fLocalNS will be free here
204 }
205 
206 ///////////////////////////////////////////////////////////////////////////////
207 /// Set program name for Python interpeter
208 ///
209 /// \param[in] name Program name
210 
212 {
213  #if PY_MAJOR_VERSION < 3
214  Py_SetProgramName(const_cast<char*>(name.Data()));
215  #else
216  Py_SetProgramName((wchar_t *)name.Data());
217  #endif
218 }
219 
220 
221 ///////////////////////////////////////////////////////////////////////////////
222 
223 size_t mystrlen(const char* s) { return strlen(s); }
224 
225 ///////////////////////////////////////////////////////////////////////////////
226 
227 size_t mystrlen(const wchar_t* s) { return wcslen(s); }
228 
229 ///////////////////////////////////////////////////////////////////////////////
230 /// Get program name from Python interpreter
231 ///
232 /// \return Program name
233 
235 {
236  auto progName = ::Py_GetProgramName();
237  return std::string(progName, progName + mystrlen(progName));
238 }
239 
240 ///////////////////////////////////////////////////////////////////////////////
241 /// Check Python interpreter initialization status
242 ///
243 /// \return Boolean whether interpreter is initialized
244 
246 {
247  if (!Py_IsInitialized()) return kFALSE;
248  if (!fEval) return kFALSE;
249  if (!fModuleBuiltin) return kFALSE;
250  if (!fPickleDumps) return kFALSE;
251  if (!fPickleLoads) return kFALSE;
252  return kTRUE;
253 }
254 
255 ///////////////////////////////////////////////////////////////////////////////
256 /// Serialize Python object
257 ///
258 /// \param[in] path Path where object is written to file
259 /// \param[in] obj Python object
260 ///
261 /// The input Python object is serialized and written to a file. The Python
262 /// module `pickle` is used to do so.
263 
265 {
266  if(!PyIsInitialized()) PyInitialize();
267 
268  PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"wb");
269  PyObject *file = PyObject_CallObject(fOpen,file_arg);
270  PyObject *model_arg = Py_BuildValue("(OO)", obj,file);
271  PyObject *model_data = PyObject_CallObject(fPickleDumps , model_arg);
272 
273  Py_DECREF(file_arg);
274  Py_DECREF(file);
275  Py_DECREF(model_arg);
276  Py_DECREF(model_data);
277 }
278 
279 ///////////////////////////////////////////////////////////////////////////////
280 /// Unserialize Python object
281 ///
282 /// \param[in] path Path to serialized Python object
283 /// \param[in] obj Python object where the unserialized Python object is loaded
284 /// \return Error code
285 
287 {
288  // Load file
289  PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"rb");
290  PyObject *file = PyObject_CallObject(fOpen,file_arg);
291  if(!file) return 1;
292 
293  // Load object from file using pickle
294  PyObject *model_arg = Py_BuildValue("(O)", file);
295  *obj = PyObject_CallObject(fPickleLoads , model_arg);
296  if(!obj) return 2;
297 
298  Py_DECREF(file_arg);
299  Py_DECREF(file);
300  Py_DECREF(model_arg);
301 
302  return 0;
303 }
304 
305 ///////////////////////////////////////////////////////////////////////////////
306 /// Execute Python code from string
307 ///
308 /// \param[in] code Python code as string
309 /// \param[in] errorMessage Error message which shall be shown if the execution fails
310 /// \param[in] start Start symbol
311 ///
312 /// Helper function to run python code from string in local namespace with
313 /// error handling
314 /// `start` defines the start symbol defined in PyRun_String (Py_eval_input,
315 /// Py_single_input, Py_file_input)
316 
317 void PyMethodBase::PyRunString(TString code, TString errorMessage, int start) {
318  fPyReturn = PyRun_String(code, start, fGlobalNS, fLocalNS);
319  if (!fPyReturn) {
320  Log() << kWARNING << "Failed to run python code: " << code << Endl;
321  Log() << kWARNING << "Python error message:" << Endl;
322  PyErr_Print();
323  Log() << kFATAL << errorMessage << Endl;
324  }
325 }
TMVA::PyMethodBase::UnSerialize
static Int_t UnSerialize(TString file, PyObject **obj)
Unserialize Python object.
Definition: PyMethodBase.cxx:286
kTRUE
const Bool_t kTRUE
Definition: RtypesCore.h:100
TMVA::Configurable::Log
MsgLogger & Log() const
Definition: Configurable.h:122
TMVA::PyMethodBase::Eval
PyObject * Eval(TString code)
Evaluate Python code.
Definition: PyMethodBase.cxx:108
TMVA::PyMethodBase
Definition: PyMethodBase.h:56
PyObject
_object PyObject
Definition: PyMethodBase.h:42
TString::Data
const char * Data() const
Definition: TString.h:369
DataSetInfo.h
ClassImp
#define ClassImp(name)
Definition: Rtypes.h:364
TGeant4Unit::s
static constexpr double s
Definition: TGeant4SystemOfUnits.h:162
TMVA::PyMethodBase::fPickleLoads
static PyObject * fPickleLoads
Definition: PyMethodBase.h:127
TMVA::PyMethodBase::fModuleBuiltin
static PyObject * fModuleBuiltin
Definition: PyMethodBase.h:120
TMVA::PyMethodBase::fOpen
static PyObject * fOpen
Definition: PyMethodBase.h:122
TMVA::PyMethodBase::fMain
static PyObject * fMain
Definition: PyMethodBase.h:129
TString
Basic string class.
Definition: TString.h:136
TMVA::PyMethodBase::fPickleDumps
static PyObject * fPickleDumps
Definition: PyMethodBase.h:126
TMVA::PyMethodBase::fLocalNS
PyObject * fLocalNS
Definition: PyMethodBase.h:131
TMVA::DataSetInfo
Class that contains all the data information.
Definition: DataSetInfo.h:62
mystrlen
size_t mystrlen(const char *s)
Definition: PyMethodBase.cxx:223
TMVA::Internal::PyGILRAII::PyGILRAII
PyGILRAII()
Definition: PyMethodBase.cxx:36
MsgLogger.h
Timer.h
TMVA::PyMethodBase::fPyReturn
PyObject * fPyReturn
Definition: PyMethodBase.h:114
TMVA::Internal::PyGILRAII::~PyGILRAII
~PyGILRAII()
Definition: PyMethodBase.cxx:37
TMVA::PyMethodBase::~PyMethodBase
virtual ~PyMethodBase()
Definition: PyMethodBase.cxx:95
TMVA::PyMethodBase::fModulePickle
static PyObject * fModulePickle
Definition: PyMethodBase.h:125
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:101
TMVA::Internal::PyGILRAII
Definition: MethodPyAdaBoost.cxx:47
TMVA::PyMethodBase::PyMethodBase
PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
Definition: PyMethodBase.cxx:59
TMVA::MethodBase
Virtual base Class for all MVA method.
Definition: MethodBase.h:111
TMVA::PyMethodBase::PyRunString
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=Py_single_input)
Execute Python code from string.
Definition: PyMethodBase.cxx:317
TMVA::PyMethodBase::fGlobalNS
static PyObject * fGlobalNS
Definition: PyMethodBase.h:130
TMVA::Endl
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:158
TMVA::MsgLogger
ostringstream derivative to redirect and format output
Definition: MsgLogger.h:59
TMVA::kFATAL
@ kFATAL
Definition: Types.h:63
file
Definition: file.py:1
TMVA::Types::EMVA
EMVA
Definition: Types.h:78
TMVA::PyMethodBase::fEval
static PyObject * fEval
Definition: PyMethodBase.h:121
name
char name[80]
Definition: TGX11.cxx:110
TMVA::PyMethodBase::PyFinalize
static void PyFinalize()
Definition: PyMethodBase.cxx:196
TMVA::PyMethodBase::Py_GetProgramName
static TString Py_GetProgramName()
Get program name from Python interpreter.
Definition: PyMethodBase.cxx:234
TMVA::PyMethodBase::Serialize
static void Serialize(TString file, PyObject *classifier)
Serialize Python object.
Definition: PyMethodBase.cxx:264
TMVA::PyMethodBase::PyIsInitialized
static int PyIsInitialized()
Check Python interpreter initialization status.
Definition: PyMethodBase.cxx:245
TMVA::PyMethodBase::PySetProgramName
static void PySetProgramName(TString name)
Set program name for Python interpeter.
Definition: PyMethodBase.cxx:211
PyMethodBase.h
Results.h
DataSet.h
TMVA::kWARNING
@ kWARNING
Definition: Types.h:61
TMVA::Internal::PyGILRAII::m_GILState
PyGILState_STATE m_GILState
Definition: MethodPyAdaBoost.cxx:48
TMVA::PyMethodBase::PyInitialize
static void PyInitialize()
Initialize Python interpreter.
Definition: PyMethodBase.cxx:125
TMVA
create variable transformations
Definition: GeneticMinimizer.h:22
int