Logo ROOT  
Reference Guide
PyMethodBase.cxx
Go to the documentation of this file.
1// @(#)root/tmva/pymva $Id$
2// Authors: Omar Zapata, Lorenzo Moneta, Sergei Gleyzer 2015, Stefan Wunsch 2017
3
4/**********************************************************************************
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : PyMethodBase *
8 * *
9 * Description: *
10 * Virtual base class for all MVA method based on python *
11 * *
12 **********************************************************************************/
13
14#include <Python.h> // Needs to be included first to avoid redefinition of _POSIX_C_SOURCE
15#include <TMVA/PyMethodBase.h>
16
17#include "TMVA/DataSet.h"
18#include "TMVA/DataSetInfo.h"
19#include "TMVA/MsgLogger.h"
20#include "TMVA/Results.h"
21#include "TMVA/Timer.h"
22
23#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
24#include <numpy/arrayobject.h>
25
26#include <cwchar>
27
28using namespace TMVA;
29
30namespace TMVA {
31namespace Internal {
32class PyGILRAII {
33 PyGILState_STATE m_GILState;
34
35public:
36 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
37 ~PyGILRAII() { PyGILState_Release(m_GILState); }
38};
39} // namespace Internal
40} // namespace TMVA
41
43
44// NOTE: Introduce here nothing that breaks if multiple instances
45// of the same method share these objects, e.g., the local namespace.
46PyObject *PyMethodBase::fModuleBuiltin = NULL;
47PyObject *PyMethodBase::fEval = NULL;
48PyObject *PyMethodBase::fOpen = NULL;
49
50PyObject *PyMethodBase::fModulePickle = NULL;
51PyObject *PyMethodBase::fPickleDumps = NULL;
52PyObject *PyMethodBase::fPickleLoads = NULL;
53
54PyObject *PyMethodBase::fMain = NULL;
55PyObject *PyMethodBase::fGlobalNS = NULL;
56
57///////////////////////////////////////////////////////////////////////////////
58
59PyMethodBase::PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi,
60 const TString &theOption)
61 : MethodBase(jobName, methodType, methodTitle, dsi, theOption),
62 fClassifier(NULL)
63{
64 if (!PyIsInitialized()) {
66 }
67
68 // Set up private local namespace for each method instance
69 fLocalNS = PyDict_New();
70 if (!fLocalNS) {
71 Log() << kFATAL << "Can't init local namespace" << Endl;
72 }
73}
74
75///////////////////////////////////////////////////////////////////////////////
76
78 DataSetInfo &dsi,
79 const TString &weightFile): MethodBase(methodType, dsi, weightFile),
80 fClassifier(NULL)
81{
82 if (!PyIsInitialized()) {
84 }
85
86 // Set up private local namespace for each method instance
87 fLocalNS = PyDict_New();
88 if (!fLocalNS) {
89 Log() << kFATAL << "Can't init local namespace" << Endl;
90 }
91}
92
93///////////////////////////////////////////////////////////////////////////////
94
96{
97}
98
99///////////////////////////////////////////////////////////////////////////////
100/// Evaluate Python code
101///
102/// \param[in] code Python code as string
103/// \return Python object from evaluation of code line
104///
105/// Take a Python code as input and evaluate it in the local namespace. Then,
106/// return the result as Python object.
107
109{
111 PyObject *pycode = Py_BuildValue("(sOO)", code.Data(), fGlobalNS, fLocalNS);
112 PyObject *result = PyObject_CallObject(fEval, pycode);
113 Py_DECREF(pycode);
114 return result;
115}
116
117///////////////////////////////////////////////////////////////////////////////
118/// Initialize Python interpreter
119///
120/// NOTE: We introduce a shared global namespace `fGlobalNS`, but using
121/// a private local namespace `fLocalNS`. This prohibits the interference
122/// of instances of the same method with the same factory, e.g., by overriding
123/// variables in the same local namespace.
124
126{
128
129 bool pyIsInitialized = PyIsInitialized();
130 if (!pyIsInitialized) {
131 Py_Initialize();
132 }
133
135 if (!pyIsInitialized) {
136 _import_array();
137 }
138
139 fMain = PyImport_AddModule("__main__");
140 if (!fMain) {
141 Log << kFATAL << "Can't import __main__" << Endl;
142 Log << Endl;
143 }
144
145 fGlobalNS = PyModule_GetDict(fMain);
146 if (!fGlobalNS) {
147 Log << kFATAL << "Can't init global namespace" << Endl;
148 Log << Endl;
149 }
150
151 #if PY_MAJOR_VERSION < 3
152 //preparing objects for eval
153 PyObject *bName = PyUnicode_FromString("__builtin__");
154 // Import the file as a Python module.
155 fModuleBuiltin = PyImport_Import(bName);
156 if (!fModuleBuiltin) {
157 Log << kFATAL << "Can't import __builtin__" << Endl;
158 Log << Endl;
159 }
160 #else
161 //preparing objects for eval
162 PyObject *bName = PyUnicode_FromString("builtins");
163 // Import the file as a Python module.
164 fModuleBuiltin = PyImport_Import(bName);
165 if (!fModuleBuiltin) {
166 Log << kFATAL << "Can't import builtins" << Endl;
167 Log << Endl;
168 }
169 #endif
170
171 PyObject *mDict = PyModule_GetDict(fModuleBuiltin);
172 fEval = PyDict_GetItemString(mDict, "eval");
173 fOpen = PyDict_GetItemString(mDict, "open");
174
175 Py_DECREF(bName);
176 Py_DECREF(mDict);
177 //preparing objects for pickle
178 PyObject *pName = PyUnicode_FromString("pickle");
179 // Import the file as a Python module.
180 fModulePickle = PyImport_Import(pName);
181 if (!fModulePickle) {
182 Log << kFATAL << "Can't import pickle" << Endl;
183 Log << Endl;
184 }
185 PyObject *pDict = PyModule_GetDict(fModulePickle);
186 fPickleDumps = PyDict_GetItemString(pDict, "dump");
187 fPickleLoads = PyDict_GetItemString(pDict, "load");
188
189 Py_DECREF(pName);
190 Py_DECREF(pDict);
191}
192
193///////////////////////////////////////////////////////////////////////////////
194// Finalize Python interpreter
195
197{
198 Py_Finalize();
199 if (fEval) Py_DECREF(fEval);
200 if (fModuleBuiltin) Py_DECREF(fModuleBuiltin);
201 if (fPickleDumps) Py_DECREF(fPickleDumps);
202 if (fPickleLoads) Py_DECREF(fPickleLoads);
203 if(fMain) Py_DECREF(fMain);//objects fGlobalNS and fLocalNS will be free here
204}
205
206///////////////////////////////////////////////////////////////////////////////
207/// Set program name for Python interpeter
208///
209/// \param[in] name Program name
210
212{
213 #if PY_MAJOR_VERSION < 3
214 Py_SetProgramName(const_cast<char*>(name.Data()));
215 #else
216 Py_SetProgramName((wchar_t *)name.Data());
217 #endif
218}
219
220
221///////////////////////////////////////////////////////////////////////////////
222
223size_t mystrlen(const char* s) { return strlen(s); }
224
225///////////////////////////////////////////////////////////////////////////////
226
227size_t mystrlen(const wchar_t* s) { return wcslen(s); }
228
229///////////////////////////////////////////////////////////////////////////////
230/// Get program name from Python interpreter
231///
232/// \return Program name
233
235{
236 auto progName = ::Py_GetProgramName();
237 return std::string(progName, progName + mystrlen(progName));
238}
239
240///////////////////////////////////////////////////////////////////////////////
241/// Check Python interpreter initialization status
242///
243/// \return Boolean whether interpreter is initialized
244
246{
247 if (!Py_IsInitialized()) return kFALSE;
248 if (!fEval) return kFALSE;
249 if (!fModuleBuiltin) return kFALSE;
250 if (!fPickleDumps) return kFALSE;
251 if (!fPickleLoads) return kFALSE;
252 return kTRUE;
253}
254
255///////////////////////////////////////////////////////////////////////////////
256/// Serialize Python object
257///
258/// \param[in] path Path where object is written to file
259/// \param[in] obj Python object
260///
261/// The input Python object is serialized and written to a file. The Python
262/// module `pickle` is used to do so.
263
265{
267
268 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"wb");
269 PyObject *file = PyObject_CallObject(fOpen,file_arg);
270 PyObject *model_arg = Py_BuildValue("(OO)", obj,file);
271 PyObject *model_data = PyObject_CallObject(fPickleDumps , model_arg);
272
273 Py_DECREF(file_arg);
274 Py_DECREF(file);
275 Py_DECREF(model_arg);
276 Py_DECREF(model_data);
277}
278
279///////////////////////////////////////////////////////////////////////////////
280/// Unserialize Python object
281///
282/// \param[in] path Path to serialized Python object
283/// \param[in] obj Python object where the unserialized Python object is loaded
284/// \return Error code
285
287{
288 // Load file
289 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"rb");
290 PyObject *file = PyObject_CallObject(fOpen,file_arg);
291 if(!file) return 1;
292
293 // Load object from file using pickle
294 PyObject *model_arg = Py_BuildValue("(O)", file);
295 *obj = PyObject_CallObject(fPickleLoads , model_arg);
296 if(!obj) return 2;
297
298 Py_DECREF(file_arg);
299 Py_DECREF(file);
300 Py_DECREF(model_arg);
301
302 return 0;
303}
304
305///////////////////////////////////////////////////////////////////////////////
306/// Execute Python code from string
307///
308/// \param[in] code Python code as string
309/// \param[in] errorMessage Error message which shall be shown if the execution fails
310/// \param[in] start Start symbol
311///
312/// Helper function to run python code from string in local namespace with
313/// error handling
314/// `start` defines the start symbol defined in PyRun_String (Py_eval_input,
315/// Py_single_input, Py_file_input)
316
317void PyMethodBase::PyRunString(TString code, TString errorMessage, int start) {
318 fPyReturn = PyRun_String(code, start, fGlobalNS, fLocalNS);
319 if (!fPyReturn) {
320 Log() << kWARNING << "Failed to run python code: " << code << Endl;
321 Log() << kWARNING << "Python error message:" << Endl;
322 PyErr_Print();
323 Log() << kFATAL << errorMessage << Endl;
324 }
325}
326
327///////////////////////////////////////////////////////////////////////////////
328/// Execute Python code from string
329///
330/// \param[in] code Python code as string
331/// \param[in] globalNS Global Namespace for Python Session
332/// \param[in] localNS Local Namespace for Python Session
333///
334/// Overloaded static Helper function to run python code
335/// from string and throw runtime error if the Python session
336/// is unable to execute the code
337
338void PyMethodBase::PyRunString(TString code, PyObject *globalNS, PyObject *localNS){
339 PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
340 if (!fPyReturn) {
341 std::cout<<"\nPython error message:\n";
342 PyErr_Print();
343 throw std::runtime_error("\nFailed to run python code: "+code);
344 }
345}
346
347///////////////////////////////////////////////////////////////////////////////
348/// Returns `const char*` from Python string in PyObject
349///
350/// \param[in] string Python String object
351/// \return String representation in `const char*`
352
354 PyObject* encodedString = PyUnicode_AsUTF8String(string);
355 const char* cstring = PyBytes_AsString(encodedString);
356 return cstring;
357}
#define PyBytes_AsString
Definition: CPyCppyy.h:86
size_t mystrlen(const char *s)
_object PyObject
Definition: PyMethodBase.h:42
#define Py_single_input
Definition: PyMethodBase.h:44
int Int_t
Definition: RtypesCore.h:45
const Bool_t kFALSE
Definition: RtypesCore.h:101
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define ClassImp(name)
Definition: Rtypes.h:364
char name[80]
Definition: TGX11.cxx:110
MsgLogger & Log() const
Definition: Configurable.h:122
Class that contains all the data information.
Definition: DataSetInfo.h:62
Virtual base Class for all MVA method.
Definition: MethodBase.h:111
ostringstream derivative to redirect and format output
Definition: MsgLogger.h:59
PyObject * fPyReturn
Definition: PyMethodBase.h:114
static int PyIsInitialized()
Check Python interpreter initialization status.
static PyObject * fOpen
Definition: PyMethodBase.h:122
static PyObject * fPickleDumps
Definition: PyMethodBase.h:126
PyObject * Eval(TString code)
Evaluate Python code.
static TString Py_GetProgramName()
Get program name from Python interpreter.
static PyObject * fMain
Definition: PyMethodBase.h:129
static void PyInitialize()
Initialize Python interpreter.
static void Serialize(TString file, PyObject *classifier)
Serialize Python object.
static void PyFinalize()
static Int_t UnSerialize(TString file, PyObject **obj)
Unserialize Python object.
static const char * PyStringAsString(PyObject *string)
Returns const char* from Python string in PyObject.
static PyObject * fPickleLoads
Definition: PyMethodBase.h:127
static void PySetProgramName(TString name)
Set program name for Python interpeter.
virtual ~PyMethodBase()
static PyObject * fGlobalNS
Definition: PyMethodBase.h:130
static PyObject * fModulePickle
Definition: PyMethodBase.h:125
static PyObject * fModuleBuiltin
Definition: PyMethodBase.h:120
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=Py_single_input)
Execute Python code from string.
PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
static PyObject * fEval
Definition: PyMethodBase.h:121
PyObject * fLocalNS
Definition: PyMethodBase.h:131
@ kWARNING
Definition: Types.h:61
@ kFATAL
Definition: Types.h:63
Basic string class.
Definition: TString.h:136
const char * Data() const
Definition: TString.h:369
static constexpr double s
create variable transformations
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:158
Definition: file.py:1