Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RModel.cxx
Go to the documentation of this file.
1#include <limits>
2#include <algorithm>
3#include <cctype>
4#include <memory>
5#include <string>
6
7#ifdef SOFIE_SUPPORT_ROOT_BINARY
8#include "TFile.h"
9#endif
10
11#include "TMVA/RModel.hxx"
12#include "TMVA/SOFIE_common.hxx"
13
14namespace TMVA {
15namespace Experimental {
16namespace SOFIE {
17
18namespace {
19const std::string SP = " ";
20}
21
22std::underlying_type_t<Options> operator|(Options opA, Options opB) {
23 return static_cast<std::underlying_type_t<Options>>(opA) | static_cast<std::underlying_type_t<Options>>(opB);
24}
25std::underlying_type_t<Options> operator|(std::underlying_type_t<Options> opA, Options opB) {
26 return opA | static_cast<std::underlying_type_t<Options>>(opB);
27}
28
29std::vector<size_t> RModel::GetTensorShape(const std::string & name) const {
30 auto f = fReadyInputTensorInfos.find(name);
31 if (f != fReadyInputTensorInfos.end()) {
32 return f->second.shape;
33 }
34 auto f2 = fInitializedTensors.find(name);
35 if (f2 != fInitializedTensors.end()) {
36 return f2->second.shape();
37 }
38 auto f3 = fInputTensorInfos.find(name);
39 if (f3 != fInputTensorInfos.end()) {
40 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] is an input tensor with unspecified dimension parameter");
41 }
42 auto f4 = fIntermediateTensorInfos.find(name);
43 if (f4 != fIntermediateTensorInfos.end()) {
44 return f4->second.shape;
45 }
46 // case of shape tensors
47 auto f5 = fShapeTensors.find(name);
48 if (f5 != fShapeTensors.end()) {
49 // shape is vector of size 1 with size of shape values or just a scalar
50 if (f5->second.second) // check scalar flag
51 return std::vector<size_t>{};
52 else
53 return std::vector<size_t>{f5->second.first.size()};
54 }
55
57 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] is a dynamic tensor. Use GetDynamicTensorShape instead of GetTensorShape");
58
61
62 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the shape is requested is not found");
63}
64
65std::vector<Dim> RModel::GetDimTensorShape(const std::string & name) const {
66 if (auto f = fDynamicTensorInfos.find(name); f != fDynamicTensorInfos.end()) {
67 return f->second.shape;
68 }
69 if (auto f = fInputTensorInfos.find(name); f != fInputTensorInfos.end()) {
70 return f->second.shape;
71 }
72 // in case is not a dynamic tensor convert normal shape to Dim one
73 // for this we need to return the vector by value
75}
76std::vector<Dim> RModel::GetDynamicTensorShape(const std::string & name) const {
77 if (auto f = fDynamicTensorInfos.find(name); f != fDynamicTensorInfos.end()) {
78 return f->second.shape;
79 }
80 if (auto f = fInputTensorInfos.find(name); f != fInputTensorInfos.end()) {
81 return f->second.shape;
82 }
83 // throw error if shape is not dynamic
85 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the shape is requested is not dynamic");
86
87 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the shape is requested is not found");
88}
89
91 auto f = fReadyInputTensorInfos.find(name);
92 if (f != fReadyInputTensorInfos.end()) {
93 return f->second.type;
94 }
95 auto f2 = fInitializedTensors.find(name);
96 if (f2 != fInitializedTensors.end()) {
97 return f2->second.type();
98 }
99 auto f3 = fInputTensorInfos.find(name);
100 if (f3 != fInputTensorInfos.end()) {
101 return f3->second.type;
102 }
103 auto f4 = fIntermediateTensorInfos.find(name);
104 if (f4 != fIntermediateTensorInfos.end()) {
105 return f4->second.type;
106 }
107 auto f5 = fDynamicTensorInfos.find(name);
108 if (f5 != fDynamicTensorInfos.end()){
109 return f5->second.type;
110 }
111 // case of shape tensor type is INT64
112 if (fShapeTensors.find(name) != fShapeTensors.end()){
113 return ETensorType::INT64;
114 }
115
118
119 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the type is requested is not found, model name: " + fName);
120}
121
122bool RModel::CheckIfTensorAlreadyExist(std::string tensor_name) {
123 if (fReadyInputTensorInfos.find(tensor_name) != fReadyInputTensorInfos.end()) return true;
124 if (fInputTensorInfos.find(tensor_name) != fInputTensorInfos.end()) return true;
125 if (fInitializedTensors.find(tensor_name) != fInitializedTensors.end()) return true;
126 if (fIntermediateTensorInfos.find(tensor_name) != fIntermediateTensorInfos.end()) return true;
127 if (fDynamicTensorInfos.find(tensor_name) != fDynamicTensorInfos.end()) return true;
128 if (fShapeTensors.find(tensor_name) != fShapeTensors.end()) return true;
130 return false;
131}
132
133void RModel::AddInputTensorInfo(std::string input_name, ETensorType type, std::vector<Dim> shape) {
136 throw std::runtime_error("TMVA-SOFIE: input tensor with name " + input_name + " already exists \n");
137 }
138
139 InputTensorInfo inputInfo { type, shape };
141}
142
143void RModel::AddInputTensorInfo(std::string input_name, ETensorType type, std::vector<size_t> shape) {
146 throw std::runtime_error("TMVA-SOFIE: input tensor with name " + input_name + " already exists \n");
147 }
148 TensorInfo inputInfo { type, shape };
150}
151
155
156void RModel::AddOperator(std::unique_ptr<ROperator> op, int order_execution) {
157 AddBlasRoutines(op->GetBlasRoutines());
158 auto libs = op->GetStdLibs();
159 auto op_input_tensors = op->GetOpInputTensors();
160 for (auto& stdlib : libs) {
162 }
163 if (order_execution >= 0) {
164 fOperators.insert(fOperators.begin() + order_execution, std::move(op));
165 } else {
166 fOperators.push_back(std::move(op));
167 }
168
169 // storing the last usage of tensors which are input to
170 // operators (but are not inputs to the model, i.e. they are intermediate
171 // tensors). This information is needed to keep a check on when a
172 // particular intermediate tensor can be flushed to free up memory for reuse.
173 for(size_t index = 0; index<op_input_tensors.size() &&
175 std::find(fInputTensorNames.begin(), fInputTensorNames.end(),
178 ++index){
180 }
181}
182
183void RModel::AddInitializedTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape, std::shared_ptr<void> data) {
184 tensor_name = UTILITY::Clean_name(tensor_name);
185 //NB: own data
186 if (CheckIfTensorAlreadyExist(tensor_name)) {
187 throw std::runtime_error("TMVA-SOFIE: initialized tensor with name " + tensor_name + " already exists \n");
188 }
190 fInitializedTensors[tensor_name] = new_tensor;
191}
192
193void RModel::AddConstantTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape, std::shared_ptr<void> data) {
194 tensor_name = UTILITY::Clean_name(tensor_name);
195 //NB: own data
196 if (CheckIfTensorAlreadyExist(tensor_name)) {
197 throw std::runtime_error("TMVA-SOFIE: constant tensor with name " + tensor_name + " already exists \n");
198 }
199 InitializedTensor new_tensor {type, shape, data, true}; // add here flag to specify is a constant tensor
200 fInitializedTensors[tensor_name] = new_tensor;
201}
202
203void RModel::AddShapeTensor(const std::string & name, const std::vector<Dim> & shape_values, bool scalar){
204 auto tensor_name = UTILITY::Clean_name(name);
205 if (fShapeTensors.count(tensor_name) != 0) {
206 throw std::runtime_error("TMVA-SOFIE: shape tensor with name " + tensor_name + " already exists \n");
207 }
208 fShapeTensors[tensor_name] = std::make_pair(shape_values, scalar);
209}
210
211bool RModel::IsShapeTensor(const std::string & tensor_name) const {
212 return fShapeTensors.count(tensor_name) != 0;
213}
214
215const std::vector<Dim> & RModel::GetShapeTensorValues(const std::string & tensor_name) const {
216 //if (!IsShapeTensor(tensor_name) ) return std::vector<Dim>{};
217 return fShapeTensors.at(tensor_name).first;
218}
219
220bool RModel::IsInitializedTensor(const std::string& tensorName) const {
221 std::string name = UTILITY::Clean_name(tensorName);
222 return fInitializedTensors.find(name) != fInitializedTensors.end();
223}
224bool RModel::IsConstantTensor(const std::string& tensorName) const {
225 std::string name = UTILITY::Clean_name(tensorName);
226 auto itr = fInitializedTensors.find(name);
227 if (itr == fInitializedTensors.end()) return false;
228 return itr->second.IsConstantTensor();
229}
230
231// dynamic tensors include also Dim input tensors
232bool RModel::IsDynamicTensor(const std::string& tensorName) const {
233 std::string name = UTILITY::Clean_name(tensorName);
235 return (ret) ? true : IsDimInputTensor(tensorName);
236}
237bool RModel::IsDimInputTensor(const std::string& tensorName) const {
238 std::string name = UTILITY::Clean_name(tensorName);
239 return fInputTensorInfos.find(name) != fInputTensorInfos.end();
240}
241bool RModel::IsReadyInputTensor(const std::string& tensorName) const {
242 std::string name = UTILITY::Clean_name(tensorName);
244}
245
246// generic addition of a tensor
247void RModel::AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector<Dim> dim_shape) {
249 if (!int_shape.empty())
250 AddIntermediateTensor(tensor_name, type, int_shape);
251 else
252 AddDynamicTensor(tensor_name, type, dim_shape);
253}
254
255void RModel::AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape) {
256 tensor_name = UTILITY::Clean_name(tensor_name);
257 if (CheckIfTensorAlreadyExist(tensor_name)) {
258 throw std::runtime_error("TMVA-SOFIE: intermediate tensor with name " + tensor_name + " already exists \n");
259 }
260 TensorInfo new_tensor {type, shape};
262}
263
264void RModel::AddDynamicTensor(std::string tensor_name, ETensorType type, std::vector<Dim> shape){
265 tensor_name = UTILITY::Clean_name(tensor_name);
266 if (CheckIfTensorAlreadyExist(tensor_name)){
267 throw std::runtime_error("TMVA-SOFIE: intermediate tensor with name " + tensor_name + " already exists \n");
268 }
270 fDynamicTensorInfos[tensor_name] = new_tensor;
271 // store shape parameter if not existing
272 for (auto &d : shape) {
273 if (d.isParam) {
274 if (d.dim != size_t(-1)) {
275 AddShapeParam(d.param, d.dim);
276 }
277 }
278 }
279}
280
281void RModel::AddShapeParam(const std::string & param, size_t default_value) {
282 if (fShapeParams.count(param) == 0) {
283 fShapeParams[param] = std::to_string(default_value);
284 // add also in the vector list (used to keep the order)
285 fDimShapeNames.push_back(param);
286 }
287}
288
290 fOutputTensorNames.clear();
291 for(auto& it : outputtensornames) {
292 fOutputTensorNames.emplace_back(UTILITY::Clean_name(it));
293 }
294}
295
296void RModel::UpdateOutputTensorList(std::vector<std::string> curr_output_tensors, std::vector<std::string> new_output_tensors) {
297 for(auto& it:curr_output_tensors) {
298 fOutputTensorNames.erase(std::remove(fOutputTensorNames.begin(), fOutputTensorNames.end(), it), fOutputTensorNames.end());
299 }
301}
302
303void RModel::UpdateInitializedTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape, std::shared_ptr<void> data) {
304 tensor_name = UTILITY::Clean_name(tensor_name);
305 if (!CheckIfTensorAlreadyExist(tensor_name)) {
306 throw std::runtime_error("TMVA-SOFIE: tensor " + tensor_name + " not found when trying to update it");
307 }
309 fInitializedTensors[tensor_name] = new_tensor;
310}
311
312std::shared_ptr<void> RModel::GetInitializedTensorData(std::string tensor_name) {
313 auto f = fInitializedTensors.find(tensor_name);
314 if (f == fInitializedTensors.end()) {
315 throw std::runtime_error("TMVA-SOFIE: tensor " + tensor_name + " not found when trying to get its data");
316 } else {
317 return f->second.sharedptr();
318 }
319}
320
321void RModel::SetNotWritableInitializedTensor(const std::string & tensor_name) {
322 auto t = fInitializedTensors.find(tensor_name);
323 if (t == fInitializedTensors.end()) {
324 throw std::runtime_error("TMVA-SOFIE: initialized tensor " + tensor_name + " not found when trying to get its info");
325 }
326 t->second.SetNotWritable();
327 }
328
329std::string RModel::AllocateIntermediateMemory(std::span<const std::string_view> op_output_tensors)
330{
331 std::stringstream code;
332
333 if (fVerbose) {
334 std::cout << "Total chunks allocated\n";
336 std::cout << "..... chunk " << chunk->first << " size " << chunk->second.tensor_size << " " << chunk->second.tensor_name << std::endl;
337 }
338 }
339
340 auto declareIntermediateTensor = [this, &code](std::string const &name, size_t size, size_t location) {
341 std::string typeName = ConvertTypeToString(GetTensorType(name));
342 code << "\n // Allocating memory for intermediate tensor " << name << " with size " << size << " bytes";
343 code << "\n"
344 << typeName << "* tensor_" << name << " = reinterpret_cast<" << typeName
345 << "*>(fIntermediateMemoryPool.data() + " << location << ");\n";
346 };
347
348 if (fVerbose) std::cout << "*** AllocateIntermediateMemory: Loop on op output tensors\n";
349 // order output tensors by size
350 std::vector<TensorMemoryInfo> ordered_output_tensors;
351
352 for (auto &it : op_output_tensors) {
353 auto name = std::string(it);
356 continue;
357
359 // important fill the pair in the ordered output tensors with the string view and not the string
360 TensorMemoryInfo tmi = {it, tensor_size};
361 ordered_output_tensors.push_back(tmi);
362 }
364 [](const TensorMemoryInfo &a, const TensorMemoryInfo &b) { return a.tensor_size > b.tensor_size; });
365
366 for (auto &it : ordered_output_tensors) {
367 bool allocated = false;
368 std::string name = std::string{it.tensor_name};
369 size_t tensor_size = it.tensor_size;
370 if (fVerbose)
371 std::cout << "output tensor " << name << " size " << tensor_size << std::endl;
372
375
376 if (fVerbose) std::cout << ".. available chunk " << chunk->first << " with size = " << chunk->second;
377 // check if available memory chunks can accommodate the tensor
378 if (chunk->second >= tensor_size) {
379 // need to use here string_view (i.e it.tensor_name)
380 // split returns the new chunk with size of new tensor. The free chunk is before the used one
381 auto new_chunk = fIntermediateMemoryInfo.total_stack[chunk->first].split(it.tensor_name, tensor_size);
382 auto new_chunk_location = chunk->first + chunk->second - tensor_size;
384
386 chunk->second -= tensor_size;
387
388 allocated = true;
389
390 if (fVerbose) std::cout << " is re-used and split in a new of size " << new_chunk.tensor_size << " at " << new_chunk_location;
391
392 if (chunk->second == 0) {
393 if (fVerbose) std::cout << " and deleted since size matches";
395 }
396 if (fVerbose) std::cout << std::endl;
397 break;
398 } else if (chunk->first == fIntermediateMemoryInfo.available_stack.rbegin()->first &&
399 fIntermediateMemoryInfo.total_stack.rbegin()->first == chunk->first) {
400 // case last available chunk is the last in the memory, we can increase that one
401 fIntermediateMemoryInfo.total_stack[chunk->first] = {it.tensor_name, tensor_size};
402 declareIntermediateTensor(name, tensor_size, chunk->first);
404 allocated = true;
405 if (fVerbose) std::cout << " is extended with a bigger one of size " << tensor_size << std::endl;
406 break;
407 }
408 ++chunk;
409 if (fVerbose) std::cout << std::endl;
410 }
411
412 if (!allocated) {
414 ? 0
415 : fIntermediateMemoryInfo.total_stack.rbegin()->first +
416 fIntermediateMemoryInfo.total_stack.rbegin()->second.tensor_size;
417
419
421
422 if (fVerbose) std::cout << "no chunk available - add in total stack a new chunk with size of tensor and idx : " << chunk_idx
423 << std::endl;
424 }
425 }
426 return code.str();
427}
428
429void RModel::CheckAndFlushIntermediateMemory(std::span<const std::string_view> op_input_tensors, const size_t& op_idx){
430 if (fVerbose) std::cout << "*** CheckAndFlushIntermediateMemory: Loop on input tensors for op " << op_idx << "\n";
431 //print available chunks
432 if (fVerbose) std::cout << "available chunks before freeing them : \n";
435 if (fVerbose) std::cout << "-- free chunk " << chunk->first << " size = " << chunk->second << std::endl;
436 }
437 for (auto &it : op_input_tensors) {
438 // last occurence of the tensor is reached => flush it from memory
439 if (fVerbose) std::cout << ".. input tensors : " << it;
441 if (fVerbose) std::cout << " flash condition is met - looping on chunks to find matching one \n";
442 for (auto chunk = fIntermediateMemoryInfo.total_stack.begin();
444 if (fVerbose) std::cout << "--- chunk " << chunk->first << " , " << chunk->second.tensor_name << " size " << chunk->second.tensor_size;
445 if (chunk->second.tensor_name == it) {
446 if (fVerbose) std::cout << " -- Found chunk corresponding to input tensor: " << chunk->first;
447 // check if nearby chunks in available memory can coalesce
449 chunk->first); // smallest element greater than the flushed chunk idx
452 : std::prev(first_greater); // largest element smaller than the flushed chunk idx
453
454 // check if the next stack entry is actually adjacent in memory
455
457 last_smaller->first + last_smaller->second == chunk->first) {
458 // merge chunk with previous one
459 last_smaller->second += chunk->second.tensor_size;
461 if (fVerbose) std::cout << " is adjacent in memory with previous one - merge ";
463 last_smaller->first + last_smaller->second == first_greater->first) {
464 // merge also with following one
465 last_smaller->second += first_greater->second;
468 // delete merged one in available stack and in total stack
471 if (fVerbose) std::cout << " merge also with following that is free ";
472 }
474 if (fVerbose) std::cout << std::endl;
475 break;
477 chunk->first + chunk->second.tensor_size == first_greater->first) {
478 // merge with first greater
479 if (fVerbose) std::cout << " is adjacent in memory with following one - merge \n";
480 // cannot modify idx of first_greter. Insert a new one and delete previous one
481 size_t new_size = chunk->second.tensor_size + first_greater->second;
482 size_t first_greater_idx = first_greater->first;
484 // cannot use anymore first_greater
489 } else {
490 fIntermediateMemoryInfo.available_stack.insert({chunk->first, chunk->second.tensor_size});
491 if (fVerbose) std::cout << " insert in the available stack the chunk with size " << chunk->second.tensor_size << std::endl;
492 }
493 chunk->second.tensor_name = "free";
494 break;
495 }
496 }
497 } else {
498 if (fVerbose) std::cout << std::endl;
499 }
500 }
501}
502
503void RModel::Initialize(int batchSize, bool verbose) {
504 std::map<std::string, size_t> inputParams;
505 if (batchSize > 0) {
506 inputParams["input_size"] = batchSize;
507 inputParams["batch_size"] = batchSize;
508 inputParams["bs"] = batchSize;
509 }
510 Initialize(inputParams, verbose);
512}
513void RModel::Initialize(const std::map<std::string, size_t> & inputParams, bool verbose) {
514
515 fVerbose = int(verbose);
516
517 if (fIsInitialized) {
518 if (verbose)
519 std::cout << "Model is already initialized - skip initialization " << std::endl;
520 return;
521 }
523 fDynamicTensorInfos.clear();
524
525 // loop on inputs and see if shape can be full specified
526 // if the batch size is provided it can be used to specify the full shape
527 // Add the full specified tensors in fReadyInputTensors collection
528 auto originalInputTensorInfos = fInputTensorInfos; // need to copy because we may delete elements
529 for (auto &input : originalInputTensorInfos) {
530 if (verbose) std::cout << "looking at the tensor " << input.first << std::endl;
531 // if a parameter (e.g. batch_size) is specified use for converting parametric shape in defined one
532 if (!inputParams.empty()) {
533 for (auto &d : input.second.shape) {
534 if (d.isParam) {
535 std::string pname = d.param;
536 if (pname == input.first + "_size") pname = "input_size";
537 auto itr = inputParams.find(pname);
538 if (itr != inputParams.end() ) {
539 d = Dim{ itr->second };
540 if (verbose)
541 std::cout << "Tensor: " << input.first << " - fix parametric shape " << itr->first << " to " << itr->second << std::endl;
542 }
543 }
544 }
545 }
546 // see if shape now is fully defined
547 auto shape = ConvertShapeToInt(input.second.shape);
548 if (verbose)
549 std::cout << "converting input shape for " << input.first << " " << ConvertShapeToString(shape) << " from "
550 << ConvertShapeToString(input.second.shape) << std::endl;
551 if (!shape.empty()) {
552 // case shape is defined (not parametric) we add the tensor in the fReadyInputTensorInfos map and
553 // we remove the tensor from the fInputTensorInfo where th eold parametric shape was stored
554 fInputTensorInfos.erase(input.first);
555 // add to the ready input tensor information the new fixed shape
556 AddInputTensorInfo(input.first, input.second.type, shape);
557 // check consistency
559 }
560 // store the parameters of the input tensors
561 else {
562 // store the found parametric shape parameters
563 for (auto &d : input.second.shape) {
564 if (d.isParam) {
565 if (fShapeParams.count(d.param) == 0) {
566 fDimShapeNames.push_back(d.param);
567 fShapeParams[d.param] = std::to_string(d.dim);
568 }
569 }
570 }
571 }
572 }
573
574 if (verbose) {
577 }
578
579 // check if there are initialized tensors to write in a weight file
580 // support for the time being only weight of FLOAT type
581 if (fUseWeightFile) {
582 bool modelHasWeights = false;
583 for (auto &i : fInitializedTensors) {
584 if (i.second.type() == ETensorType::FLOAT) {
585 modelHasWeights = true;
586 break;
587 }
588 }
589 if (!modelHasWeights)
590 fUseWeightFile = false;
591 }
592 // Go through model and initialize each operator
593 int i = 0;
594
595 std::vector<size_t> temp_available_stack; // vector stores individual chunks of available memory that maybe reused
596
597 for(size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx){
598 if (verbose) {
599 auto& r = *fOperators[op_idx].get();
600 std::cout << "Initializing operator " << i << " " << typeid(r).name() << std::endl;
601 }
602 fOperators[op_idx]->Initialize(*this);
603 for(auto &it:fOperators[op_idx]->GetOpOutputTensors()){
604 std::string name = std::string{it};
606 std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), name) == fOutputTensorNames.end() &&
610 }
611 }
612 i++;
613 }
614
615 fIsInitialized = true;
616}
617
618void RModel::InitializeSubGraph(std::shared_ptr<RModel> graph) {
619 // add the subgraph to the list
620 fSubGraphs.push_back(graph);
621 //this needs to be done before initializing
622 graph->fParentGraph = this;
623 graph->fIsSubGraph = true;
624
625 graph->Initialize(fBatchSize, fVerbose);
626 // set the same options as parent model
627 graph->fWeightFile = fWeightFile;
628 graph->fUseWeightFile = fUseWeightFile;
629 graph->fUseSession = fUseSession;
630 // add needed blas routines and libs
631 std::vector<std::string> blasRoutines;
632 for (auto & e : graph->fNeededBlasRoutines)
633 blasRoutines.push_back(e);
635 for (auto e : graph->fNeededStdLib)
637
638 // add parent input tensors to current graph
639 for (auto & name : fInputTensorNames)
640 graph->fInputTensorNames.emplace_back(name);
641
642 // clean graph name
643 graph->fName = UTILITY::Clean_name(graph->fName);
644
645}
646
647// Function to generate the code for declaring and initializing constant tensors
648// This is for tensors which are not part of weight files and can be created from the Constant operator
649template <typename T>
650std::string GenerateConstantTensorCode(const std::pair<std::string, InitializedTensor> &t)
651{
652 std::stringstream strs;
653 std::string type = ConvertTypeToString(t.second.type());
654 size_t length = ConvertShapeToLength(t.second.shape());
655 // avoid using stack sizes for constant tensors to reduce compilation time
656 bool allocateOnStack = (length > 100) ? false : true;
657
658 const T *data = t.second.data<T>();
659
660 // and check if all values are the same
661 bool sameData = false;
662 // for non stack allocation check if data are the same
663 if (!allocateOnStack && length > 1) {
664 size_t idx = 1;
665 do {
666 sameData = (data[idx] == data[idx - 1]);
667 idx++;
668 } while (sameData && idx < length);
669 }
670 if (allocateOnStack) {
671 strs << type << " tensor_" << t.first << "[" << length << "] = " << ConvertValuesToString(length, data) << ";\n";
672 } else {
673 strs << "std::vector<" << type << "> fTensor_" << t.first << " = ";
674 if (sameData)
675 strs << "std::vector<" << type << ">(" << length << ", " << ConvertValToString(data[0]) << ");\n";
676 else {
678 }
679 strs << "const " << type << " * tensor_" + t.first + " = fTensor_" + t.first + ".data();\n";
680 }
681 return strs.str();
682}
683
685{
686 if (!fInitializedTensors.empty())
687 fGC += "// initialized tensors\n";
688
689 for (auto &i : fInitializedTensors) {
690 if (!fUseWeightFile || i.second.IsConstantTensor()) {
691 if (i.second.type() == ETensorType::FLOAT) {
693 fConstantTensorSize += ConvertShapeToLength(i.second.shape()) * 4;
694 } else if (i.second.type() == ETensorType::INT64) {
696 fConstantTensorSize += ConvertShapeToLength(i.second.shape()) * 8;
697 }
698
699 } else {
700 // case of tensors which are read from a file
701 size_t length = ConvertShapeToLength(i.second.shape());
702 if (i.second.type() == ETensorType::FLOAT) {
703 fGC += "std::vector<float> fTensor_" + i.first + " = std::vector<float>(" + std::to_string(length) + ");\n";
704 fGC += "float * tensor_" + i.first + " = fTensor_" + i.first + ".data();\n";
705 fWeightsTensorSize += ConvertShapeToLength(i.second.shape()) * 4;
706 }
707 }
708 }
709}
710
712 if (fIntermediateMemoryInfo.total_stack.empty()) return;
713 fGC += "\n//--- Allocating session memory pool to be used for allocating intermediate tensors\n";
714
715 // char memory block is allocated since char takes 1 byte, thus easier to allocate tensors
716 // of other data types
718 const size_t memPoolSize = totalStack.rbegin()->first + totalStack.rbegin()->second.tensor_size;
719 fGC += "std::vector<char> fIntermediateMemoryPool = std::vector<char>(" + std::to_string(memPoolSize) + ");\n\n";
720}
721
723 if (!fIntermediateTensorInfos.empty()) {
724 std::string tensor_declaration_block = "";
725 for (auto &i : fIntermediateTensorInfos) {
726 if (i.second.type == ETensorType::BOOL) {
727 tensor_declaration_block += "std::vector<std::uint8_t> fTensor_" + i.first + " = std::vector<std::uint8_t>(" + std::to_string(ConvertShapeToLength(i.second.shape)) + ");\n";
728 tensor_declaration_block += "std::uint8_t * tensor_" + i.first + " = fTensor_" + i.first + ".data();\n";
729 continue;
730 }
732 bool not_in_freq_map =
735 (std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), i.first) == fOutputTensorNames.end());
736
738 size_t length = ConvertShapeToLength(i.second.shape);
739
740 if (i.second.type == ETensorType::FLOAT) {
741 tensor_declaration_block += "std::vector<float> fTensor_" + i.first + " = std::vector<float>(" + std::to_string(length) + ");\n";
742 tensor_declaration_block += "float * tensor_" + i.first + " = fTensor_" + i.first + ".data();\n";
744 }
745 else if (i.second.type == ETensorType::DOUBLE) {
746 tensor_declaration_block += "std::vector<double> fTensor_" + i.first + " = std::vector<double>(" + std::to_string(length) + ");\n";
747 tensor_declaration_block += "double * tensor_" + i.first + " = fTensor_" + i.first + ".data();\n";
749 }
750 else if (i.second.type == ETensorType::INT64) {
751 tensor_declaration_block += "std::vector<int64_t> fTensor_" + i.first + " = std::vector<int64_t>(" + std::to_string(length) + ");\n";
752 tensor_declaration_block += "int64_t * tensor_" + i.first + " = fTensor_" + i.first + ".data();\n";
754 }
755 }
756 }
757
758 if (tensor_declaration_block.length()) {
759 fGC += "\n//--- declare and allocate the intermediate tensors\n" + tensor_declaration_block;
760 }
761 }
762 // add also the dynamic tensors (only declarations, allocation will be done later)
763 if (!fDynamicTensorInfos.empty()) {
764 fGC += "//--- declare the dynamic tensors\n";
765 for (auto &i : fDynamicTensorInfos) {
766 if (i.second.type == ETensorType::FLOAT) {
767 fGC += "std::vector<float> fTensor_" + i.first + ";\n";
768 fGC += "float * tensor_" + i.first + " = nullptr;\n";
769 } else if (i.second.type == ETensorType::DOUBLE) {
770 fGC += "std::vector<double> fTensor_" + i.first + ";\n";
771 fGC += "double * tensor_" + i.first + " = nullptr;\n";
772 } else if (i.second.type == ETensorType::INT64) {
773 fGC += "std::vector<int64_t> fTensor_" + i.first + ";\n";
774 fGC += "int64_t * tensor_" + i.first + " = nullptr;\n";
775 }
776 }
777 }
778}
779
780// generate code for specific operator declarations to be defined in the Session class
782 std::string strcode;
783 for (auto & op : fOperators) {
784 strcode += op->GenerateDeclCode();
785 }
786 if (strcode.empty()) return;
787 fGC += "\n//---- operator declarations \n";
788 fGC += strcode;
789 fGC += "\n";
790}
791
793{
794 std::stringstream out;
795 for (auto &i : fDynamicTensorInfos) {
796 auto length = ConvertDynamicShapeToLength(i.second.shape);
797 out << SP << "if (" << length << " > 0) {\n";
798 out << SP << SP << "fTensor_" << i.first << ".resize(" << length << ");\n";
799 out << SP << SP << "tensor_" << i.first << " = fTensor_" << i.first << ".data();\n";
800 out << SP << "}\n";
801 }
802 fGC += out.str();
803}
804
806 // generate the infer signature given the inputs: eg. "float * tensor1, float * tensor2"
807 // if (decl = false) generate only calling signature (tensor1,tensor2,....)
808 std::string rGC;
809 std::unordered_map<std::string, int> inputParams;
810 int i_input = 0;
811 for (auto &name : fInputTensorNames) {
812 // if is a dynamic tensor pass initial parameters
813 if (IsDimInputTensor(name)) {
814 auto shape = GetDynamicTensorShape(name);
815 for (auto &d : shape) {
816 std::string pName = d.param;
817 // need to check if the input parameters is already existing in another input tensor
818 if (d.isParam && inputParams.count(pName) == 0) {
819 if (isdecl) rGC += "size_t ";
820 rGC += d.param + ",";
822 }
823 }
824 }
825 if (isdecl) {
827 if (type == "other")
828 throw std::runtime_error("TMVA-SOFIE: input tensor " + name +
829 " is of a data type which is not yet supported.");
830 rGC += type + " const* ";
831 }
832 rGC += "tensor_" + name + ",";
833 i_input++;
834 }
835
836 if (fInputTensorNames.size() > 0) rGC.pop_back();// remove last ","
837 return rGC;
838}
839
840namespace {
841
842std::string typeForOutput(ETensorType t) {
843 // The std::vector<bool> is a special type that is not wrapping continuous memory.
844 // We don't want to use it as a return type.
846 return ConvertTypeToString(t);
847}
848
849}
850
852{
853 size_t outputSize = fOutputTensorNames.size();
854 // assume output types are all the same
855
856 bool sameOutputTypes = true;
857 std::string inferReturnType; // type return by infer function
859 fGC += "\n\n";
860 if (outputSize == 1) {
861 fGC += "std::vector<" + typeForOutput(eFirstOutputType) + ">";
862 } else {
863 // if all output types are the same we return an std::vector - otherwise a tuple
864 for (std::string const &name : fOutputTensorNames) {
866 sameOutputTypes = false;
867 }
868 if (sameOutputTypes)
869 fGC += "std::vector<std::vector<" + typeForOutput(eFirstOutputType) + ">>";
870 else {
871 inferReturnType = "std::tuple<";
872 for (size_t i = 0; i < outputSize; i++) {
873 inferReturnType += "std::vector<" + typeForOutput(GetTensorType(fOutputTensorNames[i])) + ">";
874 if (i < outputSize - 1)
875 inferReturnType += ",";
876 }
877 inferReturnType += ">";
879 }
880 }
881
882 fGC += " infer(" + GenerateInferSignature() + "){\n";
883
884 std::string doInferArgs = GenerateInferSignature(false);
885 if (!doInferArgs.empty())
886 doInferArgs += ",";
887 for (std::string const &name : fOutputTensorNames) {
888 fGC += SP + "std::vector<" + typeForOutput(GetTensorType(name)) + " > output_tensor_" + name + ";\n";
889 doInferArgs += " output_tensor_" + name + ",";
890 }
891 if (!doInferArgs.empty())
892 doInferArgs.back() = ' ';
893
894 fGC += SP + "doInfer(" + doInferArgs + ");\n";
895
896 fGC += SP + "return {";
897 for (size_t i = 0; i < fOutputTensorNames.size(); i++) {
898 fGC += "output_tensor_" + fOutputTensorNames[i];
899 if (i < fOutputTensorNames.size() - 1)
900 fGC += ",";
901 }
902 fGC += "};\n";
903 fGC += "}\n"; // end of infer function scope
904}
905
907{
908 // Determine the signature of the actual inference function
910 if (!doInferSignature.empty())
911 doInferSignature += ", ";
912 for (auto const &name : fOutputTensorNames) {
913 doInferSignature += " std::vector<" + typeForOutput(GetTensorType(name)) + "> &output_tensor_" + name + ",";
914 }
915 doInferSignature.back() = ' ';
916
917 doInferSignature = "void doInfer(" + doInferSignature + ")";
918
919 // define the Session struct (for GNN this is generated in RModel_GNN)
921 if (!fIsSubGraph)
922 fGC += "struct Session {\n";
923 else
924 fGC += "struct Session_" + fName + " {\n";
925 }
926
927 // generate code for declaring the initialized tensors
929
931 // evaluate total intermediate memory and position intermediate tensor addresses
932 std::string intermediate_memory_alloc_string = "";
933 intermediate_memory_alloc_string += "\n// --- Positioning intermediate tensor memory --";
934 for (size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx) {
935 if (fVerbose) {
936 auto op = fOperators[op_idx].get();
937 std::cout << "\n******************\n analyzing input/output operator " << op_idx << " "
938 << typeid(*op).name() << std::endl;
939 }
942 }
943
944 // to check remaining unused fragments after memory allocation (lesser the better)
945 // for (const auto &it: fIntermediateMemoryInfo.available_stack){
946 // std::cout<<"chunk_idx: "<<it.first<<", chunk_size: "<<it.second<<"\n";
947 // }
948
949 // generate the memory pool to be used by intermediate tensors
951
952 // position intermediate tensors
954 }
955
956 // generate the declaring the intermediate tensors
958 // generate code for declarations of some specific operators
960
961
962
963 // add subgraph session
964 if (!fSubGraphs.empty()) fGC += "// subgraph sessions\n";
965 for (auto & graph : fSubGraphs) {
966 fGC += "Session_" + graph->fName + " fSession_" + graph->fName + ";\n";
967 }
968
969 // Generate code for Session constructor
970 if (fUseSession) {
971 std::string sessionName = "Session";
972 if (fIsSubGraph)
973 sessionName += "_" + fName;
974 // add here specific operator code that needs to define session data members
975 fGC += "\n";
976 for (size_t id = 0; id < fOperators.size(); id++) {
977 std::string opName = std::to_string(id);
978 fGC += fOperators[id]->GenerateSessionMembersCode(opName);
979 }
980 fGC += "\n";
981 // here add initialization and reading of weight tensors
982 if (fUseWeightFile) {
983 std::string fileName = fName;
985 fileName += ".dat";
986 }
988 fileName += ".root";
989 }
990 fGC += sessionName + "(std::string filename =\"" + fileName + "\"";
991 } else {
992 // no need to pass weight file since it is not used
993 // keep passing a string for compatibility
994 fGC += sessionName + "(std::string = \"\"";
995 }
996 // add initialization of shape parameters
997 // assume all parameters are of type size_t
998 if (!fDimShapeNames.empty()) {
999 for (auto &p : fDimShapeNames) {
1000 fGC += ",\n";
1001 fGC += " size_t " + p + " = " + fShapeParams[p];
1002 }
1003 }
1004 fGC += ") {\n";
1005
1006 if (fUseWeightFile) {
1007 fGC += "\n//--- reading weights from file\n";
1009 fGC += "\n";
1010 // fUseWeightFile = fUseWeightFile;
1011 }
1012
1013 // now we have passed the parameters we can allocate the dynamic tensors
1015
1016 // add here initialization code for operator
1017 for (size_t id = 0; id < fOperators.size(); id++) {
1018 fGC += fOperators[id]->GenerateInitCode();
1019 }
1020
1021 fGC += "}\n\n";
1022 }
1023
1024 fGC += doInferSignature + "{\n";
1025 fGC += "\n";
1026
1027 // generate the inference code
1028 if (fVerbose)
1029 std::cout << "Generating main inference code for " << fName << std::endl;
1030
1031 if (fOutputTensorNames.size() == 0)
1032 throw std::runtime_error("TMVA-SOFIE: output size=0 are not supported");
1033
1034 for (size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx) {
1035 if (fVerbose)
1036 std::cout << "Generating code for operator .... " << op_idx << std::endl;
1037 fGC += (fOperators[op_idx]->Generate(std::to_string(op_idx)));
1038 }
1039
1040 fGC += SP + "using TMVA::Experimental::SOFIE::UTILITY::FillOutput;\n\n";
1041
1042 for (std::string const &name : fOutputTensorNames) {
1043 // need to check is size is the same (don't want to return a vector with
1044 // larger size) in that case better to copy
1046 std::string n = isIntermediate ? std::to_string(ConvertShapeToLength(GetTensorShape(name)))
1048 fGC += SP + "FillOutput(tensor_" + name + ", output_tensor_" + name + ", " + n + ");\n";
1049 }
1050
1051 fGC += "}\n\n";
1052
1053 // generate the inference overload that returns an output struct
1055
1056 // end of session
1057 if (fUseSession && !fIsGNNComponent) {
1058 fGC += "}; // end of Session\n\n";
1059 }
1060}
1061
1062void RModel::Generate(std::underlying_type_t<Options> options, int batchSize, long pos, bool verbose)
1063{
1064 fVerbose = verbose;
1065 fBatchSize = batchSize;
1066 fReadPos = pos;
1067
1068 // session flag is used in operator initialize
1069 if (static_cast<std::underlying_type_t<Options>>(Options::kNoSession) & options) {
1070 fUseSession = false;
1072 }
1073 if (static_cast<std::underlying_type_t<Options>>(Options::kNoWeightFile) & options) {
1074 fUseWeightFile = false;
1076 }
1077 if (static_cast<std::underlying_type_t<Options>>(Options::kRootBinaryWeightFile) & options) {
1078 fUseWeightFile = true;
1080 }
1081 if (fUseWeightFile && !fUseSession) {
1082 throw std::runtime_error(
1083 "TMVA-SOFIE: RModel::Generate: cannot use a separate weight file without generating a Session class");
1084 }
1085
1086 if (static_cast<std::underlying_type_t<Options>>(Options::kGNN) & options)
1087 fIsGNN = true;
1088 if (static_cast<std::underlying_type_t<Options>>(Options::kGNNComponent) & options)
1089 fIsGNNComponent = true;
1090
1091 // initialize the model including all operators and sub-graphs
1092 Initialize(batchSize, verbose);
1093
1094 std::string hgname;
1095 if (!fIsGNNComponent && !fIsSubGraph) {
1096 fGC.clear();
1098 }
1099
1100 // generate first code for the subgraphs
1101 for (auto &graph : fSubGraphs) {
1102 if (fVerbose)
1103 std::cout << "generate session code for subgraph " << graph->fName << std::endl;
1104 graph->GenerateSessionCode();
1105 fGC += graph->fGC;
1106 }
1107
1108 if (fVerbose)
1109 std::cout << "generate Main session code - model " << fName << std::endl;
1110
1111 // generate main session code
1113
1114 if (!fIsGNNComponent && !fIsSubGraph) {
1115 fGC += ("} //TMVA_SOFIE_" + fName + "\n");
1116 fGC += "\n#endif // " + hgname + "\n";
1117 }
1118}
1119
1121 // generate the code to read initialized tensors from a text data file
1123 if (fInitializedTensors.empty()) return;
1124
1125 fGC += " std::ifstream f;\n";
1126 fGC += " f.open(filename);\n";
1127 fGC += " if (!f.is_open()) {\n";
1128 fGC += " throw std::runtime_error(\"tmva-sofie failed to open file \" + filename + \" for input weights\");\n";
1129 fGC += " }\n";
1130
1131 if(fIsGNNComponent) {
1132 fGC += " f.seekg(" + std::to_string(pos) + ");\n";
1133 }
1134
1135 fGC += " using TMVA::Experimental::SOFIE::ReadTensorFromStream;\n";
1136
1137 // loop on tensors and parse the file
1138 for (auto& i: fInitializedTensors) {
1139 // skip Constant and shape tensors (not written in a file)
1140 if (!i.second.IsWeightTensor()) continue;
1141 std::string tensor_name = "tensor_" + i.first;
1142 if (i.second.type() == ETensorType::FLOAT) {
1143 std::string length = std::to_string(ConvertShapeToLength(i.second.shape()));
1144 fGC += " ReadTensorFromStream(f, " + tensor_name + ", \"" + tensor_name + "\", " + length + ");\n";
1145 } else {
1146 std::runtime_error("tmva-sofie tensor " + tensor_name + " with type " + ConvertTypeToString(i.second.type()) + " cannot be read from a file");
1147 }
1148 }
1149 fGC += " f.close();\n";
1150 }
1151
1152 // generate the code to read initialized tensors from a ROOT data file
1154#ifdef SOFIE_SUPPORT_ROOT_BINARY
1155 fGC += " {\n";
1156 fGC += " std::unique_ptr<TFile> rootFile(TFile::Open(filename.c_str(), \"READ\"));\n";
1157 fGC += " if (!rootFile->IsOpen()) {\n";
1158 fGC += " throw std::runtime_error(\"tmva-sofie failed to open ROOT file for input weights\");\n";
1159 fGC += " }\n";
1160
1161 std::string dirName = fName + "_weights";
1162 fGC += " if (!rootFile->GetKey(\"" + dirName + "\")) {\n";
1163 fGC += " throw std::runtime_error(\"tmva-sofie failed to open ROOT directory for input weights\");\n";
1164 fGC += " }\n";
1165
1166 for (auto &i : fInitializedTensors) {
1167 // skip Constant and shape tensors
1168 if (!i.second.IsWeightTensor()) continue;
1169 fGC += " {\n";
1170 std::string tensor_name = "tensor_" + i.first;
1171 if (i.second.type() == ETensorType::FLOAT) {
1172 fGC += " fTensor_" + i.first + " = *reinterpret_cast<std::vector<float>*>(rootFile->Get(\"";
1173 fGC += dirName + "/" + tensor_name + "\"));\n";
1174 } else if (i.second.type() == ETensorType::DOUBLE) {
1175 fGC += " fTensor_" + i.first + " = *reinterpret_cast<std::vector<double>*>(rootFile->Get(\"";
1176 fGC += dirName + + "/" + tensor_name + "\"));\n";
1177 } else if (i.second.type() == ETensorType::INT64) {
1178 fGC += " fTensor_" + i.first + " = *reinterpret_cast<std::vector<int64_t>*>(rootFile->Get(\"";
1179 fGC += dirName + "/" + tensor_name + "\"));\n";
1180 } else {
1181 std::runtime_error("tmva-sofie tensor " + tensor_name + " with type " + ConvertTypeToString(i.second.type()) + " cannot be read from a ROOT file");
1182 }
1183 fGC += " }\n";
1184 }
1185 fGC += " }\n";
1186#else
1187 throw std::runtime_error("SOFIE was not built with ROOT file support.");
1188#endif // SOFIE_SUPPORT_ROOT_BINARY
1189 }
1190}
1191
1193 // Determine the file extension based on the weight file type
1194 std::string fileExtension;
1195 switch (fWeightFile) {
1197 fileExtension = ".dat";
1198 break;
1200 fileExtension = ".root";
1201 break;
1203 fileExtension = ".dat";
1204 break;
1205 }
1206
1207 // If filename is empty, use the model name as the base filename
1208 if (filename.empty()) {
1210 }
1211
1212 // Write the initialized tensors to the file
1214#ifdef SOFIE_SUPPORT_ROOT_BINARY
1215 if(fIsGNNComponent || fIsGNN) {
1216 throw std::runtime_error("SOFIE-GNN yet not supports writing to a ROOT file.");
1217 }
1218 std::unique_ptr<TFile> outputFile(TFile::Open(filename.c_str(), "UPDATE"));
1219
1220 std::string dirName = fName + "_weights";
1221 // check if directory exists, in case delete to replace with new one
1222 if (outputFile->GetKey(dirName.c_str()))
1223 outputFile->rmdir(dirName.c_str());
1224
1225 auto outputDir = outputFile->mkdir(dirName.c_str());
1226
1227 for (const auto& item : fInitializedTensors) {
1228 // skip Constant tensors and tensors which are not writable (e.g. shape tensors)
1229 if (!item.second.IsWeightTensor()) continue;
1230 std::string tensorName = "tensor_" + item.first;
1231 size_t length = 1;
1232 length = ConvertShapeToLength(item.second.shape());
1233 if(item.second.type() == ETensorType::FLOAT) {
1234 const float* data = item.second.data<float>();
1235 std::vector<float> tensorDataVector(data, data + length);
1236 outputDir->WriteObjectAny(&tensorDataVector, "std::vector<float>", tensorName.c_str());
1237 }
1238 else if(item.second.type() == ETensorType::DOUBLE) {
1239 const double* data = item.second.data<double>();
1240 std::vector<double> tensorDataVector(data, data + length);
1241 outputDir->WriteObjectAny(&tensorDataVector, "std::vector<double>", tensorName.c_str());
1242 }
1243 else if(item.second.type() == ETensorType::INT64) {
1244 const int64_t* data = item.second.data<int64_t>();
1245 std::vector<int64_t> tensorDataVector(data, data + length);
1246 outputDir->WriteObjectAny(&tensorDataVector, "std::vector<int64_t>", tensorName.c_str());
1247 }
1248 else {
1249 std::runtime_error("tmva-sofie tensor " + tensorName + " with type " + ConvertTypeToString(item.second.type()) +
1250 " cannot be written to a ROOT file");
1251 }
1252 }
1253 outputFile->Write(filename.c_str());
1254
1255 // this needs to be changed, similar to the text file
1256 return -1;
1257
1258#else
1259 throw std::runtime_error("SOFIE was not built with ROOT file support.");
1260#endif // SOFIE_SUPPORT_ROOT_BINARY
1261 } else if (fWeightFile == WeightFileType::Text) {
1262 std::ofstream f;
1263 if(fIsGNNComponent) {
1264 // appending all GNN components into the same file
1265 f.open(filename, std::ios::app);
1266 } else {
1267 f.open(filename);
1268 }
1269 if (!f.is_open())
1270 throw
1271 std::runtime_error("tmva-sofie failed to open file " + filename + " for tensor weight data");
1272 for (auto& i: fInitializedTensors) {
1273 // skip Constant tensors and not writable tensors (e.g. shape tensors)
1274 if (!i.second.IsWeightTensor()) {
1275 continue;
1276 }
1277 size_t length = ConvertShapeToLength(i.second.shape());
1278 std::string tensor_name = "tensor_" + i.first;
1279 f << tensor_name << " " << length << "\n";
1280 if (i.second.type() == ETensorType::FLOAT) {
1281 const float * data = i.second.data<float>();
1282 for (size_t idx = 0; idx < length; idx++) {
1283 // round to zero sub-normal values
1284 float value = data[idx];
1285 if (value != 0. && std::abs(value) < std::numeric_limits<float>::min() ) value = 0;
1286 f << std::setprecision(std::numeric_limits<float>::max_digits10) << value;
1287 f << ( (idx < length-1) ? " " : "\n" );
1288 }
1289 }
1290 else {
1291 std::runtime_error("tmva-sofie tensor " + tensor_name + " with type " + ConvertTypeToString(i.second.type()) + " cannot be written to a file");
1292 }
1293 if (f.fail())
1294 std::runtime_error("tmva-sofie failed to write tensor data to file for " + tensor_name);
1295 }
1296 long curr_pos = f.tellp();
1297 f.close();
1298 return curr_pos;
1299 } else {
1300 return -1;
1301 }
1302}
1303
1305 std::cout << "Model requires following inputs:\n";
1306 for (auto& inputInfo: fInputTensorInfos) {
1307 std::cout << "Parametrised Tensor name: " << inputInfo.first << "\t";
1308 std::cout << "type: " << ConvertTypeToString(inputInfo.second.type) << "\t";
1309 std::cout << "shape: [";
1310 for (size_t i = 0; i < inputInfo.second.shape.size(); i++) {
1311 if (inputInfo.second.shape[i].isParam) {
1312 std::cout << inputInfo.second.shape[i].param;
1313 } else {
1314 std::cout << inputInfo.second.shape[i].dim ;
1315 }
1316 if (i < inputInfo.second.shape.size() - 1) std::cout << ",";
1317 }
1318 std::cout << "]" << std::endl;
1319 }
1320
1321 for (auto& inputInfo: fReadyInputTensorInfos) {
1322 std::cout << "Fully Specified Tensor name: " << inputInfo.first << "\t";
1323 std::cout << "type: " << ConvertTypeToString(inputInfo.second.type) << "\t";
1324 std::cout << "shape: [";
1325 for (size_t i = 0; i < inputInfo.second.shape.size(); i++) {
1326 std::cout << inputInfo.second.shape[i];
1327 if (i < inputInfo.second.shape.size() - 1) std::cout << ",";
1328 }
1329 std::cout << "]" << std::endl;
1330 }
1331 std::cout << "\n";
1332}
1333
1335 std::cout << "Model initialized the following tensors:\n";
1336 for (auto& it: fInitializedTensors) {
1337 std::cout << "Tensor name: \"" << it.first << "\"\t";
1338 std::cout << "type: " << ConvertTypeToString(it.second.type()) << "\t";
1339 std::cout << "shape: [";
1340 for (size_t i = 0; i < it.second.shape().size(); i++) {
1341 std::cout << it.second.shape()[i];
1342 if (i < it.second.shape().size() - 1) std::cout << ",";
1343 }
1344 std::cout << "]";
1345 if (it.second.IsConstantTensor()) std::cout << " (Constant)";
1346 else if (!it.second.IsWeightTensor()) std::cout << " (Not Writable)";
1347 std::cout << std::endl;
1348 }
1349 std::cout << "\n";
1350}
1351
1353 std::cout << "Model specify the following intermediate tensors:\n";
1354 for (auto& it: fIntermediateTensorInfos) {
1355 std::cout << "Tensor name: \"" << it.first << "\"\t";
1356 std::cout << "type: " << ConvertTypeToString(it.second.type) << "\t";
1357 std::cout << "shape: [";
1358 for (size_t i = 0; i < it.second.shape.size(); i++) {
1359 std::cout << it.second.shape[i];
1360 if (i < it.second.shape.size() - 1) std::cout << ",";
1361 }
1362 std::cout << "]" << std::endl;
1363 }
1364 std::cout << "\n";
1365}
1366
1368 std::cout << "Model specify the following dynamic tensors:\n";
1369 for (auto& it: fDynamicTensorInfos) {
1370 std::cout << "Tensor name: \"" << it.first << "\"\t";
1371 std::cout << "type: " << ConvertTypeToString(it.second.type) << "\t";
1372 std::cout << "shape: [";
1373 for (size_t i = 0; i < it.second.shape.size(); i++) {
1374 std::cout << it.second.shape[i].GetVal();
1375 if (i < it.second.shape.size() - 1) std::cout << ",";
1376 }
1377 std::cout << "]" << std::endl;
1378 }
1379 std::cout << "\n";
1380}
1381
1383 std::cout << "Model specify the following output tensors:\n";
1384 for (auto& it: fOutputTensorNames) {
1385 std::cout << "Tensor name: \"" << it << "\"\t";
1386 if (!IsDynamicTensor(it))
1387 std::cout << "shape: " << ConvertShapeToString(GetTensorShape(it)) << std::endl;
1388 else
1389 std::cout << "shape: " << ConvertShapeToString(GetDynamicTensorShape(it)) << std::endl;
1390 }
1391 std::cout << "\n";
1392}
1393
1395 auto it = fInitializedTensors.find(name);
1396 if (it == fInitializedTensors.end()) {
1397 std::cout << "Tensor " << name << " not found in model's initialized tensor list" << std::endl;
1398 return;
1399 }
1400
1401 std::cout << "Tensor name: " << it->first << "\t";
1402 std::cout << "type: " << ConvertTypeToString(it->second.type()) << "\t";
1403 int length =1;
1404 std::cout << "shape: [";
1405 for (size_t i = 0; i < it->second.shape().size(); i++) {
1406 std::cout << it->second.shape()[i];
1407 length *= it->second.shape()[i];
1408 if (i < it->second.shape().size() - 1) std::cout << ",";
1409 }
1410 std::cout << "]" << std::endl;
1411 bool ellipsis = true;
1412 if (n_print > length) {
1413 n_print = length;
1414 ellipsis = false;
1415 }
1416
1417 std::cout << "data: [" << std::endl;
1418 if (it->second.type() == ETensorType::FLOAT) {
1419 auto converted_data = it->second.data<float>();
1420 for (int i =0; i < n_print; i++) {
1421 std::cout << converted_data[i];
1422 if (i < n_print - 1) std::cout << " ,";
1423 }
1424 }
1425 if (ellipsis) std::cout << ", ...";
1426 std::cout << "]" << std::endl;
1427
1428}
1429
1430void RModel::OutputGenerated(std::string filename, bool append) {
1431
1433
1434 // write weights in a text file
1435 if (fUseWeightFile) {
1436 if (!filename.empty()) {
1437 size_t pos = filename.find(".hxx");
1439 filename.replace(pos, 4, ".dat");
1441 filename = filename.erase(pos, 4);
1442 filename += ".root";
1443 }
1444 } else {
1445 filename = fName;
1446 filename += fWeightFile == WeightFileType::Text ? ".dat" : ".root";
1447 }
1449 }
1450}
1451
1452void RModel::Streamer(TBuffer &R__b) {
1453 if (R__b.IsReading()) {
1454 RModel::Class()->ReadBuffer(R__b, this);
1455 for (auto & i : fInitializedTensors) {
1456 i.second.CastPersistentToShared();
1457 }
1458 }
1459 else {
1460 for (auto & i : fInitializedTensors) {
1461 i.second.CastSharedToPersistent();
1462 }
1463 RModel::Class()->WriteBuffer(R__b, this);
1464 }
1465}
1466
1467}//SOFIE
1468}//Experimental
1469}//TMVA
#define d(i)
Definition RSha256.hxx:102
#define b(i)
Definition RSha256.hxx:100
#define f(i)
Definition RSha256.hxx:104
#define a(i)
Definition RSha256.hxx:99
#define e(i)
Definition RSha256.hxx:103
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void input
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize id
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:110
const_iterator begin() const
const_iterator end() const
Buffer base class used for serializing objects.
Definition TBuffer.h:43
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Definition TFile.cxx:3765
void GenerateHeaderInfo(std::string &hgname)
void OutputGenerated(std::string filename="", bool append=false)
void AddBlasRoutines(std::vector< std::string > routines)
void AddNeededStdLib(std::string libname)
void AddShapeParam(const std::string &name, size_t def_value=0)
Definition RModel.cxx:281
std::vector< size_t > GetTensorShape(const std::string &name) const
Definition RModel.cxx:29
std::vector< Dim > GetDimTensorShape(const std::string &name) const
Definition RModel.cxx:65
std::unordered_map< std::string, DynamicTensorInfo > fDynamicTensorInfos
Definition RModel.hxx:30
bool IsDynamicTensor(const std::string &name) const
Definition RModel.cxx:232
void AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector< Dim > dim_shape)
Definition RModel.cxx:247
std::string GenerateInferSignature(bool isdecl=true)
Definition RModel.cxx:805
bool CheckIfTensorAlreadyExist(std::string tensor_name)
Definition RModel.cxx:122
std::vector< std::unique_ptr< ROperator > > fOperators
Definition RModel.hxx:39
void OutputGenerated(std::string filename="", bool append=false)
Definition RModel.cxx:1430
void AddInputTensorInfo(std::string input_name, ETensorType type, std::vector< Dim > shape)
Definition RModel.cxx:133
std::unordered_map< std::string, TensorInfo > fIntermediateTensorInfos
Definition RModel.hxx:29
void AddOutputTensorNameList(std::vector< std::string > output_tensor_names)
Definition RModel.cxx:289
std::unordered_map< std::string, TensorInfo > fReadyInputTensorInfos
Definition RModel.hxx:27
void AddConstantTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
Definition RModel.cxx:193
void AddDynamicTensor(std::string tensor_name, ETensorType type, std::vector< Dim > shape)
Definition RModel.cxx:264
std::vector< std::string > fDimShapeNames
Definition RModel.hxx:33
void AddInitializedTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
Definition RModel.cxx:183
std::unordered_map< std::string_view, size_t > fIntermediateTensorFrequencyLookup
! lookup table for intermediate tensor frequency (transient)
Definition RModel.hxx:46
void AddInputTensorName(std::string name)
Definition RModel.cxx:152
std::vector< std::string > fOutputTensorNames
Definition RModel.hxx:34
bool IsDimInputTensor(const std::string &name) const
Definition RModel.cxx:237
bool IsShapeTensor(const std::string &name) const
check if a tensor is a shape tensor
Definition RModel.cxx:211
bool IsInitializedTensor(const std::string &name) const
Definition RModel.cxx:220
void CheckAndFlushIntermediateMemory(std::span< const std::string_view > op_output_tensors, const size_t &op_idx)
Definition RModel.cxx:429
void AddOperator(std::unique_ptr< ROperator > op, int order_execution=-1)
Definition RModel.cxx:156
void HeadInitializedTensors(std::string name, int n_print=50)
Definition RModel.cxx:1394
bool IsConstantTensor(const std::string &name) const
Definition RModel.cxx:224
void Initialize(int batchSize=-1, bool verbose=false)
Definition RModel.cxx:503
long WriteInitializedTensorsToFile(std::string filename="")
Definition RModel.cxx:1192
OptimizationLevel fOptimizationLevel
Definition RModel.hxx:24
void Generate(std::underlying_type_t< Options > options, int batchSize=-1, long pos=0, bool verbose=false)
Definition RModel.cxx:1062
std::vector< Dim > GetDynamicTensorShape(const std::string &name) const
Definition RModel.cxx:76
std::unordered_map< std::string, InputTensorInfo > fInputTensorInfos
Definition RModel.hxx:26
std::shared_ptr< void > GetInitializedTensorData(std::string tensor_name)
Definition RModel.cxx:312
MemoryPoolInfo fIntermediateMemoryInfo
! intermediate memory info (transient)
Definition RModel.hxx:45
std::string AllocateIntermediateMemory(std::span< const std::string_view > op_output_tensors)
Definition RModel.cxx:329
std::unordered_map< std::string, std::pair< std::vector< Dim >, bool > > fShapeTensors
Definition RModel.hxx:31
void InitializeSubGraph(std::shared_ptr< RModel > graph)
Definition RModel.cxx:618
std::unordered_map< std::string, std::string > fShapeParams
Definition RModel.hxx:32
void SetNotWritableInitializedTensor(const std::string &tensor_name)
Definition RModel.cxx:321
ETensorType GetTensorType(std::string name) const
Definition RModel.cxx:90
std::vector< std::string > fInputTensorNames
Definition RModel.hxx:35
std::unordered_map< std::string, InitializedTensor > fInitializedTensors
Definition RModel.hxx:28
void UpdateInitializedTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
Definition RModel.cxx:303
const std::vector< Dim > & GetShapeTensorValues(const std::string &tensor_name) const
Definition RModel.cxx:215
std::vector< std::shared_ptr< RModel > > fSubGraphs
! sub-graph models (transient)
Definition RModel.hxx:41
bool IsReadyInputTensor(const std::string &name) const
Definition RModel.cxx:241
void UpdateOutputTensorList(std::vector< std::string > curr_output_tensor, std::vector< std::string > modify_output_tensor)
Definition RModel.cxx:296
void AddShapeTensor(const std::string &name, const std::vector< Dim > &shapeValues, bool scalar=false)
Definition RModel.cxx:203
const Int_t n
Definition legend1.C:16
std::string Clean_name(std::string input_tensor_name)
std::size_t ConvertShapeToLength(const std::vector< size_t > &shape)
std::string ConvertDynamicShapeToLength(const std::vector< Dim > &shape)
std::vector< Dim > ConvertShapeToDim(const std::vector< size_t > &shape)
Convert shape from integer format to dynamic one (based on Dim)
constexpr size_t GetTypeSize(ETensorType type)
std::string ConvertValuesToString(size_t n, const T *data)
std::string GenerateConstantTensorCode(const std::pair< std::string, InitializedTensor > &t)
Definition RModel.cxx:650
std::vector< size_t > ConvertShapeToInt(const std::vector< Dim > &shape)
Convert shape based on Dim to integer format.
std::string ConvertTypeToString(ETensorType type)
std::underlying_type_t< Options > operator|(Options opA, Options opB)
Definition RModel.cxx:22
std::string ConvertDimShapeToLength(const std::vector< Dim > &shape)
std::string ConvertShapeToString(const std::vector< size_t > &shape)
std::string ConvertValToString(T value)
create variable transformations
std::map< size_t, TensorMemoryInfo > total_stack
std::map< size_t, size_t > available_stack