Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
Evaluator.cxx
Go to the documentation of this file.
1/*
2 * Project: RooFit
3 * Authors:
4 * Jonas Rembser, CERN 2021
5 * Emmanouil Michalainas, CERN 2021
6 *
7 * Copyright (c) 2021, CERN
8 *
9 * Redistribution and use in source and binary forms,
10 * with or without modification, are permitted according to the terms
11 * listed in LICENSE (http://roofit.sourceforge.net/license.txt)
12 */
13
14/**
15\file Evaluator.cxx
16\class RooFit::Evaluator
17\ingroup Roofitcore
18
19Evaluates a RooAbsReal object in other ways than recursive graph
20traversal. Currently, it is being used for evaluating a RooAbsReal object and
21supplying the value to the minimizer, during a fit. The class scans the
22dependencies and schedules the computations in a secure and efficient way. The
23computations take place in the RooBatchCompute library and can be carried off
24by either the CPU or a CUDA-supporting GPU. The Evaluator class takes care
25of data transfers. An instance of this class is created every time
26RooAbsPdf::fitTo() is called and gets destroyed when the fitting ends.
27**/
28
29#include <RooFit/Evaluator.h>
30
31#include <RooAbsCategory.h>
32#include <RooAbsData.h>
33#include <RooAbsReal.h>
34#include <RooRealVar.h>
35#include <RooBatchCompute.h>
36#include <RooMsgService.h>
37#include <RooNameReg.h>
38#include <RooSimultaneous.h>
39
40
42#include "RooFitImplHelpers.h"
43
44#include <atomic>
45#include <iomanip>
46#include <mutex>
47#include <numeric>
48#include <set>
49#include <unordered_set>
50
51namespace RooFit {
52
53namespace {
54
55// To avoid deleted move assignment.
56template <class T>
57void assignSpan(std::span<T> &to, std::span<T> const &from)
58{
59 to = from;
60}
61
62void logArchitectureInfo(bool useGPU)
63{
64 // We have to exit early if the message stream is not active. Otherwise it's
65 // possible that this function skips logging because it thinks it has
66 // already logged, but actually it didn't.
67 if (!RooMsgService::instance().isActive(nullptr, RooFit::Fitting, RooFit::INFO)) {
68 return;
69 }
70
71 // Don't repeat logging architecture info if the useGPU option didn't change
72 {
73 // Second element of pair tracks whether this function has already been called
74 static std::pair<bool, bool> lastUseGPU;
75 if (lastUseGPU.second && lastUseGPU.first == useGPU)
76 return;
77 lastUseGPU = {useGPU, true};
78 }
79
80 auto log = [](std::string_view message) {
81 oocxcoutI(static_cast<RooAbsArg *>(nullptr), Fitting) << message << std::endl;
82 };
83
85 log("using generic CPU library compiled with no vectorizations");
86 } else {
87 log(std::string("using CPU computation library compiled with -m") + RooBatchCompute::cpuArchitectureName());
88 }
89 if (useGPU) {
90 log("using CUDA computation library");
91 }
92}
93
94/// Advise the user to implement CUDA support for a class that had to be
95/// evaluated on the CPU even though the CUDA backend was requested. Just like
96/// for the analogous message about the missing batch evaluation interface in
97/// RooAbsReal::doEval(), the message is only printed once per class, because
98/// the computation graph is evaluated many times, e.g. in every minimizer
99/// iteration.
100void logMissingCudaSupport(RooAbsArg const &arg)
101{
103 return;
104 }
105 static std::set<std::string> warnedClasses;
106 static std::mutex warnedClassesMutex;
107 std::scoped_lock guard{warnedClassesMutex};
108 if (warnedClasses.insert(arg.ClassName()).second) {
109 oocoutI(&arg, FastEvaluations) << "The class " << arg.ClassName()
110 << " could not be evaluated on the GPU because it doesn't support it."
111 << " Consider requesting or implementing it to benefit from a speed up."
112 << std::endl;
113 }
114}
115
116} // namespace
117
118/// A struct used by the Evaluator to store information on the RooAbsArgs in
119/// the computation graph.
120struct NodeInfo {
121
122 bool isScalar() const { return outputSize == 1; }
123
124 RooAbsArg *absArg = nullptr;
126
127 std::shared_ptr<RooBatchCompute::AbsBuffer> buffer;
128 std::size_t iNode = 0;
130 bool fromArrayInput = false;
131 bool isVariable = false;
132 bool isDirty = true;
133 bool isCategory = false;
134 bool hasLogged = false;
135 bool computeInGPU = false;
136 bool isValueServer = false; // if this node is a value server to the top node
137 std::size_t outputSize = 1;
138 std::size_t lastSetValCount = std::numeric_limits<std::size_t>::max();
139 int lastCatVal = std::numeric_limits<int>::max();
140 double scalarBuffer = 0.0;
141 std::vector<NodeInfo *> serverInfos;
142 std::vector<NodeInfo *> clientInfos;
143};
144
145/// Construct a new Evaluator. The constructor analyzes and saves metadata about the graph,
146/// useful for the evaluation of it that will be done later. In case the CUDA mode is selected,
147/// there's also some CUDA-related initialization.
148///
149/// \param[in] absReal The RooAbsReal object that sits on top of the
150/// computation graph that we want to evaluate.
151/// \param[in] useGPU Whether the evaluation should be preferably done on the GPU.
152Evaluator::Evaluator(const RooAbsReal &absReal, bool useGPU)
153 : _topNode{const_cast<RooAbsReal &>(absReal)}, _useGPU{useGPU}
154{
156 if (useGPU && RooBatchCompute::initCUDA() != 0) {
157 throw std::runtime_error("Can't create Evaluator in CUDA mode because RooBatchCompute CUDA could not be loaded!");
158 }
159 // Some checks and logging of used architectures
160 logArchitectureInfo(_useGPU);
161
164
165 RooArgSet serverSet;
167
168 _evalContextCPU.resize(serverSet.size());
169 if (useGPU) {
170 _evalContextCUDA.resize(serverSet.size());
171 }
172
173 std::map<RooFit::Detail::DataKey, NodeInfo *> nodeInfos;
174
175 // Fill the ordered nodes list and initialize the node info structs.
176 _nodes.reserve(serverSet.size());
177 std::size_t iNode = 0;
178 for (RooAbsArg *arg : serverSet) {
179
180 _nodes.emplace_back();
181 auto &nodeInfo = _nodes.back();
182 _nodesMap[arg->namePtr()] = &nodeInfo;
183
184 nodeInfo.absArg = arg;
185 nodeInfo.originalOperMode = arg->operMode();
186 nodeInfo.iNode = iNode;
187 nodeInfos[arg] = &nodeInfo;
188
189 if (dynamic_cast<RooRealVar const *>(arg)) {
190 nodeInfo.isVariable = true;
191 } else {
192 arg->setDataToken(iNode, this);
193 }
194 if (dynamic_cast<RooAbsCategory const *>(arg)) {
195 nodeInfo.isCategory = true;
196 }
197
198 ++iNode;
199 }
200
201 for (NodeInfo &info : _nodes) {
202 info.serverInfos.reserve(info.absArg->servers().size());
203 for (RooAbsArg *server : info.absArg->servers()) {
204 if (server->isValueServer(*info.absArg)) {
205 auto *serverInfo = nodeInfos.at(server);
206 info.serverInfos.emplace_back(serverInfo);
207 serverInfo->clientInfos.emplace_back(&info);
208 }
209 }
210 }
211
212 // Figure out which nodes are value servers to the top node
213 _nodes.back().isValueServer = true; // the top node itself
214 for (auto iter = _nodes.rbegin(); iter != _nodes.rend(); ++iter) {
215 if (!iter->isValueServer)
216 continue;
217 for (auto &serverInfo : iter->serverInfos) {
218 serverInfo->isValueServer = true;
219 }
220 }
221
223
224 if (_useGPU) {
225 // Create the single CUDA stream on which all GPU computations and data
226 // transfers of this Evaluator are enqueued. The graph is evaluated in
227 // topological order, so ordering the operations by the stream is enough
228 // to guarantee correct results.
232 for (auto &info : _nodes) {
233 _evalContextCUDA.setConfig(info.absArg, cfg);
234 }
235 }
236
238}
239
240/// If there are servers with the same name that got de-duplicated in the
241/// `_nodes` list, we need to set their data tokens too. We find such nodes by
242/// visiting the servers of every known node.
244{
245 for (NodeInfo &info : _nodes) {
246 std::size_t iValueServer = 0;
247 for (RooAbsArg *server : info.absArg->servers()) {
248 if (server->isValueServer(*info.absArg)) {
249 auto *knownServer = info.serverInfos[iValueServer]->absArg;
250 if (knownServer->hasDataToken()) {
251 server->setDataToken(knownServer->dataToken());
252 }
253 ++iValueServer;
254 }
255 }
256 }
257}
258
259/// Request nodes to output their logarithms if their only value client takes
260/// the logarithm anyway, like the RooNLLVarNew and RooConstraintSum do. This
261/// avoids exp() -> log() roundtrips, e.g. for Gaussian constraints, and the
262/// underflow of products with many factors. Nodes that output their
263/// logarithms can request the same from their servers, e.g. the factors of a
264/// product.
266{
267 // Visit clients before their servers, because whether a client can use a
268 // logarithm can depend on its own output mode.
269 for (auto iter = _nodes.rbegin(); iter != _nodes.rend(); ++iter) {
270 NodeInfo &info = *iter;
271 bool outputsLogarithm = false;
272 if (info.clientInfos.size() == 1 && !info.fromArrayInput && info.absArg->canOutputLogarithm()) {
273 NodeInfo const &client = *info.clientInfos[0];
274 outputsLogarithm =
276 }
277 // The output of a node can be read on both devices.
278 _evalContextCPU._outputsLogarithm[info.iNode] = outputsLogarithm;
279 if (_useGPU) {
280 _evalContextCUDA._outputsLogarithm[info.iNode] = outputsLogarithm;
281 }
282 }
283}
284
285void Evaluator::setInput(std::string const &name, std::span<const double> inputArray, bool isOnDevice)
286{
287 if (isOnDevice && !_useGPU) {
288 throw std::runtime_error("Evaluator can only take device array as input in CUDA mode!");
289 }
290
291 // Check if "name" is used in the computation graph. If yes, add the span to
292 // the data map and set the node info accordingly.
293
294 auto found = _nodesMap.find(RooNameReg::ptr(name.c_str()));
295
296 if (found == _nodesMap.end())
297 return;
298
300
301 // Invalidate the caches that reducer nodes key on the input data, like the
302 // cached sum of event weights in RooNLLVarNew. The counter is global so
303 // that generation values can never alias between different Evaluators.
304 {
305 static std::atomic<std::size_t> nextInputGeneration{1};
306 const std::size_t gen = ++nextInputGeneration;
309 }
310
311 NodeInfo &info = *found->second;
312
313 info.fromArrayInput = true;
314 info.absArg->setDataToken(info.iNode);
315 info.outputSize = inputArray.size();
316
317 if (!_useGPU) {
318 _evalContextCPU.set(info.absArg, inputArray);
319 return;
320 }
321
322 if (info.outputSize <= 1) {
323 // Empty or scalar observables from the data don't need to be
324 // copied to the GPU.
325 _evalContextCPU.set(info.absArg, inputArray);
326 _evalContextCUDA.set(info.absArg, inputArray);
327 return;
328 }
329
330 // For simplicity, we put the data on both host and device for
331 // now. This could be optimized by inspecting the clients of the
332 // variable.
333 if (isOnDevice) {
334 _evalContextCUDA.set(info.absArg, inputArray);
335 auto gpuSpan = _evalContextCUDA.at(info.absArg);
336 info.buffer = _bufferManager->makeCpuBuffer(gpuSpan.size());
337 info.buffer->assignFromDevice(gpuSpan);
338 _evalContextCPU.set(info.absArg, {info.buffer->hostReadPtr(), gpuSpan.size()});
339 } else {
340 _evalContextCPU.set(info.absArg, inputArray);
341 auto cpuSpan = _evalContextCPU.at(info.absArg);
342 info.buffer = _bufferManager->makeGpuBuffer(cpuSpan.size());
343 info.buffer->assignFromHost(cpuSpan);
344 _evalContextCUDA.set(info.absArg, {info.buffer->deviceReadPtr(), cpuSpan.size()});
345 }
346}
347
349{
350 std::map<RooFit::Detail::DataKey, std::size_t> sizeMap;
351 for (auto &info : _nodes) {
352 if (info.fromArrayInput) {
353 sizeMap[info.absArg] = info.outputSize;
354 } else {
355 // any buffer for temporary results is invalidated by resetting the output sizes
356 info.buffer.reset();
357 }
358 }
359
360 auto outputSizeMap =
361 RooFit::BatchModeDataHelpers::determineOutputSizes(_topNode, [&](RooFit::Detail::DataKey key) -> int {
362 auto found = sizeMap.find(key);
363 return found != sizeMap.end() ? found->second : -1;
364 });
365
366 for (auto &info : _nodes) {
367 info.outputSize = outputSizeMap.at(info.absArg);
368 info.isDirty = true;
369 }
370
371 if (_useGPU) {
372 markGPUNodes();
373 }
374
375 // Which nodes are array inputs might have changed.
377
379}
380
382{
383 for (auto &info : _nodes) {
384 // Nodes that were deleted before the Evaluator were unregistered
385 if (!info.isVariable && info.absArg) {
386 info.absArg->resetDataToken();
387 }
388 }
389 if (_cudaStream) {
391 }
392}
393
394/// Called from RooAbsArg::~RooAbsArg() when a node of the computation graph is
395/// deleted while this Evaluator is still alive. The Evaluator can't be used
396/// anymore after that, but its destructor must not touch the deleted node.
397/// For internal use only.
399{
400 auto found = _nodesMap.find(arg.namePtr());
401 if (found != _nodesMap.end() && found->second->absArg == &arg) {
402 found->second->absArg = nullptr;
403 }
404}
405
407{
408 using namespace Detail;
409
410 const std::size_t nOut = info.outputSize;
411
412 double *buffer = nullptr;
413 if (nOut == 1) {
414 buffer = &info.scalarBuffer;
415 if (_useGPU) {
416 _evalContextCUDA.set(node, {buffer, nOut});
417 }
418 } else {
419 // Advising to implement the CUDA evaluation makes only sense if the batch was not a scalar.
420 // Otherwise, there would be no speedup benefit.
421 if (!info.hasLogged && _useGPU) {
422 logMissingCudaSupport(*info.absArg);
423 info.hasLogged = true;
424 }
425 if (!info.buffer) {
426 info.buffer = info.copyAfterEvaluation ? _bufferManager->makePinnedBuffer(nOut, _cudaStream)
427 : _bufferManager->makeCpuBuffer(nOut);
428 }
429 buffer = info.buffer->hostWritePtr();
430 }
431 assignSpan(_evalContextCPU._currentOutput, {buffer, nOut});
432 _evalContextCPU.set(node, {buffer, nOut});
433 if (nOut > 1) {
435 }
436 if (info.isCategory) {
437 auto nodeAbsCategory = static_cast<RooAbsCategory const *>(node);
438 if (nOut == 1) {
439 buffer[0] = nodeAbsCategory->getCurrentIndex();
440 } else {
441 throw std::runtime_error("RooFit::Evaluator - non-scalar category values are not supported!");
442 }
443 } else {
444 auto nodeAbsReal = static_cast<RooAbsReal const *>(node);
445 nodeAbsReal->doEval(_evalContextCPU);
446 }
449 if (info.copyAfterEvaluation) {
450 // The deviceReadPtr() call triggers the copy of the result to the GPU.
451 // The copy is ordered by the CUDA stream, so GPU clients enqueued later
452 // will see the result without any further synchronization.
453 _evalContextCUDA.set(node, {info.buffer->deviceReadPtr(), nOut});
454 }
455}
456
457/// Process a variable in the computation graph. This is a separate non-inlined
458/// function such that we can see in performance profiles how long this takes.
460{
461 RooAbsArg *node = nodeInfo.absArg;
462 auto *var = static_cast<RooRealVar const *>(node);
463 if (nodeInfo.lastSetValCount != var->valueResetCounter()) {
464 nodeInfo.lastSetValCount = var->valueResetCounter();
465 for (NodeInfo *clientInfo : nodeInfo.clientInfos) {
466 clientInfo->isDirty = true;
467 }
468 computeCPUNode(node, nodeInfo);
469 nodeInfo.isDirty = false;
470 }
471}
472
473/// Process a category in the computation graph. This is a separate non-inlined
474/// function such that we can see in performance profiles how long this takes.
476{
477 RooAbsArg *node = nodeInfo.absArg;
478 auto *cat = static_cast<RooAbsCategory const *>(node);
479 if (nodeInfo.lastCatVal != cat->getCurrentIndex()) {
480 nodeInfo.lastCatVal = cat->getCurrentIndex();
481 for (NodeInfo *clientInfo : nodeInfo.clientInfos) {
482 clientInfo->isDirty = true;
483 }
484 computeCPUNode(node, nodeInfo);
485 nodeInfo.isDirty = false;
486 }
487}
488
489/// Flags all the clients of a given node dirty. This is a separate non-inlined
490/// function such that we can see in performance profiles how long this takes.
492{
493 for (NodeInfo *clientInfo : nodeInfo.clientInfos) {
494 clientInfo->isDirty = true;
495 }
496}
497
498/// Returns the value of the top node in the computation graph
499std::span<const double> Evaluator::run()
500{
503
505
506 // Discard leftover deferred actions in case a previous evaluation was
507 // aborted by an exception.
510
511 if (_useGPU) {
512 return getValHeterogeneous();
513 }
514
515 for (auto &nodeInfo : _nodes) {
516 if (!nodeInfo.fromArrayInput) {
517 if (nodeInfo.isVariable) {
518 processVariable(nodeInfo);
519 } else if (nodeInfo.isCategory) {
520 processCategory(nodeInfo);
521 } else {
522 if (nodeInfo.isDirty) {
523 setClientsDirty(nodeInfo);
524 computeCPUNode(nodeInfo.absArg, nodeInfo);
525 nodeInfo.isDirty = false;
526 }
527 }
528 }
529 }
530
531 for (auto &action : _evalContextCPU._deferredActions) {
532 action();
533 }
535
536 // return the final output
537 return _evalContextCPU.at(&_topNode);
538}
539
540/// Returns the value of the top node in the computation graph
541std::span<const double> Evaluator::getValHeterogeneous()
542{
543 // Iterate over the nodes in topological order. Like in the CPU-only path
544 // in run(), only dirty nodes are recomputed: a node is dirty if one of its
545 // value servers was recomputed in this evaluation, or if a variable
546 // changed its value since the last evaluation. Clean nodes keep the output
547 // buffers from their last evaluation, which is why the buffers persist
548 // across evaluations in this mode instead of being released as soon as
549 // the clients of a node are computed. During a minimization, most
550 // likelihood evaluations change only one or two parameters (e.g. for the
551 // numeric gradient and the Hessian), so skipping the clean nodes saves
552 // most of the GPU work and the kernel launches. The price is the device
553 // memory for the intermediate results of the whole computation graph,
554 // which is small compared to the input data for the usual graphs where
555 // most nodes are scalars.
556 //
557 // Nodes that are computed on the GPU only enqueue their computation on the
558 // single CUDA stream and return immediately, so independent CPU nodes that
559 // come later in the ordering naturally overlap with the GPU computations.
560 // Ordering by the stream guarantees that GPU nodes see the results of
561 // their GPU servers, and host-side reads of GPU results synchronize on the
562 // stream in the buffer implementation. Since no buffer is recycled within
563 // an evaluation, the results of clean nodes can't be overwritten by
564 // enqueued work either.
565 try {
566 for (auto &info : _nodes) {
567 if (info.fromArrayInput) {
568 continue;
569 }
570 if (info.isVariable) {
571 processVariable(info);
572 } else if (info.isCategory) {
573 processCategory(info);
574 } else if (info.isDirty) {
575 setClientsDirty(info);
576 if (info.computeInGPU) {
577 assignToGPU(info);
578 } else {
579 computeCPUNode(info.absArg, info);
580 }
581 info.isDirty = false;
582 }
583 }
584 } catch (...) {
585 // The evaluation was aborted, but readbacks that compute() calls
586 // deferred may still be armed. Deliver them now, while the destination
587 // memory in the nodes of the computation graph is guaranteed to be
588 // alive, so that no armed readback survives into a later evaluation.
589 // The node that threw stays dirty and gets recomputed in the next
590 // evaluation, together with its clients that were already flagged.
591 try {
593 } catch (...) {
594 // The stream is in an unrecoverable error state. The deferred
595 // readbacks are dropped together with the scratch memory when the
596 // stream gets deleted.
597 }
600 throw;
601 }
602
603 // Ensure that all enqueued GPU work has completed when run() returns. For
604 // the usual likelihood evaluations this is mostly a no-op, because the
605 // final reduction has synchronized the stream already. It also delivers
606 // the deferred readbacks like the evaluation error counters.
608
609 // Run the deferred actions now that all results have arrived on the host,
610 // e.g. the logging of evaluation errors that were counted on the GPU.
611 // Nodes evaluated on the CPU register their actions in the CPU context,
612 // so both contexts are drained.
613 for (auto *ctx : {&_evalContextCUDA, &_evalContextCPU}) {
614 for (auto &action : ctx->_deferredActions) {
615 action();
616 }
617 ctx->_deferredActions.clear();
618 }
619
620 // return the final value
622}
623
624/// Enqueue the computation of a node on the GPU.
626{
627 using namespace Detail;
628
629 auto node = static_cast<RooAbsReal const *>(info.absArg);
630
631 const std::size_t nOut = info.outputSize;
632
633 double *buffer = nullptr;
634 if (nOut == 1) {
635 buffer = &info.scalarBuffer;
636 _evalContextCPU.set(node, {buffer, nOut});
637 } else {
638 if (!info.buffer) {
639 info.buffer = info.copyAfterEvaluation ? _bufferManager->makePinnedBuffer(nOut, _cudaStream)
640 : _bufferManager->makeGpuBuffer(nOut);
641 }
642 buffer = info.buffer->deviceWritePtr();
643 }
644 assignSpan(_evalContextCUDA._currentOutput, {buffer, nOut});
645 _evalContextCUDA.set(node, {buffer, nOut});
646 node->doEval(_evalContextCUDA);
647 if (info.copyAfterEvaluation) {
648 // The hostReadPtr() call triggers the copy of the result to the host,
649 // which waits for the enqueued computation via the CUDA stream.
650 _evalContextCPU.set(node, {info.buffer->hostReadPtr(), nOut});
651 }
652}
653
654/// Decides which nodes are assigned to the GPU in a CUDA fit.
656{
657 // Decide which nodes get evaluated on the GPU: we select nodes that support
658 // CUDA evaluation and have at least one input of size greater than one.
659 for (auto &info : _nodes) {
660 info.computeInGPU = false;
661 if (!info.absArg->canComputeBatchWithCuda()) {
662 continue;
663 }
664 for (NodeInfo const *serverInfo : info.serverInfos) {
665 if (serverInfo->outputSize > 1) {
666 info.computeInGPU = true;
667 break;
668 }
669 }
670 }
671
672 // In a second pass, figure out which nodes need to copy over their results.
673 for (auto &info : _nodes) {
674 info.copyAfterEvaluation = false;
675 // scalar nodes don't need copying
676 if (!info.isScalar()) {
677 for (auto *clientInfo : info.clientInfos) {
678 if (info.computeInGPU != clientInfo->computeInGPU) {
679 info.copyAfterEvaluation = true;
680 break;
681 }
682 }
683 }
684 }
685}
686
687/// \brief Sets the number of threads to use for the evaluation of a single node.
688///
689/// With a value greater than one, the computation functions and reductions of
690/// the CPU backend process large batches multi-threaded, using up to the
691/// given number of threads. Nodes evaluated on the CPU with fewer events than
692/// an internal threshold are still evaluated single-threaded, so requesting
693/// multiple threads never introduces scheduling overhead for small fits.
694void Evaluator::setNThreads(int nThreads)
695{
696 for (auto &info : _nodes) {
697 if (info.isVariable) {
698 continue;
699 }
701 cfg.setNThreads(nThreads);
702 _evalContextCPU.setConfig(info.absArg, cfg);
703 }
704}
705
706/// Temporarily change the operation mode of a RooAbsArg until the
707/// Evaluator gets deleted.
709{
710 if (!_operModeChanges)
711 _operModeChanges = std::make_unique<ChangeOperModeRAII>();
712 _operModeChanges->change(arg, opMode);
713}
714
715// Change the operation modes of all RooAbsArgs in the computation graph.
716// The changes are reset when the returned RAII object goes out of scope.
717//
718// We also walk transitively through value clients of the nodes to cover any
719// node that RooAbsReal::doEval (the fallback scalar implementation) might
720// inadvertently propagate the ADirty mode to via its recursive restore: that
721// helper sets servers temporarily to AClean and then calls
722// setOperMode(oldOperMode) to restore, which recurses to value clients when
723// oldOperMode is ADirty. If we did not protect those clients here, any node
724// outside the computation graph that shares a fundamental (e.g. a parameter
725// like a RooRealVar) would be left permanently in ADirty after the first
726// minimization, dramatically slowing down later scalar evaluations.
727std::unique_ptr<ChangeOperModeRAII> Evaluator::setOperModes(RooAbsArg::OperMode opMode)
728{
729 auto out = std::make_unique<ChangeOperModeRAII>();
730 std::unordered_set<RooAbsArg *> visited;
731
732 std::vector<RooAbsArg *> queue;
733 queue.reserve(_nodes.size());
734 for (auto &info : _nodes) {
735 queue.push_back(info.absArg);
736 }
737
738 while (!queue.empty()) {
739 RooAbsArg *node = queue.back();
740 queue.pop_back();
741 if (!visited.insert(node).second)
742 continue;
743
744 out->change(node, opMode);
745
746 // Only follow value-client links: that is exactly the propagation path
747 // used by RooAbsArg::setOperMode with mode==ADirty.
748 if (opMode == RooAbsArg::ADirty) {
749 for (auto *client : node->valueClients()) {
750 queue.push_back(client);
751 }
752 }
753 }
754 return out;
755}
756
757void Evaluator::print(std::ostream &os)
758{
759 std::cout << "--- RooFit BatchMode evaluation ---\n";
760
761 std::vector<int> widths{9, 37, 20, 9, 10, 4, 20};
762
763 auto printElement = [&](int iCol, auto const &t) {
764 const char separator = ' ';
765 os << separator << std::left << std::setw(widths[iCol]) << std::setfill(separator) << t;
766 os << "|";
767 };
768
769 auto printHorizontalRow = [&]() {
770 int n = 0;
771 for (int w : widths) {
772 n += w + 2;
773 }
774 for (int i = 0; i < n; i++) {
775 os << '-';
776 }
777 os << "|\n";
778 };
779
780 printHorizontalRow();
781
782 os << "|";
783 printElement(0, "Index");
784 printElement(1, "Name");
785 printElement(2, "Class");
786 printElement(3, "Size");
787 printElement(4, "From Data");
788 printElement(5, "Log");
789 printElement(6, "1st value");
790 std::cout << "\n";
791
792 printHorizontalRow();
793
794 for (std::size_t iNode = 0; iNode < _nodes.size(); ++iNode) {
795 auto &nodeInfo = _nodes[iNode];
796 RooAbsArg *node = nodeInfo.absArg;
797
798 auto span = _evalContextCPU.at(node);
799
800 os << "|";
801 printElement(0, iNode);
802 printElement(1, node->GetName());
803 printElement(2, node->ClassName());
804 printElement(3, nodeInfo.outputSize);
805 printElement(4, nodeInfo.fromArrayInput);
806 printElement(5, _evalContextCPU.outputsLogarithm(node));
807 printElement(6, span[0]);
808
809 std::cout << "\n";
810 }
811
812 printHorizontalRow();
813}
814
815/// Gets all the parameters of the RooAbsReal. This is in principle not
816/// necessary, because we can always ask the RooAbsReal itself, but the
817/// Evaluator has the cached information to get the answer quicker.
818/// Therefore, this is not meant to be used in general, just where it matters.
819/// \warning If we find another solution to get the parameters efficiently,
820/// this function might be removed without notice.
822{
823 RooArgSet parameters;
824 for (auto &nodeInfo : _nodes) {
825 if (nodeInfo.isValueServer && nodeInfo.absArg->isFundamental()) {
826 parameters.add(*nodeInfo.absArg);
827 }
828 }
829 // Just like in RooAbsArg::getParameters(), we sort the parameters alphabetically.
830 parameters.sort();
831 return parameters;
832}
833
834/// \brief Sets the offset mode for evaluation.
835///
836/// This function sets the offset mode for evaluation to the specified mode.
837/// It updates the offset mode for both CPU and CUDA evaluation contexts.
838///
839/// \param mode The offset mode to be set.
840///
841/// \note This function marks reducer nodes as dirty if the offset mode is
842/// changed, because only reducer nodes can use offsetting.
844{
846 return;
847
850
851 for (auto &nodeInfo : _nodes) {
852 if (nodeInfo.absArg->isReducerNode()) {
853 nodeInfo.isDirty = true;
854 }
855 }
856}
857
858} // namespace RooFit
std::string message
#define oocoutI(o, a)
#define oocxcoutI(o, a)
Option_t Option_t TPoint TPoint const char mode
char name[80]
Definition TGX11.cxx:142
Common abstract base class for objects that represent a value and a "shape" in RooFit.
Definition RooAbsArg.h:77
const TNamed * namePtr() const
De-duplicated pointer to this object's name.
Definition RooAbsArg.h:484
virtual bool canOutputLogarithm() const
Whether doEval() can output the logarithm of the value instead of the value.
Definition RooAbsArg.h:501
const RefCountList_t & valueClients() const
List of all value clients of this object. Value clients receive value updates.
Definition RooAbsArg.h:140
void setDataToken(std::size_t index, RooFit::Evaluator *evaluator=nullptr)
Sets the token for retrieving results in the BatchMode.
virtual bool canUseLogarithmOf(RooAbsArg const &, bool) const
Whether doEval() accepts the logarithm of the given value server instead of its value.
Definition RooAbsArg.h:506
OperMode operMode() const
Query the operation mode of this node.
Definition RooAbsArg.h:400
A space to attach TBranches.
virtual bool add(const RooAbsArg &var, bool silent=false)
Add the specified argument to list.
Storage_t::size_type size() const
void sort(bool reverse=false)
Sort collection using std::sort and name comparison.
Abstract base class for objects that represent a real value and implements functionality common to al...
Definition RooAbsReal.h:63
virtual void doEval(RooFit::EvalContext &) const
Base function for computing multiple values of a RooAbsReal.
RooArgSet is a container object that can hold multiple RooAbsArg objects.
Definition RooArgSet.h:24
Minimal configuration struct to steer the evaluation of a single node with the RooBatchCompute librar...
void setCudaStream(CudaInterface::CudaStream *cudaStream)
void setNThreads(int nThreads)
Number of threads to use for CPU batch computations and reductions.
virtual void synchronizeCudaStream(CudaInterface::CudaStream *) const =0
Wait until all work that was enqueued on the stream has completed.
virtual std::unique_ptr< AbsBufferManager > createBufferManager() const =0
virtual CudaInterface::CudaStream * newCudaStream() const =0
virtual void deleteCudaStream(CudaInterface::CudaStream *) const =0
std::size_t _inputGeneration
std::vector< std::function< void()> > _deferredActions
void set(RooAbsArg const *arg, std::span< const double > const &span)
Definition EvalContext.h:92
std::span< const double > at(RooAbsArg const *arg, RooAbsArg const *caller=nullptr)
void enableVectorBuffers(bool enable)
OffsetMode _offsetMode
bool outputsLogarithm(RooAbsArg const *arg) const
Whether the output of the given node is the logarithm of its value.
RooBatchCompute::Config config(RooAbsArg const *arg) const
void setConfig(RooAbsArg const *arg, RooBatchCompute::Config const &config)
std::vector< bool > _outputsLogarithm
std::span< double > _currentOutput
void resize(std::size_t n)
void print(std::ostream &os)
void setClientsDirty(NodeInfo &nodeInfo)
Flags all the clients of a given node dirty.
std::unique_ptr< ChangeOperModeRAII > setOperModes(RooAbsArg::OperMode opMode)
RooArgSet getParameters() const
Gets all the parameters of the RooAbsReal.
void setOffsetMode(RooFit::EvalContext::OffsetMode)
Sets the offset mode for evaluation.
void unregisterNode(RooAbsArg const &arg)
Called from RooAbsArg::~RooAbsArg() when a node of the computation graph is deleted while this Evalua...
void syncDataTokens()
If there are servers with the same name that got de-duplicated in the _nodes list,...
const bool _useGPU
Definition Evaluator.h:71
std::unordered_map< TNamed const *, NodeInfo * > _nodesMap
Definition Evaluator.h:77
std::unique_ptr< ChangeOperModeRAII > _operModeChanges
Definition Evaluator.h:78
std::vector< NodeInfo > _nodes
Definition Evaluator.h:76
bool _needToUpdateOutputSizes
Definition Evaluator.h:73
std::span< const double > getValHeterogeneous()
Returns the value of the top node in the computation graph.
std::span< const double > run()
Returns the value of the top node in the computation graph.
Evaluator(const RooAbsReal &absReal, bool useGPU=false)
Construct a new Evaluator.
void setNThreads(int nThreads)
Sets the number of threads to use for the evaluation of a single node.
void processVariable(NodeInfo &nodeInfo)
Process a variable in the computation graph.
void processCategory(NodeInfo &nodeInfo)
Process a category in the computation graph.
RooBatchCompute::CudaInterface::CudaStream * _cudaStream
Definition Evaluator.h:80
std::unique_ptr< RooBatchCompute::AbsBufferManager > _bufferManager
Definition Evaluator.h:69
void markGPUNodes()
Decides which nodes are assigned to the GPU in a CUDA fit.
void assignToGPU(NodeInfo &info)
Enqueue the computation of a node on the GPU.
void setInput(std::string const &name, std::span< const double > inputArray, bool isOnDevice)
RooFit::EvalContext _evalContextCUDA
Definition Evaluator.h:75
RooFit::EvalContext _evalContextCPU
Definition Evaluator.h:74
void computeCPUNode(const RooAbsArg *node, NodeInfo &info)
void setOperMode(RooAbsArg *arg, RooAbsArg::OperMode opMode)
Temporarily change the operation mode of a RooAbsArg until the Evaluator gets deleted.
void setupLogEvaluation()
Request nodes to output their logarithms if their only value client takes the logarithm anyway,...
RooAbsReal & _topNode
Definition Evaluator.h:70
static RooMsgService & instance()
Return reference to singleton instance.
static const TNamed * ptr(const char *stringPtr)
Return a unique TNamed pointer for given C++ string.
Variable that can be changed from the outside.
Definition RooRealVar.h:37
const char * GetName() const override
Returns name of object.
Definition TNamed.h:49
virtual const char * ClassName() const
Returns name of class to which the object belongs.
Definition TObject.cxx:226
RVec< PromoteType< T > > log(const RVec< T > &v)
Definition RVec.hxx:1821
const Int_t n
Definition legend1.C:16
R__EXTERN RooBatchComputeInterface * dispatchCUDA
std::string cpuArchitectureName()
R__EXTERN RooBatchComputeInterface * dispatchCPU
This dispatch pointer points to an implementation of the compute library, provided one has been loade...
Architecture cpuArchitecture()
int initCPU()
Inspect hardware capabilities, and load the optimal library for RooFit computations.
The namespace RooFit contains mostly switches that change the behaviour of functions of PDFs (or othe...
Definition CodegenImpl.h:73
@ FastEvaluations
void getSortedComputationGraph(RooAbsArg const &func, RooArgSet &out)
A struct used by the Evaluator to store information on the RooAbsArgs in the computation graph.
RooAbsArg * absArg
bool isScalar() const
std::size_t iNode
std::size_t lastSetValCount
std::vector< NodeInfo * > serverInfos
RooAbsArg::OperMode originalOperMode
std::size_t outputSize
std::vector< NodeInfo * > clientInfos
std::shared_ptr< RooBatchCompute::AbsBuffer > buffer