49#include <unordered_set>
57void assignSpan(std::span<T> &to, std::span<T>
const &from)
62void logArchitectureInfo(
bool useGPU)
74 static std::pair<bool, bool> lastUseGPU;
75 if (lastUseGPU.second && lastUseGPU.first == useGPU)
77 lastUseGPU = {useGPU,
true};
85 log(
"using generic CPU library compiled with no vectorizations");
90 log(
"using CUDA computation library");
100void logMissingCudaSupport(
RooAbsArg const &arg)
105 static std::set<std::string> warnedClasses;
106 static std::mutex warnedClassesMutex;
107 std::scoped_lock guard{warnedClassesMutex};
108 if (warnedClasses.insert(arg.
ClassName()).second) {
110 <<
" could not be evaluated on the GPU because it doesn't support it."
111 <<
" Consider requesting or implementing it to benefit from a speed up."
127 std::shared_ptr<RooBatchCompute::AbsBuffer>
buffer;
153 : _topNode{const_cast<
RooAbsReal &>(absReal)}, _useGPU{useGPU}
157 throw std::runtime_error(
"Can't create Evaluator in CUDA mode because RooBatchCompute CUDA could not be loaded!");
173 std::map<RooFit::Detail::DataKey, NodeInfo *> nodeInfos;
177 std::size_t iNode = 0;
181 auto &nodeInfo =
_nodes.back();
184 nodeInfo.absArg = arg;
185 nodeInfo.originalOperMode = arg->
operMode();
186 nodeInfo.iNode = iNode;
187 nodeInfos[arg] = &nodeInfo;
190 nodeInfo.isVariable =
true;
195 nodeInfo.isCategory =
true;
202 info.serverInfos.reserve(info.absArg->servers().size());
203 for (
RooAbsArg *server : info.absArg->servers()) {
204 if (server->isValueServer(*info.absArg)) {
205 auto *serverInfo = nodeInfos.at(server);
206 info.serverInfos.emplace_back(serverInfo);
207 serverInfo->clientInfos.emplace_back(&info);
213 _nodes.back().isValueServer =
true;
214 for (
auto iter =
_nodes.rbegin(); iter !=
_nodes.rend(); ++iter) {
215 if (!iter->isValueServer)
217 for (
auto &serverInfo : iter->serverInfos) {
218 serverInfo->isValueServer =
true;
232 for (
auto &info :
_nodes) {
246 std::size_t iValueServer = 0;
247 for (
RooAbsArg *server : info.absArg->servers()) {
248 if (server->isValueServer(*info.absArg)) {
249 auto *knownServer = info.serverInfos[iValueServer]->absArg;
250 if (knownServer->hasDataToken()) {
251 server->setDataToken(knownServer->dataToken());
269 for (
auto iter =
_nodes.rbegin(); iter !=
_nodes.rend(); ++iter) {
271 bool outputsLogarithm =
false;
288 throw std::runtime_error(
"Evaluator can only take device array as input in CUDA mode!");
305 static std::atomic<std::size_t> nextInputGeneration{1};
306 const std::size_t gen = ++nextInputGeneration;
337 info.
buffer->assignFromDevice(gpuSpan);
343 info.
buffer->assignFromHost(cpuSpan);
350 std::map<RooFit::Detail::DataKey, std::size_t> sizeMap;
351 for (
auto &info :
_nodes) {
352 if (info.fromArrayInput) {
353 sizeMap[info.absArg] = info.outputSize;
362 auto found = sizeMap.find(key);
363 return found != sizeMap.end() ? found->second : -1;
366 for (
auto &info :
_nodes) {
367 info.outputSize = outputSizeMap.at(info.absArg);
383 for (
auto &info :
_nodes) {
385 if (!info.isVariable && info.absArg) {
386 info.absArg->resetDataToken();
401 if (found !=
_nodesMap.end() && found->second->absArg == &arg) {
402 found->second->absArg =
nullptr;
408 using namespace Detail;
412 double *buffer =
nullptr;
422 logMissingCudaSupport(*info.
absArg);
429 buffer = info.
buffer->hostWritePtr();
439 buffer[0] = nodeAbsCategory->getCurrentIndex();
441 throw std::runtime_error(
"RooFit::Evaluator - non-scalar category values are not supported!");
444 auto nodeAbsReal =
static_cast<RooAbsReal const *
>(node);
462 auto *var =
static_cast<RooRealVar const *
>(node);
466 clientInfo->isDirty =
true;
479 if (nodeInfo.
lastCatVal != cat->getCurrentIndex()) {
482 clientInfo->isDirty =
true;
494 clientInfo->isDirty =
true;
515 for (
auto &nodeInfo :
_nodes) {
516 if (!nodeInfo.fromArrayInput) {
517 if (nodeInfo.isVariable) {
519 }
else if (nodeInfo.isCategory) {
522 if (nodeInfo.isDirty) {
525 nodeInfo.isDirty =
false;
566 for (
auto &info :
_nodes) {
567 if (info.fromArrayInput) {
570 if (info.isVariable) {
572 }
else if (info.isCategory) {
574 }
else if (info.isDirty) {
576 if (info.computeInGPU) {
581 info.isDirty =
false;
614 for (
auto &action : ctx->_deferredActions) {
617 ctx->_deferredActions.clear();
627 using namespace Detail;
633 double *buffer =
nullptr;
642 buffer = info.
buffer->deviceWritePtr();
659 for (
auto &info :
_nodes) {
660 info.computeInGPU =
false;
661 if (!info.absArg->canComputeBatchWithCuda()) {
664 for (
NodeInfo const *serverInfo : info.serverInfos) {
665 if (serverInfo->outputSize > 1) {
666 info.computeInGPU =
true;
673 for (
auto &info :
_nodes) {
674 info.copyAfterEvaluation =
false;
676 if (!info.isScalar()) {
677 for (
auto *clientInfo : info.clientInfos) {
678 if (info.computeInGPU != clientInfo->computeInGPU) {
679 info.copyAfterEvaluation =
true;
696 for (
auto &info :
_nodes) {
697 if (info.isVariable) {
729 auto out = std::make_unique<ChangeOperModeRAII>();
730 std::unordered_set<RooAbsArg *> visited;
732 std::vector<RooAbsArg *> queue;
733 queue.reserve(
_nodes.size());
734 for (
auto &info :
_nodes) {
735 queue.push_back(info.absArg);
738 while (!queue.empty()) {
741 if (!visited.insert(node).second)
744 out->change(node, opMode);
750 queue.push_back(client);
759 std::cout <<
"--- RooFit BatchMode evaluation ---\n";
761 std::vector<int> widths{9, 37, 20, 9, 10, 4, 20};
763 auto printElement = [&](
int iCol,
auto const &t) {
764 const char separator =
' ';
765 os << separator << std::left << std::setw(widths[iCol]) << std::setfill(separator) << t;
769 auto printHorizontalRow = [&]() {
771 for (
int w : widths) {
774 for (
int i = 0; i <
n; i++) {
780 printHorizontalRow();
783 printElement(0,
"Index");
784 printElement(1,
"Name");
785 printElement(2,
"Class");
786 printElement(3,
"Size");
787 printElement(4,
"From Data");
788 printElement(5,
"Log");
789 printElement(6,
"1st value");
792 printHorizontalRow();
794 for (std::size_t iNode = 0; iNode <
_nodes.size(); ++iNode) {
795 auto &nodeInfo =
_nodes[iNode];
801 printElement(0, iNode);
802 printElement(1, node->
GetName());
804 printElement(3, nodeInfo.outputSize);
805 printElement(4, nodeInfo.fromArrayInput);
807 printElement(6, span[0]);
812 printHorizontalRow();
824 for (
auto &nodeInfo :
_nodes) {
825 if (nodeInfo.isValueServer && nodeInfo.absArg->isFundamental()) {
826 parameters.
add(*nodeInfo.absArg);
851 for (
auto &nodeInfo :
_nodes) {
852 if (nodeInfo.absArg->isReducerNode()) {
853 nodeInfo.isDirty =
true;
Option_t Option_t TPoint TPoint const char mode
Common abstract base class for objects that represent a value and a "shape" in RooFit.
const TNamed * namePtr() const
De-duplicated pointer to this object's name.
virtual bool canOutputLogarithm() const
Whether doEval() can output the logarithm of the value instead of the value.
const RefCountList_t & valueClients() const
List of all value clients of this object. Value clients receive value updates.
void setDataToken(std::size_t index, RooFit::Evaluator *evaluator=nullptr)
Sets the token for retrieving results in the BatchMode.
virtual bool canUseLogarithmOf(RooAbsArg const &, bool) const
Whether doEval() accepts the logarithm of the given value server instead of its value.
OperMode operMode() const
Query the operation mode of this node.
A space to attach TBranches.
virtual bool add(const RooAbsArg &var, bool silent=false)
Add the specified argument to list.
Storage_t::size_type size() const
void sort(bool reverse=false)
Sort collection using std::sort and name comparison.
Abstract base class for objects that represent a real value and implements functionality common to al...
virtual void doEval(RooFit::EvalContext &) const
Base function for computing multiple values of a RooAbsReal.
RooArgSet is a container object that can hold multiple RooAbsArg objects.
Minimal configuration struct to steer the evaluation of a single node with the RooBatchCompute librar...
void setCudaStream(CudaInterface::CudaStream *cudaStream)
void setNThreads(int nThreads)
Number of threads to use for CPU batch computations and reductions.
virtual void synchronizeCudaStream(CudaInterface::CudaStream *) const =0
Wait until all work that was enqueued on the stream has completed.
virtual std::unique_ptr< AbsBufferManager > createBufferManager() const =0
virtual CudaInterface::CudaStream * newCudaStream() const =0
virtual void deleteCudaStream(CudaInterface::CudaStream *) const =0
std::size_t _inputGeneration
std::vector< std::function< void()> > _deferredActions
void set(RooAbsArg const *arg, std::span< const double > const &span)
std::span< const double > at(RooAbsArg const *arg, RooAbsArg const *caller=nullptr)
void resetVectorBuffers()
void enableVectorBuffers(bool enable)
bool outputsLogarithm(RooAbsArg const *arg) const
Whether the output of the given node is the logarithm of its value.
RooBatchCompute::Config config(RooAbsArg const *arg) const
void setConfig(RooAbsArg const *arg, RooBatchCompute::Config const &config)
std::vector< bool > _outputsLogarithm
std::span< double > _currentOutput
void resize(std::size_t n)
void print(std::ostream &os)
void setClientsDirty(NodeInfo &nodeInfo)
Flags all the clients of a given node dirty.
std::unique_ptr< ChangeOperModeRAII > setOperModes(RooAbsArg::OperMode opMode)
RooArgSet getParameters() const
Gets all the parameters of the RooAbsReal.
void setOffsetMode(RooFit::EvalContext::OffsetMode)
Sets the offset mode for evaluation.
void unregisterNode(RooAbsArg const &arg)
Called from RooAbsArg::~RooAbsArg() when a node of the computation graph is deleted while this Evalua...
void syncDataTokens()
If there are servers with the same name that got de-duplicated in the _nodes list,...
std::unordered_map< TNamed const *, NodeInfo * > _nodesMap
std::unique_ptr< ChangeOperModeRAII > _operModeChanges
std::vector< NodeInfo > _nodes
bool _needToUpdateOutputSizes
std::span< const double > getValHeterogeneous()
Returns the value of the top node in the computation graph.
std::span< const double > run()
Returns the value of the top node in the computation graph.
Evaluator(const RooAbsReal &absReal, bool useGPU=false)
Construct a new Evaluator.
void setNThreads(int nThreads)
Sets the number of threads to use for the evaluation of a single node.
void processVariable(NodeInfo &nodeInfo)
Process a variable in the computation graph.
void processCategory(NodeInfo &nodeInfo)
Process a category in the computation graph.
RooBatchCompute::CudaInterface::CudaStream * _cudaStream
std::unique_ptr< RooBatchCompute::AbsBufferManager > _bufferManager
void markGPUNodes()
Decides which nodes are assigned to the GPU in a CUDA fit.
void assignToGPU(NodeInfo &info)
Enqueue the computation of a node on the GPU.
void setInput(std::string const &name, std::span< const double > inputArray, bool isOnDevice)
RooFit::EvalContext _evalContextCUDA
RooFit::EvalContext _evalContextCPU
void computeCPUNode(const RooAbsArg *node, NodeInfo &info)
void setOperMode(RooAbsArg *arg, RooAbsArg::OperMode opMode)
Temporarily change the operation mode of a RooAbsArg until the Evaluator gets deleted.
void setupLogEvaluation()
Request nodes to output their logarithms if their only value client takes the logarithm anyway,...
static RooMsgService & instance()
Return reference to singleton instance.
static const TNamed * ptr(const char *stringPtr)
Return a unique TNamed pointer for given C++ string.
Variable that can be changed from the outside.
const char * GetName() const override
Returns name of object.
virtual const char * ClassName() const
Returns name of class to which the object belongs.
RVec< PromoteType< T > > log(const RVec< T > &v)
R__EXTERN RooBatchComputeInterface * dispatchCUDA
std::string cpuArchitectureName()
R__EXTERN RooBatchComputeInterface * dispatchCPU
This dispatch pointer points to an implementation of the compute library, provided one has been loade...
Architecture cpuArchitecture()
int initCPU()
Inspect hardware capabilities, and load the optimal library for RooFit computations.
The namespace RooFit contains mostly switches that change the behaviour of functions of PDFs (or othe...
void getSortedComputationGraph(RooAbsArg const &func, RooArgSet &out)
A struct used by the Evaluator to store information on the RooAbsArgs in the computation graph.
std::size_t lastSetValCount
std::vector< NodeInfo * > serverInfos
RooAbsArg::OperMode originalOperMode
std::vector< NodeInfo * > clientInfos
std::shared_ptr< RooBatchCompute::AbsBuffer > buffer