Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
HypoTestCalculatorGeneric.cxx
Go to the documentation of this file.
1// @(#)root/roostats:$Id$
2// Author: Kyle Cranmer, Sven Kreiss 23/05/10
3/*************************************************************************
4 * Copyright (C) 1995-2008, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11/** \class RooStats::HypoTestCalculatorGeneric
12 \ingroup Roostats
13
14Common base class for the Hypothesis Test Calculators.
15It is not designed to use directly but via its derived classes
16
17Same purpose as HybridCalculatorOriginal, but different implementation.
18
19This is the "generic" version that works with any TestStatSampler. The
20HybridCalculator derives from this class but explicitly uses the
21ToyMCSampler as its TestStatSampler.
22
23*/
24
30
31#include "RooAddPdf.h"
32
33#include "RooRandom.h"
34
35#include <memory>
36
37
38
39using namespace RooStats;
40using std::endl, std::string;
41
42////////////////////////////////////////////////////////////////////////////////
43/// Constructor. When test stat sampler is not provided
44/// uses ToyMCSampler and RatioOfProfiledLikelihoodsTestStat
45/// and nToys = 1000.
46/// User can : GetTestStatSampler()->SetNToys( # )
47
49 const RooAbsData &data,
50 const ModelConfig &altModel,
51 const ModelConfig &nullModel,
52 TestStatSampler *sampler
53 ) :
54 fAltModel(&altModel),
55 fNullModel(&nullModel),
56 fData(&data),
57 fTestStatSampler(sampler),
58 fDefaultSampler(nullptr),
59 fDefaultTestStat(nullptr),
60 fAltToysSeed(0)
61{
62 if(!sampler){
63 std::unique_ptr<const RooArgSet> altSnapshot{altModel.GetSnapshot()};
66 *altModel.GetPdf(),
67 altSnapshot.get());
68
69 auto toymcs = new ToyMCSampler(*fDefaultTestStat, 1000);
70 const bool dataIsBinned = dynamic_cast<const RooDataHist*>(fData);
71 if (dataIsBinned) {
72 // Ensure the ToyMCSampler generates toys with the same structure as the observed data.
73 // TODO: understand why this is needed only in the RooDataHist case,
74 // but results in backwards-incompatibility for RooDataSet, which
75 // manifests for example in the stressRooStats tests.
76 // See also: https://github.com/root-project/root/pull/20171
77 toymcs->SetProtoData(&data);
78 }
79 toymcs->SetGenerateBinned(dataIsBinned); // if observed is RooDataHist -> generate RooDataHist toys
80
81 fDefaultSampler = toymcs;
82 fTestStatSampler = toymcs;
83 }
84
85
86}
87
88////////////////////////////////////////////////////////////////////////////////
89/// common setup for both models
90
95
96 // for this model
97 model.LoadSnapshot();
98 fTestStatSampler->SetSamplingDistName(model.GetName());
99 fTestStatSampler->SetPdf(*model.GetPdf());
100 fTestStatSampler->SetNuisanceParameters(*model.GetNuisanceParameters());
101 // global observables or nuisance pdf will be set by the derived classes
102 // (e.g. Frequentist or HybridCalculator)
103}
104
105////////////////////////////////////////////////////////////////////////////////
106
111
112////////////////////////////////////////////////////////////////////////////////
113/// several possibilities:
114/// no prior nuisance given and no nuisance parameters: ok
115/// no prior nuisance given but nuisance parameters: error
116/// prior nuisance given for some nuisance parameters:
117/// - nuisance parameters are constant, so they don't float in test statistic
118/// - nuisance parameters are floating, so they do float in test statistic
119
121
122 // initial setup
123 PreHook();
124 const_cast<ModelConfig*>(fNullModel)->GuessObsAndNuisance(*fData->get());
125 const_cast<ModelConfig*>(fAltModel)->GuessObsAndNuisance(*fData->get());
126
127 std::unique_ptr<const RooArgSet> nullSnapshot {fNullModel->GetSnapshot()};
128 if(nullSnapshot == nullptr) {
129 oocoutE(nullptr,Generation) << "Null model needs a snapshot. Set using modelconfig->SetSnapshot(poi)." << std::endl;
130 return nullptr;
131 }
132
133 // CheckHook
134 if(CheckHook() != 0) {
135 oocoutE(nullptr,Generation) << "There was an error in CheckHook(). Stop." << std::endl;
136 return nullptr;
137 }
138
140 oocoutE(nullptr,InputArguments) << "Test Statistic Sampler or Test Statistics not defined. Stop." << std::endl;
141 return nullptr;
142 }
143
144 // get a big list of all variables for convenient switching
145 std::unique_ptr<RooArgSet> altParams{fAltModel->GetPdf()->getParameters(*fData)};
146 // save all parameters so we can set them back to what they were
147 std::unique_ptr<RooArgSet> bothParams{fNullModel->GetPdf()->getParameters(*fData)};
148 bothParams->add(*altParams,false);
149 std::unique_ptr<RooArgSet> saveAll {(RooArgSet*) bothParams->snapshot()};
150
151 // check whether we have a ToyMCSampler and if so, keep a pointer to it
152 ToyMCSampler* toymcs = dynamic_cast<ToyMCSampler*>(fTestStatSampler );
153
154 // evaluate test statistic on data
155 RooArgSet nullP(*nullSnapshot);
156 double obsTestStat;
157
158 std::unique_ptr<RooArgList> allTS;
159 if( toymcs ) {
160 allTS = std::unique_ptr<RooArgList>{toymcs->EvaluateAllTestStatistics(*const_cast<RooAbsData*>(fData), nullP)};
161 if (!allTS) return nullptr;
162 //oocoutP(nullptr,Generation) << "All Test Statistics on data: " << std::endl;
163 //allTS->Print("v");
164 RooRealVar* firstTS = static_cast<RooRealVar*>(allTS->at(0));
165 obsTestStat = firstTS->getVal();
166 if (allTS->size()<=1) {
167 allTS = nullptr; // don't save
168 }
169
170 }else{
171 obsTestStat = fTestStatSampler->EvaluateTestStatistic(*const_cast<RooAbsData*>(fData), nullP);
172 }
173 oocoutP(nullptr,Generation) << "Test Statistic on data: " << obsTestStat << std::endl;
174
175 // set parameters back ... in case the evaluation of the test statistic
176 // modified something (e.g. a nuisance parameter that is not randomized
177 // must be set here)
178 bothParams->assign(*saveAll);
179
180 // If the pdf is not extended, the generation of toy datasets will fail if
181 // the number of events requested per toy is not set. In that case, we
182 // generate the same number of events as in the observed data.
183 auto setNEventsIfNeeded = [&](RooFit::ModelConfig const *modelConfig) {
184 int nEventsPerToy = toymcs->nEventsPerToy();
185 auto *pdf = modelConfig->GetPdf();
186 if (nEventsPerToy == 0 && (!pdf->canBeExtended() || pdf->expectedEvents(modelConfig->GetObservables()) <= 0)) {
188 }
189 };
190
191
192 // Generate sampling distribution for null
194 RooArgSet paramPointNull(*fNullModel->GetParametersOfInterest());
195 if(PreNullHook(&paramPointNull, obsTestStat) != 0) {
196 oocoutE(nullptr,Generation) << "PreNullHook did not return 0." << std::endl;
197 }
198 SamplingDistribution* samp_null = nullptr;
199 RooDataSet* detOut_null = nullptr;
200 if(toymcs) {
201 setNEventsIfNeeded(fNullModel);
202
203 detOut_null = toymcs->GetSamplingDistributions(paramPointNull);
204 if( detOut_null ) {
205 samp_null = new SamplingDistribution( detOut_null->GetName(), detOut_null->GetTitle(), *detOut_null );
206 if (detOut_null->get()->size()<=1) {
207 delete detOut_null;
208 detOut_null= nullptr;
209 }
210 }
211 } else {
212 samp_null = fTestStatSampler->GetSamplingDistribution(paramPointNull);
213 }
214
215 // set parameters back
216 bothParams->assign(*saveAll);
217
218 // Generate sampling distribution for alternate
221 if(PreAltHook(&paramPointAlt, obsTestStat) != 0) {
222 oocoutE(nullptr,Generation) << "PreAltHook did not return 0." << std::endl;
223 }
224 SamplingDistribution* samp_alt = nullptr;
225 RooDataSet* detOut_alt = nullptr;
226 if(toymcs) {
227
228 // case of re-using same toys for every points
229 // set a given seed
230 unsigned int prevSeed = 0;
231 if (fAltToysSeed > 0) {
232 prevSeed = RooRandom::integer(std::numeric_limits<unsigned int>::max()-1)+1; // want to avoid zero value
234 }
235
236 setNEventsIfNeeded(fAltModel);
237
238 detOut_alt = toymcs->GetSamplingDistributions(paramPointAlt);
239 if( detOut_alt ) {
240 samp_alt = new SamplingDistribution( detOut_alt->GetName(), detOut_alt->GetTitle(), *detOut_alt );
241 if (detOut_alt->get()->size()<=1) {
242 delete detOut_alt;
243 detOut_alt= nullptr;
244 }
245 }
246
247 // restore the seed
248 if (prevSeed > 0) {
250 }
251
252 } else {
253 samp_alt = fTestStatSampler->GetSamplingDistribution(paramPointAlt);
254 }
255
256 // create result
257 string resultname = "HypoTestCalculator_result";
258 HypoTestResult* res = new HypoTestResult(resultname.c_str());
260 res->SetTestStatisticData(obsTestStat);
261 res->SetAltDistribution(samp_alt); // takes ownership of samp_alt
262 res->SetNullDistribution(samp_null); // takes ownership of samp_null
263 res->SetAltDetailedOutput( detOut_alt );
264 res->SetNullDetailedOutput( detOut_null );
265 res->SetAllTestStatisticsData( allTS.get() );
266
267 const RooArgSet *aset = GetFitInfo();
268 if (aset != nullptr) {
269 RooDataSet *dset = new RooDataSet("", "", *aset);
270 dset->add(*aset);
271 res->SetFitInfo( dset );
272 }
273
274 bothParams->assign(*saveAll);
275 PostHook();
276 return res;
277}
278
279////////////////////////////////////////////////////////////////////////////////
280/// to re-use same toys for alternate hypothesis
281
283 fAltToysSeed = RooRandom::integer(std::numeric_limits<unsigned int>::max()-1)+1;
284}
#define oocoutE(o, a)
#define oocoutP(o, a)
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
RooFit::OwningPtr< RooArgSet > getParameters(const RooAbsData *data, bool stripDisconnected=true) const
Create a list of leaf nodes in the arg tree starting with ourself as top node that don't match any of...
Storage_t::size_type size() const
Abstract base class for binned and unbinned datasets.
Definition RooAbsData.h:55
virtual double sumEntries() const =0
Return effective number of entries in dataset, i.e., sum all weights.
virtual const RooArgSet * get() const
Definition RooAbsData.h:99
double getVal(const RooArgSet *normalisationSet=nullptr) const
Evaluate object.
Definition RooAbsReal.h:107
RooArgSet is a container object that can hold multiple RooAbsArg objects.
Definition RooArgSet.h:24
Container class to hold N-dimensional binned data.
Definition RooDataHist.h:40
Container class to hold unbinned data.
Definition RooDataSet.h:32
const RooArgSet * get(Int_t index) const override
Return RooArgSet with coordinates of event 'index'.
void add(const RooArgSet &row, double weight, double weightError)
Add one ore more rows of data.
static UInt_t integer(UInt_t max, TRandom *generator=randomGenerator())
Return an integer uniformly distributed from [0,n-1].
Definition RooRandom.cxx:95
static TRandom * randomGenerator()
Return a pointer to a singleton random-number generator implementation.
Definition RooRandom.cxx:47
Variable that can be changed from the outside.
Definition RooRealVar.h:37
HypoTestResult * GetHypoTest() const override
inherited methods from HypoTestCalculator interface
virtual int PreNullHook(RooArgSet *, double) const
void SetupSampler(const ModelConfig &model) const
common setup for both models
virtual int PreAltHook(RooArgSet *, double) const
virtual const RooArgSet * GetFitInfo() const
void UseSameAltToys()
Set this for re-using always the same toys for alternate hypothesis in case of calls at different nul...
HypoTestCalculatorGeneric(const RooAbsData &data, const ModelConfig &altModel, const ModelConfig &nullModel, TestStatSampler *sampler=nullptr)
Constructor.
HypoTestResult is a base class for results from hypothesis tests.
void SetAltDetailedOutput(RooDataSet *d)
void SetNullDetailedOutput(RooDataSet *d)
void SetAllTestStatisticsData(const RooArgList *tsd)
void SetNullDistribution(SamplingDistribution *null)
void SetTestStatisticData(const double tsd)
void SetFitInfo(RooDataSet *d)
void SetAltDistribution(SamplingDistribution *alt)
< A class that holds configuration information for a model using a workspace as a store
Definition ModelConfig.h:34
const RooArgSet * GetParametersOfInterest() const
get RooArgSet containing the parameter of interest (return nullptr if not existing)
void LoadSnapshot() const
load the snapshot from ws if it exists
const RooArgSet * GetObservables() const
get RooArgSet for observables (return nullptr if not existing)
const RooArgSet * GetSnapshot() const
get RooArgSet for parameters for a particular hypothesis (return nullptr if not existing)
RooAbsPdf * GetPdf() const
get model PDF (return nullptr if pdf has not been specified or does not exist)
TestStatistic that returns the ratio of profiled likelihoods.
This class simply holds a sampling distribution of some test statistic.
TestStatSampler is an interface class for a tools which produce RooStats SamplingDistributions.
virtual void SetObservables(const RooArgSet &)=0
specify the observables in the dataset (needed to evaluate the test statistic)
virtual TestStatistic * GetTestStatistic() const =0
Get the TestStatistic.
virtual void SetParametersForTestStat(const RooArgSet &)=0
specify the values of parameters used when evaluating test statistic
virtual double EvaluateTestStatistic(RooAbsData &data, RooArgSet &paramsOfInterest)=0
Main interface to evaluate the test statistic on a dataset.
virtual void SetNuisanceParameters(const RooArgSet &)=0
specify the nuisance parameters (eg. the rest of the parameters)
virtual SamplingDistribution * GetSamplingDistribution(RooArgSet &paramsOfInterest)=0
Main interface to get a ConfInterval, pure virtual.
virtual void SetSamplingDistName(const char *name)=0
Set the name of the sampling distribution used for plotting.
virtual void SetPdf(RooAbsPdf &)=0
Set the Pdf, add to the workspace if not already there.
virtual bool PValueIsRightTail(void) const
Defines the sign convention of the test statistic. Overwrite function if necessary.
ToyMCSampler is an implementation of the TestStatSampler interface.
virtual RooArgList * EvaluateAllTestStatistics(RooAbsData &data, const RooArgSet &poi)
Evaluate all test statistics, returning result and any detailed output.
virtual RooDataSet * GetSamplingDistributions(RooArgSet &paramPoint)
Use for serial and parallel runs.
virtual void SetNEventsPerToy(const Int_t nevents)
Forces the generation of exactly n events even for extended PDFs.
const char * GetName() const override
Returns name of object.
Definition TNamed.h:49
const char * GetTitle() const override
Returns title of object.
Definition TNamed.h:50
virtual void SetSeed(ULong_t seed=0)
Set the random generator seed.
Definition TRandom.cxx:614
Namespace for the RooStats classes.
Definition CodegenImpl.h:67