Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
HypoTestInverter.cxx
Go to the documentation of this file.
1// @(#)root/roostats:$Id$
2// Author: Kyle Cranmer, Lorenzo Moneta, Gregory Schott, Wouter Verkerke
3// Contributions: Giovanni Petrucciani and Annapaola Decosa
4/*************************************************************************
5 * Copyright (C) 1995-2008, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12/** \class RooStats::HypoTestInverter
13 \ingroup Roostats
14
15A class for performing a hypothesis test inversion by scanning
16the hypothesis test results of a HypoTestCalculator for various values of the
17parameter of interest. By looking at the confidence level curve of the result, an
18upper limit can be derived by computing the intersection of the confidence level curve with the desired confidence level.
19The class implements the RooStats::IntervalCalculator interface, and returns a
20RooStats::HypoTestInverterResult. The result is a SimpleInterval, which
21via the method UpperLimit() returns to the user the upper limit value.
22
23## Scanning options
24The HypoTestInverter implements various options for performing the scan.
25- HypoTestInverter::RunFixedScan will scan the parameter of interest using a fixed grid.
26- HypoTestInverter::SetAutoScan will perform an automatic scan to find
27optimally the curve. It will stop when the desired precision is obtained.
28- HypoTestInverter::RunOnePoint computes the confidence level at a given point.
29
30### CLs presciption
31The class can scan the CLs+b values or alternatively CLs. For the latter,
32call HypoTestInverter::UseCLs().
33*/
34
36
47
48#include "RooAbsData.h"
49#include "RooRealVar.h"
50#include "RooArgSet.h"
51#include "RooAbsPdf.h"
52#include "RooRandom.h"
53#include "RooConstVar.h"
54#include "RooMsgService.h"
55
56#include "TMath.h"
57#include "TF1.h"
58#include "TFile.h"
59#include "TH1.h"
60#include "TLine.h"
61#include "TCanvas.h"
62#include "TGraphErrors.h"
63
64#include <cassert>
65#include <cmath>
66#include <memory>
67
68
69using namespace RooStats;
70using std::endl;
71
72// static variable definitions
74unsigned int HypoTestInverter::fgNToys = 500;
75
78std::string HypoTestInverter::fgAlgo = "logSecant";
79
80// helper class to wrap the functionality of the various HypoTestCalculators
81
82template<class HypoTestType>
84
85 static void SetToys(HypoTestType * h, int toyNull, int toyAlt) { h->SetToys(toyNull,toyAlt); }
86
87};
88
89////////////////////////////////////////////////////////////////////////////////
90/// get the variable to scan
91/// try first with null model if not go to alternate model
92
94
95 RooRealVar * varToScan = nullptr;
96 const ModelConfig * mc = hc.GetNullModel();
97 if (mc) {
98 const RooArgSet * poi = mc->GetParametersOfInterest();
99 if (poi) varToScan = dynamic_cast<RooRealVar*> (poi->first() );
100 }
101 if (!varToScan) {
102 mc = hc.GetAlternateModel();
103 if (mc) {
104 const RooArgSet * poi = mc->GetParametersOfInterest();
105 if (poi) varToScan = dynamic_cast<RooRealVar*> (poi->first() );
106 }
107 }
108 return varToScan;
109}
110
111////////////////////////////////////////////////////////////////////////////////
112/// check the model given the given hypotestcalculator
113
115 const ModelConfig * modelSB = hc.GetNullModel();
116 const ModelConfig * modelB = hc.GetAlternateModel();
117 if (!modelSB || ! modelB)
118 oocoutF(nullptr,InputArguments) << "HypoTestInverter - model are not existing" << std::endl;
119 assert(modelSB && modelB);
120
121 oocoutI(nullptr,InputArguments) << "HypoTestInverter ---- Input models: \n"
122 << "\t\t using as S+B (null) model : "
123 << modelSB->GetName() << "\n"
124 << "\t\t using as B (alternate) model : "
125 << modelB->GetName() << "\n" << std::endl;
126
127 // check if scanVariable is included in B model pdf
128 RooAbsPdf * bPdf = modelB->GetPdf();
129 const RooArgSet * bObs = modelB->GetObservables();
130 if (!bPdf || !bObs) {
131 oocoutE(nullptr,InputArguments) << "HypoTestInverter - B model has no pdf or observables defined" << std::endl;
132 return;
133 }
134 std::unique_ptr<RooArgSet> bParams{bPdf->getParameters(*bObs)};
135 if (!bParams) {
136 oocoutE(nullptr,InputArguments) << "HypoTestInverter - pdf of B model has no parameters" << std::endl;
137 return;
138 }
139 if (bParams->find(scanVariable.GetName() ) ) {
140 const RooArgSet * poiB = modelB->GetSnapshot();
141 if (!poiB || !poiB->find(scanVariable.GetName()) ||
142 (static_cast<RooRealVar *>(poiB->find(scanVariable.GetName())))->getVal() != 0) {
143 oocoutW(nullptr, InputArguments)
144 << "HypoTestInverter - using a B model with POI " << scanVariable.GetName() << " not equal to zero "
145 << " user must check input model configurations " << std::endl;
146 }
147 if (poiB) delete poiB;
148 }
149}
150
151////////////////////////////////////////////////////////////////////////////////
152/// default constructor (doesn't do anything)
153
155 fTotalToysRun(0),
156 fMaxToys(0),
157 fCalculator0(nullptr),
158 fScannedVariable(nullptr),
159 fResults(nullptr),
160 fUseCLs(false),
161 fScanLog(false),
162 fSize(0),
163 fVerbose(0),
164 fCalcType(kUndefined),
165 fNBins(0), fXmin(1), fXmax(1),
166 fNumErr(0)
167{
168}
169
170////////////////////////////////////////////////////////////////////////////////
171/// Constructor from a HypoTestCalculatorGeneric
172/// The HypoTest calculator must be a FrequentistCalculator or HybridCalculator type
173/// Other type of calculators are not supported.
174/// The calculator must be created before by using the S+B model for the null and
175/// the B model for the alt
176/// If no variable to scan are given they are assumed to be the first variable
177/// from the parameter of interests of the null model
178
180 RooRealVar* scannedVariable, double size ) :
181 fTotalToysRun(0),
182 fMaxToys(0),
183 fCalculator0(nullptr),
184 fScannedVariable(scannedVariable),
185 fResults(nullptr),
186 fUseCLs(false),
187 fScanLog(false),
188 fSize(size),
189 fVerbose(0),
190 fCalcType(kUndefined),
191 fNBins(0), fXmin(1), fXmax(1),
192 fNumErr(0)
193{
194
195 if (!fScannedVariable) {
197 }
198 if (!fScannedVariable) {
199 oocoutE(nullptr,InputArguments) << "HypoTestInverter - Cannot guess the variable to scan " << std::endl;
200 } else {
202 }
203
204 HybridCalculator * hybCalc = dynamic_cast<HybridCalculator*>(&hc);
205 if (hybCalc) {
207 fCalculator0 = hybCalc;
208 return;
209 }
210 FrequentistCalculator * freqCalc = dynamic_cast<FrequentistCalculator*>(&hc);
211 if (freqCalc) {
213 fCalculator0 = freqCalc;
214 return;
215 }
216 AsymptoticCalculator * asymCalc = dynamic_cast<AsymptoticCalculator*>(&hc);
217 if (asymCalc) {
219 fCalculator0 = asymCalc;
220 return;
221 }
222 oocoutE(nullptr,InputArguments) << "HypoTestInverter - Type of hypotest calculator is not supported " <<std::endl;
223 fCalculator0 = &hc;
224}
225
226////////////////////////////////////////////////////////////////////////////////
227/// Constructor from a reference to a HybridCalculator
228/// The calculator must be created before by using the S+B model for the null and
229/// the B model for the alt
230/// If no variable to scan are given they are assumed to be the first variable
231/// from the parameter of interests of the null model
232
234 RooRealVar* scannedVariable, double size ) :
235 fTotalToysRun(0),
236 fMaxToys(0),
237 fCalculator0(&hc),
238 fScannedVariable(scannedVariable),
239 fResults(nullptr),
240 fUseCLs(false),
241 fScanLog(false),
242 fSize(size),
243 fVerbose(0),
244 fCalcType(kHybrid),
245 fNBins(0), fXmin(1), fXmax(1),
246 fNumErr(0)
247{
248
249 if (!fScannedVariable) {
251 }
252 if (!fScannedVariable) {
253 oocoutE(nullptr,InputArguments) << "HypoTestInverter - Cannot guess the variable to scan " << std::endl;
254 } else {
256 }
257}
258
259////////////////////////////////////////////////////////////////////////////////
260/// Constructor from a reference to a FrequentistCalculator
261/// The calculator must be created before by using the S+B model for the null and
262/// the B model for the alt
263/// If no variable to scan are given they are assumed to be the first variable
264/// from the parameter of interests of the null model
265
267 RooRealVar* scannedVariable, double size ) :
268 fTotalToysRun(0),
269 fMaxToys(0),
270 fCalculator0(&hc),
271 fScannedVariable(scannedVariable),
272 fResults(nullptr),
273 fUseCLs(false),
274 fScanLog(false),
275 fSize(size),
276 fVerbose(0),
277 fCalcType(kFrequentist),
278 fNBins(0), fXmin(1), fXmax(1),
279 fNumErr(0)
280{
281
282 if (!fScannedVariable) {
284 }
285 if (!fScannedVariable) {
286 oocoutE(nullptr,InputArguments) << "HypoTestInverter - Cannot guess the variable to scan " << std::endl;
287 } else {
289 }
290}
291
292////////////////////////////////////////////////////////////////////////////////
293/// Constructor from a reference to a AsymptoticCalculator
294/// The calculator must be created before by using the S+B model for the null and
295/// the B model for the alt
296/// If no variable to scan are given they are assumed to be the first variable
297/// from the parameter of interests of the null model
298
300 RooRealVar* scannedVariable, double size ) :
301 fTotalToysRun(0),
302 fMaxToys(0),
303 fCalculator0(&hc),
304 fScannedVariable(scannedVariable),
305 fResults(nullptr),
306 fUseCLs(false),
307 fScanLog(false),
308 fSize(size),
309 fVerbose(0),
310 fCalcType(kAsymptotic),
311 fNBins(0), fXmin(1), fXmax(1),
312 fNumErr(0)
313{
314
315 if (!fScannedVariable) {
317 }
318 if (!fScannedVariable) {
319 oocoutE(nullptr,InputArguments) << "HypoTestInverter - Cannot guess the variable to scan " << std::endl;
320 } else {
322 }
323}
324
325////////////////////////////////////////////////////////////////////////////////
326/// Constructor from a model for B model and a model for S+B.
327/// An HypoTestCalculator (Hybrid of Frequentis) will be created using the
328/// S+B model as the null and the B model as the alternate
329/// If no variable to scan are given they are assumed to be the first variable
330/// from the parameter of interests of the null model
331
333 RooRealVar * scannedVariable, ECalculatorType type, double size) :
334 fTotalToysRun(0),
335 fMaxToys(0),
336 fCalculator0(nullptr),
337 fScannedVariable(scannedVariable),
338 fResults(nullptr),
339 fUseCLs(false),
340 fScanLog(false),
341 fSize(size),
342 fVerbose(0),
343 fCalcType(type),
344 fNBins(0), fXmin(1), fXmax(1),
345 fNumErr(0)
346{
347 if(fCalcType==kFrequentist) fHC = std::make_unique<FrequentistCalculator>(data, bModel, sbModel);
348 if(fCalcType==kHybrid) fHC = std::make_unique<HybridCalculator>(data, bModel, sbModel);
349 if(fCalcType==kAsymptotic) fHC = std::make_unique<AsymptoticCalculator>(data, bModel, sbModel);
350 fCalculator0 = fHC.get();
351 // get scanned variable
352 if (!fScannedVariable) {
354 }
355 if (!fScannedVariable) {
356 oocoutE(nullptr,InputArguments) << "HypoTestInverter - Cannot guess the variable to scan " << std::endl;
357 } else {
359 }
360}
361
362////////////////////////////////////////////////////////////////////////////////
363/// copy-constructor
364/// NOTE: this class does not copy the contained result and
365/// the HypoTestCalculator, but only the pointers
366/// It requires the original HTI to be alive
367
370 fTotalToysRun(0),
371 fCalculator0(nullptr), fScannedVariable(nullptr), // add these for Coverity
372 fResults(nullptr)
373{
374 (*this) = rhs;
375}
376
377////////////////////////////////////////////////////////////////////////////////
378/// assignment operator
379/// NOTE: this class does not copy the contained result and
380/// the HypoTestCalculator, but only the pointers
381/// It requires the original HTI to be alive
382
384 if (this == &rhs) return *this;
385 fTotalToysRun = 0;
386 fMaxToys = rhs.fMaxToys;
389 fUseCLs = rhs.fUseCLs;
390 fScanLog = rhs.fScanLog;
391 fSize = rhs.fSize;
392 fVerbose = rhs.fVerbose;
393 fCalcType = rhs.fCalcType;
394 fNBins = rhs.fNBins;
395 fXmin = rhs.fXmin;
396 fXmax = rhs.fXmax;
397 fNumErr = rhs.fNumErr;
398
399 return *this;
400}
401
402////////////////////////////////////////////////////////////////////////////////
403/// destructor (delete the HypoTestInverterResult)
404
406{
407 if (fResults) delete fResults;
408 fCalculator0 = nullptr;
409}
410
411////////////////////////////////////////////////////////////////////////////////
412/// return the test statistic which is or will be used by the class
413
415{
418 } else {
419 return nullptr;
420 }
421}
422
423////////////////////////////////////////////////////////////////////////////////
424/// set the test statistic to use
425
427{
430 return true;
431 } else {
432 return false;
433 }
434}
435
436////////////////////////////////////////////////////////////////////////////////
437/// delete contained result and graph
438
440 if (fResults) delete fResults;
441 fResults = nullptr;
442 fLimitPlot.reset();
443}
444
445////////////////////////////////////////////////////////////////////////////////
446/// create a new HypoTestInverterResult to hold all computed results
447
449 if (fResults == nullptr) {
450 TString results_name = "result_";
451 results_name += fScannedVariable->GetName();
453 TString title = "HypoTestInverter Result For ";
454 title += fScannedVariable->GetName();
455 fResults->SetTitle(title);
456 }
459 // check if one or two sided scan
460 if (fCalculator0) {
461 // if asymptotic calculator
463 if (ac) {
465 } else {
466 // in case of the other calculators
468 if (sampler) {
470 if (pl && pl->IsTwoSided() ) fResults->fIsTwoSided = true;
471 }
472 }
473 }
474}
475
476////////////////////////////////////////////////////////////////////////////////
477/// Run a fixed scan or the automatic scan depending on the configuration.
478/// Return if needed a copy of the result object which will be managed by the user.
479
481
482 // if having a result with at least one point return it
483 if (fResults && fResults->ArraySize() >= 1) {
484 oocoutI(nullptr,Eval) << "HypoTestInverter::GetInterval - return an already existing interval " << std::endl;
485 return static_cast<HypoTestInverterResult*>(fResults->Clone());
486 }
487
488 if (fNBins > 0) {
489 oocoutI(nullptr,Eval) << "HypoTestInverter::GetInterval - run a fixed scan" << std::endl;
490 bool ret = RunFixedScan(fNBins, fXmin, fXmax, fScanLog);
491 if (!ret)
492 oocoutE(nullptr,Eval) << "HypoTestInverter::GetInterval - error running a fixed scan " << std::endl;
493 }
494 else {
495 oocoutI(nullptr,Eval) << "HypoTestInverter::GetInterval - run an automatic scan" << std::endl;
496 double limit(0);
497 double err(0);
498 bool ret = RunLimit(limit,err);
499 if (!ret)
500 oocoutE(nullptr,Eval) << "HypoTestInverter::GetInterval - error running an auto scan " << std::endl;
501 }
502
503 return static_cast<HypoTestInverterResult*> (fResults->Clone());
504}
505
506////////////////////////////////////////////////////////////////////////////////
507/// Run the Hypothesis test at a previous configured point
508/// (internal function called by RunOnePoint)
509
510HypoTestResult * HypoTestInverter::Eval(HypoTestCalculatorGeneric &hc, bool adaptive, double clsTarget) const {
511 //for debug
512 // std::cout << ">>>>>>>>>>> " << std::endl;
513 // std::cout << "alternate model " << std::endl;
514 // hc.GetAlternateModel()->GetNuisanceParameters()->Print("V");
515 // hc.GetAlternateModel()->GetParametersOfInterest()->Print("V");
516 // std::cout << "Null model " << std::endl;
517 // hc.GetNullModel()->GetNuisanceParameters()->Print("V");
518 // hc.GetNullModel()->GetParametersOfInterest()->Print("V");
519 // std::cout << "<<<<<<<<<<<<<<< " << std::endl;
520
521 // run the hypothesis test
522 HypoTestResult * hcResult = hc.GetHypoTest();
523 if (hcResult == nullptr) {
524 oocoutE(nullptr,Eval) << "HypoTestInverter::Eval - HypoTest failed" << std::endl;
525 return hcResult;
526 }
527
528 // since the b model is the alt need to set the flag
529 hcResult->SetBackgroundAsAlt(true);
530
531
532 // bool flipPvalue = false;
533 // if (flipPValues)
534 // hcResult->SetPValueIsRightTail(!hcResult->GetPValueIsRightTail());
535
536 // adjust for some numerical error in discrete models and == is not anymore
537 if (hcResult->GetPValueIsRightTail()) {
538 hcResult->SetTestStatisticData(hcResult->GetTestStatisticData()-fNumErr); // issue with < vs <= in discrete models
539 } else {
540 hcResult->SetTestStatisticData(hcResult->GetTestStatisticData() +
541 fNumErr); // issue with < vs <= in discrete models
542 }
543
544 double clsMid = (fUseCLs ? hcResult->CLs() : hcResult->CLsplusb());
545 double clsMidErr = (fUseCLs ? hcResult->CLsError() : hcResult->CLsplusbError());
546
547 //if (fVerbose) std::cout << (fUseCLs ? "\tCLs = " : "\tCLsplusb = ") << clsMid << " +/- " << clsMidErr << std::endl;
548
549 if (adaptive) {
550
553
554 while (clsMidErr >= fgCLAccuracy && (clsTarget == -1 || std::abs(clsMid-clsTarget) < 3*clsMidErr) ) {
555 std::unique_ptr<HypoTestResult> more(hc.GetHypoTest());
556
557 // if (flipPValues)
558 // more->SetPValueIsRightTail(!more->GetPValueIsRightTail());
559
560 hcResult->Append(more.get());
561 clsMid = (fUseCLs ? hcResult->CLs() : hcResult->CLsplusb());
562 clsMidErr = (fUseCLs ? hcResult->CLsError() : hcResult->CLsplusbError());
563 if (fVerbose) oocoutP(nullptr,Eval) << (fUseCLs ? "\tCLs = " : "\tCLsplusb = ") << clsMid << " +/- " << clsMidErr << std::endl;
564 }
565
566 }
567 if (fVerbose ) {
568 oocoutP(nullptr,Eval) << "P values for " << fScannedVariable->GetName() << " = " <<
569 fScannedVariable->getVal() << "\n" <<
570 "\tCLs = " << hcResult->CLs() << " +/- " << hcResult->CLsError() << "\n" <<
571 "\tCLb = " << hcResult->CLb() << " +/- " << hcResult->CLbError() << "\n" <<
572 "\tCLsplusb = " << hcResult->CLsplusb() << " +/- " << hcResult->CLsplusbError() << "\n" <<
573 std::endl;
574 }
575
577 fTotalToysRun += (hcResult->GetAltDistribution()->GetSize() + hcResult->GetNullDistribution()->GetSize());
578
579 // set sampling distribution name
580 TString nullDistName = TString::Format("%s_%s_%4.2f",hcResult->GetNullDistribution()->GetName(),
582 TString altDistName = TString::Format("%s_%s_%4.2f",hcResult->GetAltDistribution()->GetName(),
584
585 hcResult->GetNullDistribution()->SetName(nullDistName);
586 hcResult->GetAltDistribution()->SetName(altDistName);
587 }
588
589 return hcResult;
590}
591
592////////////////////////////////////////////////////////////////////////////////
593/// Run a Fixed scan in npoints between min and max
594
595bool HypoTestInverter::RunFixedScan( int nBins, double xMin, double xMax, bool scanLog ) const
596{
597
599 // interpolate the limits
602
603 // safety checks
604 if ( nBins<=0 ) {
605 oocoutE(nullptr,InputArguments) << "HypoTestInverter::RunFixedScan - Please provide nBins>0\n";
606 return false;
607 }
608 if ( nBins==1 && xMin!=xMax ) {
609 oocoutW(nullptr,InputArguments) << "HypoTestInverter::RunFixedScan - nBins==1 -> I will run for xMin (" << xMin << ")\n";
610 }
611 if ( xMin==xMax && nBins>1 ) {
612 oocoutW(nullptr,InputArguments) << "HypoTestInverter::RunFixedScan - xMin==xMax -> I will enforce nBins==1\n";
613 nBins = 1;
614 }
615 if ( xMin>xMax ) {
616 oocoutE(nullptr,InputArguments) << "HypoTestInverter::RunFixedScan - Please provide xMin ("
617 << xMin << ") smaller than xMax (" << xMax << ")\n";
618 return false;
619 }
620
621 if (xMin < fScannedVariable->getMin()) {
622 xMin = fScannedVariable->getMin();
623 oocoutW(nullptr,InputArguments) << "HypoTestInverter::RunFixedScan - xMin < lower bound, using xmin = "
624 << xMin << std::endl;
625 }
626 if (xMax > fScannedVariable->getMax()) {
627 xMax = fScannedVariable->getMax();
628 oocoutW(nullptr,InputArguments) << "HypoTestInverter::RunFixedScan - xMax > upper bound, using xmax = "
629 << xMax << std::endl;
630 }
631
632 if (xMin <= 0. && scanLog) {
633 oocoutE(nullptr, InputArguments) << "HypoTestInverter::RunFixedScan - cannot go in log steps if xMin <= 0" << std::endl;
634 return false;
635 }
636
637 double thisX = xMin;
638 for (int i=0; i<nBins; i++) {
639
640 if (i > 0) { // avoids case of nBins = 1
641 if (scanLog) {
642 thisX = exp( log(xMin) + i*(log(xMax)-log(xMin))/(nBins-1) ); // scan in log x
643 } else {
644 thisX = xMin + i * (xMax - xMin) / (nBins - 1); // linear scan in x
645 }
646 }
647
648 const bool status = RunOnePoint(thisX);
649
650 // check if failed status
651 if ( status==false ) {
652 oocoutW(nullptr,Eval) << "HypoTestInverter::RunFixedScan - The hypo test for point " << thisX << " failed. Skipping." << std::endl;
653 }
654 }
655
656 return true;
657}
658
659////////////////////////////////////////////////////////////////////////////////
660/// run only one point at the given POI value
661
662bool HypoTestInverter::RunOnePoint( double rVal, bool adaptive, double clTarget) const
663{
664
666
667 // check if rVal is in the range specified for fScannedVariable
668 if ( rVal < fScannedVariable->getMin() ) {
669 oocoutE(nullptr,InputArguments) << "HypoTestInverter::RunOnePoint - Out of range: using the lower bound "
671 << " on the scanned variable rather than " << rVal<< "\n";
672 rVal = fScannedVariable->getMin();
673 }
674 if ( rVal > fScannedVariable->getMax() ) {
675 // print a message when you have a significative difference since rval is computed
676 if (rVal > fScannedVariable->getMax() * (1. + 1.E-12)) {
677 oocoutE(nullptr, InputArguments) << "HypoTestInverter::RunOnePoint - Out of range: using the upper bound "
678 << fScannedVariable->getMax() << " on the scanned variable rather than "
679 << rVal << "\n";
680 }
681 rVal = fScannedVariable->getMax();
682 }
683
684 // save old value
685 double oldValue = fScannedVariable->getVal();
686
687 // evaluate hybrid calculator at a single point
689 // need to set value of rval in hybridcalculator
690 // assume null model is S+B and alternate is B only
691 const ModelConfig * sbModel = fCalculator0->GetNullModel();
692 RooArgSet poi; poi.add(*sbModel->GetParametersOfInterest());
693 // set poi to right values
695 const_cast<ModelConfig*>(sbModel)->SetSnapshot(poi);
696
697 if (fVerbose > 0)
698 oocoutP(nullptr,Eval) << "Running for " << fScannedVariable->GetName() << " = " << fScannedVariable->getVal() << std::endl;
699
700 // compute the results
701 std::unique_ptr<HypoTestResult> result( Eval(*fCalculator0,adaptive,clTarget) );
702 if (!result) {
703 oocoutE(nullptr,Eval) << "HypoTestInverter - Error running point " << fScannedVariable->GetName() << " = " <<
704 fScannedVariable->getVal() << std::endl;
705 return false;
706 }
707 // in case of a dummy result
708 const double nullPV = result->NullPValue();
709 const double altPV = result->AlternatePValue();
710 if (!std::isfinite(nullPV) || nullPV < 0. || nullPV > 1. || !std::isfinite(altPV) || altPV < 0. || altPV > 1.) {
711 oocoutW(nullptr,Eval) << "HypoTestInverter - Skipping invalid result for point " << fScannedVariable->GetName() << " = " <<
712 fScannedVariable->getVal() << ". null p-value=" << nullPV << ", alternate p-value=" << altPV << std::endl;
713 return false;
714 }
715
716 double lastXtested;
717 if ( fResults->ArraySize()!=0 ) lastXtested = fResults->GetXValue(fResults->ArraySize()-1);
718 else lastXtested = -999;
719
720 if ( (std::abs(rVal) < 1 && TMath::AreEqualAbs(rVal, lastXtested,1.E-12) ) ||
721 (std::abs(rVal) >= 1 && TMath::AreEqualRel(rVal, lastXtested,1.E-12) ) ) {
722
723 oocoutI(nullptr,Eval) << "HypoTestInverter::RunOnePoint - Merge with previous result for "
724 << fScannedVariable->GetName() << " = " << rVal << std::endl;
726 if (prevResult && prevResult->GetNullDistribution() && prevResult->GetAltDistribution()) {
727 prevResult->Append(result.get());
728 }
729 else {
730 // if it was empty we re-use it
731 oocoutI(nullptr,Eval) << "HypoTestInverter::RunOnePoint - replace previous empty result\n";
732 auto oldObj = fResults->fYObjects.Remove(prevResult);
733 delete oldObj;
734
735 fResults->fYObjects.Add(result.release());
736 }
737
738 } else {
739
740 // fill the results in the HypoTestInverterResult array
741 fResults->fXValues.push_back(rVal);
742 fResults->fYObjects.Add(result.release());
743
744 }
745
746 fScannedVariable->setVal(oldValue);
747
748 return true;
749}
750
751////////////////////////////////////////////////////////////////////////////////
752/// Run an automatic scan until the desired accuracy is reached.
753/// Start by default from the full interval (min,max) of the POI and then via bisection find the line crossing
754/// the target line.
755/// Optionally, a hint can be provided and the scan will be done closer to that value.
756/// If by bisection the desired accuracy will not be reached, a fit to the points is performed.
757/// \param[out] limit The limit.
758/// \param[out] limitErr The error of the limit.
759/// \param[in] absAccuracy Desired absolute accuracy.
760/// \param[in] relAccuracy Desired relative accuracy.
761/// \param[in] hint Hint to start from or nullptr for no hint.
762
763bool HypoTestInverter::RunLimit(double &limit, double &limitErr, double absAccuracy, double relAccuracy, const double*hint) const {
764
765
766 // routine from G. Petrucciani (from HiggsCombination CMS package)
767
769
770 if ((hint != nullptr) && (*hint > r->getMin())) {
771 r->setMax(std::min<double>(3.0 * (*hint), r->getMax()));
772 r->setMin(std::max<double>(0.3 * (*hint), r->getMin()));
773 oocoutI(nullptr,InputArguments) << "HypoTestInverter::RunLimit - Use hint value " << *hint
774 << " search in interval " << r->getMin() << " , " << r->getMax() << std::endl;
775 }
776
777 // if not specified use the default values for rel and absolute accuracy
778 if (absAccuracy <= 0) absAccuracy = fgAbsAccuracy;
779 if (relAccuracy <= 0) relAccuracy = fgRelAccuracy;
780
781 typedef std::pair<double,double> CLs_t;
782 double clsTarget = fSize;
783 CLs_t clsMin(1, 0);
784 CLs_t clsMax(0, 0);
785 CLs_t clsMid(0, 0);
786 double rMin = r->getMin();
787 double rMax = r->getMax();
788 limit = 0.5*(rMax + rMin);
789 limitErr = 0.5*(rMax - rMin);
790 bool done = false;
791
792 TF1 expoFit("expoFit","[0]*exp([1]*(x-[2]))", rMin, rMax);
793
794 fLimitPlot = std::make_unique<TGraphErrors>();
795
796 if (fVerbose > 0) oocoutI(nullptr,Eval) << "Search for upper limit to the limit" << std::endl;
797 for (int tries = 0; tries < 6; ++tries) {
798 if (! RunOnePoint(rMax) ) {
799 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Hypotest failed at upper limit of scan range: " << rMax << std::endl;
800 rMax *= 0.95;
801 continue;
802 }
803 clsMax = std::make_pair( fResults->GetLastYValue(), fResults->GetLastYError() );
804 if (clsMax.first == 0 || clsMax.first + 3 * std::abs(clsMax.second) < clsTarget ) break;
805 rMax += rMax;
806 if (tries == 5) {
807 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Cannot determine upper limit of scan range. At " << r->GetName()
808 << " = " << rMax << " still getting "
809 << (fUseCLs ? "CLs" : "CLsplusb") << " = " << clsMax.first << std::endl;
810 return false;
811 }
812 }
813 if (fVerbose > 0) {
814 oocoutI(nullptr,Eval) << "HypoTestInverter::RunLimit - Search for lower limit to the limit" << std::endl;
815 }
816
817 if ( fUseCLs && rMin == 0 ) {
818 clsMin = CLs_t(1,0);
819 }
820 else {
821 if (! RunOnePoint(rMin) ) {
822 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Hypotest failed at lower limit of scan range: " << rMin << std::endl;
823 return false;
824 }
825 clsMin = std::make_pair( fResults->GetLastYValue(), fResults->GetLastYError() );
826 }
827 if (clsMin.first != 1 && clsMin.first - 3 * std::abs(clsMin.second) < clsTarget) {
828 if (fUseCLs) {
829 rMin = 0;
830 clsMin = CLs_t(1,0); // this is always true for CLs
831 } else {
832 rMin = -rMax / 4;
833 for (int tries = 0; tries < 6; ++tries) {
834 if (! RunOnePoint(rMin) ) {
835 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Hypotest failed at lower limit of scan range: " << rMin << std::endl;
836 rMin = rMin == 0. ? 0.1 : rMin * 1.1;
837 continue;
838 }
839 clsMin = std::make_pair( fResults->GetLastYValue(), fResults->GetLastYError() );
840 if (clsMin.first == 1 || clsMin.first - 3 * std::abs(clsMin.second) > clsTarget) break;
841 rMin += rMin;
842 if (tries == 5) {
843 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Cannot determine lower limit of scan range. At " << r->GetName()
844 << " = " << rMin << " still get " << (fUseCLs ? "CLs" : "CLsplusb")
845 << " = " << clsMin.first << std::endl;
846 return false;
847 }
848 }
849 }
850 }
851
852 if (fVerbose > 0)
853 oocoutI(nullptr,Eval) << "HypoTestInverter::RunLimit - Now doing proper bracketing & bisection" << std::endl;
854 do {
855
856 // break loop in case max toys is reached
857 if (fMaxToys > 0 && fTotalToysRun > fMaxToys ) {
858 oocoutW(nullptr,Eval) << "HypoTestInverter::RunLimit - maximum number of toys reached " << std::endl;
859 done = false; break;
860 }
861
862
863 // determine point by bisection or interpolation
864 limit = 0.5*(rMin+rMax); limitErr = 0.5*(rMax-rMin);
865 if (fgAlgo == "logSecant" && clsMax.first != 0) {
866 double logMin = log(clsMin.first);
867 double logMax = log(clsMax.first);
868 double logTarget = log(clsTarget);
869 limit = rMin + (rMax-rMin) * (logTarget - logMin)/(logMax - logMin);
870 if (clsMax.second != 0 && clsMin.second != 0) {
871 limitErr = hypot((logTarget-logMax) * (clsMin.second/clsMin.first), (logTarget-logMin) * (clsMax.second/clsMax.first));
872 limitErr *= (rMax-rMin)/((logMax-logMin)*(logMax-logMin));
873 }
874 }
875 r->setError(limitErr);
876
877 // exit if reached accuracy on r
878 if (limitErr < std::max(absAccuracy, relAccuracy * limit)) {
879 if (fVerbose > 1) {
880 oocoutI(nullptr, Eval) << "HypoTestInverter::RunLimit - reached accuracy " << limitErr << " below "
881 << std::max(absAccuracy, relAccuracy * limit) << std::endl;
882 }
883 done = true; break;
884 }
885
886 // evaluate point
887 if (! RunOnePoint(limit, true, clsTarget) ) {
888 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Hypo test failed at x=" << limit << " when trying to find limit." << std::endl;
889 return false;
890 }
891 clsMid = std::make_pair( fResults->GetLastYValue(), fResults->GetLastYError() );
892
893 if (clsMid.second == -1) {
894 oocoutE(nullptr,Eval) << "Hypotest failed" << std::endl;
895 return false;
896 }
897
898 // if sufficiently far away, drop one of the points
899 if (std::abs(clsMid.first-clsTarget) >= 2*clsMid.second) {
900 if ((clsMid.first>clsTarget) == (clsMax.first>clsTarget)) {
901 rMax = limit; clsMax = clsMid;
902 } else {
903 rMin = limit; clsMin = clsMid;
904 }
905 } else {
906 if (fVerbose > 0) oocoutI(nullptr,Eval) << "Trying to move the interval edges closer" << std::endl;
907 double rMinBound = rMin;
908 double rMaxBound = rMax;
909 // try to reduce the size of the interval
910 while (clsMin.second == 0 || std::abs(rMin-limit) > std::max(absAccuracy, relAccuracy * limit)) {
911 rMin = 0.5*(rMin+limit);
912 if (!RunOnePoint(rMin,true, clsTarget) ) {
913 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Hypo test failed at x=" << rMin << " when trying to find limit from below." << std::endl;
914 return false;
915 }
916 clsMin = std::make_pair( fResults->GetLastYValue(), fResults->GetLastYError() );
917 if (std::abs(clsMin.first-clsTarget) <= 2*clsMin.second) break;
918 rMinBound = rMin;
919 }
920 while (clsMax.second == 0 || std::abs(rMax-limit) > std::max(absAccuracy, relAccuracy * limit)) {
921 rMax = 0.5*(rMax+limit);
922 if (!RunOnePoint(rMax,true,clsTarget) ) {
923 oocoutE(nullptr,Eval) << "HypoTestInverter::RunLimit - Hypo test failed at x=" << rMin << " when trying to find limit from above." << std::endl;
924 return false;
925 }
926 clsMax = std::make_pair( fResults->GetLastYValue(), fResults->GetLastYError() );
927 if (std::abs(clsMax.first-clsTarget) <= 2*clsMax.second) break;
928 rMaxBound = rMax;
929 }
930 expoFit.SetRange(rMinBound,rMaxBound);
931 break;
932 }
933 } while (true);
934
935 if (!done) { // didn't reach accuracy with scan, now do fit
936 if (fVerbose) {
937 oocoutI(nullptr,Eval) << "HypoTestInverter::RunLimit - Before fit --- \n";
938 oocoutI(nullptr,Eval) << "Limit: " << r->GetName() << " < " << limit << " +/- " << limitErr << " [" << rMin << ", " << rMax << "]\n";
939 }
940
941 expoFit.FixParameter(0,clsTarget);
942 expoFit.SetParameter(1,log(clsMax.first/clsMin.first)/(rMax-rMin));
943 expoFit.SetParameter(2,limit);
944 double rMinBound;
945 double rMaxBound;
946 expoFit.GetRange(rMinBound, rMaxBound);
947 limitErr = std::max(std::abs(rMinBound-limit), std::abs(rMaxBound-limit));
948 int npoints = 0;
949
950 HypoTestInverterPlot plot("plot","plot",fResults);
951 fLimitPlot.reset(plot.MakePlot() );
952
953
954 for (int j = 0; j < fLimitPlot->GetN(); ++j) {
955 if (fLimitPlot->GetX()[j] >= rMinBound && fLimitPlot->GetX()[j] <= rMaxBound) npoints++;
956 }
957 for (int i = 0, imax = /*(readHybridResults_ ? 0 : */ 8; i <= imax; ++i, ++npoints) {
958 fLimitPlot->Sort();
959 fLimitPlot->Fit(&expoFit,(fVerbose <= 1 ? "QNR EX0" : "NR EXO"));
960 if (fVerbose) {
961 oocoutI(nullptr,Eval) << "Fit to " << npoints << " points: " << expoFit.GetParameter(2) << " +/- " << expoFit.GetParError(2) << std::endl;
962 }
963 if ((rMin < expoFit.GetParameter(2)) && (expoFit.GetParameter(2) < rMax) && (expoFit.GetParError(2) < 0.5*(rMaxBound-rMinBound))) {
964 // sanity check fit result
965 limit = expoFit.GetParameter(2);
966 limitErr = expoFit.GetParError(2);
967 if (limitErr < std::max(absAccuracy, relAccuracy * limit)) break;
968 }
969 // add one point in the interval.
970 double rTry = RooRandom::uniform()*(rMaxBound-rMinBound)+rMinBound;
971 if (i != imax) {
972 if (!RunOnePoint(rTry,true,clsTarget) ) return false;
973 //eval(w, mc_s, mc_b, data, rTry, true, clsTarget);
974 }
975
976 }
977 }
978
979//if (!plot_.empty() && fLimitPlot.get()) {
980 if (fLimitPlot.get() && fLimitPlot->GetN() > 0) {
981 //new TCanvas("c1","c1");
982 fLimitPlot->Sort();
983 fLimitPlot->SetLineWidth(2);
984 double xmin = r->getMin();
985 double xmax = r->getMax();
986 for (int j = 0; j < fLimitPlot->GetN(); ++j) {
987 if (fLimitPlot->GetY()[j] > 1.4*clsTarget || fLimitPlot->GetY()[j] < 0.6*clsTarget) continue;
988 xmin = std::min(fLimitPlot->GetX()[j], xmin);
989 xmax = std::max(fLimitPlot->GetX()[j], xmax);
990 }
991 fLimitPlot->GetXaxis()->SetRangeUser(xmin,xmax);
992 fLimitPlot->GetYaxis()->SetRangeUser(0.5*clsTarget, 1.5*clsTarget);
993 fLimitPlot->Draw("AP");
994 expoFit.Draw("SAME");
995 TLine line(fLimitPlot->GetX()[0], clsTarget, fLimitPlot->GetX()[fLimitPlot->GetN()-1], clsTarget);
997 line.DrawLine(limit, 0, limit, fLimitPlot->GetY()[0]);
999 line.DrawLine(limit-limitErr, 0, limit-limitErr, fLimitPlot->GetY()[0]);
1000 line.DrawLine(limit+limitErr, 0, limit+limitErr, fLimitPlot->GetY()[0]);
1001 //c1->Print(plot_.c_str());
1002 }
1003
1004 oocoutI(nullptr,Eval) << "HypoTestInverter::RunLimit - Result: \n"
1005 << "\tLimit: " << r->GetName() << " < " << limit << " +/- " << limitErr << " @ " << (1-fSize) * 100 << "% CL\n";
1006 if (fVerbose > 1) oocoutI(nullptr,Eval) << "Total toys: " << fTotalToysRun << std::endl;
1007
1008 // set value in results
1009 fResults->fUpperLimit = limit;
1010 fResults->fUpperLimitError = limitErr;
1012 // lower limit are always min of p value
1016
1017 return true;
1018}
1019
1020////////////////////////////////////////////////////////////////////////////////
1021/// get the distribution of lower limit
1022/// if rebuild = false (default) it will re-use the results of the scan done
1023/// for obtained the observed limit and no extra toys will be generated
1024/// if rebuild a new set of B toys will be done and the procedure will be repeated
1025/// for each toy
1026
1028 if (!rebuild) {
1029 if (!fResults) {
1030 oocoutE(nullptr,InputArguments) << "HypoTestInverter::GetLowerLimitDistribution(false) - result not existing\n";
1031 return nullptr;
1032 }
1034 }
1035
1036 TList * clsDist = nullptr;
1037 TList * clsbDist = nullptr;
1038 if (fUseCLs) clsDist = &fResults->fExpPValues;
1039 else clsbDist = &fResults->fExpPValues;
1040
1041 return RebuildDistributions(false, nToys,clsDist, clsbDist);
1042
1043}
1044
1045////////////////////////////////////////////////////////////////////////////////
1046/// get the distribution of lower limit
1047/// if rebuild = false (default) it will re-use the results of the scan done
1048/// for obtained the observed limit and no extra toys will be generated
1049/// if rebuild a new set of B toys will be done and the procedure will be repeated
1050/// for each toy
1051/// The nuisance parameter value used for rebuild is the current one in the model
1052/// so it is user responsibility to set to the desired value (nomi
1053
1055 if (!rebuild) {
1056 if (!fResults) {
1057 oocoutE(nullptr,InputArguments) << "HypoTestInverter::GetUpperLimitDistribution(false) - result not existing\n";
1058 return nullptr;
1059 }
1061 }
1062
1063 TList * clsDist = nullptr;
1064 TList * clsbDist = nullptr;
1065 if (fUseCLs) clsDist = &fResults->fExpPValues;
1066 else clsbDist = &fResults->fExpPValues;
1067
1068 return RebuildDistributions(true, nToys,clsDist, clsbDist);
1069}
1070
1071////////////////////////////////////////////////////////////////////////////////
1072
1076
1077////////////////////////////////////////////////////////////////////////////////
1078/// rebuild the sampling distributions by
1079/// generating some toys and find for each of them a new upper limit
1080/// Return the upper limit distribution and optionally also the pValue distributions for Cls, Clsb and Clbxs
1081/// as a TList for each scanned point
1082/// The method uses the present parameter value. It is user responsibility to give the current parameters to rebuild the distributions
1083/// It returns a upper or lower limit distribution depending on the isUpper flag, however it computes also the lower limit distribution and it is saved in the
1084/// output file as an histogram
1085
1086SamplingDistribution * HypoTestInverter::RebuildDistributions(bool isUpper, int nToys, TList * clsDist, TList * clsbDist, TList * clbDist, const char *outputfile) {
1087
1088 if (!fScannedVariable || !fCalculator0) return nullptr;
1089 // get first background snapshot
1090 const ModelConfig * bModel = fCalculator0->GetAlternateModel();
1091 const ModelConfig * sbModel = fCalculator0->GetNullModel();
1092 if (!bModel || ! sbModel) return nullptr;
1093 RooArgSet paramPoint;
1094 if (!sbModel->GetParametersOfInterest()) return nullptr;
1095 paramPoint.add(*sbModel->GetParametersOfInterest());
1096
1097 std::unique_ptr<const RooArgSet> poibkg{bModel->GetSnapshot()};
1098 if (!poibkg) {
1099 oocoutW(nullptr,InputArguments) << "HypoTestInverter::RebuildDistribution - background snapshot not existing"
1100 << " assume is for POI = 0" << std::endl;
1102 paramPoint.assign(RooArgSet(*fScannedVariable));
1103 } else {
1104 paramPoint.assign(*poibkg);
1105 }
1106 // generate data at bkg parameter point
1107
1108 ToyMCSampler * toymcSampler = dynamic_cast<ToyMCSampler *>(fCalculator0->GetTestStatSampler() );
1109 if (!toymcSampler) {
1110 oocoutE(nullptr,InputArguments) << "HypoTestInverter::RebuildDistribution - no toy MC sampler existing" << std::endl;
1111 return nullptr;
1112 }
1113 // set up test stat sampler in case of asymptotic calculator
1114 if (dynamic_cast<RooStats::AsymptoticCalculator*>(fCalculator0) ) {
1115 toymcSampler->SetObservables(*sbModel->GetObservables() );
1116 toymcSampler->SetParametersForTestStat(*sbModel->GetParametersOfInterest());
1117 toymcSampler->SetPdf(*sbModel->GetPdf());
1118 toymcSampler->SetNuisanceParameters(*sbModel->GetNuisanceParameters());
1119 if (sbModel->GetGlobalObservables() ) toymcSampler->SetGlobalObservables(*sbModel->GetGlobalObservables() );
1120 // set number of events
1121 if (!sbModel->GetPdf()->canBeExtended())
1122 toymcSampler->SetNEventsPerToy(1);
1123 }
1124
1125 // loop on data to generate
1126 int nPoints = fNBins;
1127
1128 bool storePValues = clsDist || clsbDist || clbDist;
1129 if (fNBins <=0 && storePValues) {
1130 oocoutW(nullptr,InputArguments) << "HypoTestInverter::RebuildDistribution - cannot return p values distribution with the auto scan" << std::endl;
1131 storePValues = false;
1132 nPoints = 0;
1133 }
1134
1135 if (storePValues) {
1136 if (fResults) nPoints = fResults->ArraySize();
1137 if (nPoints <=0) {
1138 oocoutE(nullptr,InputArguments) << "HypoTestInverter - result is not existing and number of point to scan is not set"
1139 << std::endl;
1140 return nullptr;
1141 }
1142 }
1143
1144 if (nToys <= 0) nToys = 100; // default value
1145
1146 std::vector<std::vector<double> > CLs_values(nPoints);
1147 std::vector<std::vector<double> > CLsb_values(nPoints);
1148 std::vector<std::vector<double> > CLb_values(nPoints);
1149
1150 if (storePValues) {
1151 for (int i = 0; i < nPoints; ++i) {
1152 CLs_values[i].reserve(nToys);
1153 CLb_values[i].reserve(nToys);
1154 CLsb_values[i].reserve(nToys);
1155 }
1156 }
1157
1158 std::vector<double> limit_values; limit_values.reserve(nToys);
1159
1160 oocoutI(nullptr,InputArguments) << "HypoTestInverter - rebuilding the p value distributions by generating ntoys = "
1161 << nToys << std::endl;
1162
1163
1164 oocoutI(nullptr,InputArguments) << "Rebuilding using parameter of interest point: ";
1165 RooStats::PrintListContent(paramPoint, oocoutI(nullptr,InputArguments) );
1166 if (sbModel->GetNuisanceParameters() ) {
1167 oocoutI(nullptr,InputArguments) << "And using nuisance parameters: ";
1168 RooStats::PrintListContent(*sbModel->GetNuisanceParameters(), oocoutI(nullptr,InputArguments) );
1169 }
1170 // save all parameters to restore them later
1171 assert(bModel->GetPdf() );
1172 assert(bModel->GetObservables() );
1173 std::unique_ptr<RooArgSet> allParams{bModel->GetPdf()->getParameters( *bModel->GetObservables() )};
1174 RooArgSet saveParams;
1175 allParams->snapshot(saveParams);
1176
1177 std::unique_ptr<TFile> fileOut{TFile::Open(outputfile,"RECREATE")};
1178 if (!fileOut) {
1179 oocoutE(nullptr,InputArguments) << "HypoTestInverter - RebuildDistributions - Error opening file " << outputfile
1180 << " - the resulting limits will not be stored" << std::endl;
1181 }
1182 // create temporary histograms to store the limit result
1183 TH1D * hL = new TH1D("lowerLimitDist","Rebuilt lower limit distribution",100,1.,0.);
1184 TH1D * hU = new TH1D("upperLimitDist","Rebuilt upper limit distribution",100,1.,0.);
1185 TH1D * hN = new TH1D("nObs","Observed events",100,1.,0.);
1186 hL->SetBuffer(2*nToys);
1187 hU->SetBuffer(2*nToys);
1188 std::vector<TH1*> hCLb;
1189 std::vector<TH1*> hCLsb;
1190 std::vector<TH1*> hCLs;
1191 if (storePValues) {
1192 for (int i = 0; i < nPoints; ++i) {
1193 hCLb.push_back(new TH1D(TString::Format("CLbDist_bin%d",i),"CLb distribution",100,1.,0.));
1194 hCLs.push_back(new TH1D(TString::Format("ClsDist_bin%d",i),"CLs distribution",100,1.,0.));
1195 hCLsb.push_back(new TH1D(TString::Format("CLsbDist_bin%d",i),"CLs+b distribution",100,1.,0.));
1196 }
1197 }
1198
1199
1200 // loop now on the toys
1201 for (int itoy = 0; itoy < nToys; ++itoy) {
1202
1203 oocoutP(nullptr,Eval) << "\nHypoTestInverter - RebuildDistributions - running toy # " << itoy << " / "
1204 << nToys << std::endl;
1205
1206
1207 oocoutI(nullptr,Eval) << "\n\nsnapshot of s+b model \n";
1208 std::unique_ptr<const RooArgSet>{sbModel->GetSnapshot()}->Print("v");
1209
1210 // reset parameters to initial values to be sure in case they are not reset
1211 if (itoy> 0) {
1212 allParams->assign(saveParams);
1213 }
1214
1215 // need to set th epdf to clear the cache in ToyMCSampler
1216 // pdf we must use is background pdf
1217 toymcSampler->SetPdf(*bModel->GetPdf() );
1218
1219
1220 RooAbsData * bkgdata = toymcSampler->GenerateToyData(paramPoint);
1221
1222 double nObs = bkgdata->sumEntries();
1223 // for debugging in case of number counting models
1224 if (bkgdata->numEntries() ==1 && !bModel->GetPdf()->canBeExtended()) {
1225 oocoutP(nullptr,Generation) << "Generate observables are : ";
1226 RooArgList genObs(*bkgdata->get(0));
1227 RooStats::PrintListContent(genObs, oocoutP(nullptr,Generation) );
1228 nObs = 0;
1229 for (auto *x : dynamic_range_cast<RooRealVar*>(genObs)) {
1230 if (x) nObs += x->getVal();
1231 }
1232 }
1233 hN->Fill(nObs);
1234
1235 // by copying I will have the same min/max as previous ones
1236 HypoTestInverter inverter = *this;
1237 inverter.SetData(*bkgdata);
1238
1239 // print global observables
1240 std::unique_ptr<RooArgSet> bModelVars{bModel->GetPdf()->getVariables()};
1241 std::unique_ptr<RooAbsCollection> gobs{bModelVars->selectCommon(*sbModel->GetGlobalObservables())};
1242 gobs->Print("v");
1243
1244 HypoTestInverterResult * r = inverter.GetInterval();
1245
1246 if (r == nullptr) continue;
1247
1248 double value = (isUpper) ? r->UpperLimit() : r->LowerLimit();
1249 limit_values.push_back( value );
1250 hU->Fill(r->UpperLimit() );
1251 hL->Fill(r->LowerLimit() );
1252
1253
1254 oocoutI(nullptr,Eval) << "The computed upper limit for toy #" << itoy << " is " << value << std::endl;
1255
1256 // write every 10 toys
1257 if (itoy%10 == 0 || itoy == nToys-1) {
1258 hU->Write("",TObject::kOverwrite);
1259 hL->Write("",TObject::kOverwrite);
1260 hN->Write("",TObject::kOverwrite);
1261 }
1262
1263 if (!storePValues) continue;
1264
1265 if (nPoints < r->ArraySize()) {
1266 oocoutW(nullptr,InputArguments) << "HypoTestInverter: skip extra points" << std::endl;
1267 }
1268 else if (nPoints > r->ArraySize()) {
1269 oocoutW(nullptr,InputArguments) << "HypoTestInverter: missing some points" << std::endl;
1270 }
1271
1272
1273 for (int ipoint = 0; ipoint < nPoints; ++ipoint) {
1274 HypoTestResult * hr = r->GetResult(ipoint);
1275 if (hr) {
1276 CLs_values[ipoint].push_back( hr->CLs() );
1277 CLsb_values[ipoint].push_back( hr->CLsplusb() );
1278 CLb_values[ipoint].push_back( hr->CLb() );
1279 hCLs[ipoint]->Fill( hr->CLs() );
1280 hCLb[ipoint]->Fill( hr->CLb() );
1281 hCLsb[ipoint]->Fill( hr->CLsplusb() );
1282 }
1283 else {
1284 oocoutW(nullptr,InputArguments) << "HypoTestInverter: missing result for point: x = "
1285 << fResults->GetXValue(ipoint) << std::endl;
1286 }
1287 }
1288 // write every 10 toys
1289 if (itoy%10 == 0 || itoy == nToys-1) {
1290 for (int ipoint = 0; ipoint < nPoints; ++ipoint) {
1291 hCLs[ipoint]->Write("",TObject::kOverwrite);
1292 hCLb[ipoint]->Write("",TObject::kOverwrite);
1293 hCLsb[ipoint]->Write("",TObject::kOverwrite);
1294 }
1295 }
1296
1297
1298 delete r;
1299 delete bkgdata;
1300 }
1301
1302
1303 if (storePValues) {
1304 if (clsDist) clsDist->SetOwner(true);
1305 if (clbDist) clbDist->SetOwner(true);
1306 if (clsbDist) clsbDist->SetOwner(true);
1307
1308 oocoutI(nullptr,InputArguments) << "HypoTestInverter: storing rebuilt p values " << std::endl;
1309
1310 for (int ipoint = 0; ipoint < nPoints; ++ipoint) {
1311 if (clsDist) {
1312 TString name = TString::Format("CLs_distrib_%d",ipoint);
1313 clsDist->Add( new SamplingDistribution(name,name,CLs_values[ipoint] ) );
1314 }
1315 if (clbDist) {
1316 TString name = TString::Format("CLb_distrib_%d",ipoint);
1317 clbDist->Add( new SamplingDistribution(name,name,CLb_values[ipoint] ) );
1318 }
1319 if (clsbDist) {
1320 TString name = TString::Format("CLsb_distrib_%d",ipoint);
1321 clsbDist->Add( new SamplingDistribution(name,name,CLsb_values[ipoint] ) );
1322 }
1323 }
1324 }
1325
1326 else {
1327 // delete all the histograms
1328 delete hL;
1329 delete hU;
1330 for (int i = 0; i < nPoints && storePValues; ++i) {
1331 delete hCLs[i];
1332 delete hCLb[i];
1333 delete hCLsb[i];
1334 }
1335 }
1336
1337 const char * disName = (isUpper) ? "upperLimit_dist" : "lowerLimit_dist";
1338 return new SamplingDistribution(disName, disName, limit_values);
1339}
dim_t fSize
#define h(i)
Definition RSha256.hxx:106
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
#define oocoutW(o, a)
#define oocoutE(o, a)
#define oocoutF(o, a)
#define oocoutI(o, a)
#define oocoutP(o, a)
@ kRed
Definition Rtypes.h:66
winID h TVirtualViewer3D TVirtualGLPainter char TVirtualGLPainter plot
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:142
float xmin
float xmax
RooFit::OwningPtr< RooArgSet > getParameters(const RooAbsData *data, bool stripDisconnected=true) const
Create a list of leaf nodes in the arg tree starting with ourself as top node that don't match any of...
RooFit::OwningPtr< RooArgSet > getVariables(bool stripDisconnected=true) const
Return RooArgSet with all variables (tree leaf nodes of expression tree)
virtual bool add(const RooAbsArg &var, bool silent=false)
Add the specified argument to list.
void assign(const RooAbsCollection &other) const
Sets the value, cache and constant attribute of any argument in our set that also appears in the othe...
RooAbsArg * first() const
RooAbsArg * find(const char *name) const
Find object with given name in list.
void Print(Option_t *options=nullptr) const override
This method must be overridden when a class wants to print itself.
Abstract base class for binned and unbinned datasets.
Definition RooAbsData.h:55
virtual double sumEntries() const =0
Return effective number of entries in dataset, i.e., sum all weights.
virtual const RooArgSet * get() const
Definition RooAbsData.h:99
virtual Int_t numEntries() const
Return number of entries in dataset, i.e., count unweighted entries.
Abstract interface for all probability density functions.
Definition RooAbsPdf.h:32
bool canBeExtended() const
If true, PDF can provide extended likelihood term.
Definition RooAbsPdf.h:214
virtual double getMax(const char *name=nullptr) const
Get maximum of currently defined range.
virtual double getMin(const char *name=nullptr) const
Get minimum of currently defined range.
double getVal(const RooArgSet *normalisationSet=nullptr) const
Evaluate object.
Definition RooAbsReal.h:107
RooArgList is a container object that can hold multiple RooAbsArg objects.
Definition RooArgList.h:22
RooArgSet is a container object that can hold multiple RooAbsArg objects.
Definition RooArgSet.h:24
RooArgSet * snapshot(bool deepCopy=true) const
Use RooAbsCollection::snapshot(), but return as RooArgSet.
Definition RooArgSet.h:159
static double uniform(TRandom *generator=randomGenerator())
Return a number uniformly distributed from (0,1)
Definition RooRandom.cxx:77
Variable that can be changed from the outside.
Definition RooRealVar.h:37
void setVal(double value) override
Set value of variable to 'value'.
Hypothesis Test Calculator based on the asymptotic formulae for the profile likelihood ratio.
Does a frequentist hypothesis test.
Same purpose as HybridCalculatorOriginal, but different implementation.
Common base class for the Hypothesis Test Calculators.
const ModelConfig * GetNullModel(void) const
void SetData(RooAbsData &data) override
Set the DataSet.
HypoTestResult * GetHypoTest() const override
inherited methods from HypoTestCalculator interface
const ModelConfig * GetAlternateModel(void) const
TestStatSampler * GetTestStatSampler(void) const
Returns instance of TestStatSampler.
Class to plot a HypoTestInverterResult, the output of the HypoTestInverter calculator.
HypoTestInverterResult class holds the array of hypothesis test results and compute a confidence inte...
SamplingDistribution * GetLowerLimitDistribution() const
get expected lower limit distributions implemented using interpolation The size for the sampling dist...
HypoTestResult * GetResult(int index) const
return a pointer to the i^th result object
int ArraySize() const
number of entries in the results array
void UseCLs(bool on=true)
flag to switch between using CLsb (default) or CLs as confidence level
double GetXValue(int index) const
function to return the value of the parameter of interest for the i^th entry in the results
TList fExpPValues
list of expected sampling distribution for each point
bool fIsTwoSided
two sided scan (look for lower/upper limit)
TList fYObjects
list of HypoTestResult for each point
void SetConfidenceLevel(double cl) override
set the confidence level for the interval (eg. 0.95 for a 95% Confidence Interval)
SamplingDistribution * GetUpperLimitDistribution() const
get expected upper limit distributions implemented using interpolation
A class for performing a hypothesis test inversion by scanning the hypothesis test results of a HypoT...
static void CheckInputModels(const HypoTestCalculatorGeneric &hc, const RooRealVar &scanVar)
check the model given the given hypotestcalculator
RooRealVar * fScannedVariable
pointer to the constrained variable
std::unique_ptr< HypoTestCalculatorGeneric > fHC
! pointer to the generic hypotest calculator used
bool RunLimit(double &limit, double &limitErr, double absTol=0, double relTol=0, const double *hint=nullptr) const
Run an automatic scan until the desired accuracy is reached.
void SetData(RooAbsData &) override
Set the DataSet ( add to the workspace if not already there ?)
HypoTestInverterResult * fResults
pointer to the result
SamplingDistribution * RebuildDistributions(bool isUpper=true, int nToys=100, TList *clsDist=nullptr, TList *clsbDist=nullptr, TList *clbDist=nullptr, const char *outputfile="HypoTestInverterRebuiltDist.root")
function to rebuild the distributions
int fMaxToys
maximum number of toys to run
HypoTestInverter()
default constructor (used only for I/O)
~HypoTestInverter() override
destructor
double ConfidenceLevel() const override
Get the Confidence level for the test.
bool SetTestStatistic(TestStatistic &stat)
set the test statistic
HypoTestInverterResult * GetInterval() const override
Run a fixed scan or the automatic scan depending on the configuration.
SamplingDistribution * GetUpperLimitDistribution(bool rebuild=false, int nToys=100)
get the distribution of lower limit if rebuild = false (default) it will re-use the results of the sc...
static unsigned int fgNToys
void Clear()
delete contained result and graph
TestStatistic * GetTestStatistic() const
get the test statistic
std::unique_ptr< TGraphErrors > fLimitPlot
! plot of limits
bool RunFixedScan(int nBins, double xMin, double xMax, bool scanLog=false) const
Run a fixed scan.
bool RunOnePoint(double thisX, bool adaptive=false, double clTarget=-1) const
run only one point at the given POI value
SamplingDistribution * GetLowerLimitDistribution(bool rebuild=false, int nToys=100)
get the upper/lower limit distribution
HypoTestInverter & operator=(const HypoTestInverter &rhs)
assignment
HypoTestCalculatorGeneric * fCalculator0
pointer to the calculator passed in the constructor
void CreateResults() const
create a new HypoTestInverterResult to hold all computed results
static RooRealVar * GetVariableToScan(const HypoTestCalculatorGeneric &hc)
helper functions
HypoTestResult * Eval(HypoTestCalculatorGeneric &hc, bool adaptive, double clsTarget) const
run the hybrid at a single point
HypoTestResult is a base class for results from hypothesis tests.
virtual double CLsplusb() const
Convert AlternatePValue into a "confidence level".
double GetTestStatisticData(void) const
virtual void Append(const HypoTestResult *other)
add values from another HypoTestResult
double CLsError() const
The error on the ratio .
void SetBackgroundAsAlt(bool l=true)
bool GetPValueIsRightTail(void) const
double CLbError() const
The error on the "confidence level" of the null hypothesis.
void SetTestStatisticData(const double tsd)
double CLsplusbError() const
The error on the "confidence level" of the alternative hypothesis.
SamplingDistribution * GetNullDistribution(void) const
virtual double CLs() const
is simply (not a method, but a quantity)
virtual double CLb() const
Convert NullPValue into a "confidence level".
SamplingDistribution * GetAltDistribution(void) const
IntervalCalculator is an interface class for a tools which produce RooStats ConfIntervals.
< A class that holds configuration information for a model using a workspace as a store
Definition ModelConfig.h:34
const RooArgSet * GetGlobalObservables() const
get RooArgSet for global observables (return nullptr if not existing)
const RooArgSet * GetParametersOfInterest() const
get RooArgSet containing the parameter of interest (return nullptr if not existing)
const RooArgSet * GetNuisanceParameters() const
get RooArgSet containing the nuisance parameters (return nullptr if not existing)
const RooArgSet * GetObservables() const
get RooArgSet for observables (return nullptr if not existing)
const RooArgSet * GetSnapshot() const
get RooArgSet for parameters for a particular hypothesis (return nullptr if not existing)
RooAbsPdf * GetPdf() const
get model PDF (return nullptr if pdf has not been specified or does not exist)
ProfileLikelihoodTestStat is an implementation of the TestStatistic interface that calculates the pro...
This class simply holds a sampling distribution of some test statistic.
Int_t GetSize() const
size of samples
double fUpperLimit
upper interval limit
double fLowerLimit
lower interval limit
TestStatSampler is an interface class for a tools which produce RooStats SamplingDistributions.
virtual TestStatistic * GetTestStatistic() const =0
Get the TestStatistic.
virtual void SetTestStatistic(TestStatistic *testStatistic)=0
Set the TestStatistic (want the argument to be a function of the data & parameter points.
TestStatistic is an interface class to provide a facility for construction test statistics distributi...
ToyMCSampler is an implementation of the TestStatSampler interface.
void SetParametersForTestStat(const RooArgSet &nullpoi) override
Set the Pdf, add to the workspace if not already there.
void SetObservables(const RooArgSet &o) override
specify the observables in the dataset (needed to evaluate the test statistic)
virtual RooAbsData * GenerateToyData(RooArgSet &paramPoint, RooAbsPdf &pdf) const
generates toy data without weight
void SetNuisanceParameters(const RooArgSet &np) override
specify the nuisance parameters (eg. the rest of the parameters)
void SetPdf(RooAbsPdf &pdf) override
Set the Pdf, add to the workspace if not already there.
void SetGlobalObservables(const RooArgSet &o) override
specify the conditional observables
virtual void SetNEventsPerToy(const Int_t nevents)
Forces the generation of exactly n events even for extended PDFs.
virtual void SetLineStyle(Style_t lstyle)
Set the line style.
Definition TAttLine.h:46
virtual void SetLineWidth(Width_t lwidth)
Set the line width.
Definition TAttLine.h:47
virtual void SetLineColor(Color_t lcolor)
Set the line color.
Definition TAttLine.h:44
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
1-Dim function class
Definition TF1.h:182
virtual Double_t GetParError(Int_t ipar) const
Return value of parameter number ipar.
Definition TF1.cxx:1981
virtual void SetRange(Double_t xmin, Double_t xmax)
Initialize the upper and lower bounds to draw the function.
Definition TF1.cxx:3584
void Draw(Option_t *option="") override
Draw this function with its current attributes.
Definition TF1.cxx:1341
virtual void GetRange(Double_t *xmin, Double_t *xmax) const
Return range of a generic N-D function.
Definition TF1.cxx:2330
virtual void SetParameter(Int_t param, Double_t value)
Definition TF1.h:608
virtual void FixParameter(Int_t ipar, Double_t value)
Fix the value of a parameter for a fit operation The specified value will be used in the fit and the ...
Definition TF1.cxx:1608
virtual Double_t GetParameter(Int_t ipar) const
Definition TF1.h:477
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Definition TFile.cxx:3802
1-D histogram with a double per channel (see TH1 documentation)
Definition TH1.h:955
virtual Int_t Fill(Double_t x)
Increment bin with abscissa X by 1.
Definition TH1.cxx:3489
virtual void SetBuffer(Int_t bufsize, Option_t *option="")
Set the maximum number of entries to be kept in the buffer.
Definition TH1.cxx:8687
Use the TLine constructor to create a simple line.
Definition TLine.h:22
virtual TLine * DrawLine(Double_t x1, Double_t y1, Double_t x2, Double_t y2)
Draw this line with new coordinates.
Definition TLine.cxx:103
A doubly linked list.
Definition TList.h:38
void Add(TObject *obj) override
Definition TList.h:81
TObject * Remove(TObject *obj) override
Remove object from the list.
Definition TList.cxx:952
TObject * Clone(const char *newname="") const override
Make a clone of an object using the Streamer facility.
Definition TNamed.cxx:73
virtual void SetTitle(const char *title="")
Set the title of the TNamed.
Definition TNamed.cxx:173
const char * GetName() const override
Returns name of object.
Definition TNamed.h:49
virtual void SetName(const char *name)
Set the name of the TNamed.
Definition TNamed.cxx:149
virtual Int_t Write(const char *name=nullptr, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Definition TObject.cxx:987
@ kOverwrite
overwrite existing object with same name
Definition TObject.h:101
virtual void Draw(Option_t *option="")
Default Draw method for all objects.
Definition TObject.cxx:292
Basic string class.
Definition TString.h:137
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString.
Definition TString.cxx:2460
TLine * line
Double_t x[n]
Definition legend1.C:17
Namespace for the RooStats classes.
Definition CodegenImpl.h:67
void PrintListContent(const RooArgList &l, std::ostream &os=std::cout)
useful function to print in one line the content of a set with their values
Bool_t AreEqualRel(Double_t af, Double_t bf, Double_t relPrec)
Comparing floating points.
Definition TMath.h:429
Bool_t AreEqualAbs(Double_t af, Double_t bf, Double_t epsilon)
Comparing floating points.
Definition TMath.h:421
static void SetToys(HypoTestType *h, int toyNull, int toyAlt)