RooUnitTest.cxx

Go to the documentation of this file.

/// \cond ROOFIT_INTERNAL
 
/*****************************************************************************
 * Project: RooFit                                                           *
 * Package: RooFitCore                                                       *
 * @(#)root/roofitcore:$Id$
 * Authors:                                                                  *
 *   WV, Wouter Verkerke, NIKHEF, verkerke@nikhef.nl                         *
 *                                                                           *
 * Copyright (c) 2000-2011, Regents of the University of California          *
 *                          and Stanford University. All rights reserved.    *
 *                                                                           *
 * Redistribution and use in source and binary forms,                        *
 * with or without modification, are permitted according to the terms        *
 * listed in LICENSE (http://roofit.sourceforge.net/license.txt)             *
 *****************************************************************************/
 
/**
\file RooUnitTest.cxx
\class RooUnitTest
\ingroup Roofitcore
 
RooUnit test is an abstract base class for unit regression tests for
RooFit and RooStats tests performed in stressRooFit and stressRooStats
Implementations of this class must implement abstract method testCode()
which defines the regression test to be performed. Inside testCode()
the regression test can define objects on which the regression is performed.
These are:
Object          | function
----------------|------------
   RooPlot      | regPlot()
   RooFitResult | regResult()
   double       | regValue()
   RooTable     | regTable()
   TH1/2/3      | regTH()
   RooWorkspace | regWS()
**/
 
#include "RooUnitTest.h"
 
#include "RooCurve.h"
#include "RooHist.h"
#include "RooMsgService.h"
#include "RooDouble.h"
#include "RooTrace.h"
#include "RooRandom.h"
 
#include <TClass.h>
#include <TDirectory.h>
#include <TFile.h>
 
#include <cmath>
 
TDirectory* RooUnitTest::gMemDir = nullptr;
 
 
////////////////////////////////////////////////////////////////////////////////
 
RooUnitTest::RooUnitTest(const char* name, TFile* refFile, bool writeRef, Int_t verbose) : TNamed(name,name),
                                  _refFile(refFile), _debug(false), _write(writeRef), _verb(verbose)
{
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::regPlot(RooPlot* frame, const char* refName)
{
  if (_refFile) {
    std::string refNameStr(refName) ;
    frame->SetName(refName) ;
    // Since ROOT 6.28, the RooPlot doesn't clone the plot variable by default
    // anymore. This is a problem for registering the RooPlots, because they
    // need to survive without dangling pointers even after the RooUnitTest is
    // done. For that reason, we ask the RooPlot to create an internal plot
    // variable clone for itself.
    frame->createInternalPlotVarClone();
    _regPlots.emplace_back(frame,refNameStr);
  } else {
    delete frame ;
  }
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::regResult(std::unique_ptr<RooFitResult> r, const char* refName)
{
  if (_refFile) {
    _regResults.emplace_back(r.release(),refName);
  }
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::regValue(double d, const char* refName)
{
  if (_refFile) {
    _regValues.emplace_back(d,refName);
  }
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::regTable(RooTable* t, const char* refName)
{
  if (_refFile) {
    _regTables.emplace_back(t,refName) ;
  } else {
    delete t ;
  }
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::regWS(RooWorkspace* ws, const char* refName)
{
  if (_refFile) {
    _regWS.emplace_back(ws,refName) ;
  } else {
    delete ws ;
  }
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::regTH(TH1* th, const char* refName)
{
  if (_refFile) {
    _regTH.emplace_back(th,refName) ;
  } else {
    delete th ;
  }
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
RooWorkspace* RooUnitTest::getWS(const char* refName)
{
  RooWorkspace* ws = dynamic_cast<RooWorkspace*>(_refFile->Get(refName)) ;
  if (!ws) {
    if(_verb >= 0) std::cout << "RooUnitTest ERROR: cannot retrieve RooWorkspace " << refName
                             << " from reference file, skipping " << std::endl ;
    return nullptr ;
  }
 
  return ws ;
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
bool RooUnitTest::areTHidentical(TH1* htest, TH1* href)
{
  if (htest->GetDimension() != href->GetDimension()) {
    return false ;
  }
 
  // Use Kolmogorov distance as metric rather than probability
  // because we expect histograms to be identical rather
  // than drawn from the same parent distribution
  double kmax = htest->KolmogorovTest(href,"M") ;
 
  if (kmax>htol()) {
 
    if(_verb >= 0) std::cout << "KS distances = " << kmax << std::endl;
 
    Int_t ntest = htest->GetNbinsX() +2 ;
    Int_t nref  = href->GetNbinsX() +2 ;
    if (htest->GetDimension()>1) {
      ntest *= htest->GetNbinsY() + 2 ;
      nref *= href->GetNbinsY() + 2 ;
    }
    if (htest->GetDimension()>2) {
      ntest *= htest->GetNbinsZ() + 2 ;
      nref *= href->GetNbinsZ() + 2 ;
    }
 
    if (ntest != nref) {
      return false ;
    }
 
    for (Int_t i=0 ; i<ntest ; i++) {
      if (std::abs(htest->GetBinContent(i)-href->GetBinContent(i))>htol()) {
        if(_verb >= 0) std::cout << "htest[" << i << "] = " << htest->GetBinContent(i) << " href[" << i << "] = " << href->GetBinContent(i) << std::endl;
      }
    }
 
    return false ;
  }
 
  return true ;
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
 
bool RooUnitTest::runCompTests()
{
  bool ret = true ;
 
  auto iter = _regPlots.begin() ;
  while (iter!=_regPlots.end()) {
 
    if (!_write) {
 
      // Comparison mode
 
      // Retrieve benchmark
      RooPlot* bmark = dynamic_cast<RooPlot*>(_refFile->Get(iter->second.c_str())) ;
      if (!bmark) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: cannot retrieve RooPlot " << iter->second
                                 << " from reference file, skipping " << std::endl;
        ret = false ;
        ++iter ;
        continue ;
      }
 
      if (_verb > 0) {
   std::cout << "comparing RooPlot " << iter->first << " to benchmark " << iter->second << " = " << bmark << std::endl ;
   std::cout << "reference: " ; iter->first->Print() ;
   std::cout << "benchmark: " ; bmark->Print() ;
      }
 
      RooPlot* compPlot = _debug ? iter->first->emptyClone(Form("%s_comparison",iter->first->GetName())) : nullptr ;
      bool anyFail=false ;
 
      Stat_t nItems = iter->first->numItems() ;
      for (Stat_t i=0 ; i<nItems ; i++) {
   // coverity[NULL_RETURNS]
   TObject* obj = iter->first->getObject((Int_t)i) ;
 
   // Retrieve corresponding object from reference frame
   TObject* objRef = bmark->findObject(obj->GetName()) ;
 
   if (!objRef) {
     if(_verb >= 0) std::cout << "RooUnitTest ERROR: cannot retrieve reference object " << obj->GetName()
                               << " from benchmark RooPlot " << iter->second << ", skipping" << std::endl;
     ret = false ;
     break ;
   }
 
   // Histogram comparisons
   if (obj->IsA()==RooHist::Class()) {
     RooHist* testHist = static_cast<RooHist*>(obj) ;
     RooHist* refHist = static_cast<RooHist*>(objRef) ;
     if (!testHist->isIdentical(*refHist,htol(),_verb >= 0)) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: comparison of object " << obj->ClassName() << "::" << obj->GetName()
                                 <<   " fails comparison with counterpart in reference RooPlot " << bmark->GetName() << std::endl;
 
       if (compPlot) {
         compPlot->addPlotable(static_cast<RooHist*>(testHist->Clone()),"P") ;
         compPlot->getAttLine()->SetLineColor(kRed) ;
         compPlot->getAttMarker()->SetMarkerColor(kRed) ;
         compPlot->getAttLine()->SetLineWidth(1) ;
 
         compPlot->addPlotable(static_cast<RooHist*>(refHist->Clone()),"P") ;
         compPlot->getAttLine()->SetLineColor(kBlue) ;
         compPlot->getAttMarker()->SetMarkerColor(kBlue) ;
         compPlot->getAttLine()->SetLineWidth(1) ;
       }
 
       anyFail=true ;
       ret = false ;
     }
   } else if (obj->IsA()==RooCurve::Class()) {
     RooCurve* testCurve = static_cast<RooCurve*>(obj) ;
     RooCurve* refCurve = static_cast<RooCurve*>(objRef) ;
     if (!testCurve->isIdentical(*refCurve,ctol(),_verb >= 0)) {
       if(_verb >= 0) std::cout << "RooUnitTest ERROR: comparison of object " << obj->ClassName() << "::" << obj->GetName()
                                 <<   " fails comparison with counterpart in reference RooPlot " << bmark->GetName() << std::endl;
 
       if (compPlot) {
         compPlot->addPlotable(static_cast<RooCurve*>(testCurve->Clone())) ;
         compPlot->getAttLine()->SetLineColor(kRed) ;
         compPlot->getAttLine()->SetLineWidth(1) ;
         compPlot->getAttLine()->SetLineStyle(kSolid) ;
 
         compPlot->addPlotable(static_cast<RooCurve*>(refCurve->Clone())) ;
         compPlot->getAttLine()->SetLineColor(kBlue) ;
         compPlot->getAttLine()->SetLineWidth(1) ;
         compPlot->getAttLine()->SetLineStyle(kDashed) ;
       }
 
       anyFail=true ;
       ret = false ;
     }
 
   }
 
      }
 
      if (anyFail && compPlot) {
   std::cout << "RooUnitTest INFO: writing comparison plot " << compPlot->GetName() << " of failed test to RooUnitTest_DEBUG.root" << std::endl ;
   TFile fdbg("RooUnitTest_DEBUG.root","UPDATE") ;
   compPlot->Write() ;
   fdbg.Close() ;
      } else {
   delete compPlot ;
      }
 
      // Delete RooPlot when comparison is finished to avoid noise in leak checking
      delete iter->first ;
 
    } else {
 
      // Writing mode
 
      std::cout <<"RooUnitTest: Writing reference RooPlot " << iter->first << " as benchmark " << iter->second << std::endl ;
      _refFile->cd() ;
      iter->first->Write(iter->second.c_str()) ;
      gMemDir->cd() ;
    }
 
    ++iter ;
  }
 
 
  auto iter2 = _regResults.begin() ;
  while (iter2!=_regResults.end()) {
 
    if (!_write) {
 
      // Comparison mode
 
     // Retrieve benchmark
      RooFitResult* bmark = dynamic_cast<RooFitResult*>(_refFile->Get(iter2->second.c_str())) ;
      if (!bmark) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: cannot retrieve RooFitResult "
                                 << iter2->second << " from reference file, skipping " << std::endl ;
        ++iter2 ;
        ret = false ;
        continue ;
      }
 
      if (_verb > 0) {
   std::cout << "comparing RooFitResult " << iter2->first << " to benchmark " << iter2->second << " = " << bmark << std::endl ;
      }
 
      if (!iter2->first->isIdentical(*bmark,fptol(),fctol(),_verb >= 0)) {
   if (_verb >= 0) {
     std::cout << "RooUnitTest ERROR: comparison of object " << iter2->first->ClassName()
               << "::" << iter2->first->GetName() << " from result " << iter2->second
               << " fails comparison with counterpart in reference RooFitResult " << bmark->GetName() << std::endl;
   }
        ret = false ;
      }
 
      // Delete RooFitResult when comparison is finished to avoid noise in leak checking
      delete iter2->first ;
 
 
    } else {
 
      // Writing mode
 
      std::cout <<"RooUnitTest: Writing reference RooFitResult " << iter2->first << " as benchmark " << iter2->second << std::endl ;
      _refFile->cd() ;
      iter2->first->Write(iter2->second.c_str()) ;
      gMemDir->cd() ;
    }
 
    ++iter2 ;
  }
 
  auto iter3 = _regValues.begin() ;
  while (iter3!=_regValues.end()) {
 
    if (!_write) {
 
      // Comparison mode
 
     // Retrieve benchmark
      RooDouble* ref = dynamic_cast<RooDouble*>(_refFile->Get(iter3->second.c_str())) ;
      if (!ref) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: cannot retrieve RooDouble " << iter3->second << " from reference file, skipping " << std::endl;
        ++iter3 ;
        ret = false ;
        continue ;
      }
 
      if (_verb > 0) {
   std::cout << "comparing value " << iter3->first << " to benchmark " << iter3->second << " = " << (double)(*ref) << std::endl ;
      }
 
      if (std::abs(iter3->first - (double)(*ref))>vtol() ) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: comparison of value " << iter3->first <<   " fails comparison with reference " << ref->GetName() << std::endl ;
        ret = false ;
      }
 
 
    } else {
 
      // Writing mode
 
      std::cout <<"RooUnitTest: Writing reference double " << iter3->first << " as benchmark " << iter3->second << std::endl ;
      _refFile->cd() ;
      RooDouble* rd = new RooDouble(iter3->first) ;
      rd->Write(iter3->second.c_str()) ;
      gMemDir->cd() ;
    }
 
    ++iter3 ;
  }
 
 
  auto iter4 = _regTables.begin() ;
  while (iter4!=_regTables.end()) {
 
    if (!_write) {
 
      // Comparison mode
 
     // Retrieve benchmark
      RooTable* bmark = dynamic_cast<RooTable*>(_refFile->Get(iter4->second.c_str())) ;
      if (!bmark) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: cannot retrieve RooTable " << iter4->second << " from reference file, skipping " << std::endl ;
        ++iter4 ;
        ret = false ;
        continue ;
      }
 
      if (_verb > 0) {
   std::cout << "comparing RooTable " << iter4->first << " to benchmark " << iter4->second << " = " << bmark << std::endl ;
      }
 
      if (!iter4->first->isIdentical(*bmark, _verb >= 0)) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: comparison of object " << iter4->first->ClassName() << "::" << iter4->first->GetName()
            <<   " fails comparison with counterpart in reference RooTable " << bmark->GetName() << std::endl ;
        if (_verb > 0) {
          iter4->first->Print("V");
          bmark->Print("V");
        }
        ret = false;
      }
 
      // Delete RooTable when comparison is finished to avoid noise in leak checking
      delete iter4->first ;
 
 
    } else {
 
      // Writing mode
 
      std::cout <<"RooUnitTest: Writing reference RooTable " << iter4->first << " as benchmark " << iter4->second << std::endl ;
      _refFile->cd() ;
      iter4->first->Write(iter4->second.c_str()) ;
      gMemDir->cd() ;
    }
 
    ++iter4 ;
  }
 
 
  auto iter5 = _regWS.begin() ;
  while (iter5!=_regWS.end()) {
 
    if (_write) {
 
      // Writing mode
 
      std::cout <<"RooUnitTest: Writing reference RooWorkspace " << iter5->first << " as benchmark " << iter5->second << std::endl ;
      _refFile->cd() ;
      iter5->first->Write(iter5->second.c_str()) ;
      gMemDir->cd() ;
    }
 
    ++iter5 ;
  }
 
  /////////////////
  auto iter6 = _regTH.begin() ;
  while (iter6!=_regTH.end()) {
 
    if (!_write) {
 
      // Comparison mode
 
     // Retrieve benchmark
      TH1* bmark = dynamic_cast<TH1*>(_refFile->Get(iter6->second.c_str())) ;
      if (!bmark) {
        if(_verb >= 0) std::cout << "RooUnitTest ERROR: cannot retrieve TH1 " << iter6->second << " from reference file, skipping " << std::endl ;
        ++iter6 ;
        ret = false ;
        continue ;
      }
 
      if (_verb > 0) {
   std::cout << "comparing TH1 " << iter6->first << " to benchmark " << iter6->second << " = " << bmark << std::endl ;
      }
 
      if (!areTHidentical(iter6->first,bmark)) {
   // coverity[NULL_RETURNS]
   if(_verb >= 0) std::cout << "RooUnitTest ERROR: comparison of object " << iter6->first->ClassName() << "::" << iter6->first->GetName()
        <<   " fails comparison with counterpart in reference TH1 " << bmark->GetName() << std::endl ;
 
 
      if (_debug) {
   std::cout << "RooUnitTest INFO: writing THx " << iter6->first->GetName() << " and " << bmark->GetName()
        << " of failed test to RooUnitTest_DEBUG.root" << std::endl ;
   TFile fdbg("RooUnitTest_DEBUG.root","UPDATE") ;
   iter6->first->SetName(Form("%s_test",iter6->first->GetName())) ;
   iter6->first->Write() ;
   bmark->SetName(Form("%s_ref",bmark->GetName())) ;
   bmark->Write() ;
   fdbg.Close() ;
      }
 
   ret = false ;
      }
 
      // Delete TH1 when comparison is finished to avoid noise in leak checking
      delete iter6->first ;
 
 
    } else {
 
      // Writing mode
 
      std::cout <<"RooUnitTest: Writing reference TH1 " << iter6->first << " as benchmark " << iter6->second << std::endl ;
      _refFile->cd() ;
      iter6->first->Write(iter6->second.c_str()) ;
      gMemDir->cd() ;
    }
 
    ++iter6 ;
  }
 
 
  /////////////////
 
  return ret ;
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::setSilentMode()
{
  RooMsgService::instance().setSilentMode(true) ;
  for (Int_t i=0 ; i<RooMsgService::instance().numStreams() ; i++) {
    if (RooMsgService::instance().getStream(i).minLevel<RooFit::ERROR) {
      RooMsgService::instance().setStreamStatus(i,false) ;
    }
  }
}
 
 
////////////////////////////////////////////////////////////////////////////////
 
void RooUnitTest::clearSilentMode()
{
  RooMsgService::instance().setSilentMode(false) ;
  for (Int_t i=0 ; i<RooMsgService::instance().numStreams() ; i++) {
    RooMsgService::instance().setStreamStatus(i,true) ;
  }
}
 
 
 
////////////////////////////////////////////////////////////////////////////////
 
bool RooUnitTest::runTest()
{
  gMemDir->cd() ;
 
  if (_verb<2) {
    setSilentMode() ;
  } else {
    std::cout << "*** Begin of output of Unit Test at normal verbosity *************" << std::endl ;
  }
 
  RooMsgService::instance().clearErrorCount() ;
 
  // Reset random generator seed to make results independent of test ordering
  gRandom->SetSeed(12345) ;
  RooRandom::randomGenerator()->SetSeed(12345) ;
 
  RooTrace::callgrind_zero() ;
  if (!testCode()) return false ;
  RooTrace::callgrind_dump() ;
 
  if (_verb<2) {
    clearSilentMode() ;
  } else {
    std::cout << "*** End of output of Unit Test at normal verbosity ***************" << std::endl ;
  }
 
  if (RooMsgService::instance().errorCount()>0) {
    if(_verb >= 0) std::cout << "RooUnitTest: ERROR messages were logged, failing test" << std::endl ;
    return false ;
  }
 
  return runCompTests() ;
}
 
 
////////////////////////////////////////////////////////////////////////////////
/// Set gMemDir to memDir
 
void RooUnitTest::setMemDir(TDirectory* memDir) {
   gMemDir = memDir ;
}
 
/// \endcond