Logo ROOT   6.18/05
Reference Guide
MethodRuleFit.cxx
Go to the documentation of this file.
1// @(#)root/tmva $Id$
2// Author: Fredrik Tegenfeldt
3
4/**********************************************************************************
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : MethodRuleFit *
8 * Web : http://tmva.sourceforge.net *
9 * *
10 * Description: *
11 * Implementation (see header file for description) *
12 * *
13 * Authors (alphabetical): *
14 * Fredrik Tegenfeldt <Fredrik.Tegenfeldt@cern.ch> - Iowa State U., USA *
15 * *
16 * Copyright (c) 2005: *
17 * CERN, Switzerland *
18 * Iowa State U. *
19 * MPI-K Heidelberg, Germany *
20 * *
21 * Redistribution and use in source and binary forms, with or without *
22 * modification, are permitted according to the terms listed in LICENSE *
23 * (http://tmva.sourceforge.net/LICENSE) *
24 **********************************************************************************/
25
26/*! \class TMVA::MethodRuleFit
27\ingroup TMVA
28J Friedman's RuleFit method
29*/
30
31#include "TMVA/MethodRuleFit.h"
32
34#include "TMVA/Config.h"
35#include "TMVA/Configurable.h"
36#include "TMVA/CrossEntropy.h"
37#include "TMVA/DataSet.h"
38#include "TMVA/DecisionTree.h"
39#include "TMVA/GiniIndex.h"
40#include "TMVA/IMethod.h"
41#include "TMVA/MethodBase.h"
43#include "TMVA/MsgLogger.h"
44#include "TMVA/Ranking.h"
45#include "TMVA/RuleFitAPI.h"
46#include "TMVA/SdivSqrtSplusB.h"
47#include "TMVA/SeparationBase.h"
48#include "TMVA/Timer.h"
49#include "TMVA/Tools.h"
50#include "TMVA/Types.h"
51
52#include "Riostream.h"
53#include "TRandom3.h"
54#include "TMath.h"
55#include "TMatrix.h"
56#include "TDirectory.h"
57
58#include <algorithm>
59#include <list>
60#include <random>
61
62using std::min;
63
64REGISTER_METHOD(RuleFit)
65
67
68////////////////////////////////////////////////////////////////////////////////
69/// standard constructor
70
72 const TString& methodTitle,
73 DataSetInfo& theData,
74 const TString& theOption) :
75 MethodBase( jobName, Types::kRuleFit, methodTitle, theData, theOption)
76 , fSignalFraction(0)
77 , fNTImportance(0)
78 , fNTCoefficient(0)
79 , fNTSupport(0)
80 , fNTNcuts(0)
81 , fNTNvars(0)
82 , fNTPtag(0)
83 , fNTPss(0)
84 , fNTPsb(0)
85 , fNTPbs(0)
86 , fNTPbb(0)
87 , fNTSSB(0)
88 , fNTType(0)
89 , fUseRuleFitJF(kFALSE)
90 , fRFNrules(0)
91 , fRFNendnodes(0)
92 , fNTrees(0)
93 , fTreeEveFrac(0)
94 , fSepType(0)
95 , fMinFracNEve(0)
96 , fMaxFracNEve(0)
97 , fNCuts(0)
98 , fPruneMethod(TMVA::DecisionTree::kCostComplexityPruning)
99 , fPruneStrength(0)
100 , fUseBoost(kFALSE)
101 , fGDPathEveFrac(0)
102 , fGDValidEveFrac(0)
103 , fGDTau(0)
104 , fGDTauPrec(0)
105 , fGDTauMin(0)
106 , fGDTauMax(0)
107 , fGDTauScan(0)
108 , fGDPathStep(0)
109 , fGDNPathSteps(0)
110 , fGDErrScale(0)
111 , fMinimp(0)
112 , fRuleMinDist(0)
113 , fLinQuantile(0)
114{
115 fMonitorNtuple = NULL;
116}
117
118////////////////////////////////////////////////////////////////////////////////
119/// constructor from weight file
120
122 const TString& theWeightFile) :
123 MethodBase( Types::kRuleFit, theData, theWeightFile)
124 , fSignalFraction(0)
125 , fNTImportance(0)
126 , fNTCoefficient(0)
127 , fNTSupport(0)
128 , fNTNcuts(0)
129 , fNTNvars(0)
130 , fNTPtag(0)
131 , fNTPss(0)
132 , fNTPsb(0)
133 , fNTPbs(0)
134 , fNTPbb(0)
135 , fNTSSB(0)
136 , fNTType(0)
137 , fUseRuleFitJF(kFALSE)
138 , fRFNrules(0)
139 , fRFNendnodes(0)
140 , fNTrees(0)
141 , fTreeEveFrac(0)
142 , fSepType(0)
143 , fMinFracNEve(0)
144 , fMaxFracNEve(0)
145 , fNCuts(0)
146 , fPruneMethod(TMVA::DecisionTree::kCostComplexityPruning)
147 , fPruneStrength(0)
148 , fUseBoost(kFALSE)
149 , fGDPathEveFrac(0)
150 , fGDValidEveFrac(0)
151 , fGDTau(0)
152 , fGDTauPrec(0)
153 , fGDTauMin(0)
154 , fGDTauMax(0)
155 , fGDTauScan(0)
156 , fGDPathStep(0)
157 , fGDNPathSteps(0)
158 , fGDErrScale(0)
159 , fMinimp(0)
160 , fRuleMinDist(0)
161 , fLinQuantile(0)
162{
163 fMonitorNtuple = NULL;
164}
165
166////////////////////////////////////////////////////////////////////////////////
167/// destructor
168
170{
171 for (UInt_t i=0; i<fEventSample.size(); i++) delete fEventSample[i];
172 for (UInt_t i=0; i<fForest.size(); i++) delete fForest[i];
173}
174
175////////////////////////////////////////////////////////////////////////////////
176/// RuleFit can handle classification with 2 classes
177
179{
180 if (type == Types::kClassification && numberClasses == 2) return kTRUE;
181 return kFALSE;
182}
183
184////////////////////////////////////////////////////////////////////////////////
185/// define the options (their key words) that can be set in the option string
186/// know options.
187///
188/// #### general
189///
190/// - RuleFitModule <string>
191/// available values are:
192/// - RFTMVA - use TMVA implementation
193/// - RFFriedman - use Friedmans original implementation
194///
195/// #### Path search (fitting)
196///
197/// - GDTau <float> gradient-directed path: fit threshold, default
198/// - GDTauPrec <float> gradient-directed path: precision of estimated tau
199/// - GDStep <float> gradient-directed path: step size
200/// - GDNSteps <float> gradient-directed path: number of steps
201/// - GDErrScale <float> stop scan when error>scale*errmin
202///
203/// #### Tree generation
204///
205/// - fEventsMin <float> minimum fraction of events in a splittable node
206/// - fEventsMax <float> maximum fraction of events in a splittable node
207/// - nTrees <float> number of trees in forest.
208/// - ForestType <string>
209/// available values are:
210/// - Random - create forest using random subsample and only random variables subset at each node
211/// - AdaBoost - create forest with boosted events
212///
213/// #### Model creation
214///
215/// - RuleMinDist <float> min distance allowed between rules
216/// - MinImp <float> minimum rule importance accepted
217/// - Model <string> model to be used
218/// available values are:
219/// - ModRuleLinear <default>
220/// - ModRule
221/// - ModLinear
222///
223/// #### Friedmans module
224///
225/// - RFWorkDir <string> directory where Friedmans module (rf_go.exe) is installed
226/// - RFNrules <int> maximum number of rules allowed
227/// - RFNendnodes <int> average number of end nodes in the forest of trees
228
230{
231 DeclareOptionRef(fGDTau=-1, "GDTau", "Gradient-directed (GD) path: default fit cut-off");
232 DeclareOptionRef(fGDTauPrec=0.01, "GDTauPrec", "GD path: precision of tau");
233 DeclareOptionRef(fGDPathStep=0.01, "GDStep", "GD path: step size");
234 DeclareOptionRef(fGDNPathSteps=10000, "GDNSteps", "GD path: number of steps");
235 DeclareOptionRef(fGDErrScale=1.1, "GDErrScale", "Stop scan when error > scale*errmin");
236 DeclareOptionRef(fLinQuantile, "LinQuantile", "Quantile of linear terms (removes outliers)");
237 DeclareOptionRef(fGDPathEveFrac=0.5, "GDPathEveFrac", "Fraction of events used for the path search");
238 DeclareOptionRef(fGDValidEveFrac=0.5, "GDValidEveFrac", "Fraction of events used for the validation");
239 // tree options
240 DeclareOptionRef(fMinFracNEve=0.1, "fEventsMin", "Minimum fraction of events in a splittable node");
241 DeclareOptionRef(fMaxFracNEve=0.9, "fEventsMax", "Maximum fraction of events in a splittable node");
242 DeclareOptionRef(fNTrees=20, "nTrees", "Number of trees in forest.");
243
244 DeclareOptionRef(fForestTypeS="AdaBoost", "ForestType", "Method to use for forest generation (AdaBoost or RandomForest)");
245 AddPreDefVal(TString("AdaBoost"));
246 AddPreDefVal(TString("Random"));
247 // rule cleanup options
248 DeclareOptionRef(fRuleMinDist=0.001, "RuleMinDist", "Minimum distance between rules");
249 DeclareOptionRef(fMinimp=0.01, "MinImp", "Minimum rule importance accepted");
250 // rule model option
251 DeclareOptionRef(fModelTypeS="ModRuleLinear", "Model", "Model to be used");
252 AddPreDefVal(TString("ModRule"));
253 AddPreDefVal(TString("ModRuleLinear"));
254 AddPreDefVal(TString("ModLinear"));
255 DeclareOptionRef(fRuleFitModuleS="RFTMVA", "RuleFitModule","Which RuleFit module to use");
256 AddPreDefVal(TString("RFTMVA"));
257 AddPreDefVal(TString("RFFriedman"));
258
259 DeclareOptionRef(fRFWorkDir="./rulefit", "RFWorkDir", "Friedman\'s RuleFit module (RFF): working dir");
260 DeclareOptionRef(fRFNrules=2000, "RFNrules", "RFF: Mximum number of rules");
261 DeclareOptionRef(fRFNendnodes=4, "RFNendnodes", "RFF: Average number of end nodes");
262}
263
264////////////////////////////////////////////////////////////////////////////////
265/// process the options specified by the user
266
268{
269 if (IgnoreEventsWithNegWeightsInTraining()) {
270 Log() << kFATAL << "Mechanism to ignore events with negative weights in training not yet available for method: "
271 << GetMethodTypeName()
272 << " --> please remove \"IgnoreNegWeightsInTraining\" option from booking string."
273 << Endl;
274 }
275
276 fRuleFitModuleS.ToLower();
277 if (fRuleFitModuleS == "rftmva") fUseRuleFitJF = kFALSE;
278 else if (fRuleFitModuleS == "rffriedman") fUseRuleFitJF = kTRUE;
279 else fUseRuleFitJF = kTRUE;
280
281 fSepTypeS.ToLower();
282 if (fSepTypeS == "misclassificationerror") fSepType = new MisClassificationError();
283 else if (fSepTypeS == "giniindex") fSepType = new GiniIndex();
284 else if (fSepTypeS == "crossentropy") fSepType = new CrossEntropy();
285 else fSepType = new SdivSqrtSplusB();
286
287 fModelTypeS.ToLower();
288 if (fModelTypeS == "modlinear" ) fRuleFit.SetModelLinear();
289 else if (fModelTypeS == "modrule" ) fRuleFit.SetModelRules();
290 else fRuleFit.SetModelFull();
291
292 fPruneMethodS.ToLower();
293 if (fPruneMethodS == "expectederror" ) fPruneMethod = DecisionTree::kExpectedErrorPruning;
294 else if (fPruneMethodS == "costcomplexity" ) fPruneMethod = DecisionTree::kCostComplexityPruning;
295 else fPruneMethod = DecisionTree::kNoPruning;
296
297 fForestTypeS.ToLower();
298 if (fForestTypeS == "random" ) fUseBoost = kFALSE;
299 else if (fForestTypeS == "adaboost" ) fUseBoost = kTRUE;
300 else fUseBoost = kTRUE;
301 //
302 // if creating the forest by boosting the events
303 // the full training sample is used per tree
304 // -> only true for the TMVA version of RuleFit.
305 if (fUseBoost && (!fUseRuleFitJF)) fTreeEveFrac = 1.0;
306
307 // check event fraction for tree generation
308 // if <0 set to automatic number
309 if (fTreeEveFrac<=0) {
310 Int_t nevents = Data()->GetNTrainingEvents();
311 Double_t n = static_cast<Double_t>(nevents);
312 fTreeEveFrac = min( 0.5, (100.0 +6.0*sqrt(n))/n);
313 }
314 // verify ranges of options
315 VerifyRange(Log(), "nTrees", fNTrees,0,100000,20);
316 VerifyRange(Log(), "MinImp", fMinimp,0.0,1.0,0.0);
317 VerifyRange(Log(), "GDTauPrec", fGDTauPrec,1e-5,5e-1);
318 VerifyRange(Log(), "GDTauMin", fGDTauMin,0.0,1.0);
319 VerifyRange(Log(), "GDTauMax", fGDTauMax,fGDTauMin,1.0);
320 VerifyRange(Log(), "GDPathStep", fGDPathStep,0.0,100.0,0.01);
321 VerifyRange(Log(), "GDErrScale", fGDErrScale,1.0,100.0,1.1);
322 VerifyRange(Log(), "GDPathEveFrac", fGDPathEveFrac,0.01,0.9,0.5);
323 VerifyRange(Log(), "GDValidEveFrac",fGDValidEveFrac,0.01,1.0-fGDPathEveFrac,1.0-fGDPathEveFrac);
324 VerifyRange(Log(), "fEventsMin", fMinFracNEve,0.0,1.0);
325 VerifyRange(Log(), "fEventsMax", fMaxFracNEve,fMinFracNEve,1.0);
326
327 fRuleFit.GetRuleEnsemblePtr()->SetLinQuantile(fLinQuantile);
328 fRuleFit.GetRuleFitParamsPtr()->SetGDTauRange(fGDTauMin,fGDTauMax);
329 fRuleFit.GetRuleFitParamsPtr()->SetGDTau(fGDTau);
330 fRuleFit.GetRuleFitParamsPtr()->SetGDTauPrec(fGDTauPrec);
331 fRuleFit.GetRuleFitParamsPtr()->SetGDTauScan(fGDTauScan);
332 fRuleFit.GetRuleFitParamsPtr()->SetGDPathStep(fGDPathStep);
333 fRuleFit.GetRuleFitParamsPtr()->SetGDNPathSteps(fGDNPathSteps);
334 fRuleFit.GetRuleFitParamsPtr()->SetGDErrScale(fGDErrScale);
335 fRuleFit.SetImportanceCut(fMinimp);
336 fRuleFit.SetRuleMinDist(fRuleMinDist);
337
338
339 // check if Friedmans module is used.
340 // print a message concerning the options.
341 if (fUseRuleFitJF) {
342 Log() << kINFO << "" << Endl;
343 Log() << kINFO << "--------------------------------------" <<Endl;
344 Log() << kINFO << "Friedmans RuleFit module is selected." << Endl;
345 Log() << kINFO << "Only the following options are used:" << Endl;
346 Log() << kINFO << Endl;
347 Log() << kINFO << gTools().Color("bold") << " Model" << gTools().Color("reset") << Endl;
348 Log() << kINFO << gTools().Color("bold") << " RFWorkDir" << gTools().Color("reset") << Endl;
349 Log() << kINFO << gTools().Color("bold") << " RFNrules" << gTools().Color("reset") << Endl;
350 Log() << kINFO << gTools().Color("bold") << " RFNendnodes" << gTools().Color("reset") << Endl;
351 Log() << kINFO << gTools().Color("bold") << " GDNPathSteps" << gTools().Color("reset") << Endl;
352 Log() << kINFO << gTools().Color("bold") << " GDPathStep" << gTools().Color("reset") << Endl;
353 Log() << kINFO << gTools().Color("bold") << " GDErrScale" << gTools().Color("reset") << Endl;
354 Log() << kINFO << "--------------------------------------" <<Endl;
355 Log() << kINFO << Endl;
356 }
357
358 // Select what weight to use in the 'importance' rule visualisation plots.
359 // Note that if UseCoefficientsVisHists() is selected, the following weight is used:
360 // w = rule coefficient * rule support
361 // The support is a positive number which is 0 if no events are accepted by the rule.
362 // Normally the importance gives more useful information.
363 //
364 //fRuleFit.UseCoefficientsVisHists();
365 fRuleFit.UseImportanceVisHists();
366
367 fRuleFit.SetMsgType( Log().GetMinType() );
368
369 if (HasTrainingTree()) InitEventSample();
370
371}
372
373////////////////////////////////////////////////////////////////////////////////
374/// initialize the monitoring ntuple
375
377{
378 BaseDir()->cd();
379 fMonitorNtuple= new TTree("MonitorNtuple_RuleFit","RuleFit variables");
380 fMonitorNtuple->Branch("importance",&fNTImportance,"importance/D");
381 fMonitorNtuple->Branch("support",&fNTSupport,"support/D");
382 fMonitorNtuple->Branch("coefficient",&fNTCoefficient,"coefficient/D");
383 fMonitorNtuple->Branch("ncuts",&fNTNcuts,"ncuts/I");
384 fMonitorNtuple->Branch("nvars",&fNTNvars,"nvars/I");
385 fMonitorNtuple->Branch("type",&fNTType,"type/I");
386 fMonitorNtuple->Branch("ptag",&fNTPtag,"ptag/D");
387 fMonitorNtuple->Branch("pss",&fNTPss,"pss/D");
388 fMonitorNtuple->Branch("psb",&fNTPsb,"psb/D");
389 fMonitorNtuple->Branch("pbs",&fNTPbs,"pbs/D");
390 fMonitorNtuple->Branch("pbb",&fNTPbb,"pbb/D");
391 fMonitorNtuple->Branch("soversb",&fNTSSB,"soversb/D");
392}
393
394////////////////////////////////////////////////////////////////////////////////
395/// default initialization
396
398{
399 // the minimum requirement to declare an event signal-like
400 SetSignalReferenceCut( 0.0 );
401
402 // set variables that used to be options
403 // any modifications are then made in ProcessOptions()
404 fLinQuantile = 0.025; // Quantile of linear terms (remove outliers)
405 fTreeEveFrac = -1.0; // Fraction of events used to train each tree
406 fNCuts = 20; // Number of steps during node cut optimisation
407 fSepTypeS = "GiniIndex"; // Separation criterion for node splitting; see BDT
408 fPruneMethodS = "NONE"; // Pruning method; see BDT
409 fPruneStrength = 3.5; // Pruning strength; see BDT
410 fGDTauMin = 0.0; // Gradient-directed path: min fit threshold (tau)
411 fGDTauMax = 1.0; // Gradient-directed path: max fit threshold (tau)
412 fGDTauScan = 1000; // Gradient-directed path: number of points scanning for best tau
413
414}
415
416////////////////////////////////////////////////////////////////////////////////
417/// write all Events from the Tree into a vector of Events, that are
418/// more easily manipulated.
419/// This method should never be called without existing trainingTree, as it
420/// the vector of events from the ROOT training tree
421
423{
424 if (Data()->GetNEvents()==0) Log() << kFATAL << "<Init> Data().TrainingTree() is zero pointer" << Endl;
425
426 Int_t nevents = Data()->GetNEvents();
427 for (Int_t ievt=0; ievt<nevents; ievt++){
428 const Event * ev = GetEvent(ievt);
429 fEventSample.push_back( new Event(*ev));
430 }
431 if (fTreeEveFrac<=0) {
432 Double_t n = static_cast<Double_t>(nevents);
433 fTreeEveFrac = min( 0.5, (100.0 +6.0*sqrt(n))/n);
434 }
435 if (fTreeEveFrac>1.0) fTreeEveFrac=1.0;
436 //
437 std::shuffle(fEventSample.begin(), fEventSample.end(), std::default_random_engine{});
438 //
439 Log() << kDEBUG << "Set sub-sample fraction to " << fTreeEveFrac << Endl;
440}
441
442////////////////////////////////////////////////////////////////////////////////
443
445{
447 // training of rules
448
449 if(!IsSilentFile()) InitMonitorNtuple();
450
451 // fill the STL Vector with the event sample
452 this->InitEventSample();
453
454 if (fUseRuleFitJF) {
455 TrainJFRuleFit();
456 }
457 else {
458 TrainTMVARuleFit();
459 }
460 fRuleFit.GetRuleEnsemblePtr()->ClearRuleMap();
462 ExitFromTraining();
463}
464
465////////////////////////////////////////////////////////////////////////////////
466/// training of rules using TMVA implementation
467
469{
470 if (IsNormalised()) Log() << kFATAL << "\"Normalise\" option cannot be used with RuleFit; "
471 << "please remove the option from the configuration string, or "
472 << "use \"!Normalise\""
473 << Endl;
474
475 // timer
476 Timer timer( 1, GetName() );
477
478 // test tree nmin cut -> for debug purposes
479 // the routine will generate trees with stopping cut on N(eve) given by
480 // a fraction between [20,N(eve)-1].
481 //
482 // MakeForestRnd();
483 // exit(1);
484 //
485
486 // Init RuleFit object and create rule ensemble
487 // + make forest & rules
488 fRuleFit.Initialize( this );
489
490 // Make forest of decision trees
491 // if (fRuleFit.GetRuleEnsemble().DoRules()) fRuleFit.MakeForest();
492
493 // Fit the rules
494 Log() << kDEBUG << "Fitting rule coefficients ..." << Endl;
495 fRuleFit.FitCoefficients();
496
497 // Calculate importance
498 Log() << kDEBUG << "Computing rule and variable importance" << Endl;
499 fRuleFit.CalcImportance();
500
501 // Output results and fill monitor ntuple
502 fRuleFit.GetRuleEnsemblePtr()->Print();
503 //
504 if(!IsSilentFile())
505 {
506 Log() << kDEBUG << "Filling rule ntuple" << Endl;
507 UInt_t nrules = fRuleFit.GetRuleEnsemble().GetRulesConst().size();
508 const Rule *rule;
509 for (UInt_t i=0; i<nrules; i++ ) {
510 rule = fRuleFit.GetRuleEnsemble().GetRulesConst(i);
511 fNTImportance = rule->GetRelImportance();
512 fNTSupport = rule->GetSupport();
513 fNTCoefficient = rule->GetCoefficient();
514 fNTType = (rule->IsSignalRule() ? 1:-1 );
515 fNTNvars = rule->GetRuleCut()->GetNvars();
516 fNTNcuts = rule->GetRuleCut()->GetNcuts();
517 fNTPtag = fRuleFit.GetRuleEnsemble().GetRulePTag(i); // should be identical with support
518 fNTPss = fRuleFit.GetRuleEnsemble().GetRulePSS(i);
519 fNTPsb = fRuleFit.GetRuleEnsemble().GetRulePSB(i);
520 fNTPbs = fRuleFit.GetRuleEnsemble().GetRulePBS(i);
521 fNTPbb = fRuleFit.GetRuleEnsemble().GetRulePBB(i);
522 fNTSSB = rule->GetSSB();
523 fMonitorNtuple->Fill();
524 }
525
526 fRuleFit.MakeVisHists();
527 fRuleFit.MakeDebugHists();
528 }
529 Log() << kDEBUG << "Training done" << Endl;
530
531}
532
533////////////////////////////////////////////////////////////////////////////////
534/// training of rules using Jerome Friedmans implementation
535
537{
538 fRuleFit.InitPtrs( this );
539 Data()->SetCurrentType(Types::kTraining);
540 UInt_t nevents = Data()->GetNTrainingEvents();
541 std::vector<const TMVA::Event*> tmp;
542 for (Long64_t ievt=0; ievt<nevents; ievt++) {
543 const Event *event = GetEvent(ievt);
544 tmp.push_back(event);
545 }
546 fRuleFit.SetTrainingEvents( tmp );
547
548 RuleFitAPI *rfAPI = new RuleFitAPI( this, &fRuleFit, Log().GetMinType() );
549
550 rfAPI->WelcomeMessage();
551
552 // timer
553 Timer timer( 1, GetName() );
554
555 Log() << kINFO << "Training ..." << Endl;
556 rfAPI->TrainRuleFit();
557
558 Log() << kDEBUG << "reading model summary from rf_go.exe output" << Endl;
559 rfAPI->ReadModelSum();
560
561 // fRuleFit.GetRuleEnsemblePtr()->MakeRuleMap();
562
563 Log() << kDEBUG << "calculating rule and variable importance" << Endl;
564 fRuleFit.CalcImportance();
565
566 // Output results and fill monitor ntuple
567 fRuleFit.GetRuleEnsemblePtr()->Print();
568 //
569 if(!IsSilentFile())fRuleFit.MakeVisHists();
570
571 delete rfAPI;
572
573 Log() << kDEBUG << "done training" << Endl;
574}
575
576////////////////////////////////////////////////////////////////////////////////
577/// computes ranking of input variables
578
580{
581 // create the ranking object
582 fRanking = new Ranking( GetName(), "Importance" );
583
584 for (UInt_t ivar=0; ivar<GetNvar(); ivar++) {
585 fRanking->AddRank( Rank( GetInputLabel(ivar), fRuleFit.GetRuleEnsemble().GetVarImportance(ivar) ) );
586 }
587
588 return fRanking;
589}
590
591////////////////////////////////////////////////////////////////////////////////
592/// add the rules to XML node
593
594void TMVA::MethodRuleFit::AddWeightsXMLTo( void* parent ) const
595{
596 fRuleFit.GetRuleEnsemble().AddXMLTo( parent );
597}
598
599////////////////////////////////////////////////////////////////////////////////
600/// read rules from an std::istream
601
603{
604 fRuleFit.GetRuleEnsemblePtr()->ReadRaw( istr );
605}
606
607////////////////////////////////////////////////////////////////////////////////
608/// read rules from XML node
609
611{
612 fRuleFit.GetRuleEnsemblePtr()->ReadFromXML( wghtnode );
613}
614
615////////////////////////////////////////////////////////////////////////////////
616/// returns MVA value for given event
617
619{
620 // cannot determine error
621 NoErrorCalc(err, errUpper);
622
623 return fRuleFit.EvalEvent( *GetEvent() );
624}
625
626////////////////////////////////////////////////////////////////////////////////
627/// write special monitoring histograms to file (here ntuple)
628
630{
631 BaseDir()->cd();
632 Log() << kINFO << "Write monitoring ntuple to file: " << BaseDir()->GetPath() << Endl;
633 fMonitorNtuple->Write();
634}
635
636////////////////////////////////////////////////////////////////////////////////
637/// write specific classifier response
638
639void TMVA::MethodRuleFit::MakeClassSpecific( std::ostream& fout, const TString& className ) const
640{
641 Int_t dp = fout.precision();
642 fout << " // not implemented for class: \"" << className << "\"" << std::endl;
643 fout << "};" << std::endl;
644 fout << "void " << className << "::Initialize(){}" << std::endl;
645 fout << "void " << className << "::Clear(){}" << std::endl;
646 fout << "double " << className << "::GetMvaValue__( const std::vector<double>& inputValues ) const {" << std::endl;
647 fout << " double rval=" << std::setprecision(10) << fRuleFit.GetRuleEnsemble().GetOffset() << ";" << std::endl;
648 MakeClassRuleCuts(fout);
649 MakeClassLinear(fout);
650 fout << " return rval;" << std::endl;
651 fout << "}" << std::endl;
652 fout << std::setprecision(dp);
653}
654
655////////////////////////////////////////////////////////////////////////////////
656/// print out the rule cuts
657
658void TMVA::MethodRuleFit::MakeClassRuleCuts( std::ostream& fout ) const
659{
660 Int_t dp = fout.precision();
661 if (!fRuleFit.GetRuleEnsemble().DoRules()) {
662 fout << " //" << std::endl;
663 fout << " // ==> MODEL CONTAINS NO RULES <==" << std::endl;
664 fout << " //" << std::endl;
665 return;
666 }
667 const RuleEnsemble *rens = &(fRuleFit.GetRuleEnsemble());
668 const std::vector< Rule* > *rules = &(rens->GetRulesConst());
669 const RuleCut *ruleCut;
670 //
671 std::list< std::pair<Double_t,Int_t> > sortedRules;
672 for (UInt_t ir=0; ir<rules->size(); ir++) {
673 sortedRules.push_back( std::pair<Double_t,Int_t>( (*rules)[ir]->GetImportance()/rens->GetImportanceRef(),ir ) );
674 }
675 sortedRules.sort();
676 //
677 fout << " //" << std::endl;
678 fout << " // here follows all rules ordered in importance (most important first)" << std::endl;
679 fout << " // at the end of each line, the relative importance of the rule is given" << std::endl;
680 fout << " //" << std::endl;
681 //
682 for ( std::list< std::pair<double,int> >::reverse_iterator itpair = sortedRules.rbegin();
683 itpair != sortedRules.rend(); ++itpair ) {
684 UInt_t ir = itpair->second;
685 Double_t impr = itpair->first;
686 ruleCut = (*rules)[ir]->GetRuleCut();
687 if (impr<rens->GetImportanceCut()) fout << " //" << std::endl;
688 fout << " if (" << std::flush;
689 for (UInt_t ic=0; ic<ruleCut->GetNvars(); ic++) {
690 Double_t sel = ruleCut->GetSelector(ic);
691 Double_t valmin = ruleCut->GetCutMin(ic);
692 Double_t valmax = ruleCut->GetCutMax(ic);
693 Bool_t domin = ruleCut->GetCutDoMin(ic);
694 Bool_t domax = ruleCut->GetCutDoMax(ic);
695 //
696 if (ic>0) fout << "&&" << std::flush;
697 if (domin) {
698 fout << "(" << std::setprecision(10) << valmin << std::flush;
699 fout << "<inputValues[" << sel << "])" << std::flush;
700 }
701 if (domax) {
702 if (domin) fout << "&&" << std::flush;
703 fout << "(inputValues[" << sel << "]" << std::flush;
704 fout << "<" << std::setprecision(10) << valmax << ")" <<std::flush;
705 }
706 }
707 fout << ") rval+=" << std::setprecision(10) << (*rules)[ir]->GetCoefficient() << ";" << std::flush;
708 fout << " // importance = " << Form("%3.3f",impr) << std::endl;
709 }
710 fout << std::setprecision(dp);
711}
712
713////////////////////////////////////////////////////////////////////////////////
714/// print out the linear terms
715
716void TMVA::MethodRuleFit::MakeClassLinear( std::ostream& fout ) const
717{
718 if (!fRuleFit.GetRuleEnsemble().DoLinear()) {
719 fout << " //" << std::endl;
720 fout << " // ==> MODEL CONTAINS NO LINEAR TERMS <==" << std::endl;
721 fout << " //" << std::endl;
722 return;
723 }
724 fout << " //" << std::endl;
725 fout << " // here follows all linear terms" << std::endl;
726 fout << " // at the end of each line, the relative importance of the term is given" << std::endl;
727 fout << " //" << std::endl;
728 const RuleEnsemble *rens = &(fRuleFit.GetRuleEnsemble());
729 UInt_t nlin = rens->GetNLinear();
730 for (UInt_t il=0; il<nlin; il++) {
731 if (rens->IsLinTermOK(il)) {
732 Double_t norm = rens->GetLinNorm(il);
733 Double_t imp = rens->GetLinImportance(il)/rens->GetImportanceRef();
734 fout << " rval+="
735 // << std::setprecision(10) << rens->GetLinCoefficients(il)*norm << "*std::min(" << setprecision(10) << rens->GetLinDP(il)
736 // << ", std::max( inputValues[" << il << "]," << std::setprecision(10) << rens->GetLinDM(il) << "));"
737 << std::setprecision(10) << rens->GetLinCoefficients(il)*norm
738 << "*std::min( double(" << std::setprecision(10) << rens->GetLinDP(il)
739 << "), std::max( double(inputValues[" << il << "]), double(" << std::setprecision(10) << rens->GetLinDM(il) << ")));"
740 << std::flush;
741 fout << " // importance = " << Form("%3.3f",imp) << std::endl;
742 }
743 }
744}
745
746////////////////////////////////////////////////////////////////////////////////
747/// get help message text
748///
749/// typical length of text line:
750/// "|--------------------------------------------------------------|"
751
753{
754 TString col = gConfig().WriteOptionsReference() ? TString() : gTools().Color("bold");
755 TString colres = gConfig().WriteOptionsReference() ? TString() : gTools().Color("reset");
756 TString brk = gConfig().WriteOptionsReference() ? "<br>" : "";
757
758 Log() << Endl;
759 Log() << col << "--- Short description:" << colres << Endl;
760 Log() << Endl;
761 Log() << "This method uses a collection of so called rules to create a" << Endl;
762 Log() << "discriminating scoring function. Each rule consists of a series" << Endl;
763 Log() << "of cuts in parameter space. The ensemble of rules are created" << Endl;
764 Log() << "from a forest of decision trees, trained using the training data." << Endl;
765 Log() << "Each node (apart from the root) corresponds to one rule." << Endl;
766 Log() << "The scoring function is then obtained by linearly combining" << Endl;
767 Log() << "the rules. A fitting procedure is applied to find the optimum" << Endl;
768 Log() << "set of coefficients. The goal is to find a model with few rules" << Endl;
769 Log() << "but with a strong discriminating power." << Endl;
770 Log() << Endl;
771 Log() << col << "--- Performance optimisation:" << colres << Endl;
772 Log() << Endl;
773 Log() << "There are two important considerations to make when optimising:" << Endl;
774 Log() << Endl;
775 Log() << " 1. Topology of the decision tree forest" << brk << Endl;
776 Log() << " 2. Fitting of the coefficients" << Endl;
777 Log() << Endl;
778 Log() << "The maximum complexity of the rules is defined by the size of" << Endl;
779 Log() << "the trees. Large trees will yield many complex rules and capture" << Endl;
780 Log() << "higher order correlations. On the other hand, small trees will" << Endl;
781 Log() << "lead to a smaller ensemble with simple rules, only capable of" << Endl;
782 Log() << "modeling simple structures." << Endl;
783 Log() << "Several parameters exists for controlling the complexity of the" << Endl;
784 Log() << "rule ensemble." << Endl;
785 Log() << Endl;
786 Log() << "The fitting procedure searches for a minimum using a gradient" << Endl;
787 Log() << "directed path. Apart from step size and number of steps, the" << Endl;
788 Log() << "evolution of the path is defined by a cut-off parameter, tau." << Endl;
789 Log() << "This parameter is unknown and depends on the training data." << Endl;
790 Log() << "A large value will tend to give large weights to a few rules." << Endl;
791 Log() << "Similarly, a small value will lead to a large set of rules" << Endl;
792 Log() << "with similar weights." << Endl;
793 Log() << Endl;
794 Log() << "A final point is the model used; rules and/or linear terms." << Endl;
795 Log() << "For a given training sample, the result may improve by adding" << Endl;
796 Log() << "linear terms. If best performance is obtained using only linear" << Endl;
797 Log() << "terms, it is very likely that the Fisher discriminant would be" << Endl;
798 Log() << "a better choice. Ideally the fitting procedure should be able to" << Endl;
799 Log() << "make this choice by giving appropriate weights for either terms." << Endl;
800 Log() << Endl;
801 Log() << col << "--- Performance tuning via configuration options:" << colres << Endl;
802 Log() << Endl;
803 Log() << "I. TUNING OF RULE ENSEMBLE:" << Endl;
804 Log() << Endl;
805 Log() << " " << col << "ForestType " << colres
806 << ": Recommended is to use the default \"AdaBoost\"." << brk << Endl;
807 Log() << " " << col << "nTrees " << colres
808 << ": More trees leads to more rules but also slow" << Endl;
809 Log() << " performance. With too few trees the risk is" << Endl;
810 Log() << " that the rule ensemble becomes too simple." << brk << Endl;
811 Log() << " " << col << "fEventsMin " << colres << brk << Endl;
812 Log() << " " << col << "fEventsMax " << colres
813 << ": With a lower min, more large trees will be generated" << Endl;
814 Log() << " leading to more complex rules." << Endl;
815 Log() << " With a higher max, more small trees will be" << Endl;
816 Log() << " generated leading to more simple rules." << Endl;
817 Log() << " By changing this range, the average complexity" << Endl;
818 Log() << " of the rule ensemble can be controlled." << brk << Endl;
819 Log() << " " << col << "RuleMinDist " << colres
820 << ": By increasing the minimum distance between" << Endl;
821 Log() << " rules, fewer and more diverse rules will remain." << Endl;
822 Log() << " Initially it is a good idea to keep this small" << Endl;
823 Log() << " or zero and let the fitting do the selection of" << Endl;
824 Log() << " rules. In order to reduce the ensemble size," << Endl;
825 Log() << " the value can then be increased." << Endl;
826 Log() << Endl;
827 // "|--------------------------------------------------------------|"
828 Log() << "II. TUNING OF THE FITTING:" << Endl;
829 Log() << Endl;
830 Log() << " " << col << "GDPathEveFrac " << colres
831 << ": fraction of events in path evaluation" << Endl;
832 Log() << " Increasing this fraction will improve the path" << Endl;
833 Log() << " finding. However, a too high value will give few" << Endl;
834 Log() << " unique events available for error estimation." << Endl;
835 Log() << " It is recommended to use the default = 0.5." << brk << Endl;
836 Log() << " " << col << "GDTau " << colres
837 << ": cutoff parameter tau" << Endl;
838 Log() << " By default this value is set to -1.0." << Endl;
839 // "|----------------|---------------------------------------------|"
840 Log() << " This means that the cut off parameter is" << Endl;
841 Log() << " automatically estimated. In most cases" << Endl;
842 Log() << " this should be fine. However, you may want" << Endl;
843 Log() << " to fix this value if you already know it" << Endl;
844 Log() << " and want to reduce on training time." << brk << Endl;
845 Log() << " " << col << "GDTauPrec " << colres
846 << ": precision of estimated tau" << Endl;
847 Log() << " Increase this precision to find a more" << Endl;
848 Log() << " optimum cut-off parameter." << brk << Endl;
849 Log() << " " << col << "GDNStep " << colres
850 << ": number of steps in path search" << Endl;
851 Log() << " If the number of steps is too small, then" << Endl;
852 Log() << " the program will give a warning message." << Endl;
853 Log() << Endl;
854 Log() << "III. WARNING MESSAGES" << Endl;
855 Log() << Endl;
856 Log() << col << "Risk(i+1)>=Risk(i) in path" << colres << brk << Endl;
857 Log() << col << "Chaotic behaviour of risk evolution." << colres << Endl;
858 // "|----------------|---------------------------------------------|"
859 Log() << " The error rate was still decreasing at the end" << Endl;
860 Log() << " By construction the Risk should always decrease." << Endl;
861 Log() << " However, if the training sample is too small or" << Endl;
862 Log() << " the model is overtrained, such warnings can" << Endl;
863 Log() << " occur." << Endl;
864 Log() << " The warnings can safely be ignored if only a" << Endl;
865 Log() << " few (<3) occur. If more warnings are generated," << Endl;
866 Log() << " the fitting fails." << Endl;
867 Log() << " A remedy may be to increase the value" << brk << Endl;
868 Log() << " "
869 << col << "GDValidEveFrac" << colres
870 << " to 1.0 (or a larger value)." << brk << Endl;
871 Log() << " In addition, if "
872 << col << "GDPathEveFrac" << colres
873 << " is too high" << Endl;
874 Log() << " the same warnings may occur since the events" << Endl;
875 Log() << " used for error estimation are also used for" << Endl;
876 Log() << " path estimation." << Endl;
877 Log() << " Another possibility is to modify the model - " << Endl;
878 Log() << " See above on tuning the rule ensemble." << Endl;
879 Log() << Endl;
880 Log() << col << "The error rate was still decreasing at the end of the path"
881 << colres << Endl;
882 Log() << " Too few steps in path! Increase "
883 << col << "GDNSteps" << colres << "." << Endl;
884 Log() << Endl;
885 Log() << col << "Reached minimum early in the search" << colres << Endl;
886
887 Log() << " Minimum was found early in the fitting. This" << Endl;
888 Log() << " may indicate that the used step size "
889 << col << "GDStep" << colres << "." << Endl;
890 Log() << " was too large. Reduce it and rerun." << Endl;
891 Log() << " If the results still are not OK, modify the" << Endl;
892 Log() << " model either by modifying the rule ensemble" << Endl;
893 Log() << " or add/remove linear terms" << Endl;
894}
#define REGISTER_METHOD(CLASS)
for example
#define e(i)
Definition: RSha256.hxx:103
int Int_t
Definition: RtypesCore.h:41
unsigned int UInt_t
Definition: RtypesCore.h:42
const Bool_t kFALSE
Definition: RtypesCore.h:88
bool Bool_t
Definition: RtypesCore.h:59
double Double_t
Definition: RtypesCore.h:55
long long Long64_t
Definition: RtypesCore.h:69
const Bool_t kTRUE
Definition: RtypesCore.h:87
#define ClassImp(name)
Definition: Rtypes.h:365
int type
Definition: TGX11.cxx:120
double sqrt(double)
char * Form(const char *fmt,...)
Bool_t WriteOptionsReference() const
Definition: Config.h:67
Implementation of the CrossEntropy as separation criterion.
Definition: CrossEntropy.h:43
Class that contains all the data information.
Definition: DataSetInfo.h:60
Implementation of a Decision Tree.
Definition: DecisionTree.h:64
Implementation of the GiniIndex as separation criterion.
Definition: GiniIndex.h:63
Virtual base Class for all MVA method.
Definition: MethodBase.h:109
J Friedman's RuleFit method.
Definition: MethodRuleFit.h:47
Double_t GetMvaValue(Double_t *err=0, Double_t *errUpper=0)
returns MVA value for given event
void MakeClassSpecific(std::ostream &, const TString &) const
write specific classifier response
void MakeClassLinear(std::ostream &) const
print out the linear terms
void GetHelpMessage() const
get help message text
void TrainJFRuleFit()
training of rules using Jerome Friedmans implementation
virtual Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t)
RuleFit can handle classification with 2 classes.
void ProcessOptions()
process the options specified by the user
void ReadWeightsFromStream(std::istream &istr)
read rules from an std::istream
void AddWeightsXMLTo(void *parent) const
add the rules to XML node
void InitEventSample(void)
write all Events from the Tree into a vector of Events, that are more easily manipulated.
void MakeClassRuleCuts(std::ostream &) const
print out the rule cuts
void InitMonitorNtuple()
initialize the monitoring ntuple
virtual ~MethodRuleFit(void)
destructor
void Init(void)
default initialization
void WriteMonitoringHistosToFile(void) const
write special monitoring histograms to file (here ntuple)
MethodRuleFit(const TString &jobName, const TString &methodTitle, DataSetInfo &theData, const TString &theOption="")
standard constructor
void ReadWeightsFromXML(void *wghtnode)
read rules from XML node
void DeclareOptions()
define the options (their key words) that can be set in the option string know options.
const Ranking * CreateRanking()
computes ranking of input variables
void TrainTMVARuleFit()
training of rules using TMVA implementation
Implementation of the MisClassificationError as separation criterion.
Ranking for variables in method (implementation)
Definition: Ranking.h:48
A class describing a 'rule cut'.
Definition: RuleCut.h:34
UInt_t GetNvars() const
Definition: RuleCut.h:70
Double_t GetCutMin(Int_t is) const
Definition: RuleCut.h:72
UInt_t GetSelector(Int_t is) const
Definition: RuleCut.h:71
Char_t GetCutDoMin(Int_t is) const
Definition: RuleCut.h:74
Char_t GetCutDoMax(Int_t is) const
Definition: RuleCut.h:75
UInt_t GetNcuts() const
get number of cuts
Definition: RuleCut.cxx:164
Double_t GetCutMax(Int_t is) const
Definition: RuleCut.h:73
Double_t GetLinDP(int i) const
Definition: RuleEnsemble.h:284
Double_t GetLinDM(int i) const
Definition: RuleEnsemble.h:283
const std::vector< Double_t > & GetLinCoefficients() const
Definition: RuleEnsemble.h:269
Double_t GetImportanceRef() const
Definition: RuleEnsemble.h:264
const std::vector< Double_t > & GetLinNorm() const
Definition: RuleEnsemble.h:270
UInt_t GetNLinear() const
Definition: RuleEnsemble.h:273
const std::vector< TMVA::Rule * > & GetRulesConst() const
Definition: RuleEnsemble.h:267
const std::vector< Double_t > & GetLinImportance() const
Definition: RuleEnsemble.h:271
Bool_t IsLinTermOK(int i) const
Definition: RuleEnsemble.h:293
J Friedman's RuleFit method.
Definition: RuleFitAPI.h:50
Bool_t ReadModelSum()
read model from rulefit.sum
Definition: RuleFitAPI.cxx:546
void WelcomeMessage()
welcome message
Definition: RuleFitAPI.cxx:78
void TrainRuleFit()
Definition: RuleFitAPI.h:201
Implementation of a rule.
Definition: Rule.h:48
Double_t GetSupport() const
Definition: Rule.h:140
const RuleCut * GetRuleCut() const
Definition: Rule.h:137
Bool_t IsSignalRule() const
Definition: Rule.h:117
Double_t GetCoefficient() const
Definition: Rule.h:139
const RuleEnsemble * GetRuleEnsemble() const
Definition: Rule.h:138
Double_t GetSSB() const
Definition: Rule.h:115
Double_t GetRelImportance() const
Definition: Rule.h:100
Implementation of the SdivSqrtSplusB as separation criterion.
Timing information for training and evaluation of MVA methods.
Definition: Timer.h:58
const TString & Color(const TString &)
human readable color strings
Definition: Tools.cxx:840
Singleton class for Global types used by TMVA.
Definition: Types.h:73
EAnalysisType
Definition: Types.h:127
@ kClassification
Definition: Types.h:128
@ kTraining
Definition: Types.h:144
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Definition: TObject.cxx:785
virtual void Print(Option_t *option="") const
This method must be overridden when a class wants to print itself.
Definition: TObject.cxx:550
Basic string class.
Definition: TString.h:131
A TTree represents a columnar dataset.
Definition: TTree.h:71
const Int_t n
Definition: legend1.C:16
std::string GetName(const std::string &scope_name)
Definition: Cppyy.cxx:146
create variable transformations
Config & gConfig()
Tools & gTools()
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:158
Double_t Log(Double_t x)
Definition: TMath.h:748