Logo ROOT   6.10/09
Reference Guide
Functions.h
Go to the documentation of this file.
1 // @(#)root/tmva/tmva/dnn:$Id$
2 // Author: Simon Pfreundschuh 20/06/16
3 
4 /*************************************************************************
5  * Copyright (C) 2016, Simon Pfreundschuh *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /////////////////////////////////////////////////////////////////////
13 // Contains function enums for activation and output functions, as //
14 // well as generic evaluation functions, that delegate the call to //
15 // the corresponding evaluation kernel. //
16 /////////////////////////////////////////////////////////////////////
17 
18 #ifndef TMVA_DNN_FUNCTIONS
19 #define TMVA_DNN_FUNCTIONS
20 
21 namespace TMVA
22 {
23 namespace DNN
24 {
25 //______________________________________________________________________________
26 //
27 // Enum Definitions
28 //______________________________________________________________________________
29 
30 /*! Enum that represents layer activation functions. */
32 {
33  kIdentity = 0,
34  kRelu = 1,
35  kSigmoid = 2,
36  kTanh = 3,
37  kSymmRelu = 4,
38  kSoftSign = 5,
39  kGauss = 6
40 };
41 
42 /*! Enum that represents output functions */
43 enum class EOutputFunction
44 {
45  kIdentity = 'I',
46  kSigmoid = 'S',
47  kSoftmax = 'M'
48 };
49 
50 /*! Enum that represents objective functions for the net, i.e. functions
51 * that take the output from the last layer in the net together with the
52 * truths and return the objective function values that is to be minimized
53 * in the training process. */
54 enum class ELossFunction
55 {
56  kCrossEntropy = 'C',
57  kMeanSquaredError = 'R',
59 };
60 
61 /*! Enum representing the regularization type applied for a given layer */
62 enum class ERegularization
63 {
64  kNone = '0',
65  kL1 = '1',
66  kL2 = '2'
67  };
68 
69 /* Enum represnting the initialization method used for this layer. */
70 enum class EInitialization {
71  kGauss = 'G',
72  kUniform = 'U',
73  kIdentity = 'I',
74  kZero = 'Z'
75 };
76 
77 //______________________________________________________________________________
78 //
79 // Activation Functions
80 //______________________________________________________________________________
81 
82 /*! Apply the given activation function to each value in the given
83 * matrix A. */
84 template<typename Architecture_t>
85 inline void evaluate(typename Architecture_t::Matrix_t &A,
87 {
88  switch(f)
89  {
90  case EActivationFunction::kIdentity : break;
91  case EActivationFunction::kRelu : Architecture_t::Relu(A);
92  break;
94  break;
96  break;
97  case EActivationFunction::kSymmRelu : Architecture_t::SymmetricRelu(A);
98  break;
100  break;
102  break;
103  }
104 }
105 
106 
107 /*! Compute the first partial derivative of the activation function for
108 * the values given in matrix A and write the results into B. */
109 //______________________________________________________________________________
110 template<typename Architecture_t>
111 inline void evaluateDerivative(typename Architecture_t::Matrix_t & B,
113  const typename Architecture_t::Matrix_t & A)
114 {
115  switch(f)
116  {
117  case EActivationFunction::kIdentity : Architecture_t::IdentityDerivative(B, A);
118  break;
119  case EActivationFunction::kRelu : Architecture_t::ReluDerivative(B, A);
120  break;
121  case EActivationFunction::kSigmoid : Architecture_t::SigmoidDerivative(B, A);
122  break;
123  case EActivationFunction::kTanh : Architecture_t::TanhDerivative(B, A);
124  break;
125  case EActivationFunction::kSymmRelu : Architecture_t::SymmetricReluDerivative(B, A);
126  break;
127  case EActivationFunction::kSoftSign : Architecture_t::SoftSignDerivative(B, A);
128  break;
129  case EActivationFunction::kGauss : Architecture_t::GaussDerivative(B, A);
130  break;
131  }
132 }
133 
134 //______________________________________________________________________________
135 //
136 // Output Functions
137 //______________________________________________________________________________
138 
139 /*! Apply the given output function to each value in the given
140 * matrix A. */
141 template<typename Architecture_t>
142 inline void evaluate(typename Architecture_t::Matrix_t &A,
144  const typename Architecture_t::Matrix_t &X)
145 {
146  switch(f)
147  {
149  break;
151  break;
152  case EOutputFunction::kSoftmax : Architecture_t::Softmax(A, X);
153  break;
154  }
155 }
156 
157 //______________________________________________________________________________
158 //
159 // Loss Functions
160 //______________________________________________________________________________
161 
162 /*! Compute the value of the objective function f for given activations
163 * of the ouput layer and the truth Y. */
164 template<typename Architecture_t>
166  const typename Architecture_t::Matrix_t & Y,
167  const typename Architecture_t::Matrix_t & output)
168 -> decltype(Architecture_t::CrossEntropy(Y,output))
169 {
170  switch(f)
171  {
173  return Architecture_t::CrossEntropy(Y, output);
175  return Architecture_t::MeanSquaredError(Y, output);
177  return Architecture_t::SoftmaxCrossEntropy(Y, output);
178  }
179  return 0.0;
180 }
181 
182 /*! Compute the gradient of the given output function f for given activations
183 * output of the output layer and truth Y and write the results into dY. */
184 //______________________________________________________________________________
185 template<typename Architecture_t>
186 inline void evaluateGradients(typename Architecture_t::Matrix_t & dY,
188  const typename Architecture_t::Matrix_t &Y,
189  const typename Architecture_t::Matrix_t &output)
190 {
191  switch(f)
192  {
194  Architecture_t::CrossEntropyGradients(dY, Y, output);
195  break;
197  Architecture_t::MeanSquaredErrorGradients(dY, Y, output);
198  break;
200  Architecture_t::SoftmaxCrossEntropyGradients(dY, Y, output);
201  break;
202  }
203 }
204 
205 
206 //______________________________________________________________________________
207 //
208 // Regularization
209 //______________________________________________________________________________
210 
211 /*! Evaluate the regularization functional for a given weight matrix. */
212 template<typename Architecture_t>
213 inline auto regularization(const typename Architecture_t::Matrix_t &A,
215 -> decltype(Architecture_t::L1Regularization(A))
216 {
217  switch(R)
218  {
220  return 0.0;
221  case ERegularization::kL1 :
222  return Architecture_t::L1Regularization(A);
223  case ERegularization::kL2 :
224  return Architecture_t::L2Regularization(A);
225  }
226  return 0.0;
227 }
228 
229 /*! Add the regularization gradient corresponding to weight matrix W, to
230 * the matrix A. */
231 //______________________________________________________________________________
232 template<typename Architecture_t>
233 inline void addRegularizationGradients(typename Architecture_t::Matrix_t &A,
234  const typename Architecture_t::Matrix_t &W,
235  typename Architecture_t::Scalar_t weightDecay,
237 {
238  switch(R)
239  {
241  break;
242  case ERegularization::kL1 :
243  Architecture_t::AddL1RegularizationGradients(A, W, weightDecay);
244  break;
245  case ERegularization::kL2 :
246  Architecture_t::AddL2RegularizationGradients(A, W, weightDecay);
247  break;
248  }
249 }
250 
251 //______________________________________________________________________________
252 //
253 // Initialization
254 //______________________________________________________________________________
255 
256 template<typename Architecture_t>
257 inline void initialize(typename Architecture_t::Matrix_t & A,
259 {
260  switch(m) {
261  case EInitialization::kGauss : Architecture_t::InitializeGauss(A);
262  break;
263  case EInitialization::kUniform : Architecture_t::InitializeUniform(A);
264  break;
265  case EInitialization::kIdentity : Architecture_t::InitializeIdentity(A);
266  break;
267  case EInitialization::kZero : Architecture_t::InitializeZero(A);
268  break;
269  }
270 }
271 
272 } // namespace DNN
273 } // namespace TMVA
274 
275 #endif
static double B[]
void evaluateDerivative(typename Architecture_t::Matrix_t &B, EActivationFunction f, const typename Architecture_t::Matrix_t &A)
Compute the first partial derivative of the activation function for the values given in matrix A and ...
Definition: Functions.h:111
static std::shared_ptr< std::function< double(double)> > Tanh
Definition: NeuralNet.icc:55
static double A[]
void evaluate(typename Architecture_t::Matrix_t &A, EActivationFunction f)
Apply the given activation function to each value in the given matrix A.
Definition: Functions.h:85
EInitialization
Definition: Functions.h:70
static std::shared_ptr< std::function< double(double)> > Sigmoid
Definition: NeuralNet.icc:52
void evaluateGradients(typename Architecture_t::Matrix_t &dY, ELossFunction f, const typename Architecture_t::Matrix_t &Y, const typename Architecture_t::Matrix_t &output)
Compute the gradient of the given output function f for given activations output of the output layer ...
Definition: Functions.h:186
double weightDecay(double error, ItWeight itWeight, ItWeight itWeightEnd, double factorWeightDecay, EnumRegularization eRegularization)
compute the weight decay for regularization (L1 or L2)
Definition: NeuralNet.icc:496
TMarker * m
Definition: textangle.C:8
auto regularization(const typename Architecture_t::Matrix_t &A, ERegularization R) -> decltype(Architecture_t::L1Regularization(A))
Evaluate the regularization functional for a given weight matrix.
Definition: Functions.h:213
static std::shared_ptr< std::function< double(double)> > SoftSign
Definition: NeuralNet.icc:73
void Copy(void *source, void *dest)
void addRegularizationGradients(typename Architecture_t::Matrix_t &A, const typename Architecture_t::Matrix_t &W, typename Architecture_t::Scalar_t weightDecay, ERegularization R)
Add the regularization gradient corresponding to weight matrix W, to the matrix A.
Definition: Functions.h:233
double f(double x)
EOutputFunction
Enum that represents output functions.
Definition: Functions.h:43
ELossFunction
Enum that represents objective functions for the net, i.e.
Definition: Functions.h:54
static std::shared_ptr< std::function< double(double)> > Gauss
Definition: NeuralNet.icc:76
Abstract ClassifierFactory template that handles arbitrary types.
ERegularization
Enum representing the regularization type applied for a given layer.
Definition: Functions.h:62
EActivationFunction
Enum that represents layer activation functions.
Definition: Functions.h:31
void initialize(typename Architecture_t::Matrix_t &A, EInitialization m)
Definition: Functions.h:257
TRandom3 R
a TMatrixD.
Definition: testIO.cxx:28