Logo ROOT  
Reference Guide
ActivationFunctions.hxx
Go to the documentation of this file.
1// @(#)root/tmva/tmva/dnn:$Id$
2// Author: Simon Pfreundschuh 19/07/16
3
4/*************************************************************************
5 * Copyright (C) 2016, Simon Pfreundschuh *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12 ///////////////////////////////////////////////////////////////////
13 // Implementation of the activation functions for multi-threaded //
14 // CPU architectures using Roots TThreadExecutor and BLAS. //
15 ///////////////////////////////////////////////////////////////////
16
18#include <math.h>
19
20#ifdef R__HAS_VDT
21#include "vdt/tanh.h"
22#endif
23
24
25namespace TMVA
26{
27namespace DNN
28{
29
30//______________________________________________________________________________
31template<typename AFloat>
33 const ActivationDescriptor_t /* activationDescr */,
34 const double /* coef */, const AFloat /*alpha */, const AFloat /*beta*/)
35{
36 // scaling and translation is not yet implemented
37 TMVA::DNN::evaluate<TCpu<AFloat>>( X, activFunct);
38}
39//______________________________________________________________________________
40template<typename AFloat>
42 const Tensor_t & dY, const Tensor_t & X,
43 EActivationFunction activFunct,
44 const ActivationDescriptor_t /* activationDescr */,
45 const AFloat /* alpha */, const AFloat /* beta */)
46{
47 // scaling and translation not yet implemented
48 // output tensor (Y) could also be used to speed up derivative calculation
49 // compute dx = f'(x)
50 TMVA::DNN::evaluateDerivative<TCpu<AFloat>>(dX, activFunct, X);
51 // Compute element-wise product. dx = f'(x) * dY
52 Hadamard(dX, dY);
53}
54//______________________________________________________________________________
55template<typename AFloat>
57 const TCpuTensor<AFloat> &/*A*/)
58{
59 auto f = [](AFloat) {return 1.0;};
60 B.Map(f);
61}
62
63//______________________________________________________________________________
64template<typename AFloat>
66{
67 auto f = [](AFloat x) {return (x < 0.0) ? 0.0 : x;};
68 B.Map(f);
69}
70
71//______________________________________________________________________________
72template<typename AFloat>
74 const TCpuTensor<AFloat> &A)
75{
76 auto f = [](AFloat x) {return (x < 0.0) ? 0.0 : 1.0;};
77 B.MapFrom(f, A);
78}
79
80//______________________________________________________________________________
81template<typename AFloat>
83{
84 auto f = [](AFloat x) {return 1.0 / (1.0 + exp(-x));};
85 B.Map(f);
86}
87
88//______________________________________________________________________________
89template<typename AFloat>
91 const TCpuTensor<AFloat> &A)
92{
93 auto f = [](AFloat x) {
94 AFloat sig = 1.0 / (1.0 + exp(-x));
95 return sig * (1.0 - sig);
96 };
97 B.MapFrom(f, A);
98}
99
100//______________________________________________________________________________
101template<typename AFloat>
103{
104 auto f = [](AFloat x) {return tanh(x);};
105 B.Map(f);
106}
107
108//______________________________________________________________________________
109template<typename AFloat>
111 const TCpuTensor<AFloat> &A)
112{
113 auto f = [](AFloat x) {
114 AFloat t = tanh(x);
115 return 1 - t * t;
116 };
117 B.MapFrom(f, A);
118}
119
120#ifdef R__HAS_VDT
121//______________________________________________________________________________
122template <>
124{
125 auto f = [](float x) { return vdt::fast_tanhf(x); };
126 B.Map(f);
127}
128template <>
129void TCpu<double>::FastTanh(TCpuTensor<double> &B)
130{
131 auto f = [](double x) { return vdt::fast_tanh(x); };
132 B.Map(f);
133}
134
135//______________________________________________________________________________
136template <>
137void TCpu<float>::FastTanhDerivative(TCpuTensor<float> &B, const TCpuTensor<float> &A)
138{
139 auto f = [](float x) {
140 double t = vdt::fast_tanhf(x);
141 return 1 - t * t;
142 };
143 B.MapFrom(f, A);
144}
145template <>
146void TCpu<double>::FastTanhDerivative(TCpuTensor<double> &B, const TCpuTensor<double> &A)
147{
148 auto f = [](double x) {
149 double t = vdt::fast_tanh(x);
150 return 1 - t * t;
151 };
152 B.MapFrom(f, A);
153}
154
155#else // when VDT is not available
156//______________________________________________________________________________
157template <typename AFloat>
159{
161}
162
163//______________________________________________________________________________
164template <typename AFloat>
166{
168}
169#endif
170
171//______________________________________________________________________________
172template<typename AFloat>
174{
175 auto f = [](AFloat x) {return fabs(x);};
176 B.Map(f);
177}
178
179//______________________________________________________________________________
180template<typename AFloat>
182 const TCpuTensor<AFloat> &A)
183{
184 auto f = [](AFloat x) {
185 return (x < 0.0) ? -1.0 : 1.0;
186 };
187 B.MapFrom(f, A);
188}
189
190//______________________________________________________________________________
191template<typename AFloat>
193{
194 auto f = [](AFloat x) {return x / (1 + fabs(x));};
195 B.Map(f);
196}
197
198//______________________________________________________________________________
199template<typename AFloat>
201 const TCpuTensor<AFloat> &A)
202{
203 auto f = [](AFloat x) {
204 x = 1.0 + fabs(x);
205 x = 1.0 / (x * x);
206 return x;
207 };
208 B.MapFrom(f, A);
209}
210
211//______________________________________________________________________________
212template<typename AFloat>
214{
215 auto f = [](AFloat x) {return exp(- x * x);};
216 B.Map(f);
217}
218
219//______________________________________________________________________________
220template<typename AFloat>
222 const TCpuTensor<AFloat> &A)
223{
224 auto f = [](AFloat x) {return - 2.0 * x * exp(- x * x);};
225 B.MapFrom(f, A);
226}
227
228} // namespace DNN
229} // namespace TMVA
#define f(i)
Definition: RSha256.hxx:104
static void FastTanh(Tensor_t &B)
static void Gauss(Tensor_t &B)
static void Sigmoid(Tensor_t &B)
static void SoftSign(Tensor_t &B)
static void SymmetricReluDerivative(Tensor_t &B, const Tensor_t &A)
static void SymmetricRelu(Tensor_t &B)
static void TanhDerivative(Tensor_t &B, const Tensor_t &A)
static void Tanh(Tensor_t &B)
static void ActivationFunctionForward(Tensor_t &X, EActivationFunction activFunct, const ActivationDescriptor_t activationDescr, const double coef=0.0, const Scalar_t alpha=1, const Scalar_t beta=0)
static void SoftSignDerivative(Tensor_t &B, const Tensor_t &A)
static void IdentityDerivative(Tensor_t &B, const Tensor_t &A)
static void Relu(Tensor_t &B)
static void ActivationFunctionBackward(Tensor_t &dX, const Tensor_t &Y, const Tensor_t &dY, const Tensor_t &X, EActivationFunction activFunct, const ActivationDescriptor_t activationDescr, const Scalar_t alpha=1, const Scalar_t beta=0)
Computes the gradient of the activation function.
static void GaussDerivative(Tensor_t &B, const Tensor_t &A)
static void SigmoidDerivative(Tensor_t &B, const Tensor_t &A)
static void FastTanhDerivative(Tensor_t &B, const Tensor_t &A)
static void ReluDerivative(Tensor_t &B, const Tensor_t &A)
RVec< PromoteType< T > > exp(const RVec< T > &v)
Definition: RVec.hxx:1744
RVec< PromoteType< T > > tanh(const RVec< T > &v)
Definition: RVec.hxx:1768
Double_t x[n]
Definition: legend1.C:17
static double B[]
static double A[]
VecExpr< UnaryOp< Fabs< T >, VecExpr< A, T, D >, T >, T, D > fabs(const VecExpr< A, T, D > &rhs)
EActivationFunction
Enum that represents layer activation functions.
Definition: Functions.h:32
create variable transformations