Namespaces | |
namespace | Blas |
namespace | CNN |
namespace | Cuda |
namespace | RNN |
Typedefs | |
typedef std::vector< char > | DropContainer |
using | IndexIterator_t = typename std::vector<size_t>::iterator |
using | MatrixInput_t = std::tuple<const TMatrixT<Double_t> &, const TMatrixT<Double_t> &, const TMatrixT<Double_t> &> |
using | MemoryLayout = TMVA::Experimental::MemoryLayout |
typedef std::tuple< Settings &, Batch &, DropContainer & > | pass_through_type |
using | TensorInput |
using | TMVAInput_t |
Enumerations | |
enum class | EActivationFunction { kIdentity = 0 , kRelu = 1 , kSigmoid = 2 , kTanh = 3 , kSymmRelu = 4 , kSoftSign = 5 , kGauss = 6 , kFastTanh = 7 } |
Enum that represents layer activation functions. More... | |
enum class | EInitialization { kGauss = 'G' , kUniform = 'U' , kIdentity = 'I' , kZero = 'Z' , kGlorotNormal = 'X' , kGlorotUniform = 'F' } |
enum class | ELossFunction { kCrossEntropy = 'C' , kMeanSquaredError = 'R' , kSoftmaxCrossEntropy = 'S' } |
Enum that represents objective functions for the net, i.e. More... | |
enum class | EnumFunction { ZERO = '0' , LINEAR = 'L' , TANH = 'T' , RELU = 'R' , SYMMRELU = 'r' , TANHSHIFT = 't' , SIGMOID = 's' , SOFTSIGN = 'S' , GAUSS = 'G' , GAUSSCOMPLEMENT = 'C' } |
enum class | EnumRegularization { NONE , L1 , L2 , L1MAX } |
enum class | EOptimizer { kSGD = 0 , kAdam = 1 , kAdagrad = 2 , kRMSProp = 3 , kAdadelta = 4 } |
Enum representing the optimizer used for training. More... | |
enum class | EOutputFunction { kIdentity = 'I' , kSigmoid = 'S' , kSoftmax = 'M' } |
Enum that represents output functions. More... | |
enum class | ERegularization { kNone = '0' , kL1 = '1' , kL2 = '2' } |
Enum representing the regularization type applied for a given layer. More... | |
enum | MinimizerType { fSteepest } |
< list all the minimizer types More... | |
enum class | ModeErrorFunction { SUMOFSQUARES = 'S' , CROSSENTROPY = 'C' , CROSSENTROPY_MUTUALEXCLUSIVE = 'M' } |
error functions to be chosen from More... | |
enum class | ModeOutput { FETCH } |
enum class | ModeOutputValues : int { DIRECT = 0x01 , SIGMOID = 0x02 , SOFTMAX = 0x04 , BATCHNORMALIZATION = 0x08 } |
enum class | WeightInitializationStrategy { XAVIER , TEST , LAYERSIZE , XAVIERUNIFORM } |
weight initialization strategies to be chosen from More... | |
Variables | |
std::shared_ptr< std::function< double(double)> > | Gauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return exp (-std::pow(value*s,2.0)); }) |
std::shared_ptr< std::function< double(double)> > | GaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return 1.0 - exp (-std::pow(value*s,2.0)); }) |
std::shared_ptr< std::function< double(double)> > | InvGauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return -2.0 * value * s*s * (*Gauss.get ()) (value); }) |
std::shared_ptr< std::function< double(double)> > | InvGaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return +2.0 * value * s*s * (*GaussComplement.get ()) (value); }) |
std::shared_ptr< std::function< double(double)> > | InvLinear = std::make_shared<std::function<double(double)>> ([](double ){ return 1.0; }) |
std::shared_ptr< std::function< double(double)> > | InvReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? 1.0 : 0; }) |
std::shared_ptr< std::function< double(double)> > | InvSigmoid = std::make_shared<std::function<double(double)>> ([](double value){ double s = (*Sigmoid.get ()) (value); return s*(1.0-s); }) |
std::shared_ptr< std::function< double(double)> > | InvSoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 / (1.0 + std::exp (-value)); }) |
std::shared_ptr< std::function< double(double)> > | InvSoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return std::pow ((1.0 - fabs (value)),2.0); }) |
std::shared_ptr< std::function< double(double)> > | InvSymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? 1.0 : value < -margin ? 1.0 : 0; }) |
std::shared_ptr< std::function< double(double)> > | InvTanh = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 - std::pow (value, 2.0); }) |
std::shared_ptr< std::function< double(double)> > | InvTanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return 0.3 + (1.0 - std::pow (value, 2.0)); }) |
std::shared_ptr< std::function< double(double)> > | Linear = std::make_shared<std::function<double(double)>> ([](double value){ return value; }) |
std::shared_ptr< std::function< double(double)> > | ReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? value-margin : 0; }) |
std::shared_ptr< std::function< double(double)> > | Sigmoid = std::make_shared<std::function<double(double)>> ([](double value){ value = std::max (-100.0, std::min (100.0,value)); return 1.0/(1.0 + std::exp (-value)); }) |
std::shared_ptr< std::function< double(double)> > | SoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return std::log (1.0+ std::exp (value)); }) |
std::shared_ptr< std::function< double(double)> > | SoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return value / (1.0 + fabs (value)); }) |
std::shared_ptr< std::function< double(double)> > | SymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? value-margin : value < -margin ? value+margin : 0; }) |
std::shared_ptr< std::function< double(double)> > | Tanh = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value); }) |
std::shared_ptr< std::function< double(double)> > | TanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value-0.3); }) |
std::shared_ptr< std::function< double(double)> > | ZeroFnc = std::make_shared<std::function<double(double)>> ([](double ){ return 0; }) |
typedef std::vector<char> TMVA::DNN::DropContainer |
Definition at line 227 of file NeuralNet.h.
typedef typename std::vector< size_t >::iterator TMVA::DNN::IndexIterator_t = typename std::vector<size_t>::iterator |
Definition at line 42 of file DataLoader.h.
using TMVA::DNN::MatrixInput_t = std::tuple<const TMatrixT<Double_t> &, const TMatrixT<Double_t> &, const TMatrixT<Double_t> &> |
Definition at line 38 of file DataLoader.h.
Definition at line 56 of file CudaTensor.h.
typedef std::tuple<Settings&, Batch&, DropContainer&> TMVA::DNN::pass_through_type |
Definition at line 1294 of file NeuralNet.h.
Definition at line 43 of file TensorDataLoader.h.
typedef std::tuple< const std::vector< Event * > &, const DataSetInfo & > TMVA::DNN::TMVAInput_t |
Definition at line 39 of file DataLoader.h.
Enum that represents layer activation functions.
Enumerator | |
---|---|
kIdentity | |
kRelu | |
kSigmoid | |
kTanh | |
kSymmRelu | |
kSoftSign | |
kGauss | |
kFastTanh |
Definition at line 31 of file Functions.h.
|
strong |
Enumerator | |
---|---|
kGauss | |
kUniform | |
kIdentity | |
kZero | |
kGlorotNormal | |
kGlorotUniform |
Definition at line 72 of file Functions.h.
|
strong |
Enum that represents objective functions for the net, i.e.
functions that take the output from the last layer in the net together with the truths and return the objective function values that is to be minimized in the training process.
Enumerator | |
---|---|
kCrossEntropy | |
kMeanSquaredError | |
kSoftmaxCrossEntropy |
Definition at line 56 of file Functions.h.
|
strong |
Enumerator | |
---|---|
ZERO | |
LINEAR | |
TANH | |
RELU | |
SYMMRELU | |
TANHSHIFT | |
SIGMOID | |
SOFTSIGN | |
GAUSS | |
GAUSSCOMPLEMENT |
Definition at line 156 of file NeuralNet.h.
Enumerator | |
---|---|
NONE | |
L1 | |
L2 | |
L1MAX |
Definition at line 172 of file NeuralNet.h.
|
strong |
Enum representing the optimizer used for training.
Enumerator | |
---|---|
kSGD | |
kAdam | |
kAdagrad | |
kRMSProp | |
kAdadelta |
Definition at line 82 of file Functions.h.
|
strong |
Enum that represents output functions.
Enumerator | |
---|---|
kIdentity | |
kSigmoid | |
kSoftmax |
Definition at line 45 of file Functions.h.
|
strong |
Enum representing the regularization type applied for a given layer.
Enumerator | |
---|---|
kNone | |
kL1 | |
kL2 |
Definition at line 64 of file Functions.h.
error functions to be chosen from
Enumerator | |
---|---|
SUMOFSQUARES | |
CROSSENTROPY | |
CROSSENTROPY_MUTUALEXCLUSIVE |
Definition at line 1038 of file NeuralNet.h.
|
strong |
Enumerator | |
---|---|
FETCH |
Definition at line 1029 of file NeuralNet.h.
|
strong |
Enumerator | |
---|---|
DIRECT | |
SIGMOID | |
SOFTMAX | |
BATCHNORMALIZATION |
Definition at line 178 of file NeuralNet.h.
weight initialization strategies to be chosen from
Enumerator | |
---|---|
XAVIER | |
TEST | |
LAYERSIZE | |
XAVIERUNIFORM |
Definition at line 1049 of file NeuralNet.h.
|
inline |
Add the regularization gradient corresponding to weight matrix W, to the matrix A.
Definition at line 258 of file Functions.h.
apply the activation functions
Definition at line 147 of file NeuralNet.icc.
void TMVA::DNN::applyFunctions | ( | ItValue | itValue, |
ItValue | itValueEnd, | ||
Fnc | fnc, | ||
InvFnc | invFnc, | ||
ItGradient | itGradient ) |
apply the activation functions and compute the gradient
Definition at line 164 of file NeuralNet.icc.
void TMVA::DNN::applyFunctions | ( | ItValue | itValue, |
ItValue | itValueEnd, | ||
ItFunction | itFunction ) |
void TMVA::DNN::applyFunctions | ( | ItValue | itValue, |
ItValue | itValueEnd, | ||
ItFunction | itFunction, | ||
ItInverseFunction | itInverseFunction, | ||
ItGradient | itGradient ) |
void TMVA::DNN::applyWeights | ( | ItSource | itSourceBegin, |
ItSource | itSourceEnd, | ||
ItWeight | itWeight, | ||
ItTarget | itTargetBegin, | ||
ItTarget | itTargetEnd ) |
void TMVA::DNN::applyWeights | ( | ItSource | itSourceBegin, |
ItSource | itSourceEnd, | ||
ItWeight | itWeight, | ||
ItTarget | itTargetBegin, | ||
ItTarget | itTargetEnd, | ||
ItDrop | itDrop ) |
apply weights using drop-out; for no drop out, provide (&bool = true) to itDrop such that *itDrop becomes "true"
itDrop correlates with itSourceBegin
Definition at line 91 of file NeuralNet.icc.
void TMVA::DNN::applyWeightsBackwards | ( | ItSource | itCurrBegin, |
ItSource | itCurrEnd, | ||
ItWeight | itWeight, | ||
ItPrev | itPrevBegin, | ||
ItPrev | itPrevEnd ) |
void TMVA::DNN::applyWeightsBackwards | ( | ItSource | itCurrBegin, |
ItSource | itCurrEnd, | ||
ItWeight | itWeight, | ||
ItPrev | itPrevBegin, | ||
ItPrev | itPrevEnd, | ||
ItDrop | itDrop ) |
apply weights backwards (for backprop); for no drop out, provide (&bool = true) to itDrop such that *itDrop becomes "true"
itDrop correlates with itPrev (to be in agreement with "applyWeights" where it correlates with itSources (same node as itTarget here in applyBackwards)
Definition at line 118 of file NeuralNet.icc.
backward application of the weights (back-propagation of the error)
Definition at line 572 of file NeuralNet.icc.
|
inline |
compute the regularization (L1, L2)
Definition at line 209 of file NeuralNet.icc.
|
inline |
Definition at line 219 of file NeuralNet.icc.
|
inline |
Definition at line 226 of file NeuralNet.icc.
double TMVA::DNN::crossEntropy | ( | ItProbability | itProbabilityBegin, |
ItProbability | itProbabilityEnd, | ||
ItTruth | itTruthBegin, | ||
ItTruth | itTruthEnd, | ||
ItDelta | itDelta, | ||
ItDelta | itDeltaEnd, | ||
ItInvActFnc | itInvActFnc, | ||
double | patternWeight ) |
cross entropy error function
Definition at line 412 of file NeuralNet.icc.
|
inline |
Function to check cuda return code.
Taken from http://stackoverflow.com/questions/14038589/
Definition at line 198 of file CudaMatrix.h.
__global__ void TMVA::DNN::CurandInitializationKernel | ( | unsigned long long | seed, |
curandState_t * | state ) |
Definition at line 26 of file CudaMatrix.cu.
auto TMVA::DNN::debugTensor | ( | const typename Architecture::Tensor_t & | A, |
const std::string | name = "tensor" ) -> void |
Definition at line 582 of file GeneralLayer.h.
|
inline |
Compute the value of the objective function f for given activations of the ouput layer and the truth Y.
Definition at line 198 of file Functions.h.
|
inline |
Apply the given output function to each value in the given tensor A.
Definition at line 175 of file Functions.h.
|
inline |
Apply the given activation function to each value in the given tensor A.
Definition at line 98 of file Functions.h.
|
inline |
Compute the first partial derivative of the activation function for the values given in tensor A and write the results into B.
Definition at line 125 of file Functions.h.
|
inline |
Definition at line 160 of file Functions.h.
|
inline |
Compute the gradient of the given output function f for given activations output of the output layer and truth Y and write the results into dY.
Definition at line 215 of file Functions.h.
|
inline |
Definition at line 152 of file Functions.h.
apply the weights (and functions) in forward direction of the DNN
Definition at line 546 of file NeuralNet.icc.
Definition at line 35 of file NeuralNet.cxx.
|
inline |
Definition at line 282 of file Functions.h.
Definition at line 212 of file NeuralNet.h.
|
inline |
Definition at line 199 of file NeuralNet.h.
|
inline |
Definition at line 204 of file NeuralNet.h.
|
inline |
Definition at line 188 of file NeuralNet.h.
|
inline |
Definition at line 193 of file NeuralNet.h.
Definition at line 52 of file NeuralNet.cxx.
|
inline |
Evaluate the regularization functional for a given weight matrix.
Definition at line 238 of file Functions.h.
double TMVA::DNN::softMaxCrossEntropy | ( | ItOutput | itProbabilityBegin, |
ItOutput | itProbabilityEnd, | ||
ItTruth | itTruthBegin, | ||
ItTruth | itTruthEnd, | ||
ItDelta | itDelta, | ||
ItDelta | itDeltaEnd, | ||
ItInvActFnc | itInvActFnc, | ||
double | patternWeight ) |
soft-max-cross-entropy error function (for mutual exclusive cross-entropy)
Definition at line 458 of file NeuralNet.icc.
Definition at line 60 of file NeuralNet.cxx.
double TMVA::DNN::sumOfSquares | ( | ItOutput | itOutputBegin, |
ItOutput | itOutputEnd, | ||
ItTruth | itTruthBegin, | ||
ItTruth | itTruthEnd, | ||
ItDelta | itDelta, | ||
ItDelta | itDeltaEnd, | ||
ItInvActFnc | itInvActFnc, | ||
double | patternWeight ) |
double TMVA::DNN::sumOfSquares | ( | ItOutput | itOutputBegin, |
ItOutput | itOutputEnd, | ||
ItTruth | itTruthBegin, | ||
ItTruth | , | ||
ItDelta | itDelta, | ||
ItDelta | itDeltaEnd, | ||
InvFnc | invFnc, | ||
double | patternWeight ) |
sum of squares error function
Definition at line 382 of file NeuralNet.icc.
void TMVA::DNN::uniformDouble | ( | Container & | container, |
T | maxValue ) |
Definition at line 42 of file NeuralNet.icc.
Definition at line 43 of file NeuralNet.cxx.
T TMVA::DNN::uniformFromTo | ( | T | from, |
T | to ) |
Definition at line 34 of file NeuralNet.icc.
void TMVA::DNN::update | ( | const LAYERDATA & | prevLayerData, |
LAYERDATA & | currLayerData, | ||
double | weightDecay, | ||
EnumRegularization | regularization ) |
update the node values
Definition at line 600 of file NeuralNet.icc.
void TMVA::DNN::update | ( | ItSource | itSource, |
ItSource | itSourceEnd, | ||
ItDelta | itTargetDeltaBegin, | ||
ItDelta | itTargetDeltaEnd, | ||
ItTargetGradient | itTargetGradientBegin, | ||
ItGradient | itGradient ) |
update the gradients
Definition at line 183 of file NeuralNet.icc.
void TMVA::DNN::update | ( | ItSource | itSource, |
ItSource | itSourceEnd, | ||
ItDelta | itTargetDeltaBegin, | ||
ItDelta | itTargetDeltaEnd, | ||
ItTargetGradient | itTargetGradientBegin, | ||
ItGradient | itGradient, | ||
ItWeight | itWeight, | ||
double | weightDecay ) |
update the gradients, using regularization
Definition at line 237 of file NeuralNet.icc.
double TMVA::DNN::weightDecay | ( | double | error, |
ItWeight | itWeight, | ||
ItWeight | itWeightEnd, | ||
double | factorWeightDecay, | ||
EnumRegularization | eRegularization ) |
compute the weight decay for regularization (L1 or L2)
Definition at line 498 of file NeuralNet.icc.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::Gauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return exp (-std::pow(value*s,2.0)); }) |
Definition at line 12 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::GaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return 1.0 - exp (-std::pow(value*s,2.0)); }) |
Definition at line 13 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvGauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return -2.0 * value * s*s * (*Gauss.get ()) (value); }) |
Definition at line 14 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvGaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return +2.0 * value * s*s * (*GaussComplement.get ()) (value); }) |
Definition at line 15 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvLinear = std::make_shared<std::function<double(double)>> ([](double ){ return 1.0; }) |
Definition at line 16 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? 1.0 : 0; }) |
Definition at line 17 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvSigmoid = std::make_shared<std::function<double(double)>> ([](double value){ double s = (*Sigmoid.get ()) (value); return s*(1.0-s); }) |
Definition at line 18 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvSoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 / (1.0 + std::exp (-value)); }) |
Definition at line 19 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvSoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return std::pow ((1.0 - fabs (value)),2.0); }) |
Definition at line 20 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvSymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? 1.0 : value < -margin ? 1.0 : 0; }) |
Definition at line 21 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvTanh = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 - std::pow (value, 2.0); }) |
Definition at line 22 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::InvTanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return 0.3 + (1.0 - std::pow (value, 2.0)); }) |
Definition at line 23 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::Linear = std::make_shared<std::function<double(double)>> ([](double value){ return value; }) |
Definition at line 24 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::ReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? value-margin : 0; }) |
Definition at line 25 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::Sigmoid = std::make_shared<std::function<double(double)>> ([](double value){ value = std::max (-100.0, std::min (100.0,value)); return 1.0/(1.0 + std::exp (-value)); }) |
Definition at line 26 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::SoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return std::log (1.0+ std::exp (value)); }) |
Definition at line 27 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::SoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return value / (1.0 + fabs (value)); }) |
Definition at line 32 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::SymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? value-margin : value < -margin ? value+margin : 0; }) |
Definition at line 30 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::Tanh = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value); }) |
Definition at line 29 of file NeuralNet.cxx.
std::shared_ptr< std::function< double(double)> > TMVA::DNN::TanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value-0.3); }) |
Definition at line 31 of file NeuralNet.cxx.