template<typename AReal = Real_t>
class TMVA::DNN::TCpu< AReal >
The TCpu architecture class.
Low-level interface class for multi-threaded CPU architectures. Contains as public types the declaration of the scalar, matrix and data loader types for this architecture as well as the remaining functions in the low-level interface in the form of static members.
Definition at line 44 of file Cpu.h.
|
static void | ConvLayerForward (std::vector< TCpuMatrix< Scalar_t > > &output, std::vector< TCpuMatrix< Scalar_t > > &derivatives, const std::vector< TCpuMatrix< Scalar_t > > &input, const TCpuMatrix< Scalar_t > &weights, const TCpuMatrix< Scalar_t > &biases, const DNN::CNN::TConvParams ¶ms, EActivationFunction activFunc, std::vector< TCpuMatrix< Scalar_t > > &) |
| Forward propagation in the Convolutional layer. More...
|
|
static void | PrepareInternals (std::vector< TCpuMatrix< Scalar_t > > &) |
| Dummy placeholder - preparation is currently only required for the CUDA architecture. More...
|
|
|
Low-level functions required for the forward propagation of activations through the network.
|
static void | MultiplyTranspose (TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &input, const TCpuMatrix< Scalar_t > &weights) |
| Matrix-multiply input with the transpose of \pweights and write the results into output . More...
|
|
static void | AddRowWise (TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &biases) |
| Add the vectors biases row-wise to the matrix output. More...
|
|
|
Low-level functions required for the forward propagation of activations through the network.
|
static void | Backward (TCpuMatrix< Scalar_t > &activationGradientsBackward, TCpuMatrix< Scalar_t > &weightGradients, TCpuMatrix< Scalar_t > &biasGradients, TCpuMatrix< Scalar_t > &df, const TCpuMatrix< Scalar_t > &activationGradients, const TCpuMatrix< Scalar_t > &weights, const TCpuMatrix< Scalar_t > &activationBackward) |
| Perform the complete backward propagation step. More...
|
|
static Matrix_t & | RecurrentLayerBackward (TCpuMatrix< Scalar_t > &state_gradients_backward, TCpuMatrix< Scalar_t > &input_weight_gradients, TCpuMatrix< Scalar_t > &state_weight_gradients, TCpuMatrix< Scalar_t > &bias_gradients, TCpuMatrix< Scalar_t > &df, const TCpuMatrix< Scalar_t > &state, const TCpuMatrix< Scalar_t > &weights_input, const TCpuMatrix< Scalar_t > &weights_state, const TCpuMatrix< Scalar_t > &input, TCpuMatrix< Scalar_t > &input_gradient) |
| Backward pass for Recurrent Networks. More...
|
|
static void | ScaleAdd (TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &B, Scalar_t beta=1.0) |
| Adds a the elements in matrix B scaled by c to the elements in the matrix A. More...
|
|
static void | Copy (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
template<typename AMatrix_t > |
static void | CopyDiffArch (TCpuMatrix< Scalar_t > &B, const AMatrix_t &A) |
|
static void | ScaleAdd (std::vector< TCpuMatrix< Scalar_t > > &A, const std::vector< TCpuMatrix< Scalar_t > > &B, Scalar_t beta=1.0) |
| Above functions extended to vectors. More...
|
|
static void | Copy (std::vector< TCpuMatrix< Scalar_t > > &A, const std::vector< TCpuMatrix< Scalar_t > > &B) |
|
template<typename AMatrix_t > |
static void | CopyDiffArch (std::vector< TCpuMatrix< Scalar_t > > &A, const std::vector< AMatrix_t > &B) |
|
|
For each activation function, the low-level interface contains two routines.
One that applies the acitvation function to a matrix and one that evaluate the derivatives of the activation function at the elements of a given matrix and writes the results into the result matrix.
|
static void | IdentityDerivative (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
static void | Relu (TCpuMatrix< Scalar_t > &B) |
|
static void | ReluDerivative (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
static void | Sigmoid (TCpuMatrix< Scalar_t > &B) |
|
static void | SigmoidDerivative (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
static void | Tanh (TCpuMatrix< Scalar_t > &B) |
|
static void | TanhDerivative (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
static void | SymmetricRelu (TCpuMatrix< Scalar_t > &B) |
|
static void | SymmetricReluDerivative (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
static void | SoftSign (TCpuMatrix< Scalar_t > &B) |
|
static void | SoftSignDerivative (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
static void | Gauss (TCpuMatrix< Scalar_t > &B) |
|
static void | GaussDerivative (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A) |
|
|
Loss functions compute a scalar value given the output of the network for a given training input and the expected network prediction Y that quantifies the quality of the prediction.
For each function also a routing that computes the gradients (suffixed by Gradients) must be provided for the starting of the backpropagation algorithm.
|
static Scalar_t | MeanSquaredError (const TCpuMatrix< Scalar_t > &Y, const TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &weights) |
|
static void | MeanSquaredErrorGradients (TCpuMatrix< Scalar_t > &dY, const TCpuMatrix< Scalar_t > &Y, const TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &weights) |
|
static Scalar_t | CrossEntropy (const TCpuMatrix< Scalar_t > &Y, const TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &weights) |
| Sigmoid transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net. More...
|
|
static void | CrossEntropyGradients (TCpuMatrix< Scalar_t > &dY, const TCpuMatrix< Scalar_t > &Y, const TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &weights) |
|
static Scalar_t | SoftmaxCrossEntropy (const TCpuMatrix< Scalar_t > &Y, const TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &weights) |
| Softmax transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net. More...
|
|
static void | SoftmaxCrossEntropyGradients (TCpuMatrix< Scalar_t > &dY, const TCpuMatrix< Scalar_t > &Y, const TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &weights) |
|
|
Output functions transform the activations output of the output layer in the network to a valid prediction YHat for the desired usage of the network, e.g.
the identity function for regression or the sigmoid transformation for two-class classification.
|
static void | Sigmoid (TCpuMatrix< Scalar_t > &YHat, const TCpuMatrix< Scalar_t > &) |
|
static void | Softmax (TCpuMatrix< Scalar_t > &YHat, const TCpuMatrix< Scalar_t > &) |
|
|
For each regularization type two functions are required, one named <Type>Regularization that evaluates the corresponding regularization functional for a given weight matrix and the Add<Type>RegularizationGradients , that adds the regularization component in the gradients to the provided matrix.
|
static Scalar_t | L1Regularization (const TCpuMatrix< Scalar_t > &W) |
|
static void | AddL1RegularizationGradients (TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &W, Scalar_t weightDecay) |
|
static Scalar_t | L2Regularization (const TCpuMatrix< Scalar_t > &W) |
|
static void | AddL2RegularizationGradients (TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &W, Scalar_t weightDecay) |
|
|
For each initialization method, one function in the low-level interface is provided.
The naming scheme is
Initialize<Type>
for a given initialization method Type.
|
static void | InitializeGauss (TCpuMatrix< Scalar_t > &A) |
|
static void | InitializeUniform (TCpuMatrix< Scalar_t > &A) |
|
static void | InitializeIdentity (TCpuMatrix< Scalar_t > &A) |
|
static void | InitializeZero (TCpuMatrix< Scalar_t > &A) |
|
static void | InitializeGlorotNormal (TCpuMatrix< Scalar_t > &A) |
| Truncated normal initialization (Glorot, called also Xavier normal) The values are sample with a normal distribution with stddev = sqrt(2/N_input + N_output) and values larger than 2 * stddev are discarded See Glorot & Bengio, AISTATS 2010 - http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf. More...
|
|
static void | InitializeGlorotUniform (TCpuMatrix< Scalar_t > &A) |
| Sample from a uniform distribution in range [ -lim,+lim] where lim = sqrt(6/N_in+N_out). More...
|
|
static TRandom & | GetRandomGenerator () |
|
static void | SetRandomSeed (size_t seed) |
|
|
static void | Dropout (TCpuMatrix< Scalar_t > &A, Scalar_t p) |
| Apply dropout with activation probability p to the given matrix A and scale the result by reciprocal of p . More...
|
|
|
static size_t | calculateDimension (size_t imgDim, size_t fltDim, size_t padding, size_t stride) |
| Calculate how many neurons "fit" in the output layer, given the input as well as the layer's hyperparameters. More...
|
|
static void | Im2col (TCpuMatrix< AReal > &A, const TCpuMatrix< AReal > &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth) |
| Transform the matrix B in local view format, suitable for convolution, and store it in matrix A. More...
|
|
static void | Im2colIndices (std::vector< int > &V, const TCpuMatrix< AReal > &B, size_t nLocalViews, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth) |
|
static void | Im2colFast (TCpuMatrix< AReal > &A, const TCpuMatrix< AReal > &B, const std::vector< int > &V) |
|
static void | RotateWeights (TCpuMatrix< AReal > &A, const TCpuMatrix< AReal > &B, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t numFilters) |
| Rotates the matrix B , which is representing a weights, and stores them in the matrix A . More...
|
|
static void | AddConvBiases (TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &biases) |
| Add the biases in the Convolutional Layer. More...
|
|
|
static void | ConvLayerBackward (std::vector< TCpuMatrix< Scalar_t > > &activationGradientsBackward, TCpuMatrix< Scalar_t > &weightGradients, TCpuMatrix< Scalar_t > &biasGradients, std::vector< TCpuMatrix< Scalar_t > > &df, const std::vector< TCpuMatrix< Scalar_t > > &activationGradients, const TCpuMatrix< Scalar_t > &weights, const std::vector< TCpuMatrix< Scalar_t > > &activationBackward, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews) |
| Perform the complete backward propagation step in a Convolutional Layer. More...
|
|
static void | CalculateConvActivationGradients (std::vector< TCpuMatrix< Scalar_t > > &activationGradientsBackward, const std::vector< TCpuMatrix< Scalar_t > > &df, const TCpuMatrix< Scalar_t > &weights, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth) |
| Utility function for calculating the activation gradients of the layer before the convolutional layer. More...
|
|
static void | CalculateConvWeightGradients (TCpuMatrix< Scalar_t > &weightGradients, const std::vector< TCpuMatrix< Scalar_t > > &df, const std::vector< TCpuMatrix< Scalar_t > > &activations_backward, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews) |
| Utility function for calculating the weight gradients of the convolutional layer. More...
|
|
static void | CalculateConvBiasGradients (TCpuMatrix< Scalar_t > &biasGradients, const std::vector< TCpuMatrix< Scalar_t > > &df, size_t batchSize, size_t depth, size_t nLocalViews) |
| Utility function for calculating the bias gradients of the convolutional layer. More...
|
|
|
static void | Downsample (TCpuMatrix< AReal > &A, TCpuMatrix< AReal > &B, const TCpuMatrix< AReal > &C, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols) |
| Downsample the matrix C to the matrix A , using max operation, such that the winning indices are stored in matrix B . More...
|
|
|
static void | MaxPoolLayerBackward (TCpuMatrix< AReal > &activationGradientsBackward, const TCpuMatrix< AReal > &activationGradients, const TCpuMatrix< AReal > &indexMatrix, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t nLocalViews) |
| Perform the complete backward propagation step in a Pooling Layer. More...
|
|
|
static void | Reshape (TCpuMatrix< AReal > &A, const TCpuMatrix< AReal > &B) |
| Transform the matrix B to a matrix with different dimensions A . More...
|
|
static void | Flatten (TCpuMatrix< AReal > &A, const std::vector< TCpuMatrix< AReal > > &B, size_t size, size_t nRows, size_t nCols) |
| Flattens the tensor B , such that each matrix, is stretched in one row, resulting with a matrix A . More...
|
|
static void | Deflatten (std::vector< TCpuMatrix< AReal > > &A, const TCpuMatrix< AReal > &B, size_t index, size_t nRows, size_t nCols) |
| Transforms each row of B to a matrix and stores it in the tensor B . More...
|
|
static void | Rearrange (std::vector< TCpuMatrix< AReal > > &out, const std::vector< TCpuMatrix< AReal > > &in) |
| Rearrage data accoring to time fill B x T x D out with T x B x D matrix in. More...
|
|
|
Additional arithmetic on CUDA matrices used to implement the low-level interface.
|
static void | Multiply (TCpuMatrix< Scalar_t > &C, const TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &B) |
| Standard multiplication of two matrices A and B with the result being written into C. More...
|
|
static void | TransposeMultiply (TCpuMatrix< Scalar_t > &output, const TCpuMatrix< Scalar_t > &input, const TCpuMatrix< Scalar_t > &Weights, Scalar_t alpha=1.0, Scalar_t beta=0.) |
| Matrix multiplication of two matrices A and B^T (transposed) with the result being written into C. More...
|
|
static void | Hadamard (TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &B) |
| In-place Hadamard (element-wise) product of matrices A and B with the result being written into A . More...
|
|
static void | SumColumns (TCpuMatrix< Scalar_t > &B, const TCpuMatrix< Scalar_t > &A, Scalar_t alpha=1.0, Scalar_t beta=0.) |
| Sum columns of (m x n) matrixx A and write the results into the first m elements in A . More...
|
|
static Scalar_t | Sum (const TCpuMatrix< Scalar_t > &A) |
| Compute the sum of all elements in A . More...
|
|
static bool | AlmostEquals (const TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &B, double epsilon=0.1) |
| Check two matrices for equality, taking floating point arithmetic errors into account. More...
|
|
static void | ConstAdd (TCpuMatrix< Scalar_t > &A, Scalar_t beta) |
| Add the constant beta to all the elements of matrix A and write the result into A . More...
|
|
static void | ConstMult (TCpuMatrix< Scalar_t > &A, Scalar_t beta) |
| Multiply the constant beta to all the elements of matrix A and write the result into A . More...
|
|
static void | ReciprocalElementWise (TCpuMatrix< Scalar_t > &A) |
| Reciprocal each element of the matrix A and write the result into A . More...
|
|
static void | SquareElementWise (TCpuMatrix< Scalar_t > &A) |
| Square each element of the matrix A and write the result into A . More...
|
|
static void | SqrtElementWise (TCpuMatrix< Scalar_t > &A) |
| Square root each element of the matrix A and write the result into A . More...
|
|
static void | AdamUpdate (TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &M, const TCpuMatrix< Scalar_t > &V, Scalar_t alpha, Scalar_t eps) |
| Adam updates. More...
|
|
static void | AdamUpdateFirstMom (TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &B, Scalar_t beta) |
|
static void | AdamUpdateSecondMom (TCpuMatrix< Scalar_t > &A, const TCpuMatrix< Scalar_t > &B, Scalar_t beta) |
|