template<typename AFloat = Real_t>
class TMVA::DNN::TCuda< AFloat >
The TCuda architecture class.
Low-level interface class for CUDA computing architectures. Contains as public types the declaration of the scalar, matrix and buffer types for this architecture as well as the remaining functions in the low-level interface in the form of static members.
Definition at line 46 of file Cuda.h.
|
static void | ConvLayerForward (std::vector< TCudaMatrix< AFloat >> &output, std::vector< TCudaMatrix< AFloat >> &derivatives, const std::vector< TCudaMatrix< AFloat >> &input, const TCudaMatrix< Scalar_t > &weights, const TCudaMatrix< Scalar_t > &biases, EActivationFunction func, const std::vector< int > &vIndices, size_t nlocalViews, size_t nlocalViewPixels, Scalar_t dropoutProbability, bool applyDropout) |
| Forward propagation in the Convolutional layer. More...
|
|
|
Low-level functions required for the forward propagation of activations through the network.
|
static void | MultiplyTranspose (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &input, const TCudaMatrix< AFloat > &weights) |
| Matrix-multiply input with the transpose of and write the results into output . More...
|
|
static void | AddRowWise (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &biases) |
| Add the vectors biases row-wise to the matrix output. More...
|
|
|
Low-level functions required for the forward propagation of activations through the network.
|
static void | Backward (TCudaMatrix< AFloat > &activationGradientsBackward, TCudaMatrix< AFloat > &weightGradients, TCudaMatrix< AFloat > &biasGradients, TCudaMatrix< AFloat > &df, const TCudaMatrix< AFloat > &activationGradients, const TCudaMatrix< AFloat > &weights, const TCudaMatrix< AFloat > &activationBackward) |
| Perform the complete backward propagation step. More...
|
|
static Matrix_t & | RecurrentLayerBackward (TCudaMatrix< AFloat > &state_gradients_backward, TCudaMatrix< AFloat > &input_weight_gradients, TCudaMatrix< AFloat > &state_weight_gradients, TCudaMatrix< AFloat > &bias_gradients, TCudaMatrix< AFloat > &df, const TCudaMatrix< AFloat > &state, const TCudaMatrix< AFloat > &weights_input, const TCudaMatrix< AFloat > &weights_state, const TCudaMatrix< AFloat > &input, TCudaMatrix< AFloat > &input_gradient) |
| Backward pass for Recurrent Networks. More...
|
|
static void | ScaleAdd (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, Scalar_t beta=1.0) |
| Adds a the elements in matrix B scaled by c to the elements in the matrix A. More...
|
|
static void | Copy (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
| Copy the elements of matrix A into matrix B. More...
|
|
template<typename AMatrix_t > |
static void | CopyDiffArch (TCudaMatrix< Scalar_t > &B, const AMatrix_t &A) |
|
static void | ScaleAdd (std::vector< TCudaMatrix< Scalar_t >> &A, const std::vector< TCudaMatrix< Scalar_t >> &B, Scalar_t beta=1.0) |
| Above functions extended to vectors. More...
|
|
static void | Copy (std::vector< TCudaMatrix< Scalar_t >> &A, const std::vector< TCudaMatrix< Scalar_t >> &B) |
|
template<typename AMatrix_t > |
static void | CopyDiffArch (std::vector< TCudaMatrix< Scalar_t >> &A, const std::vector< AMatrix_t > &B) |
|
|
For each activation function, the low-level interface contains two routines.
One that applies the acitvation function to a matrix and one that evaluate the derivatives of the activation function at the elements of a given matrix and writes the results into the result matrix.
|
static void | Identity (TCudaMatrix< AFloat > &B) |
|
static void | IdentityDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
|
static void | Relu (TCudaMatrix< AFloat > &B) |
|
static void | ReluDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
|
static void | Sigmoid (TCudaMatrix< AFloat > &B) |
|
static void | SigmoidDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
|
static void | Tanh (TCudaMatrix< AFloat > &B) |
|
static void | TanhDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
|
static void | SymmetricRelu (TCudaMatrix< AFloat > &B) |
|
static void | SymmetricReluDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
|
static void | SoftSign (TCudaMatrix< AFloat > &B) |
|
static void | SoftSignDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
|
static void | Gauss (TCudaMatrix< AFloat > &B) |
|
static void | GaussDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
|
|
Loss functions compute a scalar value given the output of the network for a given training input and the expected network prediction Y that quantifies the quality of the prediction.
For each function also a routing that computes the gradients (suffixed by Gradients) must be provided for the starting of the backpropagation algorithm.
|
static AFloat | MeanSquaredError (const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights) |
|
static void | MeanSquaredErrorGradients (TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights) |
|
static AFloat | CrossEntropy (const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights) |
| Sigmoid transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net. More...
|
|
static void | CrossEntropyGradients (TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights) |
|
static AFloat | SoftmaxCrossEntropy (const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights) |
| Softmax transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net. More...
|
|
static void | SoftmaxCrossEntropyGradients (TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights) |
|
|
Output functions transform the activations output of the output layer in the network to a valid prediction YHat for the desired usage of the network, e.g.
the identity function for regression or the sigmoid transformation for two-class classification.
|
static void | Sigmoid (TCudaMatrix< AFloat > &YHat, const TCudaMatrix< AFloat > &) |
|
static void | Softmax (TCudaMatrix< AFloat > &YHat, const TCudaMatrix< AFloat > &) |
|
|
For each regularization type two functions are required, one named <Type>Regularization that evaluates the corresponding regularization functional for a given weight matrix and the Add<Type>RegularizationGradients , that adds the regularization component in the gradients to the provided matrix.
|
static AFloat | L1Regularization (const TCudaMatrix< AFloat > &W) |
|
static void | AddL1RegularizationGradients (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &W, AFloat weightDecay) |
|
static AFloat | L2Regularization (const TCudaMatrix< AFloat > &W) |
|
static void | AddL2RegularizationGradients (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &W, AFloat weightDecay) |
|
|
For each initialization method, one function in the low-level interface is provided.
The naming scheme is
Initialize<Type>
for a given initialization method Type.
|
static void | InitializeGauss (TCudaMatrix< AFloat > &A) |
|
static void | InitializeUniform (TCudaMatrix< AFloat > &A) |
|
static void | InitializeIdentity (TCudaMatrix< AFloat > &A) |
|
static void | InitializeZero (TCudaMatrix< AFloat > &A) |
|
static void | InitializeGlorotUniform (TCudaMatrix< AFloat > &A) |
|
static void | InitializeGlorotNormal (TCudaMatrix< AFloat > &A) |
|
static TRandom & | GetRandomGenerator () |
|
static void | SetRandomSeed (size_t seed) |
|
|
static void | Dropout (TCudaMatrix< AFloat > &A, AFloat p) |
| Apply dropout with activation probability p to the given matrix A and scale the result by reciprocal of p . More...
|
|
|
static void | Im2col (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth) |
| Transform the matrix B in local view format, suitable for convolution, and store it in matrix A . More...
|
|
static void | Im2colIndices (std::vector< int > &V, const TCudaMatrix< AFloat > &B, size_t nLocalViews, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth) |
|
static void | Im2colFast (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, const std::vector< int > &V) |
|
static void | RotateWeights (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t numFilters) |
| Rotates the matrix B , which is representing a weights, and stores them in the matrix A . More...
|
|
static void | AddConvBiases (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &biases) |
| Add the biases in the Convolutional Layer. More...
|
|
|
static void | ConvLayerBackward (std::vector< TCudaMatrix< AFloat >> &activationGradientsBackward, TCudaMatrix< AFloat > &weightGradients, TCudaMatrix< AFloat > &biasGradients, std::vector< TCudaMatrix< AFloat >> &df, const std::vector< TCudaMatrix< AFloat >> &activationGradients, const TCudaMatrix< AFloat > &weights, const std::vector< TCudaMatrix< AFloat >> &activationBackward, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews) |
| Perform the complete backward propagation step in a Convolutional Layer. More...
|
|
static void | CalculateConvActivationGradients (std::vector< TCudaMatrix< AFloat >> &activationGradientsBackward, std::vector< TCudaMatrix< AFloat >> &df, const TCudaMatrix< AFloat > &weights, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth) |
| Utility function for calculating the activation gradients of the layer before the convolutional layer. More...
|
|
static void | CalculateConvWeightGradients (TCudaMatrix< AFloat > &weightGradients, std::vector< TCudaMatrix< AFloat >> &df, const std::vector< TCudaMatrix< AFloat >> &activations_backward, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews) |
| Utility function for calculating the weight gradients of the convolutional layer. More...
|
|
static void | CalculateConvBiasGradients (TCudaMatrix< AFloat > &biasGradients, std::vector< TCudaMatrix< AFloat >> &df, size_t batchSize, size_t depth, size_t nLocalViews) |
| Utility function for calculating the bias gradients of the convolutional layer. More...
|
|
|
static void | Downsample (TCudaMatrix< AFloat > &A, TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &C, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols) |
| Downsample the matrix C to the matrix A , using max operation, such that the winning indices are stored in matrix B . More...
|
|
|
static void | MaxPoolLayerBackward (std::vector< TCudaMatrix< AFloat >> &activationGradientsBackward, const std::vector< TCudaMatrix< AFloat >> &activationGradients, const std::vector< TCudaMatrix< AFloat >> &indexMatrix, size_t batchSize, size_t depth, size_t nLocalViews) |
| Perform the complete backward propagation step in a Pooling Layer. More...
|
|
|
static void | Reshape (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B) |
| Transform the matrix B to a matrix with different dimensions A . More...
|
|
static void | Flatten (TCudaMatrix< AFloat > &A, const std::vector< TCudaMatrix< AFloat >> &B, size_t size, size_t nRows, size_t nCols) |
| Flattens the tensor B , such that each matrix, is stretched in one row, resulting with a matrix A . More...
|
|
static void | Deflatten (std::vector< TCudaMatrix< AFloat >> &A, const TCudaMatrix< AFloat > &B, size_t index, size_t nRows, size_t nCols) |
| Transforms each row of B to a matrix and stores it in the tensor B . More...
|
|
static void | Rearrange (std::vector< TCudaMatrix< AFloat >> &out, const std::vector< TCudaMatrix< AFloat >> &in) |
| Rearrage data accoring to time fill B x T x D out with T x B x D matrix in. More...
|
|
|
Additional arithmetic on CUDA matrices used to implement the low-level interface.
|
static void | Multiply (TCudaMatrix< AFloat > &C, const TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B) |
| Standard multiplication of two matrices A and B with the result being written into C. More...
|
|
static void | TransposeMultiply (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &input, const TCudaMatrix< AFloat > &Weights) |
| Matrix multiplication of two matrices A and B^T (transposed) with the result being written into C. More...
|
|
static void | Hadamard (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B) |
| In-place Hadamard (element-wise) product of matrices A and B with the result being written into A . More...
|
|
static void | SumColumns (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A) |
| Sum columns of (m x n) matrixx A and write the results into the first m elements in A . More...
|
|
static AFloat | Sum (const TCudaMatrix< AFloat > &A) |
| Compute the sum of all elements in A . More...
|
|