template<typename AFloat = Real_t>
class TMVA::DNN::TCuda< AFloat >

The TCuda architecture class.

Low-level interface class for CUDA computing architectures. Contains as public types the declaration of the scalar, matrix and buffer types for this architecture as well as the remaining functions in the low-level interface in the form of static members.

Definition at line 46 of file Cuda.h.

Public Types
using	DeviceBuffer_t = TCudaDeviceBuffer< AFloat >

using	HostBuffer_t = TCudaHostBuffer< AFloat >

using	Matrix_t = TCudaMatrix< AFloat >

using	Scalar_t = AFloat

Static Public Member Functions
static void	ConvLayerForward (std::vector< TCudaMatrix< AFloat > > &output, std::vector< TCudaMatrix< AFloat > > &derivatives, const std::vector< TCudaMatrix< AFloat > > &input, const TCudaMatrix< AFloat > &weights, const TCudaMatrix< AFloat > &biases, const DNN::CNN::TConvParams &params, EActivationFunction activFunc, std::vector< TCudaMatrix< AFloat > > &inputPrime)
	Forward propagation in the Convolutional layer. More...

Forward Propagation
Low-level functions required for the forward propagation of activations through the network.
static void	MultiplyTranspose (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &input, const TCudaMatrix< AFloat > &weights)
	Matrix-multiply `input` with the transpose of \pweights and write the results into `output`. More...

static void	AddRowWise (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &biases)
	Add the vectors biases row-wise to the matrix output. More...

Backward Propagation
Low-level functions required for the forward propagation of activations through the network.
static void	Backward (TCudaMatrix< AFloat > &activationGradientsBackward, TCudaMatrix< AFloat > &weightGradients, TCudaMatrix< AFloat > &biasGradients, TCudaMatrix< AFloat > &df, const TCudaMatrix< AFloat > &activationGradients, const TCudaMatrix< AFloat > &weights, const TCudaMatrix< AFloat > &activationBackward)
	Perform the complete backward propagation step. More...

static Matrix_t &	RecurrentLayerBackward (TCudaMatrix< AFloat > &state_gradients_backward, TCudaMatrix< AFloat > &input_weight_gradients, TCudaMatrix< AFloat > &state_weight_gradients, TCudaMatrix< AFloat > &bias_gradients, TCudaMatrix< AFloat > &df, const TCudaMatrix< AFloat > &state, const TCudaMatrix< AFloat > &weights_input, const TCudaMatrix< AFloat > &weights_state, const TCudaMatrix< AFloat > &input, TCudaMatrix< AFloat > &input_gradient)
	Backward pass for Recurrent Networks. More...

static void	ScaleAdd (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, Scalar_t beta=1.0)
	Adds a the elements in matrix B scaled by c to the elements in the matrix A. More...

static void	Copy (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
	Copy the elements of matrix A into matrix B. More...

template<typename AMatrix_t >
static void	CopyDiffArch (TCudaMatrix< Scalar_t > &B, const AMatrix_t &A)

static void	ScaleAdd (std::vector< TCudaMatrix< Scalar_t > > &A, const std::vector< TCudaMatrix< Scalar_t > > &B, Scalar_t beta=1.0)
	Above functions extended to vectors. More...

static void	Copy (std::vector< TCudaMatrix< Scalar_t > > &A, const std::vector< TCudaMatrix< Scalar_t > > &B)

template<typename AMatrix_t >
static void	CopyDiffArch (std::vector< TCudaMatrix< Scalar_t > > &A, const std::vector< AMatrix_t > &B)

Activation Functions
For each activation function, the low-level interface contains two routines. One that applies the acitvation function to a matrix and one that evaluate the derivatives of the activation function at the elements of a given matrix and writes the results into the result matrix.
static void	Identity (TCudaMatrix< AFloat > &B)

static void	IdentityDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)

static void	Relu (TCudaMatrix< AFloat > &B)

static void	ReluDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)

static void	Sigmoid (TCudaMatrix< AFloat > &B)

static void	SigmoidDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)

static void	Tanh (TCudaMatrix< AFloat > &B)

static void	TanhDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)

static void	SymmetricRelu (TCudaMatrix< AFloat > &B)

static void	SymmetricReluDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)

static void	SoftSign (TCudaMatrix< AFloat > &B)

static void	SoftSignDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)

static void	Gauss (TCudaMatrix< AFloat > &B)

static void	GaussDerivative (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)

Loss Functions
Loss functions compute a scalar value given the `output` of the network for a given training input and the expected network prediction `Y` that quantifies the quality of the prediction. For each function also a routing that computes the gradients (suffixed by Gradients) must be provided for the starting of the backpropagation algorithm.
static AFloat	MeanSquaredError (const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights)

static void	MeanSquaredErrorGradients (TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights)

static AFloat	CrossEntropy (const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights)
	Sigmoid transformation is implicitly applied, thus `output` should hold the linear activations of the last layer in the net. More...

static void	CrossEntropyGradients (TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights)

static AFloat	SoftmaxCrossEntropy (const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights)
	Softmax transformation is implicitly applied, thus `output` should hold the linear activations of the last layer in the net. More...

static void	SoftmaxCrossEntropyGradients (TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &weights)

Output Functions
Output functions transform the activations `output` of the output layer in the network to a valid prediction `YHat` for the desired usage of the network, e.g. the identity function for regression or the sigmoid transformation for two-class classification.
static void	Sigmoid (TCudaMatrix< AFloat > &YHat, const TCudaMatrix< AFloat > &)

static void	Softmax (TCudaMatrix< AFloat > &YHat, const TCudaMatrix< AFloat > &)

Regularization
For each regularization type two functions are required, one named `<Type>Regularization` that evaluates the corresponding regularization functional for a given weight matrix and the `Add<Type>RegularizationGradients`, that adds the regularization component in the gradients to the provided matrix.
static AFloat	L1Regularization (const TCudaMatrix< AFloat > &W)

static void	AddL1RegularizationGradients (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &W, AFloat weightDecay)

static AFloat	L2Regularization (const TCudaMatrix< AFloat > &W)

static void	AddL2RegularizationGradients (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &W, AFloat weightDecay)

Initialization
For each initialization method, one function in the low-level interface is provided. The naming scheme is Initialize<Type> for a given initialization method Type.
static void	InitializeGauss (TCudaMatrix< AFloat > &A)

static void	InitializeUniform (TCudaMatrix< AFloat > &A)

static void	InitializeIdentity (TCudaMatrix< AFloat > &A)

static void	InitializeZero (TCudaMatrix< AFloat > &A)

static void	InitializeGlorotUniform (TCudaMatrix< AFloat > &A)

static void	InitializeGlorotNormal (TCudaMatrix< AFloat > &A)

static TRandom &	GetRandomGenerator ()

static void	SetRandomSeed (size_t seed)

Dropout
static void	Dropout (TCudaMatrix< AFloat > &A, AFloat p)
	Apply dropout with activation probability `p` to the given matrix `A` and scale the result by reciprocal of `p`. More...

Forward Propagation in Convolutional Layer
static void	PrepareInternals (std::vector< TCudaMatrix< AFloat > > &inputPrime)
	Attaches a cuda stream to each matrix in order to accomodate parallel kernel launches. More...

static size_t	calculateDimension (size_t imgDim, size_t fltDim, size_t padding, size_t stride)
	Calculate how many neurons "fit" in the output layer, given the input as well as the layer's hyperparameters. More...

static void	Im2col (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth)
	Transform the matrix `B` in local view format, suitable for convolution, and store it in matrix `A`. More...

static void	Im2colIndices (std::vector< int > &, const TCudaMatrix< AFloat > &, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t)

static void	Im2colFast (TCudaMatrix< AFloat > &, const TCudaMatrix< AFloat > &, const std::vector< int > &)

static void	RotateWeights (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t numFilters)
	Rotates the matrix `B`, which is representing a weights, and stores them in the matrix `A`. More...

static void	AddConvBiases (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &biases)
	Add the biases in the Convolutional Layer. More...

Backward Propagation in Convolutional Layer
static void	ConvLayerBackward (std::vector< TCudaMatrix< AFloat > > &activationGradientsBackward, TCudaMatrix< AFloat > &weightGradients, TCudaMatrix< AFloat > &biasGradients, std::vector< TCudaMatrix< AFloat > > &df, const std::vector< TCudaMatrix< AFloat > > &activationGradients, const TCudaMatrix< AFloat > &weights, const std::vector< TCudaMatrix< AFloat > > &activationBackward, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews)
	Perform the complete backward propagation step in a Convolutional Layer. More...

static void	CalculateConvActivationGradients (std::vector< TCudaMatrix< AFloat > > &activationGradientsBackward, std::vector< TCudaMatrix< AFloat > > &df, const TCudaMatrix< AFloat > &weights, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth)
	Utility function for calculating the activation gradients of the layer before the convolutional layer. More...

static void	CalculateConvWeightGradients (TCudaMatrix< AFloat > &weightGradients, std::vector< TCudaMatrix< AFloat > > &df, const std::vector< TCudaMatrix< AFloat > > &activations_backward, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews)
	Utility function for calculating the weight gradients of the convolutional layer. More...

static void	CalculateConvBiasGradients (TCudaMatrix< AFloat > &biasGradients, std::vector< TCudaMatrix< AFloat > > &df, size_t batchSize, size_t depth, size_t nLocalViews)
	Utility function for calculating the bias gradients of the convolutional layer. More...

Forward Propagation in Max Pooling Layer
static void	Downsample (TCudaMatrix< AFloat > &A, TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &C, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols)
	Downsample the matrix `C` to the matrix `A`, using max operation, such that the winning indices are stored in matrix `B`. More...

Backward Propagation in Max Pooling Layer
static void	MaxPoolLayerBackward (TCudaMatrix< AFloat > &activationGradientsBackward, const TCudaMatrix< AFloat > &activationGradients, const TCudaMatrix< AFloat > &indexMatrix, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t nLocalViews)
	Perform the complete backward propagation step in a Pooling Layer. More...

Forward and Backward Propagation in Reshape Layer
static void	Reshape (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B)
	Transform the matrix `B` to a matrix with different dimensions `A`. More...

static void	Flatten (TCudaMatrix< AFloat > &A, const std::vector< TCudaMatrix< AFloat > > &B, size_t size, size_t nRows, size_t nCols)
	Flattens the tensor `B`, such that each matrix, is stretched in one row, resulting with a matrix `A`. More...

static void	Deflatten (std::vector< TCudaMatrix< AFloat > > &A, const TCudaMatrix< AFloat > &B, size_t index, size_t nRows, size_t nCols)
	Transforms each row of `B` to a matrix and stores it in the tensor `B`. More...

static void	Rearrange (std::vector< TCudaMatrix< AFloat > > &out, const std::vector< TCudaMatrix< AFloat > > &in)
	Rearrage data accoring to time fill B x T x D out with T x B x D matrix in. More...

Additional Arithmetic Functions
Additional arithmetic on CUDA matrices used to implement the low-level interface.
static void	Multiply (TCudaMatrix< AFloat > &C, const TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B)
	Standard multiplication of two matrices `A` and `B` with the result being written into C. More...

static void	TransposeMultiply (TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &input, const TCudaMatrix< AFloat > &Weights)
	Matrix multiplication of two matrices `A` and `B^T` (transposed) with the result being written into C. More...

static void	Hadamard (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B)
	In-place Hadamard (element-wise) product of matrices `A` and `B` with the result being written into `A`. More...

static void	SumColumns (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
	Sum columns of (m x n) matrix `A` and write the results into the first m elements in `B`. More...

static void	SumRows (TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
	Sum rows of (m x n) matrix `A` and write the results into the first m elements in `B`. More...

static AFloat	Sum (const TCudaMatrix< AFloat > &A)
	Compute the sum of all elements in `A`. More...

static bool	AlmostEquals (const TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, double epsilon=0.1)
	Check two matrices for equality, taking floating point arithmetic errors into account. More...

static void	ConstAdd (TCudaMatrix< AFloat > &A, AFloat beta)
	Add the constant `beta` to all the elements of matrix `A` and write the result into `A`. More...

static void	ConstMult (TCudaMatrix< AFloat > &A, AFloat beta)
	Multiply the constant `beta` to all the elements of matrix `A` and write the result into `A`. More...

static void	ReciprocalElementWise (TCudaMatrix< AFloat > &A)
	Reciprocal each element of the matrix `A` and write the result into `A`. More...

static void	SquareElementWise (TCudaMatrix< AFloat > &A)
	Square each element of the matrix `A` and write the result into `A`. More...

static void	SqrtElementWise (TCudaMatrix< AFloat > &A)
	Square root each element of the matrix `A` and write the result into `A`. More...

static void	AdamUpdate (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &M, const TCudaMatrix< AFloat > &V, AFloat alpha, AFloat eps)

static void	AdamUpdateFirstMom (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, AFloat beta)

static void	AdamUpdateSecondMom (TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, AFloat beta)

Static Private Attributes
static TRandom *	fgRandomGen

#include <TMVA/DNN/Architectures/Cuda.h>

Member Typedef Documentation

◆ DeviceBuffer_t

template<typename AFloat = Real_t>

using TMVA::DNN::TCuda< AFloat >::DeviceBuffer_t = TCudaDeviceBuffer<AFloat>

Definition at line 54 of file Cuda.h.

◆ HostBuffer_t

template<typename AFloat = Real_t>

using TMVA::DNN::TCuda< AFloat >::HostBuffer_t = TCudaHostBuffer<AFloat>

Definition at line 55 of file Cuda.h.

◆ Matrix_t

template<typename AFloat = Real_t>

using TMVA::DNN::TCuda< AFloat >::Matrix_t = TCudaMatrix<AFloat>

Definition at line 53 of file Cuda.h.

◆ Scalar_t

template<typename AFloat = Real_t>

using TMVA::DNN::TCuda< AFloat >::Scalar_t = AFloat

Definition at line 52 of file Cuda.h.

Member Function Documentation

◆ AdamUpdate()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::AdamUpdate	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	M,
		const TCudaMatrix< AFloat > &	V,
		AFloat	alpha,
		AFloat	eps
	)

static

◆ AdamUpdateFirstMom()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::AdamUpdateFirstMom	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B,
		AFloat	beta
	)

static

◆ AdamUpdateSecondMom()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::AdamUpdateSecondMom	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B,
		AFloat	beta
	)

static

◆ AddConvBiases()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::AddConvBiases	(	TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	biases
	)

static

Add the biases in the Convolutional Layer.

◆ AddL1RegularizationGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::AddL1RegularizationGradients	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	W,
		AFloat	weightDecay
	)

static

◆ AddL2RegularizationGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::AddL2RegularizationGradients	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	W,
		AFloat	weightDecay
	)

static

◆ AddRowWise()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::AddRowWise	(	TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	biases
	)

static

Add the vectors biases row-wise to the matrix output.

◆ AlmostEquals()

template<typename AFloat = Real_t>

static bool TMVA::DNN::TCuda< AFloat >::AlmostEquals	(	const TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B,
		double	epsilon = `0.1`
	)

static

Check two matrices for equality, taking floating point arithmetic errors into account.

◆ Backward()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Backward	(	TCudaMatrix< AFloat > &	activationGradientsBackward,
		TCudaMatrix< AFloat > &	weightGradients,
		TCudaMatrix< AFloat > &	biasGradients,
		TCudaMatrix< AFloat > &	df,
		const TCudaMatrix< AFloat > &	activationGradients,
		const TCudaMatrix< AFloat > &	weights,
		const TCudaMatrix< AFloat > &	activationBackward
	)

static

Perform the complete backward propagation step.

If the provided activationGradientsBackward matrix is not empty, compute the gradients of the objective function with respect to the activations of the previous layer (backward direction). Also compute the weight and the bias gradients. Modifies the values in df and thus produces only a valid result, if it is applied the first time after the corresponding forward propagation has been per- formed.

◆ CalculateConvActivationGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::CalculateConvActivationGradients	(	std::vector< TCudaMatrix< AFloat > > &	activationGradientsBackward,
		std::vector< TCudaMatrix< AFloat > > &	df,
		const TCudaMatrix< AFloat > &	weights,
		size_t	batchSize,
		size_t	inputHeight,
		size_t	inputWidth,
		size_t	depth,
		size_t	height,
		size_t	width,
		size_t	filterDepth,
		size_t	filterHeight,
		size_t	filterWidth
	)

static

Utility function for calculating the activation gradients of the layer before the convolutional layer.

◆ CalculateConvBiasGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::CalculateConvBiasGradients	(	TCudaMatrix< AFloat > &	biasGradients,
		std::vector< TCudaMatrix< AFloat > > &	df,
		size_t	batchSize,
		size_t	depth,
		size_t	nLocalViews
	)

static

Utility function for calculating the bias gradients of the convolutional layer.

◆ CalculateConvWeightGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::CalculateConvWeightGradients	(	TCudaMatrix< AFloat > &	weightGradients,
		std::vector< TCudaMatrix< AFloat > > &	df,
		const std::vector< TCudaMatrix< AFloat > > &	activations_backward,
		size_t	batchSize,
		size_t	inputHeight,
		size_t	inputWidth,
		size_t	depth,
		size_t	height,
		size_t	width,
		size_t	filterDepth,
		size_t	filterHeight,
		size_t	filterWidth,
		size_t	nLocalViews
	)

static

Utility function for calculating the weight gradients of the convolutional layer.

◆ calculateDimension()

template<typename AFloat = Real_t>

static size_t TMVA::DNN::TCuda< AFloat >::calculateDimension	(	size_t	imgDim,
		size_t	fltDim,
		size_t	padding,
		size_t	stride
	)

static

Calculate how many neurons "fit" in the output layer, given the input as well as the layer's hyperparameters.

◆ ConstAdd()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ConstAdd	(	TCudaMatrix< AFloat > &	A,
		AFloat	beta
	)

static

Add the constant beta to all the elements of matrix A and write the result into A.

◆ ConstMult()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ConstMult	(	TCudaMatrix< AFloat > &	A,
		AFloat	beta
	)

static

Multiply the constant beta to all the elements of matrix A and write the result into A.

◆ ConvLayerBackward()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ConvLayerBackward	(	std::vector< TCudaMatrix< AFloat > > &	activationGradientsBackward,
		TCudaMatrix< AFloat > &	weightGradients,
		TCudaMatrix< AFloat > &	biasGradients,
		std::vector< TCudaMatrix< AFloat > > &	df,
		const std::vector< TCudaMatrix< AFloat > > &	activationGradients,
		const TCudaMatrix< AFloat > &	weights,
		const std::vector< TCudaMatrix< AFloat > > &	activationBackward,
		size_t	batchSize,
		size_t	inputHeight,
		size_t	inputWidth,
		size_t	depth,
		size_t	height,
		size_t	width,
		size_t	filterDepth,
		size_t	filterHeight,
		size_t	filterWidth,
		size_t	nLocalViews
	)

static

Perform the complete backward propagation step in a Convolutional Layer.

If the provided activationGradientsBackward matrix is not empty, compute the gradients of the objective function with respect to the activations of the previous layer (backward direction). Also compute the weight and the bias gradients. Modifies the values in df and thus produces only a valid result, if it is applied the first time after the corresponding forward propagation has been per- formed.

◆ ConvLayerForward()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ConvLayerForward	(	std::vector< TCudaMatrix< AFloat > > &	output,
		std::vector< TCudaMatrix< AFloat > > &	derivatives,
		const std::vector< TCudaMatrix< AFloat > > &	input,
		const TCudaMatrix< AFloat > &	weights,
		const TCudaMatrix< AFloat > &	biases,
		const DNN::CNN::TConvParams &	params,
		EActivationFunction	activFunc,
		std::vector< TCudaMatrix< AFloat > > &	inputPrime
	)

static

Forward propagation in the Convolutional layer.

◆ Copy() [1/2]

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Copy	(	std::vector< TCudaMatrix< Scalar_t > > &	A,
		const std::vector< TCudaMatrix< Scalar_t > > &	B
	)

static

◆ Copy() [2/2]

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Copy	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

Copy the elements of matrix A into matrix B.

◆ CopyDiffArch() [1/2]

template<typename AFloat >

template<typename AMatrix_t >

void TMVA::DNN::TCuda< AFloat >::CopyDiffArch	(	std::vector< TCudaMatrix< Scalar_t > > &	A,
		const std::vector< AMatrix_t > &	B
	)

static

Definition at line 551 of file Cuda.h.

◆ CopyDiffArch() [2/2]

template<typename AFloat >

template<typename AMatrix_t >

void TMVA::DNN::TCuda< AFloat >::CopyDiffArch	(	TCudaMatrix< Scalar_t > &	B,
		const AMatrix_t &	A
	)

static

Definition at line 539 of file Cuda.h.

◆ CrossEntropy()

template<typename AFloat = Real_t>

static AFloat TMVA::DNN::TCuda< AFloat >::CrossEntropy	(	const TCudaMatrix< AFloat > &	Y,
		const TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	weights
	)

static

Sigmoid transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net.

◆ CrossEntropyGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::CrossEntropyGradients	(	TCudaMatrix< AFloat > &	dY,
		const TCudaMatrix< AFloat > &	Y,
		const TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	weights
	)

static

◆ Deflatten()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Deflatten	(	std::vector< TCudaMatrix< AFloat > > &	A,
		const TCudaMatrix< AFloat > &	B,
		size_t	index,
		size_t	nRows,
		size_t	nCols
	)

static

Transforms each row of B to a matrix and stores it in the tensor B.

◆ Downsample()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Downsample	(	TCudaMatrix< AFloat > &	A,
		TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	C,
		size_t	imgHeight,
		size_t	imgWidth,
		size_t	fltHeight,
		size_t	fltWidth,
		size_t	strideRows,
		size_t	strideCols
	)

static

Downsample the matrix C to the matrix A, using max operation, such that the winning indices are stored in matrix B.

◆ Dropout()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Dropout	(	TCudaMatrix< AFloat > &	A,
		AFloat	p
	)

static

Apply dropout with activation probability p to the given matrix A and scale the result by reciprocal of p.

◆ Flatten()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Flatten	(	TCudaMatrix< AFloat > &	A,
		const std::vector< TCudaMatrix< AFloat > > &	B,
		size_t	size,
		size_t	nRows,
		size_t	nCols
	)

static

Flattens the tensor B, such that each matrix, is stretched in one row, resulting with a matrix A.

◆ Gauss()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Gauss ( TCudaMatrix< AFloat > & B )

static

◆ GaussDerivative()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::GaussDerivative	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

◆ GetRandomGenerator()

template<typename AFloat = Real_t>

static TRandom & TMVA::DNN::TCuda< AFloat >::GetRandomGenerator ( )

static

◆ Hadamard()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Hadamard	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B
	)

static

In-place Hadamard (element-wise) product of matrices A and B with the result being written into A.

◆ Identity()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Identity ( TCudaMatrix< AFloat > & B )

static

◆ IdentityDerivative()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::IdentityDerivative	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

◆ Im2col()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Im2col	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B,
		size_t	imgHeight,
		size_t	imgWidth,
		size_t	fltHeight,
		size_t	fltWidth,
		size_t	strideRows,
		size_t	strideCols,
		size_t	zeroPaddingHeight,
		size_t	zeroPaddingWidth
	)

static

Transform the matrix B in local view format, suitable for convolution, and store it in matrix A.

◆ Im2colFast()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Im2colFast	(	TCudaMatrix< AFloat > &	,
		const TCudaMatrix< AFloat > &	,
		const std::vector< int > &
	)

inlinestatic

Definition at line 333 of file Cuda.h.

◆ Im2colIndices()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Im2colIndices	(	std::vector< int > &	,
		const TCudaMatrix< AFloat > &	,
		size_t	,
		size_t	,
		size_t	,
		size_t	,
		size_t	,
		size_t	,
		size_t	,
		size_t	,
		size_t
	)

inlinestatic

Definition at line 329 of file Cuda.h.

◆ InitializeGauss()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::InitializeGauss ( TCudaMatrix< AFloat > & A )

static

◆ InitializeGlorotNormal()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::InitializeGlorotNormal ( TCudaMatrix< AFloat > & A )

static

◆ InitializeGlorotUniform()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::InitializeGlorotUniform ( TCudaMatrix< AFloat > & A )

static

◆ InitializeIdentity()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::InitializeIdentity ( TCudaMatrix< AFloat > & A )

static

◆ InitializeUniform()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::InitializeUniform ( TCudaMatrix< AFloat > & A )

static

◆ InitializeZero()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::InitializeZero ( TCudaMatrix< AFloat > & A )

static

◆ L1Regularization()

template<typename AFloat = Real_t>

static AFloat TMVA::DNN::TCuda< AFloat >::L1Regularization ( const TCudaMatrix< AFloat > & W )

static

◆ L2Regularization()

template<typename AFloat = Real_t>

static AFloat TMVA::DNN::TCuda< AFloat >::L2Regularization ( const TCudaMatrix< AFloat > & W )

static

◆ MaxPoolLayerBackward()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::MaxPoolLayerBackward	(	TCudaMatrix< AFloat > &	activationGradientsBackward,
		const TCudaMatrix< AFloat > &	activationGradients,
		const TCudaMatrix< AFloat > &	indexMatrix,
		size_t	imgHeight,
		size_t	imgWidth,
		size_t	fltHeight,
		size_t	fltWidth,
		size_t	strideRows,
		size_t	strideCols,
		size_t	nLocalViews
	)

static

Perform the complete backward propagation step in a Pooling Layer.

Based on the winning idices stored in the index matrix, it just forwards the actiovation gradients to the previous layer.

◆ MeanSquaredError()

template<typename AFloat = Real_t>

static AFloat TMVA::DNN::TCuda< AFloat >::MeanSquaredError	(	const TCudaMatrix< AFloat > &	Y,
		const TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	weights
	)

static

◆ MeanSquaredErrorGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::MeanSquaredErrorGradients	(	TCudaMatrix< AFloat > &	dY,
		const TCudaMatrix< AFloat > &	Y,
		const TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	weights
	)

static

◆ Multiply()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Multiply	(	TCudaMatrix< AFloat > &	C,
		const TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B
	)

static

Standard multiplication of two matrices A and B with the result being written into C.

◆ MultiplyTranspose()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::MultiplyTranspose	(	TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	input,
		const TCudaMatrix< AFloat > &	weights
	)

static

Matrix-multiply input with the transpose of \pweights and write the results into output.

◆ PrepareInternals()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::PrepareInternals ( std::vector< TCudaMatrix< AFloat > > & inputPrime )

static

Attaches a cuda stream to each matrix in order to accomodate parallel kernel launches.

◆ Rearrange()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Rearrange	(	std::vector< TCudaMatrix< AFloat > > &	out,
		const std::vector< TCudaMatrix< AFloat > > &	in
	)

static

Rearrage data accoring to time fill B x T x D out with T x B x D matrix in.

◆ ReciprocalElementWise()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ReciprocalElementWise ( TCudaMatrix< AFloat > & A )

static

Reciprocal each element of the matrix A and write the result into A.

◆ RecurrentLayerBackward()

template<typename AFloat = Real_t>

static Matrix_t & TMVA::DNN::TCuda< AFloat >::RecurrentLayerBackward	(	TCudaMatrix< AFloat > &	state_gradients_backward,
		TCudaMatrix< AFloat > &	input_weight_gradients,
		TCudaMatrix< AFloat > &	state_weight_gradients,
		TCudaMatrix< AFloat > &	bias_gradients,
		TCudaMatrix< AFloat > &	df,
		const TCudaMatrix< AFloat > &	state,
		const TCudaMatrix< AFloat > &	weights_input,
		const TCudaMatrix< AFloat > &	weights_state,
		const TCudaMatrix< AFloat > &	input,
		TCudaMatrix< AFloat > &	input_gradient
	)

static

Backward pass for Recurrent Networks.

◆ Relu()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Relu ( TCudaMatrix< AFloat > & B )

static

◆ ReluDerivative()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ReluDerivative	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

◆ Reshape()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Reshape	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B
	)

static

Transform the matrix B to a matrix with different dimensions A.

◆ RotateWeights()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::RotateWeights	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B,
		size_t	filterDepth,
		size_t	filterHeight,
		size_t	filterWidth,
		size_t	numFilters
	)

static

Rotates the matrix B, which is representing a weights, and stores them in the matrix A.

◆ ScaleAdd() [1/2]

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ScaleAdd	(	std::vector< TCudaMatrix< Scalar_t > > &	A,
		const std::vector< TCudaMatrix< Scalar_t > > &	B,
		Scalar_t	beta = `1.0`
	)

static

Above functions extended to vectors.

◆ ScaleAdd() [2/2]

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::ScaleAdd	(	TCudaMatrix< AFloat > &	A,
		const TCudaMatrix< AFloat > &	B,
		Scalar_t	beta = `1.0`
	)

static

Adds a the elements in matrix B scaled by c to the elements in the matrix A.

This is required for the weight update in the gradient descent step.

◆ SetRandomSeed()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SetRandomSeed ( size_t seed )

static

◆ Sigmoid() [1/2]

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Sigmoid ( TCudaMatrix< AFloat > & B )

static

◆ Sigmoid() [2/2]

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Sigmoid	(	TCudaMatrix< AFloat > &	YHat,
		const TCudaMatrix< AFloat > &
	)

static

◆ SigmoidDerivative()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SigmoidDerivative	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

◆ Softmax()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Softmax	(	TCudaMatrix< AFloat > &	YHat,
		const TCudaMatrix< AFloat > &
	)

static

◆ SoftmaxCrossEntropy()

template<typename AFloat = Real_t>

static AFloat TMVA::DNN::TCuda< AFloat >::SoftmaxCrossEntropy	(	const TCudaMatrix< AFloat > &	Y,
		const TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	weights
	)

static

Softmax transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net.

◆ SoftmaxCrossEntropyGradients()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SoftmaxCrossEntropyGradients	(	TCudaMatrix< AFloat > &	dY,
		const TCudaMatrix< AFloat > &	Y,
		const TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	weights
	)

static

◆ SoftSign()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SoftSign ( TCudaMatrix< AFloat > & B )

static

◆ SoftSignDerivative()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SoftSignDerivative	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

◆ SqrtElementWise()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SqrtElementWise ( TCudaMatrix< AFloat > & A )

static

Square root each element of the matrix A and write the result into A.

◆ SquareElementWise()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SquareElementWise ( TCudaMatrix< AFloat > & A )

static

Square each element of the matrix A and write the result into A.

◆ Sum()

template<typename AFloat = Real_t>

static AFloat TMVA::DNN::TCuda< AFloat >::Sum ( const TCudaMatrix< AFloat > & A )

static

Compute the sum of all elements in A.

◆ SumColumns()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SumColumns	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

Sum columns of (m x n) matrix A and write the results into the first m elements in B.

◆ SumRows()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SumRows	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

Sum rows of (m x n) matrix A and write the results into the first m elements in B.

◆ SymmetricRelu()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SymmetricRelu ( TCudaMatrix< AFloat > & B )

static

◆ SymmetricReluDerivative()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::SymmetricReluDerivative	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

◆ Tanh()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::Tanh ( TCudaMatrix< AFloat > & B )

static

◆ TanhDerivative()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::TanhDerivative	(	TCudaMatrix< AFloat > &	B,
		const TCudaMatrix< AFloat > &	A
	)

static

◆ TransposeMultiply()

template<typename AFloat = Real_t>

static void TMVA::DNN::TCuda< AFloat >::TransposeMultiply	(	TCudaMatrix< AFloat > &	output,
		const TCudaMatrix< AFloat > &	input,
		const TCudaMatrix< AFloat > &	Weights
	)

static

Matrix multiplication of two matrices A and B^T (transposed) with the result being written into C.

Member Data Documentation

◆ fgRandomGen

template<typename AFloat = Real_t>

TRandom* TMVA::DNN::TCuda< AFloat >::fgRandomGen

staticprivate

Definition at line 49 of file Cuda.h.

The documentation for this class was generated from the following file:

tmva/tmva/inc/TMVA/DNN/Architectures/Cuda.h

Public Types

Static Public Member Functions

Static Private Attributes

Member Typedef Documentation

◆ DeviceBuffer_t

◆ HostBuffer_t

◆ Matrix_t

◆ Scalar_t

Member Function Documentation

◆ AdamUpdate()

◆ AdamUpdateFirstMom()

◆ AdamUpdateSecondMom()

◆ AddConvBiases()

◆ AddL1RegularizationGradients()

◆ AddL2RegularizationGradients()

◆ AddRowWise()

◆ AlmostEquals()

◆ Backward()

◆ CalculateConvActivationGradients()

◆ CalculateConvBiasGradients()

◆ CalculateConvWeightGradients()

◆ calculateDimension()

◆ ConstAdd()

◆ ConstMult()

◆ ConvLayerBackward()

◆ ConvLayerForward()

◆ Copy() [1/2]

◆ Copy() [2/2]

◆ CopyDiffArch() [1/2]

◆ CopyDiffArch() [2/2]

◆ CrossEntropy()

◆ CrossEntropyGradients()

◆ Deflatten()

◆ Downsample()

◆ Dropout()

◆ Flatten()

◆ Gauss()

◆ GaussDerivative()

◆ GetRandomGenerator()

◆ Hadamard()

◆ Identity()

◆ IdentityDerivative()

◆ Im2col()

◆ Im2colFast()

◆ Im2colIndices()

◆ InitializeGauss()

◆ InitializeGlorotNormal()

◆ InitializeGlorotUniform()

◆ InitializeIdentity()

◆ InitializeUniform()

◆ InitializeZero()

◆ L1Regularization()

◆ L2Regularization()

◆ MaxPoolLayerBackward()

◆ MeanSquaredError()

◆ MeanSquaredErrorGradients()

◆ Multiply()

◆ MultiplyTranspose()

◆ PrepareInternals()

◆ Rearrange()

◆ ReciprocalElementWise()

◆ RecurrentLayerBackward()

◆ Relu()

◆ ReluDerivative()

◆ Reshape()

◆ RotateWeights()

◆ ScaleAdd() [1/2]

◆ ScaleAdd() [2/2]

◆ SetRandomSeed()

◆ Sigmoid() [1/2]

◆ Sigmoid() [2/2]

◆ SigmoidDerivative()

◆ Softmax()

◆ SoftmaxCrossEntropy()

◆ SoftmaxCrossEntropyGradients()

◆ SoftSign()

◆ SoftSignDerivative()

◆ SqrtElementWise()

◆ SquareElementWise()

◆ Sum()