template<typename Architecture_t>
class TMVA::DNN::TDenseLayer< Architecture_t >

Generic layer class.

This generic layer class represents a dense layer of a neural network with a given width n and activation function f. The activation function of each layer is given by \(\mathbf{u} = \mathbf{W}\mathbf{x} + \boldsymbol{\theta}\).

In addition to the weight and bias matrices, each layer allocates memory for its activations and the corresponding input tensor before evaluation of the activation function as well as the gradients of the weights and biases.

The layer provides member functions for the forward propagation of activations through the given layer.

Definition at line 59 of file DenseLayer.h.

Public Types
using	Matrix_t = typename Architecture_t::Matrix_t

using	Scalar_t = typename Architecture_t::Scalar_t

using	Tensor_t = typename Architecture_t::Tensor_t

Public Member Functions
	TDenseLayer (const TDenseLayer &)
	Copy Constructor.

	TDenseLayer (size_t BatchSize, size_t InputWidth, size_t Width, EInitialization init, Scalar_t DropoutProbability, EActivationFunction f, ERegularization reg, Scalar_t weightDecay)
	Constructor.

	TDenseLayer (TDenseLayer< Architecture_t > *layer)
	Copy the dense layer provided as a pointer.

	~TDenseLayer ()
	Destructor.

virtual void	AddWeightsXMLTo (void *parent)
	Writes the information and the weights about the layer in an XML node.

void	Backward (Tensor_t &gradients_backward, const Tensor_t &activations_backward)
	Compute weight, bias and activation gradients.

void	Forward (Tensor_t &input, bool applyDropout=false)
	Compute activation of the layer for the given input.

EActivationFunction	GetActivationFunction () const

Scalar_t	GetDropoutProbability () const
	Getters.

Tensor_t &	GetInputActivation ()

const Tensor_t &	GetInputActivation () const

ERegularization	GetRegularization () const

Scalar_t	GetWeightDecay () const

void	Print () const
	std::vector<Matrix_t> &inp1, std::vector<Matrix_t> &inp2);

virtual void	ReadWeightsFromXML (void *parent)
	Read the information and the weights about the layer from XML node.

virtual void	SetDropoutProbability (Scalar_t dropoutProbability)
	Set dropout probabilities.

Public Member Functions inherited from TMVA::DNN::VGeneralLayer< Architecture_t >
	VGeneralLayer (const VGeneralLayer &)
	Copy Constructor.

	VGeneralLayer (size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, size_t Height, size_t Width, size_t WeightsNSlices, size_t WeightsNRows, size_t WeightsNCols, size_t BiasesNSlices, size_t BiasesNRows, size_t BiasesNCols, size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, EInitialization Init)
	Constructor.

	VGeneralLayer (size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, size_t Height, size_t Width, size_t WeightsNSlices, std::vector< size_t > WeightsNRows, std::vector< size_t > WeightsNCols, size_t BiasesNSlices, std::vector< size_t > BiasesNRows, std::vector< size_t > BiasesNCols, size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, EInitialization Init)
	General Constructor with different weights dimension.

	VGeneralLayer (VGeneralLayer< Architecture_t > *layer)
	Copy the layer provided as a pointer.

virtual	~VGeneralLayer ()
	Virtual Destructor.

void	CopyBiases (const std::vector< Matrix_t > &otherBiases)
	Copies the biases provided as an input.

template<typename Arch >
void	CopyParameters (const VGeneralLayer< Arch > &layer)
	Copy all trainable weight and biases from another equivalent layer but with different architecture The function can copy also extra parameters in addition to weights and biases if they are return by the function GetExtraLayerParameters.

void	CopyWeights (const std::vector< Matrix_t > &otherWeights)
	Copies the weights provided as an input.

Tensor_t &	GetActivationGradients ()

const Tensor_t &	GetActivationGradients () const

Matrix_t	GetActivationGradientsAt (size_t i)

const Matrix_t &	GetActivationGradientsAt (size_t i) const

size_t	GetBatchSize () const
	Getters.

std::vector< Matrix_t > &	GetBiases ()

const std::vector< Matrix_t > &	GetBiases () const

Matrix_t &	GetBiasesAt (size_t i)

const Matrix_t &	GetBiasesAt (size_t i) const

std::vector< Matrix_t > &	GetBiasGradients ()

const std::vector< Matrix_t > &	GetBiasGradients () const

Matrix_t &	GetBiasGradientsAt (size_t i)

const Matrix_t &	GetBiasGradientsAt (size_t i) const

size_t	GetDepth () const

virtual std::vector< Matrix_t >	GetExtraLayerParameters () const

size_t	GetHeight () const

EInitialization	GetInitialization () const

size_t	GetInputDepth () const

size_t	GetInputHeight () const

size_t	GetInputWidth () const

Tensor_t &	GetOutput ()

const Tensor_t &	GetOutput () const

Matrix_t	GetOutputAt (size_t i)

const Matrix_t &	GetOutputAt (size_t i) const

std::vector< Matrix_t > &	GetWeightGradients ()

const std::vector< Matrix_t > &	GetWeightGradients () const

Matrix_t &	GetWeightGradientsAt (size_t i)

const Matrix_t &	GetWeightGradientsAt (size_t i) const

std::vector< Matrix_t > &	GetWeights ()

const std::vector< Matrix_t > &	GetWeights () const

Matrix_t &	GetWeightsAt (size_t i)

const Matrix_t &	GetWeightsAt (size_t i) const

size_t	GetWidth () const

virtual void	Initialize ()
	Initialize the weights and biases according to the given initialization method.

bool	IsTraining () const

void	ReadMatrixXML (void node, const char name, Matrix_t &matrix)

virtual void	ResetTraining ()
	Reset some training flags after a loop on all batches Some layer (e.g.

void	SetBatchSize (size_t batchSize)
	Setters.

void	SetDepth (size_t depth)

virtual void	SetExtraLayerParameters (const std::vector< Matrix_t > &)

void	SetHeight (size_t height)

void	SetInputDepth (size_t inputDepth)

void	SetInputHeight (size_t inputHeight)

void	SetInputWidth (size_t inputWidth)

void	SetIsTraining (bool isTraining)

void	SetWidth (size_t width)

void	Update (const Scalar_t learningRate)
	Updates the weights and biases, given the learning rate.

void	UpdateBiases (const std::vector< Matrix_t > &biasGradients, const Scalar_t learningRate)
	Updates the biases, given the gradients and the learning rate.

void	UpdateBiasGradients (const std::vector< Matrix_t > &biasGradients, const Scalar_t learningRate)
	Updates the bias gradients, given some other weight gradients and learning rate.

void	UpdateWeightGradients (const std::vector< Matrix_t > &weightGradients, const Scalar_t learningRate)
	Updates the weight gradients, given some other weight gradients and learning rate.

void	UpdateWeights (const std::vector< Matrix_t > &weightGradients, const Scalar_t learningRate)
	Updates the weights, given the gradients and the learning rate,.

void	WriteMatrixToXML (void node, const char name, const Matrix_t &matrix)

void	WriteTensorToXML (void node, const char name, const std::vector< Matrix_t > &tensor)
	helper functions for XML

Private Attributes
Architecture_t::ActivationDescriptor_t	fActivationDesc

Tensor_t	fDerivatives
	output of GEMM and input to activation function

Scalar_t	fDropoutProbability
	activation functgion gradient

EActivationFunction	fF
	Activation function of the layer.

Tensor_t	fInputActivation

ERegularization	fReg
	The regularization method.

Scalar_t	fWeightDecay
	The weight decay.

Additional Inherited Members
Protected Attributes inherited from TMVA::DNN::VGeneralLayer< Architecture_t >
Tensor_t	fActivationGradients
	Gradients w.r.t. the activations of this layer.

size_t	fBatchSize
	Batch size used for training and evaluation.

std::vector< Matrix_t >	fBiases
	The biases associated to the layer.

std::vector< Matrix_t >	fBiasGradients
	Gradients w.r.t. the bias values of the layer.

size_t	fDepth
	The depth of the layer.

size_t	fHeight
	The height of the layer.

EInitialization	fInit
	The initialization method.

size_t	fInputDepth
	The depth of the previous layer or input.

size_t	fInputHeight
	The height of the previous layer or input.

size_t	fInputWidth
	The width of the previous layer or input.

bool	fIsTraining
	Flag indicating the mode.

Tensor_t	fOutput
	Activations of this layer.

std::vector< Matrix_t >	fWeightGradients
	Gradients w.r.t. the weights of the layer.

std::vector< Matrix_t >	fWeights
	The weights associated to the layer.

size_t	fWidth
	The width of this layer.

#include <TMVA/DNN/DenseLayer.h>

Inheritance diagram for TMVA::DNN::TDenseLayer< Architecture_t >:

[legend]

Member Typedef Documentation

◆ Matrix_t

template<typename Architecture_t >

using TMVA::DNN::TDenseLayer< Architecture_t >::Matrix_t = typename Architecture_t::Matrix_t

Definition at line 63 of file DenseLayer.h.

◆ Scalar_t

template<typename Architecture_t >

using TMVA::DNN::TDenseLayer< Architecture_t >::Scalar_t = typename Architecture_t::Scalar_t

Definition at line 62 of file DenseLayer.h.

◆ Tensor_t

template<typename Architecture_t >

using TMVA::DNN::TDenseLayer< Architecture_t >::Tensor_t = typename Architecture_t::Tensor_t

Definition at line 64 of file DenseLayer.h.

Constructor & Destructor Documentation

◆ TDenseLayer() [1/3]

template<typename Architecture_t >

TMVA::DNN::TDenseLayer< Architecture_t >::TDenseLayer	(	size_t	BatchSize,
		size_t	InputWidth,
		size_t	Width,
		EInitialization	init,
		Scalar_t	DropoutProbability,
		EActivationFunction	f,
		ERegularization	reg,
		Scalar_t	weightDecay
	)

Constructor.

Definition at line 136 of file DenseLayer.h.

◆ TDenseLayer() [2/3]

template<typename Architecture_t >

TMVA::DNN::TDenseLayer< Architecture_t >::TDenseLayer ( TDenseLayer< Architecture_t > * layer )

Copy the dense layer provided as a pointer.

Definition at line 152 of file DenseLayer.h.

◆ TDenseLayer() [3/3]

template<typename Architecture_t >

TMVA::DNN::TDenseLayer< Architecture_t >::TDenseLayer ( const TDenseLayer< Architecture_t > & layer )

Copy Constructor.

Definition at line 164 of file DenseLayer.h.

◆ ~TDenseLayer()

template<typename Architecture_t >

TMVA::DNN::TDenseLayer< Architecture_t >::~TDenseLayer

Destructor.

Definition at line 176 of file DenseLayer.h.

Member Function Documentation

◆ AddWeightsXMLTo()

template<typename Architecture_t >

void TMVA::DNN::TDenseLayer< Architecture_t >::AddWeightsXMLTo ( void * parent )

virtual

Writes the information and the weights about the layer in an XML node.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 249 of file DenseLayer.h.

◆ Backward()

template<typename Architecture_t >

auto TMVA::DNN::TDenseLayer< Architecture_t >::Backward	(	Tensor_t &	gradients_backward,
		const Tensor_t &	activations_backward
	)

virtual

Compute weight, bias and activation gradients.

Uses the precomputed first partial derviatives of the activation function computed during forward propagation and modifies them. Must only be called directly a the corresponding call to Forward(...).

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 206 of file DenseLayer.h.

◆ Forward()

template<typename Architecture_t >

auto TMVA::DNN::TDenseLayer< Architecture_t >::Forward	(	Tensor_t &	input,
		bool	applyDropout = `false`
	)

virtual

Compute activation of the layer for the given input.

The input must be in 3D tensor form with the different matrices corresponding to different events in the batch. Computes activations as well as the first partial derivative of the activation function at those activations.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 187 of file DenseLayer.h.

◆ GetActivationFunction()

template<typename Architecture_t >

EActivationFunction TMVA::DNN::TDenseLayer< Architecture_t >::GetActivationFunction ( ) const

inline

Definition at line 126 of file DenseLayer.h.

◆ GetDropoutProbability()

template<typename Architecture_t >

Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::GetDropoutProbability ( ) const

inline

Getters.

Definition at line 120 of file DenseLayer.h.

◆ GetInputActivation() [1/2]

template<typename Architecture_t >

Tensor_t & TMVA::DNN::TDenseLayer< Architecture_t >::GetInputActivation ( )

inline

Definition at line 124 of file DenseLayer.h.

◆ GetInputActivation() [2/2]

template<typename Architecture_t >

const Tensor_t & TMVA::DNN::TDenseLayer< Architecture_t >::GetInputActivation ( ) const

inline

Definition at line 123 of file DenseLayer.h.

◆ GetRegularization()

template<typename Architecture_t >

ERegularization TMVA::DNN::TDenseLayer< Architecture_t >::GetRegularization ( ) const

inline

Definition at line 127 of file DenseLayer.h.

◆ GetWeightDecay()

template<typename Architecture_t >

Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::GetWeightDecay ( ) const

inline

Definition at line 128 of file DenseLayer.h.

◆ Print()

template<typename Architecture_t >

void TMVA::DNN::TDenseLayer< Architecture_t >::Print

virtual

std::vector<Matrix_t> &inp1, std::vector<Matrix_t> &inp2);

Printing the layer info.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 231 of file DenseLayer.h.

◆ ReadWeightsFromXML()

template<typename Architecture_t >

void TMVA::DNN::TDenseLayer< Architecture_t >::ReadWeightsFromXML ( void * parent )

virtual

Read the information and the weights about the layer from XML node.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 267 of file DenseLayer.h.

◆ SetDropoutProbability()

template<typename Architecture_t >

virtual void TMVA::DNN::TDenseLayer< Architecture_t >::SetDropoutProbability ( Scalar_t dropoutProbability )

inlinevirtual

Set dropout probabilities.

Reimplemented from TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 117 of file DenseLayer.h.

Member Data Documentation

◆ fActivationDesc

template<typename Architecture_t >

Architecture_t::ActivationDescriptor_t TMVA::DNN::TDenseLayer< Architecture_t >::fActivationDesc

private

Definition at line 77 of file DenseLayer.h.

◆ fDerivatives

template<typename Architecture_t >

Tensor_t TMVA::DNN::TDenseLayer< Architecture_t >::fDerivatives

private

output of GEMM and input to activation function

Definition at line 69 of file DenseLayer.h.

◆ fDropoutProbability

template<typename Architecture_t >

Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::fDropoutProbability

private

activation functgion gradient

Probability that an input is active.

Definition at line 71 of file DenseLayer.h.

◆ fF

template<typename Architecture_t >

EActivationFunction TMVA::DNN::TDenseLayer< Architecture_t >::fF

private

Activation function of the layer.

Definition at line 73 of file DenseLayer.h.

◆ fInputActivation

template<typename Architecture_t >

Tensor_t TMVA::DNN::TDenseLayer< Architecture_t >::fInputActivation

private

Definition at line 68 of file DenseLayer.h.

◆ fReg

template<typename Architecture_t >

ERegularization TMVA::DNN::TDenseLayer< Architecture_t >::fReg

private

The regularization method.

Definition at line 74 of file DenseLayer.h.

◆ fWeightDecay

template<typename Architecture_t >

Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::fWeightDecay

private

The weight decay.

Definition at line 75 of file DenseLayer.h.

tmva/tmva/inc/TMVA/DNN/DenseLayer.h

Public Types

Public Member Functions

Private Attributes

Additional Inherited Members

Member Typedef Documentation

◆ Matrix_t

◆ Scalar_t

◆ Tensor_t

Constructor & Destructor Documentation

◆ TDenseLayer() [1/3]

◆ TDenseLayer() [2/3]

◆ TDenseLayer() [3/3]

◆ ~TDenseLayer()

Member Function Documentation

◆ AddWeightsXMLTo()

◆ Backward()

◆ Forward()

◆ GetActivationFunction()

◆ GetDropoutProbability()

◆ GetInputActivation() [1/2]

◆ GetInputActivation() [2/2]

◆ GetRegularization()

◆ GetWeightDecay()

◆ Print()

◆ ReadWeightsFromXML()

◆ SetDropoutProbability()

Member Data Documentation

◆ fActivationDesc

◆ fDerivatives

◆ fDropoutProbability

◆ fF

◆ fInputActivation

◆ fReg

◆ fWeightDecay