template<typename Architecture_t, typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>
class TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >

Adam Optimizer class.

This class represents the Adam Optimizer.

Definition at line 44 of file Adam.h.

Public Types
using	Matrix_t = typename Architecture_t::Matrix_t

using	Scalar_t = typename Architecture_t::Scalar_t

Public Types inherited from TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >
using	Matrix_t = typename Architecture_t::Matrix_t

using	Scalar_t = typename Architecture_t::Scalar_t

Public Member Functions
	TAdam (DeepNet_t &deepNet, Scalar_t learningRate=0.001, Scalar_t beta1=0.9, Scalar_t beta2=0.999, Scalar_t epsilon=1e-7)
	Constructor. More...

	~TAdam ()=default
	Destructor. More...

Scalar_t	GetBeta1 () const
	Getters. More...

Scalar_t	GetBeta2 () const

Scalar_t	GetEpsilon () const

std::vector< std::vector< Matrix_t > > &	GetFirstMomentBiases ()

std::vector< Matrix_t > &	GetFirstMomentBiasesAt (size_t i)

std::vector< std::vector< Matrix_t > > &	GetFirstMomentWeights ()

std::vector< Matrix_t > &	GetFirstMomentWeightsAt (size_t i)

std::vector< std::vector< Matrix_t > > &	GetSecondMomentBiases ()

std::vector< Matrix_t > &	GetSecondMomentBiasesAt (size_t i)

std::vector< std::vector< Matrix_t > > &	GetSecondMomentWeights ()

std::vector< Matrix_t > &	GetSecondMomentWeightsAt (size_t i)

Public Member Functions inherited from TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >
	VOptimizer (Scalar_t learningRate, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > &deepNet)
	Constructor. More...

virtual	~VOptimizer ()=default
	Virtual Destructor. More...

size_t	GetGlobalStep () const

VGeneralLayer< Architecture_t > *	GetLayerAt (size_t i)

std::vector< VGeneralLayer< Architecture_t > * > &	GetLayers ()

Scalar_t	GetLearningRate () const
	Getters. More...

void	IncrementGlobalStep ()
	Increments the global step. More...

void	SetLearningRate (size_t learningRate)
	Setters. More...

void	Step ()
	Performs one step of optimization. More...

Protected Member Functions
void	UpdateBiases (size_t layerIndex, std::vector< Matrix_t > &biases, const std::vector< Matrix_t > &biasGradients)
	Update the biases, given the current bias gradients. More...

void	UpdateWeights (size_t layerIndex, std::vector< Matrix_t > &weights, const std::vector< Matrix_t > &weightGradients)
	Update the weights, given the current weight gradients. More...

virtual void	UpdateBiases (size_t layerIndex, std::vector< Matrix_t > &biases, const std::vector< Matrix_t > &biasGradients)=0
	Update the biases, given the current bias gradients. More...

virtual void	UpdateWeights (size_t layerIndex, std::vector< Matrix_t > &weights, const std::vector< Matrix_t > &weightGradients)=0
	Update the weights, given the current weight gradients. More...

Protected Attributes
Scalar_t	fBeta1
	The Beta1 constant used by the optimizer. More...

Scalar_t	fBeta2
	The Beta2 constant used by the optimizer. More...

Scalar_t	fEpsilon
	The Smoothing term used to avoid division by zero. More...

std::vector< std::vector< Matrix_t > >	fFirstMomentBiases
	The decaying average of the first moment of the past bias gradients associated with the deep net. More...

std::vector< std::vector< Matrix_t > >	fFirstMomentWeights
	The decaying average of the first moment of the past weight gradients associated with the deep net. More...

std::vector< std::vector< Matrix_t > >	fSecondMomentBiases
	The decaying average of the second moment of the past bias gradients associated with the deep net. More...

std::vector< std::vector< Matrix_t > >	fSecondMomentWeights
	The decaying average of the second moment of the past weight gradients associated with the deep net. More...

Protected Attributes inherited from TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >
TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > &	fDeepNet
	The reference to the deep net. More...

size_t	fGlobalStep
	The current global step count during training. More...

Scalar_t	fLearningRate
	The learning rate used for training. More...

#include <TMVA/DNN/Adam.h>

Inheritance diagram for TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >:

[legend]

Member Typedef Documentation

◆ Matrix_t

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

using TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::Matrix_t = typename Architecture_t::Matrix_t

Definition at line 46 of file Adam.h.

◆ Scalar_t

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

using TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::Scalar_t = typename Architecture_t::Scalar_t

Definition at line 47 of file Adam.h.

Constructor & Destructor Documentation

◆ TAdam()

template<typename Architecture_t , typename Layer_t , typename DeepNet_t >

TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::TAdam	(	DeepNet_t &	deepNet,
		Scalar_t	learningRate = `0.001`,
		Scalar_t	beta1 = `0.9`,
		Scalar_t	beta2 = `0.999`,
		Scalar_t	epsilon = `1e-7`
	)

Constructor.

Definition at line 101 of file Adam.h.

◆ ~TAdam()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::~TAdam ( )

default

Destructor.

Member Function Documentation

◆ GetBeta1()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetBeta1 ( ) const

inline

Getters.

Definition at line 79 of file Adam.h.

◆ GetBeta2()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetBeta2 ( ) const

inline

Definition at line 80 of file Adam.h.

◆ GetEpsilon()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetEpsilon ( ) const

inline

Definition at line 81 of file Adam.h.

◆ GetFirstMomentBiases()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentBiases ( )

inline

Definition at line 86 of file Adam.h.

◆ GetFirstMomentBiasesAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentBiasesAt ( size_t i )

inline

Definition at line 87 of file Adam.h.

◆ GetFirstMomentWeights()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentWeights ( )

inline

Definition at line 83 of file Adam.h.

◆ GetFirstMomentWeightsAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentWeightsAt ( size_t i )

inline

Definition at line 84 of file Adam.h.

◆ GetSecondMomentBiases()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentBiases ( )

inline

Definition at line 92 of file Adam.h.

◆ GetSecondMomentBiasesAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentBiasesAt ( size_t i )

inline

Definition at line 93 of file Adam.h.

◆ GetSecondMomentWeights()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentWeights ( )

inline

Definition at line 89 of file Adam.h.

◆ GetSecondMomentWeightsAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentWeightsAt ( size_t i )

inline

Definition at line 90 of file Adam.h.

◆ UpdateBiases()

template<typename Architecture_t , typename Layer_t , typename DeepNet_t >

auto TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::UpdateBiases	(	size_t	layerIndex,
		std::vector< Matrix_t > &	biases,
		const std::vector< Matrix_t > &	biasGradients
	)

protectedvirtual

Update the biases, given the current bias gradients.

Implements TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >.

Definition at line 168 of file Adam.h.

◆ UpdateWeights()

template<typename Architecture_t , typename Layer_t , typename DeepNet_t >

auto TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::UpdateWeights	(	size_t	layerIndex,
		std::vector< Matrix_t > &	weights,
		const std::vector< Matrix_t > &	weightGradients
	)

protectedvirtual

Update the weights, given the current weight gradients.

Implements TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >.

Definition at line 140 of file Adam.h.

Member Data Documentation

◆ fBeta1

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fBeta1

protected

The Beta1 constant used by the optimizer.

Definition at line 50 of file Adam.h.

◆ fBeta2

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fBeta2

protected

The Beta2 constant used by the optimizer.

Definition at line 51 of file Adam.h.

◆ fEpsilon

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fEpsilon

protected

The Smoothing term used to avoid division by zero.

Definition at line 52 of file Adam.h.

◆ fFirstMomentBiases

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fFirstMomentBiases

protected

The decaying average of the first moment of the past bias gradients associated with the deep net.

Definition at line 56 of file Adam.h.

◆ fFirstMomentWeights

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fFirstMomentWeights

protected

The decaying average of the first moment of the past weight gradients associated with the deep net.

Definition at line 54 of file Adam.h.

◆ fSecondMomentBiases

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fSecondMomentBiases

protected

The decaying average of the second moment of the past bias gradients associated with the deep net.

Definition at line 61 of file Adam.h.

◆ fSecondMomentWeights

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fSecondMomentWeights

protected

The decaying average of the second moment of the past weight gradients associated with the deep net.

Definition at line 59 of file Adam.h.

The documentation for this class was generated from the following file:

tmva/tmva/inc/TMVA/DNN/Adam.h

Public Types

Public Member Functions

Protected Member Functions

Protected Attributes

Member Typedef Documentation

◆ Matrix_t

◆ Scalar_t

Constructor & Destructor Documentation

◆ TAdam()

◆ ~TAdam()

Member Function Documentation

◆ GetBeta1()

◆ GetBeta2()

◆ GetEpsilon()

◆ GetFirstMomentBiases()

◆ GetFirstMomentBiasesAt()

◆ GetFirstMomentWeights()

◆ GetFirstMomentWeightsAt()

◆ GetSecondMomentBiases()

◆ GetSecondMomentBiasesAt()

◆ GetSecondMomentWeights()

◆ GetSecondMomentWeightsAt()

◆ UpdateBiases()

◆ UpdateWeights()

Member Data Documentation

◆ fBeta1

◆ fBeta2

◆ fEpsilon

◆ fFirstMomentBiases

◆ fFirstMomentWeights

◆ fSecondMomentBiases

◆ fSecondMomentWeights