27#ifndef TMVA_DNN_RMSPROP
28#define TMVA_DNN_RMSPROP
42template <
typename Architecture_t,
typename Layer_t = VGeneralLayer<Architecture_t>,
43 typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>
46 using Matrix_t =
typename Architecture_t::Matrix_t;
47 using Scalar_t =
typename Architecture_t::Scalar_t;
53 std::vector<std::vector<Matrix_t>>
55 std::vector<std::vector<Matrix_t>>
60 std::vector<std::vector<Matrix_t>>
62 std::vector<std::vector<Matrix_t>>
64 std::vector<std::vector<Matrix_t>>
66 std::vector<std::vector<Matrix_t>>
70 void UpdateWeights(
size_t layerIndex, std::vector<Matrix_t> &weights,
const std::vector<Matrix_t> &weightGradients);
73 void UpdateBiases(
size_t layerIndex, std::vector<Matrix_t> &biases,
const std::vector<Matrix_t> &biasGradients);
105template <
typename Architecture_t,
typename Layer_t,
typename DeepNet_t>
108 :
VOptimizer<Architecture_t, Layer_t, DeepNet_t>(learningRate, deepNet), fMomentum(momentum), fRho(rho),
111 std::vector<Layer_t *> &layers = deepNet.
GetLayers();
112 const size_t layersNSlices = layers.size();
122 for (
size_t i = 0; i < layersNSlices; i++) {
123 const size_t weightsNSlices = (layers[i]->GetWeights()).size();
126 Architecture_t::CreateWeightTensors(
fWeightUpdates[i], layers[i]->GetWeights());
128 for (
size_t j = 0; j < weightsNSlices; j++) {
133 const size_t biasesNSlices = (layers[i]->GetBiases()).size();
136 Architecture_t::CreateWeightTensors(
fBiasUpdates[i], layers[i]->GetBiases());
138 for (
size_t j = 0; j < biasesNSlices; j++) {
143 Architecture_t::CreateWeightTensors(
fWorkBiasTensor1[i], layers[i]->GetBiases());
145 Architecture_t::CreateWeightTensors(
fWorkBiasTensor2[i], layers[i]->GetBiases());
150template <
typename Architecture_t,
typename Layer_t,
typename DeepNet_t>
152 const std::vector<Matrix_t> &weightGradients) ->
void
154 std::vector<Matrix_t> ¤tLayerPastSquaredWeightGradients = this->GetPastSquaredWeightGradientsAt(layerIndex);
155 std::vector<Matrix_t> ¤tLayerWeightUpdates = this->GetWeightUpdatesAt(layerIndex);
157 for (
size_t k = 0; k < currentLayerPastSquaredWeightGradients.size(); k++) {
160 auto &accumulation = fWorkWeightTensor1[layerIndex][k];
161 auto ¤tSquaredWeightGradients = fWorkWeightTensor2[layerIndex][k];
167 Architecture_t::SquareElementWise(currentSquaredWeightGradients);
168 Architecture_t::ScaleAdd(accumulation, currentLayerPastSquaredWeightGradients[k], this->GetRho());
169 Architecture_t::ScaleAdd(accumulation, currentSquaredWeightGradients, 1 - (this->GetRho()));
174 auto &
dummy = fWorkWeightTensor2[layerIndex][k];
176 Architecture_t::ConstAdd(
dummy, this->GetEpsilon());
177 Architecture_t::SqrtElementWise(
dummy);
178 Architecture_t::ReciprocalElementWise(
dummy);
179 Architecture_t::Hadamard(
dummy, weightGradients[k]);
181 Architecture_t::ScaleAdd(accumulation, currentLayerWeightUpdates[k], this->GetMomentum());
182 Architecture_t::ScaleAdd(accumulation,
dummy, this->GetLearningRate());
188 for (
size_t i = 0; i < weights.size(); i++) {
189 Architecture_t::ScaleAdd(weights[i], currentLayerWeightUpdates[i], -1.0);
194template <
typename Architecture_t,
typename Layer_t,
typename DeepNet_t>
196 const std::vector<Matrix_t> &biasGradients) ->
void
198 std::vector<Matrix_t> ¤tLayerPastSquaredBiasGradients = this->GetPastSquaredBiasGradientsAt(layerIndex);
199 std::vector<Matrix_t> ¤tLayerBiasUpdates = this->GetBiasUpdatesAt(layerIndex);
201 for (
size_t k = 0; k < currentLayerPastSquaredBiasGradients.size(); k++) {
204 auto &accumulation = fWorkBiasTensor1[layerIndex][k];
205 auto ¤tSquaredBiasGradients = fWorkBiasTensor2[layerIndex][k];
210 Architecture_t::SquareElementWise(currentSquaredBiasGradients);
211 Architecture_t::ScaleAdd(accumulation, currentLayerPastSquaredBiasGradients[k], this->GetRho());
212 Architecture_t::ScaleAdd(accumulation, currentSquaredBiasGradients, 1 - (this->GetRho()));
217 auto &
dummy = fWorkBiasTensor2[layerIndex][k];
220 Architecture_t::ConstAdd(
dummy, this->GetEpsilon());
221 Architecture_t::SqrtElementWise(
dummy);
222 Architecture_t::ReciprocalElementWise(
dummy);
223 Architecture_t::Hadamard(
dummy, biasGradients[k]);
225 Architecture_t::ScaleAdd(accumulation, currentLayerBiasUpdates[k], this->GetMomentum());
226 Architecture_t::ScaleAdd(accumulation,
dummy, this->GetLearningRate());
232 for (
size_t i = 0; i < biases.size(); i++) {
233 Architecture_t::ScaleAdd(biases[i], currentLayerBiasUpdates[i], -1.0);
static RooMathCoreReg dummy
Scalar_t fRho
The Rho constant used by the optimizer.
typename Architecture_t::Scalar_t Scalar_t
void UpdateWeights(size_t layerIndex, std::vector< Matrix_t > &weights, const std::vector< Matrix_t > &weightGradients)
Update the weights, given the current weight gradients.
~TRMSProp()=default
Destructor.
std::vector< Matrix_t > & GetPastSquaredWeightGradientsAt(size_t i)
std::vector< std::vector< Matrix_t > > fWorkBiasTensor2
working tensor used to keep a temporary copy of bias or bias gradients
std::vector< std::vector< Matrix_t > > fPastSquaredWeightGradients
The sum of the square of the past weight gradients associated with the deep net.
std::vector< std::vector< Matrix_t > > & GetBiasUpdates()
std::vector< std::vector< Matrix_t > > fWorkWeightTensor2
working tensor used to keep a temporary copy of weights or weight gradients
Scalar_t GetEpsilon() const
std::vector< std::vector< Matrix_t > > fWorkBiasTensor1
working tensor used to keep a temporary copy of bias or bias gradients
Scalar_t fMomentum
The momentum used for training.
std::vector< std::vector< Matrix_t > > & GetPastSquaredBiasGradients()
Scalar_t fEpsilon
The Smoothing term used to avoid division by zero.
TRMSProp(DeepNet_t &deepNet, Scalar_t learningRate=0.001, Scalar_t momentum=0.0, Scalar_t rho=0.9, Scalar_t epsilon=1e-7)
Constructor.
std::vector< std::vector< Matrix_t > > fPastSquaredBiasGradients
The sum of the square of the past bias gradients associated with the deep net.
std::vector< std::vector< Matrix_t > > fWeightUpdates
The accumulation of the past Weights for performing updates.
typename Architecture_t::Matrix_t Matrix_t
void UpdateBiases(size_t layerIndex, std::vector< Matrix_t > &biases, const std::vector< Matrix_t > &biasGradients)
Update the biases, given the current bias gradients.
std::vector< Matrix_t > & GetBiasUpdatesAt(size_t i)
std::vector< std::vector< Matrix_t > > & GetWeightUpdates()
std::vector< std::vector< Matrix_t > > fWorkWeightTensor1
working tensor used to keep a temporary copy of weights or weight gradients
std::vector< Matrix_t > & GetWeightUpdatesAt(size_t i)
std::vector< std::vector< Matrix_t > > & GetPastSquaredWeightGradients()
std::vector< std::vector< Matrix_t > > fBiasUpdates
The accumulation of the past Biases for performing updates.
Scalar_t GetMomentum() const
Getters.
std::vector< Matrix_t > & GetPastSquaredBiasGradientsAt(size_t i)
std::vector< Layer_t * > & GetLayers()
void Copy(void *source, void *dest)
create variable transformations