34 template <
typename Architecture>
36 typename Architecture::Matrix_t &
X,
37 const typename Architecture::Matrix_t &
Y,
41 typename Architecture::Scalar_t dx)
42 ->
typename Architecture::Scalar_t
44 using Scalar_t =
typename Architecture::Scalar_t;
46 net.GetLayer(
l).GetWeights().operator()(i,j) += dx;
47 Scalar_t res = net.Loss(
X,
Y);
48 net.GetLayer(
l).GetWeights().operator()(i,j) -= dx;
55 template <
typename Architecture>
57 typename Architecture::Matrix_t &
X,
58 const typename Architecture::Matrix_t &
Y,
61 typename Architecture::Scalar_t dx)
62 ->
typename Architecture::Scalar_t
64 using Scalar_t =
typename Architecture::Scalar_t;
66 net.GetLayer(
l).GetBiases().operator()(i,0) += dx;
67 Scalar_t res = net.Loss(
X,
Y);
68 net.GetLayer(
l).GetBiases().operator()(i,0) -= dx;
76 template <
typename Architecture>
78 ->
typename Architecture::Scalar_t
80 using Scalar_t =
typename Architecture::Scalar_t;
81 using Matrix_t =
typename Architecture::Matrix_t;
95 Matrix_t
Y(50, net.GetOutputWidth());
101 Scalar_t maximum_error = 0.0;
105 for (
size_t l = 0;
l < net.GetDepth();
l++)
107 std::cout <<
"\rTesting weight gradients: layer: " 108 <<
l <<
" / " << net.GetDepth();
109 std::cout << std::flush;
110 auto & layer = net.GetLayer(
l);
111 auto & W = layer.GetWeightGradients();
113 for (
size_t i = 0; i < layer.GetWidth(); i++)
115 for (
size_t j = 0; j < layer.GetInputWidth(); j++)
117 auto f = [& net, &
X, &
Y,
l, i, j](Scalar_t
x)
122 Scalar_t dy_ref = W(i,j);
128 error =
std::fabs((dy - dy_ref) / dy_ref);
135 maximum_error = std::max(error, maximum_error);
140 std::cout <<
"\rTesting weight gradients: ";
141 std::cout <<
"maximum relative error: " <<
print_error(maximum_error) << std::endl;
142 return maximum_error;
150 template <
typename Architecture>
152 ->
typename Architecture::Scalar_t
154 using Scalar_t =
typename Architecture::Scalar_t;
155 using Matrix_t =
typename Architecture::Matrix_t;
168 Matrix_t
Y(50, net.GetOutputWidth());
174 Scalar_t maximum_error = 0.0;
178 for (
size_t l = 0;
l < net.GetDepth();
l++)
180 std::cout <<
"\rTesting weight gradients (L1): layer: " 181 <<
l <<
" / " << net.GetDepth();
182 std::cout << std::flush;
183 auto & layer = net.GetLayer(
l);
184 auto & W = layer.GetWeights();
185 auto & dW = layer.GetWeightGradients();
187 for (
size_t i = 0; i < layer.GetWidth(); i++) {
188 for (
size_t j = 0; j < layer.GetInputWidth(); j++) {
190 if (std::abs(W(i,j)) > dx) {
191 auto f = [& net, &
X, &
Y,
l, i, j](Scalar_t
x)
196 Scalar_t dy_ref = dW(i,j);
202 error =
std::fabs((dy - dy_ref) / dy_ref);
209 maximum_error = std::max(error, maximum_error);
215 std::cout <<
"\rTesting weight gradients (L1): ";
216 std::cout <<
"maximum relative error: " <<
print_error(maximum_error) << std::endl;
217 return maximum_error;
225 template <
typename Architecture>
227 ->
typename Architecture::Scalar_t
229 using Scalar_t =
typename Architecture::Scalar_t;
230 using Matrix_t =
typename Architecture::Matrix_t;
243 Matrix_t
Y(50, net.GetOutputWidth());
249 Scalar_t maximum_error = 0.0;
253 for (
size_t l = 0;
l < net.GetDepth();
l++)
255 std::cout <<
"\rTesting weight gradients (L2): layer: " 256 <<
l <<
" / " << net.GetDepth();
257 std::cout << std::flush;
258 auto & layer = net.GetLayer(
l);
259 auto & W = layer.GetWeightGradients();
261 for (
size_t i = 0; i < layer.GetWidth(); i++)
263 for (
size_t j = 0; j < layer.GetInputWidth(); j++)
265 auto f = [& net, &
X, &
Y,
l, i, j](Scalar_t
x)
270 Scalar_t dy_ref = W(i,j);
276 error =
std::fabs((dy - dy_ref) / dy_ref);
283 maximum_error = std::max(error, maximum_error);
288 std::cout <<
"\rTesting weight gradients (L2): ";
289 std::cout <<
"maximum relative error: " <<
print_error(maximum_error) << std::endl;
290 return maximum_error;
297 template <
typename Architecture>
299 ->
typename Architecture::Scalar_t
302 using Scalar_t =
typename Architecture::Scalar_t;
303 using Matrix_t =
typename Architecture::Matrix_t;
316 Matrix_t
Y(50, net.GetOutputWidth());
322 Scalar_t maximum_error = 0.0;
326 for (
size_t l = 0;
l < net.GetDepth();
l++)
328 std::cout <<
"\rTesting bias gradients: layer: " 329 <<
l <<
" / " << net.GetDepth();
330 std::cout << std::flush;
331 auto & layer = net.GetLayer(
l);
332 auto & dtheta = layer.GetBiasGradients();
334 for (
size_t i = 0; i < layer.GetWidth(); i++)
336 auto f = [& net, &
X, &
Y,
l, i](Scalar_t
x)
341 Scalar_t dy_ref = dtheta(i,0) * 2.0 * dx;
347 error =
std::fabs((dy - dy_ref) / dy_ref);
354 maximum_error = std::max(error, maximum_error);
358 std::cout <<
"\rTesting bias gradients: ";
359 std::cout <<
"maximum relative error: " <<
print_error(maximum_error) << std::endl;
360 return maximum_error;
void randomMatrix(AMatrix &X)
Fill matrix with random, Gaussian-distributed values.
void constructRandomLinearNet(TNet< AArchitecture > &net)
Construct a random linear neural network with up to five layers.
auto evaluate_net_bias(TNet< Architecture > &net, typename Architecture::Matrix_t &X, const typename Architecture::Matrix_t &Y, size_t l, size_t i, typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t
Compute the loss of the net as a function of the weight at index i in layer l.
std::string print_error(AFloat &e)
Color code error.
AFloat finiteDifference(F f, AFloat dx)
Numerically compute the derivative of the functional f using finite differences.
Generic neural network class.
auto testBackpropagationL2Regularization(typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t
Generate a random, linear net, perform forward and backward propagation with L2 regularization and ch...
VecExpr< UnaryOp< Fabs< T >, VecExpr< A, T, D >, T >, T, D > fabs(const VecExpr< A, T, D > &rhs)
void randomBatch(AMatrix &X)
Generate a random batch as input for a neural net.
auto testBackpropagationL1Regularization(typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t
Generate a random, linear net, perform forward and backward propagation with L1 regularization and ch...
auto testBackpropagationBiasesLinear(typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t
Generate a random net, perform forward and backward propagation and check the bias gradients using nu...
you should not use this method at all Int_t Int_t Double_t Double_t Double_t e
auto testBackpropagationWeightsLinear(typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t
Generate a random net, perform forward and backward propagation and check the weight gradients using ...
auto evaluate_net_weight(TNet< Architecture > &net, typename Architecture::Matrix_t &X, const typename Architecture::Matrix_t &Y, size_t l, size_t i, size_t j, typename Architecture::Scalar_t dx) -> typename Architecture::Scalar_t
Compute the loss of the net as a function of the weight at index (i,j) in layer l.