32template <
typename AFloat>
41 if ((
int)
output.GetNrows() !=
m) {
42 Error(
"MultiplyTranspose",
"Invalid input - output rows - input: %d != output : %d",
m, (
int)
output.GetNrows());
45 if ((
int)
output.GetNcols() !=
n) {
46 Error(
"MultiplyTranspose",
"Invalid output cols or weight rows - output cols: %d != weight rows : %d",(
int)
output.GetNcols(),
n);
49 if ((
int)
Weights.GetNcols() != k) {
50 Error(
"MultiplyTranspose",
"Invalid input cols or weight cols - input cols: %d != weight cols : %d", k, (
int)
Weights.GetNcols());
62 const AFloat *A =
input.GetRawDataPointer();
63 const AFloat *B =
Weights.GetRawDataPointer();
64 AFloat *
C =
output.GetRawDataPointer();
66 ::TMVA::DNN::Blas::Gemm(&
transa, &
transb, &
m, &
n, &k, &alpha, A, &
m, B, &
n, &beta, C, &
m);
74template <
typename AFloat>
84 AFloat *A =
output.GetRawDataPointer();
86 const AFloat *
y =
biases.GetRawDataPointer();
99template <
typename AFloat>
131template <
typename AFloat>
182template <
typename AFloat>
234template <
typename AFloat>
238 R__ASSERT(
n == A.GetNcols() * A.GetNrows() );
239 AFloat *
a = A.GetRawDataPointer();
240 const AFloat *
b = B.GetRawDataPointer();
253 if (idx >= 0)
a[
ii] =
b[idx];
263 for (
size_t ii = 0;
ii <
n; ++
ii) {
265 if (idx >= 0)
a[
ii] =
b[idx];
272template <
typename AFloat>
274 size_t filterHeight,
size_t filterWidth,
size_t numFilters)
276 size_t jump = filterHeight * filterWidth;
279 for (
size_t i = 0; i <
jump; i++) {
280 A(
j, k *
jump + i) = B(k, ((
j + 1) *
jump - 1) - i);
288template <
typename AFloat>
298 AFloat *A =
output.GetRawDataPointer();
299 const AFloat *
x =
biases.GetRawDataPointer();
313template<
typename AFloat>
318 Fatal(
"calculateDimension",
"Not compatible hyper parameters for layer - (imageDim, filterDim, padding, stride) "
325template <
typename AFloat>
379template <
typename AFloat>
390 size_t batchSize,
size_t inputHeight,
391 size_t inputWidth,
size_t depth,
424template <
typename AFloat>
428 size_t inputHeight,
size_t inputWidth,
size_t depth,
size_t height,
468 R__ASSERT(batchSize == df.GetFirstSize() );
493template <
typename AFloat>
497 size_t batchSize,
size_t inputHeight,
size_t inputWidth,
size_t depth,
504 const size_t filterSize = filterHeight * filterWidth;
536 auto fmap = [&](
int i) {
554 Multiply(
mres, df.At(i).GetMatrix(),
xTr);
565 for (
size_t i = 0; i < batchSize; i++) {
568 for (
size_t j = 0;
j <
depth;
j++) {
586template <
typename AFloat>
591 for (
size_t i = 0; i <
depth; i++) {
594 for (
size_t k = 0; k < batchSize; k++) {
604template <
typename AFloat>
613 assert(
tA.GetFirstSize() ==
tC.GetFirstSize());
629 for (
int m = 0;
m < (
Int_t)C.GetNrows();
m++) {
630 AFloat
value = -std::numeric_limits<AFloat>::max();
649template <
typename AFloat>
675 for (
size_t j = 0;
j <
depth;
j++) {
692template <
typename AFloat>
703 return x.Reshape( {
x.GetShape().front(),
x.GetSize()/
x.GetShape().front()});
708template <
typename AFloat>
731 size_t n =
input.GetShape()[0];
732 size_t d =
input.GetShape()[1];
739 auto f = [&] (
size_t k)
747 for (
size_t i = 0; i <
n; i++) {
754 for (
size_t i = 0; i <
n; i++) {
765 double gK = gamma(0, k);
766 double bK = beta(0, k);
767 for (
size_t i = 0; i <
n; i++) {
779 double decay = momentum;
793template <
typename AFloat>
807 size_t n =
input.GetShape()[0];
808 size_t d =
input.GetShape()[1];
813 auto f = [&] (
size_t k) {
818 double gK = gamma(0, k);
819 double bK = beta(0, k);
824 for (
size_t i = 0; i <
n; i++) {
834template <
typename AFloat>
860 auto f = [&] (
size_t k) {
866 auto meanK = mean(0, k);
867 for (
size_t i = 0; i <
n; i++) {
880 for (
size_t i = 0; i <
n; i++) {
892template <
typename AFloat>
895 size_t nColsA = A.GetNcols();
896 size_t nColsB = B.GetNcols();
898 for (
size_t i = 0; i < A.GetNrows(); i++) {
899 for (
size_t j = 0;
j < A.GetNcols();
j++) {
907template <
typename AFloat>
914 assert( B.GetShape().size() == 3 );
915 assert( A.GetShape().size() == 3 );
918 size_t bsize = B.GetFirstSize();
919 size_t nRows = B.GetHSize();
920 size_t nCols = B.GetWSize();
922 assert ( A.GetFirstSize() == 1);
926 for (
size_t i = 0; i <
bsize; i++) {
927 for (
size_t j = 0;
j <
nRows;
j++) {
928 for (
size_t k = 0; k <
nCols; k++) {
929 A( 0, i,
j *
nCols + k) = B(i,
j, k);
943template <
typename AFloat>
947 assert( B.GetShape().size() == 3 );
948 assert( A.GetShape().size() == 3 );
950 size_t size = A.GetFirstSize();
951 size_t nRows = A.GetHSize();
952 size_t nCols = A.GetWSize();
954 assert ( B.GetFirstSize() == 1);
957 for (
size_t i = 0; i < (size_t)
size; i++) {
958 for (
size_t j = 0;
j < (size_t)
nRows;
j++) {
959 for (
size_t k = 0; k < (size_t)
nCols; k++) {
960 A(i,
j, k) = B(0, i,
j *
nCols + k);
967template <
typename AFloat>
971 assert ( out.GetShape().size() == 3 && in.
GetShape().size() == 3);
974 size_t B = out.GetFirstSize();
975 size_t T = out.GetCSize();
976 size_t D = out.GetWSize();
978 std::cout <<
"Incompatible Dimensions\n"
984 for (
size_t i = 0; i < B; ++i) {
985 for (
size_t j = 0;
j < T; ++
j) {
986 for (
size_t k = 0; k < D; ++k) {
987 out( i,
j, k ) = in(
j, i, k);
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
void Error(const char *location, const char *msgfmt,...)
Use this function in case an error occurred.
void Fatal(const char *location, const char *msgfmt,...)
Use this function in case of a fatal error. It will abort the program.
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void input
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t height
static size_t GetOnePointerSize()
static const AFloat * GetOnePointer()
static size_t GetNWorkItems(size_t nelements)
static void InitializeOneVector(size_t n)
static Executor & GetThreadExecutor()
size_t GetFirstSize() const
typename TMVA::Experimental::RTensor< AFloat >::Shape_t Shape_t
static void CalculateConvBiasGradients(Matrix_t &biasGradients, const Tensor_t &df, size_t batchSize, size_t depth, size_t nLocalViews)
Utility function for calculating the bias gradients of the convolutional layer.
static void Deflatten(Tensor_t &A, const Tensor_t &B)
Transforms each row of B to a matrix and stores it in the tensor B.
static void MaxPoolLayerBackward(Tensor_t &activationGradientsBackward, const Tensor_t &activationGradients, const Tensor_t &indexMatrix, const Tensor_t &, const Tensor_t &, const PoolingDescriptors_t &, PoolingWorkspace_t &, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t nLocalViews)
Perform the complete backward propagation step in a Pooling Layer.
static void ConvLayerForward(Tensor_t &output, Tensor_t &inputActivationFunc, const Tensor_t &input, const Matrix_t &weights, const Matrix_t &biases, const DNN::CNN::TConvParams ¶ms, EActivationFunction activFunc, Tensor_t &, const ConvDescriptors_t &, ConvWorkspace_t &)
Forward propagation in the Convolutional layer.
static void Im2colFast(Matrix_t &A, const Matrix_t &B, const std::vector< int > &V)
static void AddRowWise(Matrix_t &output, const Matrix_t &biases)
Add the vectors biases row-wise to the matrix output.
static void CalculateConvActivationGradients(Tensor_t &activationGradientsBackward, const Tensor_t &df, const Matrix_t &weights, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth)
Utility function for calculating the activation gradients of the layer before the convolutional layer...
static void BatchNormLayerForwardTraining(int axis, const Tensor_t &x, Tensor_t &y, Matrix_t &gamma, Matrix_t &beta, Matrix_t &mean, Matrix_t &, Matrix_t &iVariance, Matrix_t &runningMeans, Matrix_t &runningVars, Scalar_t nTrainedBatches, Scalar_t momentum, Scalar_t epsilon, const TensorDescriptor_t &bnParDescriptor)
The input from each batch are normalized during training to have zero mean and unit variance and they...
static void Backward(Tensor_t &activationGradientsBackward, Matrix_t &weightGradients, Matrix_t &biasGradients, const Tensor_t &df, const Tensor_t &activationGradients, const Matrix_t &weights, const Tensor_t &activationBackward)
Perform the complete backward propagation step.
static void Reshape(Matrix_t &A, const Matrix_t &B)
Transform the matrix B to a matrix with different dimensions A.
static void Rearrange(Tensor_t &out, const Tensor_t &in)
Rearrage data according to time fill B x T x D out with T x B x D matrix in.
static void MultiplyTranspose(Matrix_t &output, const Matrix_t &input, const Matrix_t &weights)
Matrix-multiply input with the transpose of weights and write the results into output.
static void CalculateConvWeightGradients(Matrix_t &weightGradients, const Tensor_t &df, const Tensor_t &activations_backward, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews)
Utility function for calculating the weight gradients of the convolutional layer.
static size_t calculateDimension(size_t imgDim, size_t fltDim, size_t padding, size_t stride)
Calculate how many neurons "fit" in the output layer, given the input as well as the layer's hyperpar...
static void BatchNormLayerBackward(int axis, const Tensor_t &x, const Tensor_t &dy, Tensor_t &dx, Matrix_t &gamma, Matrix_t &dgamma, Matrix_t &dbeta, const Matrix_t &mean, const Matrix_t &variance, const Matrix_t &iVariance, Scalar_t epsilon, const TensorDescriptor_t &)
static void ConvLayerBackward(Tensor_t &activationGradientsBackward, Matrix_t &weightGradients, Matrix_t &biasGradients, Tensor_t &df, Tensor_t &activationGradients, const Matrix_t &weights, const Tensor_t &activationBackward, const Tensor_t &outputTensor, EActivationFunction activFunc, const ConvDescriptors_t &, ConvWorkspace_t &, size_t batchSize, size_t inputHeight, size_t inputWidth, size_t depth, size_t height, size_t width, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t nLocalViews)
Perform the complete backward propagation step in a Convolutional Layer.
static Tensor_t BatchNormLayerReshapeTensor(int axis, const Tensor_t &x)
static void Im2colIndices(std::vector< int > &V, const Matrix_t &B, size_t nLocalViews, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth)
static void Flatten(Tensor_t &A, const Tensor_t &B)
Flattens the tensor B, such that each matrix, is stretched in one row, resulting with a matrix A.
static void AddConvBiases(Matrix_t &output, const Matrix_t &biases)
Add the biases in the Convolutional Layer.
static void Im2col(Matrix_t &A, const Matrix_t &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth)
Transform the matrix B in local view format, suitable for convolution, and store it in matrix A.
static void BatchNormLayerForwardInference(int axis, const Tensor_t &x, Matrix_t &gamma, Matrix_t &beta, Tensor_t &y, const Matrix_t &runningMeans, const Matrix_t &runningVars, Scalar_t epsilon, const TensorDescriptor_t &)
During inference the inputs are not normalized using the batch mean but the previously computed at ru...
static void Downsample(Tensor_t &A, Tensor_t &B, const Tensor_t &C, const PoolingDescriptors_t &, PoolingWorkspace_t &, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols)
Downsample the matrix C to the matrix A, using max operation, such that the winning indices are store...
TCpuMatrix< AReal > Matrix_t
static void RotateWeights(Matrix_t &A, const Matrix_t &B, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t numFilters)
Rotates the matrix B, which is representing a weights, and stores them in the matrix A.
static void AddRowWise(TMatrixT< Scalar_t > &output, const TMatrixT< Scalar_t > &biases)
Add the vectors biases row-wise to the matrix output.
static void AddConvBiases(TMatrixT< AReal > &output, const TMatrixT< AReal > &biases)
Add the biases in the Convolutional Layer.
const Shape_t & GetShape() const
double beta(double x, double y)
Calculates the beta function.
void Gemm(const char *transa, const char *transb, const int *m, const int *n, const int *k, const AReal *alpha, const AReal *A, const int *lda, const AReal *B, const int *ldb, const AReal *beta, AReal *C, const int *ldc)
Multiply the matrix A with the matrix B and store the result in C.
void Ger(const int *m, const int *n, const AReal *alpha, const AReal *x, const int *incx, const AReal *y, const int *incy, AReal *A, const int *lda)
Add the outer product of x and y to the matrix A.
EActivationFunction
Enum that represents layer activation functions.
create variable transformations
constexpr Double_t C()
Velocity of light in .
size_t strideRows
The number of row pixels to slid the filter each step.
size_t filterHeight
The height of the filter.
size_t inputHeight
The height of the previous layer or input.
size_t paddingWidth
The number of zero layers left and right of the input.
size_t filterWidth
The width of the filter.
size_t paddingHeight
The number of zero layers added top and bottom of the input.
size_t inputWidth
The width of the previous layer or input.
size_t inputDepth
The depth of the previous layer or input.
size_t strideCols
The number of column pixels to slid the filter each step.
static uint64_t sum(uint64_t i)