34#ifndef EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
35#define EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
38#include "../InternalHeaderCheck.h"
54#define EIGEN_BLAS_GEMM_SPECIALIZATION(EIGTYPE, EIGPREFIX, BLASTYPE, BLASFUNC) \
55 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
56 struct general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
57 ConjugateRhs, ColMajor, 1> { \
58 using Traits = gebp_traits<EIGTYPE, EIGTYPE>; \
59 using Scalar = EIGTYPE; \
61 static void run(Index rows, Index cols, Index depth, const EIGTYPE* lhs_, Index lhsStride, const EIGTYPE* rhs_, \
62 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
63 level3_blocking<EIGTYPE, EIGTYPE>& blocking, GemmParallelInfo<Index>* info) { \
64 if (rows == 0 || cols == 0 || depth == 0 || numext::is_exactly_zero(alpha)) return; \
65 if (!blas_compatible_stride<LhsStorageOrder>(rows, depth, lhsStride) || \
66 !blas_compatible_stride<RhsStorageOrder>(depth, cols, rhsStride) || \
67 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
68 general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
69 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, depth, lhs_, lhsStride, rhs_, \
70 rhsStride, res, resIncr, resStride, alpha, \
74 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
75 eigen_assert(resIncr == 1); \
76 char transa, transb; \
77 BlasIndex m, n, k, lda, ldb, ldc; \
78 const EIGTYPE *a, *b; \
80 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
81 ColMajorMatrix a_tmp, b_tmp; \
84 transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N'; \
85 transb = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N'; \
88 m = convert_index<BlasIndex>(rows); \
89 n = convert_index<BlasIndex>(cols); \
90 k = convert_index<BlasIndex>(depth); \
93 lda = blas_leading_dim<LhsStorageOrder>(rows, depth, lhsStride); \
94 ldb = blas_leading_dim<RhsStorageOrder>(depth, cols, rhsStride); \
95 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
98 EIGEN_IF_CONSTEXPR ((LhsStorageOrder == ColMajor) && (ConjugateLhs)) { \
99 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride)); \
100 a_tmp = lhs.conjugate(); \
102 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
106 EIGEN_IF_CONSTEXPR ((RhsStorageOrder == ColMajor) && (ConjugateRhs)) { \
107 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride)); \
108 b_tmp = rhs.conjugate(); \
110 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
114 BLASFUNC(&transa, &transb, &m, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
115 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
120EIGEN_BLAS_GEMM_SPECIALIZATION(
double, d,
double, dgemm)
121EIGEN_BLAS_GEMM_SPECIALIZATION(
float, f,
float, sgemm)
122EIGEN_BLAS_GEMM_SPECIALIZATION(dcomplex, cd, MKL_Complex16, zgemm)
123EIGEN_BLAS_GEMM_SPECIALIZATION(scomplex, cf, MKL_Complex8, cgemm)
125EIGEN_BLAS_GEMM_SPECIALIZATION(
double, d,
double, EIGEN_BLAS_SYM(dgemm))
126EIGEN_BLAS_GEMM_SPECIALIZATION(
float, f,
float, EIGEN_BLAS_SYM(sgemm))
127EIGEN_BLAS_GEMM_SPECIALIZATION(dcomplex, cd,
double, EIGEN_BLAS_SYM(zgemm))
128EIGEN_BLAS_GEMM_SPECIALIZATION(scomplex, cf,
float, EIGEN_BLAS_SYM(cgemm))
133#if defined(EIGEN_USE_OPENBLAS_BFLOAT16) && EIGEN_USE_OPENBLAS_BFLOAT16
137void EIGEN_BLAS_SYM(sbgemm)(
const char* trans_a,
const char* trans_b,
const EIGEN_BLAS_INT* M,
const EIGEN_BLAS_INT* N,
138 const EIGEN_BLAS_INT* K,
const float* alpha,
const Eigen::bfloat16* A,
139 const EIGEN_BLAS_INT* lda,
const Eigen::bfloat16* B,
const EIGEN_BLAS_INT* ldb,
140 const float* beta,
float* C,
const EIGEN_BLAS_INT* ldc);
143template <
typename Index,
int LhsStorageOrder,
bool ConjugateLhs,
int RhsStorageOrder,
bool ConjugateRhs>
144struct general_matrix_matrix_product<Index, Eigen::bfloat16, LhsStorageOrder, ConjugateLhs, Eigen::bfloat16,
145 RhsStorageOrder, ConjugateRhs,
ColMajor, 1> {
146 using Traits = gebp_traits<Eigen::bfloat16, Eigen::bfloat16>;
148 static void run(Index rows, Index cols, Index depth,
const Eigen::bfloat16* lhs_, Index lhsStride,
149 const Eigen::bfloat16* rhs_, Index rhsStride, Eigen::bfloat16* res, Index resIncr, Index resStride,
150 Eigen::bfloat16 alpha, level3_blocking<Eigen::bfloat16, Eigen::bfloat16>& blocking,
151 GemmParallelInfo<Index>* info) {
152 if (rows == 0 || cols == 0 || depth == 0 || numext::is_exactly_zero(alpha))
return;
153 if (!blas_compatible_stride<LhsStorageOrder>(rows, depth, lhsStride) ||
154 !blas_compatible_stride<RhsStorageOrder>(depth, cols, rhsStride)) {
155 general_matrix_matrix_product<Index, Eigen::bfloat16, LhsStorageOrder, ConjugateLhs, Eigen::bfloat16,
156 RhsStorageOrder, ConjugateRhs,
ColMajor, Dynamic>::run(rows, cols, depth, lhs_,
157 lhsStride, rhs_, rhsStride,
158 res, resIncr, resStride,
159 alpha, blocking, info);
162 EIGEN_ONLY_USED_FOR_DEBUG(resIncr);
163 eigen_assert(resIncr == 1);
165 BlasIndex m, n, k, lda, ldb, ldc;
166 const Eigen::bfloat16 *a, *b;
168 float falpha =
static_cast<float>(alpha);
169 float fbeta = float(1.0);
172 using MatrixXbf16 = Matrix<Eigen::bfloat16, Dynamic, Dynamic, ColMajor>;
173 MatrixXbf16 a_tmp, b_tmp;
176 transa = (LhsStorageOrder ==
RowMajor) ? ((ConjugateLhs) ?
'C' :
'T') :
'N';
177 transb = (RhsStorageOrder ==
RowMajor) ? ((ConjugateRhs) ?
'C' :
'T') :
'N';
180 m = convert_index<BlasIndex>(rows);
181 n = convert_index<BlasIndex>(cols);
182 k = convert_index<BlasIndex>(depth);
185 lda = blas_leading_dim<LhsStorageOrder>(rows, depth, lhsStride);
186 ldb = blas_leading_dim<RhsStorageOrder>(depth, cols, rhsStride);
187 ldc = convert_index<BlasIndex>(m);
190 EIGEN_IF_CONSTEXPR ((LhsStorageOrder ==
ColMajor) && (ConjugateLhs)) {
191 Map<const MatrixXbf16, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride));
192 a_tmp = lhs.conjugate();
194 lda = convert_index<BlasIndex>(a_tmp.outerStride());
199 EIGEN_IF_CONSTEXPR ((RhsStorageOrder ==
ColMajor) && (ConjugateRhs)) {
200 Map<const MatrixXbf16, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride));
201 b_tmp = rhs.conjugate();
203 ldb = convert_index<BlasIndex>(b_tmp.outerStride());
209 Map<MatrixXbf16, 0, OuterStride<> > result(res, m, n, OuterStride<>(resStride));
210 Matrix<float, Dynamic, Dynamic, ColMajor> r_tmp = result.cast<
float>();
212 EIGEN_BLAS_SYM(sbgemm)
213 (&transa, &transb, &m, &n, &k, (
const float*)&numext::real_ref(falpha), a, &lda, b, &ldb,
214 (
const float*)&numext::real_ref(fbeta), r_tmp.data(), &ldc);
216 result = r_tmp.cast<Eigen::bfloat16>();
222#undef EIGEN_BLAS_GEMM_SPECIALIZATION
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321