34#ifndef EIGEN_SELFADJOINT_MATRIX_MATRIX_BLAS_H
35#define EIGEN_SELFADJOINT_MATRIX_MATRIX_BLAS_H
38#include "../InternalHeaderCheck.h"
47#define EIGEN_BLAS_SYMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
48 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
49 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
50 ConjugateRhs, ColMajor, 1> { \
51 using Scalar = EIGTYPE; \
52 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
53 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
54 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
55 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
56 if (!blas_compatible_stride<LhsStorageOrder>(rows, rows, lhsStride) || \
57 !blas_compatible_stride<RhsStorageOrder>(rows, cols, rhsStride) || \
58 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
59 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
60 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
61 res, resIncr, resStride, alpha, blocking); \
64 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
65 eigen_assert(resIncr == 1); \
66 char side = 'L', uplo = 'L'; \
67 BlasIndex m, n, lda, ldb, ldc; \
68 const EIGTYPE *a, *b; \
70 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
71 ColMajorMatrix b_tmp; \
75 m = convert_index<BlasIndex>(rows); \
76 n = convert_index<BlasIndex>(cols); \
79 lda = blas_leading_dim<LhsStorageOrder>(rows, rows, lhsStride); \
80 ldb = blas_leading_dim<RhsStorageOrder>(rows, cols, rhsStride); \
81 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
84 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
89 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
90 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, n, m, OuterStride<>(rhsStride)); \
91 b_tmp = rhs.adjoint(); \
93 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
97 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
98 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
102#define EIGEN_BLAS_HEMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
103 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
104 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
105 ConjugateRhs, ColMajor, 1> { \
106 using Scalar = EIGTYPE; \
107 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
108 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
109 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
110 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
111 if (!blas_compatible_stride<LhsStorageOrder>(rows, rows, lhsStride) || \
112 !blas_compatible_stride<RhsStorageOrder>(rows, cols, rhsStride) || \
113 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
114 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
115 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
116 res, resIncr, resStride, alpha, blocking); \
119 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
120 eigen_assert(resIncr == 1); \
121 char side = 'L', uplo = 'L'; \
122 BlasIndex m, n, lda, ldb, ldc; \
123 const EIGTYPE *a, *b; \
125 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
126 ColMajorMatrix b_tmp; \
127 Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder> a_tmp; \
131 m = convert_index<BlasIndex>(rows); \
132 n = convert_index<BlasIndex>(cols); \
135 lda = blas_leading_dim<LhsStorageOrder>(rows, rows, lhsStride); \
136 ldb = blas_leading_dim<RhsStorageOrder>(rows, cols, rhsStride); \
137 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
140 EIGEN_IF_CONSTEXPR (((LhsStorageOrder == ColMajor) && ConjugateLhs) || \
141 ((LhsStorageOrder == RowMajor) && (!ConjugateLhs))) { \
142 Map<const Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder>, 0, OuterStride<> > lhs( \
143 _lhs, m, m, OuterStride<>(lhsStride)); \
144 a_tmp = lhs.conjugate(); \
146 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
149 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
153 EIGEN_IF_CONSTEXPR (RhsStorageOrder == ColMajor && (!ConjugateRhs)) { \
156 EIGEN_IF_CONSTEXPR (RhsStorageOrder == ColMajor && ConjugateRhs) { \
157 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, m, n, OuterStride<>(rhsStride)); \
158 b_tmp = rhs.conjugate(); \
159 } else EIGEN_IF_CONSTEXPR (ConjugateRhs) { \
160 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, n, m, OuterStride<>(rhsStride)); \
161 b_tmp = rhs.adjoint(); \
163 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, n, m, OuterStride<>(rhsStride)); \
164 b_tmp = rhs.transpose(); \
167 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
170 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
171 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
176EIGEN_BLAS_SYMM_L(
double,
double, d, dsymm)
177EIGEN_BLAS_SYMM_L(
float,
float, f, ssymm)
178EIGEN_BLAS_HEMM_L(dcomplex, MKL_Complex16, cd, zhemm)
179EIGEN_BLAS_HEMM_L(scomplex, MKL_Complex8, cf, chemm)
181EIGEN_BLAS_SYMM_L(
double,
double, d, EIGEN_BLAS_SYM(dsymm))
182EIGEN_BLAS_SYMM_L(
float,
float, f, EIGEN_BLAS_SYM(ssymm))
183EIGEN_BLAS_HEMM_L(dcomplex,
double, cd, EIGEN_BLAS_SYM(zhemm))
184EIGEN_BLAS_HEMM_L(scomplex,
float, cf, EIGEN_BLAS_SYM(chemm))
189#define EIGEN_BLAS_SYMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
190 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
191 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
192 ConjugateRhs, ColMajor, 1> { \
193 using Scalar = EIGTYPE; \
194 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
195 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
196 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
197 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
198 if (!blas_compatible_stride<LhsStorageOrder>(rows, cols, lhsStride) || \
199 !blas_compatible_stride<RhsStorageOrder>(cols, cols, rhsStride) || \
200 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
201 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
202 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
203 res, resIncr, resStride, alpha, blocking); \
206 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
207 eigen_assert(resIncr == 1); \
208 char side = 'R', uplo = 'L'; \
209 BlasIndex m, n, lda, ldb, ldc; \
210 const EIGTYPE *a, *b; \
212 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
213 ColMajorMatrix b_tmp; \
216 m = convert_index<BlasIndex>(rows); \
217 n = convert_index<BlasIndex>(cols); \
220 lda = blas_leading_dim<RhsStorageOrder>(cols, cols, rhsStride); \
221 ldb = blas_leading_dim<LhsStorageOrder>(rows, cols, lhsStride); \
222 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
225 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
230 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
231 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, n, m, OuterStride<>(lhsStride)); \
232 b_tmp = lhs.adjoint(); \
234 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
238 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
239 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
243#define EIGEN_BLAS_HEMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
244 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
245 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
246 ConjugateRhs, ColMajor, 1> { \
247 using Scalar = EIGTYPE; \
248 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
249 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
250 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
251 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
252 if (!blas_compatible_stride<LhsStorageOrder>(rows, cols, lhsStride) || \
253 !blas_compatible_stride<RhsStorageOrder>(cols, cols, rhsStride) || \
254 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
255 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
256 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
257 res, resIncr, resStride, alpha, blocking); \
260 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
261 eigen_assert(resIncr == 1); \
262 char side = 'R', uplo = 'L'; \
263 BlasIndex m, n, lda, ldb, ldc; \
264 const EIGTYPE *a, *b; \
266 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
267 ColMajorMatrix b_tmp; \
268 Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder> a_tmp; \
271 m = convert_index<BlasIndex>(rows); \
272 n = convert_index<BlasIndex>(cols); \
275 lda = blas_leading_dim<RhsStorageOrder>(cols, cols, rhsStride); \
276 ldb = blas_leading_dim<LhsStorageOrder>(rows, cols, lhsStride); \
277 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
280 EIGEN_IF_CONSTEXPR (((RhsStorageOrder == ColMajor) && ConjugateRhs) || \
281 ((RhsStorageOrder == RowMajor) && (!ConjugateRhs))) { \
282 Map<const Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder>, 0, OuterStride<> > rhs( \
283 _rhs, n, n, OuterStride<>(rhsStride)); \
284 a_tmp = rhs.conjugate(); \
286 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
289 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
293 EIGEN_IF_CONSTEXPR (LhsStorageOrder == ColMajor && (!ConjugateLhs)) { \
296 EIGEN_IF_CONSTEXPR (LhsStorageOrder == ColMajor && ConjugateLhs) { \
297 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, m, n, OuterStride<>(lhsStride)); \
298 b_tmp = lhs.conjugate(); \
299 } else EIGEN_IF_CONSTEXPR (ConjugateLhs) { \
300 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, n, m, OuterStride<>(lhsStride)); \
301 b_tmp = lhs.adjoint(); \
303 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, n, m, OuterStride<>(lhsStride)); \
304 b_tmp = lhs.transpose(); \
307 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
310 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
311 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
316EIGEN_BLAS_SYMM_R(
double,
double, d, dsymm)
317EIGEN_BLAS_SYMM_R(
float,
float, f, ssymm)
318EIGEN_BLAS_HEMM_R(dcomplex, MKL_Complex16, cd, zhemm)
319EIGEN_BLAS_HEMM_R(scomplex, MKL_Complex8, cf, chemm)
321EIGEN_BLAS_SYMM_R(
double,
double, d, EIGEN_BLAS_SYM(dsymm))
322EIGEN_BLAS_SYMM_R(
float,
float, f, EIGEN_BLAS_SYM(ssymm))
323EIGEN_BLAS_HEMM_R(dcomplex,
double, cd, EIGEN_BLAS_SYM(zhemm))
324EIGEN_BLAS_HEMM_R(scomplex,
float, cf, EIGEN_BLAS_SYM(chemm))
327#undef EIGEN_BLAS_SYMM_L
328#undef EIGEN_BLAS_SYMM_R
329#undef EIGEN_BLAS_HEMM_L
330#undef EIGEN_BLAS_HEMM_R