34#ifndef EIGEN_TRIANGULAR_MATRIX_MATRIX_BLAS_H
35#define EIGEN_TRIANGULAR_MATRIX_MATRIX_BLAS_H
38#include "../InternalHeaderCheck.h"
44template <
typename Scalar,
typename Index,
int Mode,
bool LhsIsTriangular,
int LhsStorageOrder,
bool ConjugateLhs,
45 int RhsStorageOrder,
bool ConjugateRhs,
int ResStorageOrder>
46struct product_triangular_matrix_matrix_trmm
47 : product_triangular_matrix_matrix<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs,
48 RhsStorageOrder, ConjugateRhs, ResStorageOrder, 1, BuiltIn> {};
51#define EIGEN_BLAS_TRMM_SPECIALIZE(Scalar, LhsIsTriangular) \
52 template <typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
53 struct product_triangular_matrix_matrix<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs, \
54 RhsStorageOrder, ConjugateRhs, ColMajor, 1, Specialized> { \
55 static inline void run(Index _rows, Index _cols, Index _depth, const Scalar* _lhs, Index lhsStride, \
56 const Scalar* _rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride, \
57 Scalar alpha, level3_blocking<Scalar, Scalar>& blocking) { \
58 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
59 eigen_assert(resIncr == 1); \
60 product_triangular_matrix_matrix_trmm<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs, \
61 RhsStorageOrder, ConjugateRhs, ColMajor>::run(_rows, _cols, _depth, _lhs, \
62 lhsStride, _rhs, rhsStride, \
63 res, resStride, alpha, \
68EIGEN_BLAS_TRMM_SPECIALIZE(
double,
true)
69EIGEN_BLAS_TRMM_SPECIALIZE(
double, false)
70EIGEN_BLAS_TRMM_SPECIALIZE(dcomplex, true)
71EIGEN_BLAS_TRMM_SPECIALIZE(dcomplex, false)
72EIGEN_BLAS_TRMM_SPECIALIZE(
float, true)
73EIGEN_BLAS_TRMM_SPECIALIZE(
float, false)
74EIGEN_BLAS_TRMM_SPECIALIZE(scomplex, true)
75EIGEN_BLAS_TRMM_SPECIALIZE(scomplex, false)
79#define EIGEN_BLAS_TRMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
80 template <typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
81 struct product_triangular_matrix_matrix_trmm<EIGTYPE, Index, Mode, true, LhsStorageOrder, ConjugateLhs, \
82 RhsStorageOrder, ConjugateRhs, ColMajor> { \
84 IsLower = (Mode & Lower) == Lower, \
85 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1, \
86 IsUnitDiag = (Mode & UnitDiag) ? 1 : 0, \
87 IsZeroDiag = (Mode & ZeroDiag) ? 1 : 0, \
88 LowUp = IsLower ? Lower : Upper, \
89 conjA = ((LhsStorageOrder == ColMajor) && ConjugateLhs) ? 1 : 0 \
92 static void run(Index _rows, Index _cols, Index _depth, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
93 Index rhsStride, EIGTYPE* res, Index resStride, EIGTYPE alpha, \
94 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
95 if (_rows == 0 || _cols == 0 || _depth == 0 || numext::is_exactly_zero(alpha)) return; \
96 if (!blas_compatible_stride<LhsStorageOrder>(_rows, _depth, lhsStride) || \
97 !blas_compatible_stride<RhsStorageOrder>(_depth, _cols, rhsStride) || \
98 !blas_compatible_stride<ColMajor>(_rows, _cols, resStride)) { \
99 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, \
100 ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, _depth, _lhs, \
101 lhsStride, _rhs, rhsStride, res, \
102 Index(1), resStride, alpha, \
106 Index diagSize = (std::min)(_rows, _depth); \
107 Index rows = IsLower ? _rows : diagSize; \
108 Index depth = IsLower ? diagSize : _depth; \
109 Index cols = _cols; \
111 using MatrixLhs = Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder>; \
112 using MatrixRhs = Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder>; \
115 if (rows != depth) { \
119 if (((nthr == 1) && (((std::max)(rows, depth) - diagSize) / (double)diagSize < 0.5))) { \
121 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, \
122 ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, _depth, _lhs, \
123 lhsStride, _rhs, rhsStride, res, \
124 1, resStride, alpha, blocking); \
128 Map<const MatrixLhs, 0, OuterStride<> > lhsMap(_lhs, rows, depth, OuterStride<>(lhsStride)); \
129 MatrixLhs aa_tmp = lhsMap.template triangularView<Mode>(); \
130 BlasIndex aStride = convert_index<BlasIndex>(aa_tmp.outerStride()); \
131 gemm_blocking_space<ColMajor, EIGTYPE, EIGTYPE, Dynamic, Dynamic, Dynamic> gemm_blocking(_rows, _cols, \
133 general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
134 ConjugateRhs, ColMajor, 1>::run(rows, cols, depth, aa_tmp.data(), aStride, \
135 _rhs, rhsStride, res, 1, resStride, alpha, \
142 char side = 'L', transa, uplo, diag = 'N'; \
145 BlasIndex m, n, lda, ldb; \
148 m = convert_index<BlasIndex>(diagSize); \
149 n = convert_index<BlasIndex>(cols); \
152 transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N'; \
155 Map<const MatrixRhs, 0, OuterStride<> > rhs(_rhs, depth, cols, OuterStride<>(rhsStride)); \
156 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
157 ColMajorMatrix b_tmp; \
159 EIGEN_IF_CONSTEXPR (ConjugateRhs) \
160 b_tmp = rhs.conjugate(); \
164 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
167 uplo = IsLower ? 'L' : 'U'; \
168 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
169 uplo = (uplo == 'L') ? 'U' : 'L'; \
172 Map<const MatrixLhs, 0, OuterStride<> > lhs(_lhs, rows, depth, OuterStride<>(lhsStride)); \
175 EIGEN_IF_CONSTEXPR ((conjA != 0) || (SetDiag == 0)) { \
176 EIGEN_IF_CONSTEXPR (conjA) \
177 a_tmp = lhs.conjugate(); \
180 EIGEN_IF_CONSTEXPR (IsZeroDiag) \
181 a_tmp.diagonal().setZero(); \
182 else EIGEN_IF_CONSTEXPR (IsUnitDiag) \
183 a_tmp.diagonal().setOnes(); \
185 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
188 lda = blas_leading_dim<LhsStorageOrder>(rows, depth, lhsStride); \
192 BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, \
193 &lda, (BLASTYPE*)b, &ldb); \
196 Map<ColMajorMatrix, 0, OuterStride<> > res_tmp(res, rows, cols, OuterStride<>(resStride)); \
197 res_tmp = res_tmp + b_tmp; \
202EIGEN_BLAS_TRMM_L(
double,
double, d, dtrmm)
203EIGEN_BLAS_TRMM_L(dcomplex, MKL_Complex16, cd, ztrmm)
204EIGEN_BLAS_TRMM_L(
float,
float, f, strmm)
205EIGEN_BLAS_TRMM_L(scomplex, MKL_Complex8, cf, ctrmm)
207EIGEN_BLAS_TRMM_L(
double,
double, d, EIGEN_BLAS_SYM(dtrmm))
208EIGEN_BLAS_TRMM_L(dcomplex,
double, cd, EIGEN_BLAS_SYM(ztrmm))
209EIGEN_BLAS_TRMM_L(
float,
float, f, EIGEN_BLAS_SYM(strmm))
210EIGEN_BLAS_TRMM_L(scomplex,
float, cf, EIGEN_BLAS_SYM(ctrmm))
214#define EIGEN_BLAS_TRMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
215 template <typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
216 struct product_triangular_matrix_matrix_trmm<EIGTYPE, Index, Mode, false, LhsStorageOrder, ConjugateLhs, \
217 RhsStorageOrder, ConjugateRhs, ColMajor> { \
219 IsLower = (Mode & Lower) == Lower, \
220 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1, \
221 IsUnitDiag = (Mode & UnitDiag) ? 1 : 0, \
222 IsZeroDiag = (Mode & ZeroDiag) ? 1 : 0, \
223 LowUp = IsLower ? Lower : Upper, \
224 conjA = ((RhsStorageOrder == ColMajor) && ConjugateRhs) ? 1 : 0 \
227 static void run(Index _rows, Index _cols, Index _depth, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
228 Index rhsStride, EIGTYPE* res, Index resStride, EIGTYPE alpha, \
229 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
230 if (_rows == 0 || _cols == 0 || _depth == 0 || numext::is_exactly_zero(alpha)) return; \
231 if (!blas_compatible_stride<LhsStorageOrder>(_rows, _depth, lhsStride) || \
232 !blas_compatible_stride<RhsStorageOrder>(_depth, _cols, rhsStride) || \
233 !blas_compatible_stride<ColMajor>(_rows, _cols, resStride)) { \
234 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, false, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, \
235 ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, _depth, _lhs, \
236 lhsStride, _rhs, rhsStride, res, \
237 Index(1), resStride, alpha, \
241 Index diagSize = (std::min)(_cols, _depth); \
242 Index rows = _rows; \
243 Index depth = IsLower ? _depth : diagSize; \
244 Index cols = IsLower ? diagSize : _cols; \
246 using MatrixLhs = Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder>; \
247 using MatrixRhs = Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder>; \
250 if (cols != depth) { \
253 if ((nthr == 1) && (((std::max)(cols, depth) - diagSize) / (double)diagSize < 0.5)) { \
255 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, false, LhsStorageOrder, ConjugateLhs, \
256 RhsStorageOrder, ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, \
265 Map<const MatrixRhs, 0, OuterStride<> > rhsMap(_rhs, depth, cols, OuterStride<>(rhsStride)); \
266 MatrixRhs aa_tmp = rhsMap.template triangularView<Mode>(); \
267 BlasIndex aStride = convert_index<BlasIndex>(aa_tmp.outerStride()); \
268 gemm_blocking_space<ColMajor, EIGTYPE, EIGTYPE, Dynamic, Dynamic, Dynamic> gemm_blocking(_rows, _cols, \
270 general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
271 ConjugateRhs, ColMajor, 1>::run(rows, cols, depth, _lhs, lhsStride, \
272 aa_tmp.data(), aStride, res, 1, resStride, \
273 alpha, gemm_blocking, 0); \
279 char side = 'R', transa, uplo, diag = 'N'; \
282 BlasIndex m, n, lda, ldb; \
285 m = convert_index<BlasIndex>(rows); \
286 n = convert_index<BlasIndex>(diagSize); \
289 transa = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N'; \
292 Map<const MatrixLhs, 0, OuterStride<> > lhs(_lhs, rows, depth, OuterStride<>(lhsStride)); \
293 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
294 ColMajorMatrix b_tmp; \
296 EIGEN_IF_CONSTEXPR (ConjugateLhs) \
297 b_tmp = lhs.conjugate(); \
301 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
304 uplo = IsLower ? 'L' : 'U'; \
305 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
306 uplo = (uplo == 'L') ? 'U' : 'L'; \
309 Map<const MatrixRhs, 0, OuterStride<> > rhs(_rhs, depth, cols, OuterStride<>(rhsStride)); \
312 EIGEN_IF_CONSTEXPR ((conjA != 0) || (SetDiag == 0)) { \
313 EIGEN_IF_CONSTEXPR (conjA) \
314 a_tmp = rhs.conjugate(); \
317 EIGEN_IF_CONSTEXPR (IsZeroDiag) \
318 a_tmp.diagonal().setZero(); \
319 else EIGEN_IF_CONSTEXPR (IsUnitDiag) \
320 a_tmp.diagonal().setOnes(); \
322 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
325 lda = blas_leading_dim<RhsStorageOrder>(depth, cols, rhsStride); \
329 BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, \
330 &lda, (BLASTYPE*)b, &ldb); \
333 Map<ColMajorMatrix, 0, OuterStride<> > res_tmp(res, rows, cols, OuterStride<>(resStride)); \
334 res_tmp = res_tmp + b_tmp; \
339EIGEN_BLAS_TRMM_R(
double,
double, d, dtrmm)
340EIGEN_BLAS_TRMM_R(dcomplex, MKL_Complex16, cd, ztrmm)
341EIGEN_BLAS_TRMM_R(
float,
float, f, strmm)
342EIGEN_BLAS_TRMM_R(scomplex, MKL_Complex8, cf, ctrmm)
344EIGEN_BLAS_TRMM_R(
double,
double, d, EIGEN_BLAS_SYM(dtrmm))
345EIGEN_BLAS_TRMM_R(dcomplex,
double, cd, EIGEN_BLAS_SYM(ztrmm))
346EIGEN_BLAS_TRMM_R(
float,
float, f, EIGEN_BLAS_SYM(strmm))
347EIGEN_BLAS_TRMM_R(scomplex,
float, cf, EIGEN_BLAS_SYM(ctrmm))
350#undef EIGEN_BLAS_TRMM_SPECIALIZE
351#undef EIGEN_BLAS_TRMM_L
352#undef EIGEN_BLAS_TRMM_R