Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TriangularMatrixMatrix_BLAS.h
1/*
2 Copyright (c) 2011, Intel Corporation. All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without modification,
5 are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice, this
8 list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright notice,
10 this list of conditions and the following disclaimer in the documentation
11 and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors may
13 be used to endorse or promote products derived from this software without
14 specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
23 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 ********************************************************************************
28 * Content : Eigen bindings to BLAS F77
29 * Triangular matrix * matrix product functionality based on ?TRMM.
30 ********************************************************************************
31*/
32// SPDX-License-Identifier: BSD-3-Clause
33
34#ifndef EIGEN_TRIANGULAR_MATRIX_MATRIX_BLAS_H
35#define EIGEN_TRIANGULAR_MATRIX_MATRIX_BLAS_H
36
37// IWYU pragma: private
38#include "../InternalHeaderCheck.h"
39
40namespace Eigen {
41
42namespace internal {
43
44template <typename Scalar, typename Index, int Mode, bool LhsIsTriangular, int LhsStorageOrder, bool ConjugateLhs,
45 int RhsStorageOrder, bool ConjugateRhs, int ResStorageOrder>
46struct product_triangular_matrix_matrix_trmm
47 : product_triangular_matrix_matrix<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs,
48 RhsStorageOrder, ConjugateRhs, ResStorageOrder, 1, BuiltIn> {};
49
50// try to go to BLAS specialization
51#define EIGEN_BLAS_TRMM_SPECIALIZE(Scalar, LhsIsTriangular) \
52 template <typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
53 struct product_triangular_matrix_matrix<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs, \
54 RhsStorageOrder, ConjugateRhs, ColMajor, 1, Specialized> { \
55 static inline void run(Index _rows, Index _cols, Index _depth, const Scalar* _lhs, Index lhsStride, \
56 const Scalar* _rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride, \
57 Scalar alpha, level3_blocking<Scalar, Scalar>& blocking) { \
58 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
59 eigen_assert(resIncr == 1); \
60 product_triangular_matrix_matrix_trmm<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs, \
61 RhsStorageOrder, ConjugateRhs, ColMajor>::run(_rows, _cols, _depth, _lhs, \
62 lhsStride, _rhs, rhsStride, \
63 res, resStride, alpha, \
64 blocking); \
65 } \
66 };
67
68EIGEN_BLAS_TRMM_SPECIALIZE(double, true)
69EIGEN_BLAS_TRMM_SPECIALIZE(double, false)
70EIGEN_BLAS_TRMM_SPECIALIZE(dcomplex, true)
71EIGEN_BLAS_TRMM_SPECIALIZE(dcomplex, false)
72EIGEN_BLAS_TRMM_SPECIALIZE(float, true)
73EIGEN_BLAS_TRMM_SPECIALIZE(float, false)
74EIGEN_BLAS_TRMM_SPECIALIZE(scomplex, true)
75EIGEN_BLAS_TRMM_SPECIALIZE(scomplex, false)
76
77// implements col-major += alpha * op(triangular) * op(general)
78// Temporaries are ColMajor, also under EIGEN_DEFAULT_TO_ROW_MAJOR: BLAS takes column-major storage.
79#define EIGEN_BLAS_TRMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
80 template <typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
81 struct product_triangular_matrix_matrix_trmm<EIGTYPE, Index, Mode, true, LhsStorageOrder, ConjugateLhs, \
82 RhsStorageOrder, ConjugateRhs, ColMajor> { \
83 enum { \
84 IsLower = (Mode & Lower) == Lower, \
85 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1, \
86 IsUnitDiag = (Mode & UnitDiag) ? 1 : 0, \
87 IsZeroDiag = (Mode & ZeroDiag) ? 1 : 0, \
88 LowUp = IsLower ? Lower : Upper, \
89 conjA = ((LhsStorageOrder == ColMajor) && ConjugateLhs) ? 1 : 0 \
90 }; \
91 \
92 static void run(Index _rows, Index _cols, Index _depth, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
93 Index rhsStride, EIGTYPE* res, Index resStride, EIGTYPE alpha, \
94 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
95 if (_rows == 0 || _cols == 0 || _depth == 0 || numext::is_exactly_zero(alpha)) return; \
96 if (!blas_compatible_stride<LhsStorageOrder>(_rows, _depth, lhsStride) || \
97 !blas_compatible_stride<RhsStorageOrder>(_depth, _cols, rhsStride) || \
98 !blas_compatible_stride<ColMajor>(_rows, _cols, resStride)) { \
99 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, \
100 ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, _depth, _lhs, \
101 lhsStride, _rhs, rhsStride, res, \
102 Index(1), resStride, alpha, \
103 blocking); \
104 return; \
105 } \
106 Index diagSize = (std::min)(_rows, _depth); \
107 Index rows = IsLower ? _rows : diagSize; \
108 Index depth = IsLower ? diagSize : _depth; \
109 Index cols = _cols; \
110 \
111 using MatrixLhs = Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder>; \
112 using MatrixRhs = Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder>; \
113 \
114 /* Non-square case - doesn't fit to BLAS ?TRMM. Fall to default triangular product or call BLAS ?GEMM*/ \
115 if (rows != depth) { \
116 /* FIXME handle mkl_domain_get_max_threads */ \
117 /*int nthr = mkl_domain_get_max_threads(EIGEN_BLAS_DOMAIN_BLAS);*/ int nthr = 1; \
118 \
119 if (((nthr == 1) && (((std::max)(rows, depth) - diagSize) / (double)diagSize < 0.5))) { \
120 /* Most likely no benefit to call TRMM or GEMM from BLAS */ \
121 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, \
122 ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, _depth, _lhs, \
123 lhsStride, _rhs, rhsStride, res, \
124 1, resStride, alpha, blocking); \
125 /*std::cout << "TRMM_L: A is not square! Go to Eigen TRMM implementation!\n";*/ \
126 } else { \
127 /* Make sense to call GEMM */ \
128 Map<const MatrixLhs, 0, OuterStride<> > lhsMap(_lhs, rows, depth, OuterStride<>(lhsStride)); \
129 MatrixLhs aa_tmp = lhsMap.template triangularView<Mode>(); \
130 BlasIndex aStride = convert_index<BlasIndex>(aa_tmp.outerStride()); \
131 gemm_blocking_space<ColMajor, EIGTYPE, EIGTYPE, Dynamic, Dynamic, Dynamic> gemm_blocking(_rows, _cols, \
132 _depth, 1, true); \
133 general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
134 ConjugateRhs, ColMajor, 1>::run(rows, cols, depth, aa_tmp.data(), aStride, \
135 _rhs, rhsStride, res, 1, resStride, alpha, \
136 gemm_blocking, 0); \
137 \
138 /*std::cout << "TRMM_L: A is not square! Go to BLAS GEMM implementation! " << nthr<<" \n";*/ \
139 } \
140 return; \
141 } \
142 char side = 'L', transa, uplo, diag = 'N'; \
143 EIGTYPE* b; \
144 const EIGTYPE* a; \
145 BlasIndex m, n, lda, ldb; \
146 \
147 /* Set m, n */ \
148 m = convert_index<BlasIndex>(diagSize); \
149 n = convert_index<BlasIndex>(cols); \
150 \
151 /* Set trans */ \
152 transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N'; \
153 \
154 /* Set b, ldb */ \
155 Map<const MatrixRhs, 0, OuterStride<> > rhs(_rhs, depth, cols, OuterStride<>(rhsStride)); \
156 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
157 ColMajorMatrix b_tmp; \
158 \
159 EIGEN_IF_CONSTEXPR (ConjugateRhs) \
160 b_tmp = rhs.conjugate(); \
161 else \
162 b_tmp = rhs; \
163 b = b_tmp.data(); \
164 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
165 \
166 /* Set uplo */ \
167 uplo = IsLower ? 'L' : 'U'; \
168 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
169 uplo = (uplo == 'L') ? 'U' : 'L'; \
170 } \
171 /* Set a, lda */ \
172 Map<const MatrixLhs, 0, OuterStride<> > lhs(_lhs, rows, depth, OuterStride<>(lhsStride)); \
173 MatrixLhs a_tmp; \
174 \
175 EIGEN_IF_CONSTEXPR ((conjA != 0) || (SetDiag == 0)) { \
176 EIGEN_IF_CONSTEXPR (conjA) \
177 a_tmp = lhs.conjugate(); \
178 else \
179 a_tmp = lhs; \
180 EIGEN_IF_CONSTEXPR (IsZeroDiag) \
181 a_tmp.diagonal().setZero(); \
182 else EIGEN_IF_CONSTEXPR (IsUnitDiag) \
183 a_tmp.diagonal().setOnes(); \
184 a = a_tmp.data(); \
185 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
186 } else { \
187 a = _lhs; \
188 lda = blas_leading_dim<LhsStorageOrder>(rows, depth, lhsStride); \
189 } \
190 /*std::cout << "TRMM_L: A is square! Go to BLAS TRMM implementation! \n";*/ \
191 /* call ?trmm*/ \
192 BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, \
193 &lda, (BLASTYPE*)b, &ldb); \
194 \
195 /* Add op(a_triangular)*b into res*/ \
196 Map<ColMajorMatrix, 0, OuterStride<> > res_tmp(res, rows, cols, OuterStride<>(resStride)); \
197 res_tmp = res_tmp + b_tmp; \
198 } \
199 };
200
201#ifdef EIGEN_USE_MKL
202EIGEN_BLAS_TRMM_L(double, double, d, dtrmm)
203EIGEN_BLAS_TRMM_L(dcomplex, MKL_Complex16, cd, ztrmm)
204EIGEN_BLAS_TRMM_L(float, float, f, strmm)
205EIGEN_BLAS_TRMM_L(scomplex, MKL_Complex8, cf, ctrmm)
206#else
207EIGEN_BLAS_TRMM_L(double, double, d, EIGEN_BLAS_SYM(dtrmm))
208EIGEN_BLAS_TRMM_L(dcomplex, double, cd, EIGEN_BLAS_SYM(ztrmm))
209EIGEN_BLAS_TRMM_L(float, float, f, EIGEN_BLAS_SYM(strmm))
210EIGEN_BLAS_TRMM_L(scomplex, float, cf, EIGEN_BLAS_SYM(ctrmm))
211#endif
212
213// implements col-major += alpha * op(general) * op(triangular)
214#define EIGEN_BLAS_TRMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
215 template <typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
216 struct product_triangular_matrix_matrix_trmm<EIGTYPE, Index, Mode, false, LhsStorageOrder, ConjugateLhs, \
217 RhsStorageOrder, ConjugateRhs, ColMajor> { \
218 enum { \
219 IsLower = (Mode & Lower) == Lower, \
220 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1, \
221 IsUnitDiag = (Mode & UnitDiag) ? 1 : 0, \
222 IsZeroDiag = (Mode & ZeroDiag) ? 1 : 0, \
223 LowUp = IsLower ? Lower : Upper, \
224 conjA = ((RhsStorageOrder == ColMajor) && ConjugateRhs) ? 1 : 0 \
225 }; \
226 \
227 static void run(Index _rows, Index _cols, Index _depth, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
228 Index rhsStride, EIGTYPE* res, Index resStride, EIGTYPE alpha, \
229 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
230 if (_rows == 0 || _cols == 0 || _depth == 0 || numext::is_exactly_zero(alpha)) return; \
231 if (!blas_compatible_stride<LhsStorageOrder>(_rows, _depth, lhsStride) || \
232 !blas_compatible_stride<RhsStorageOrder>(_depth, _cols, rhsStride) || \
233 !blas_compatible_stride<ColMajor>(_rows, _cols, resStride)) { \
234 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, false, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, \
235 ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, _depth, _lhs, \
236 lhsStride, _rhs, rhsStride, res, \
237 Index(1), resStride, alpha, \
238 blocking); \
239 return; \
240 } \
241 Index diagSize = (std::min)(_cols, _depth); \
242 Index rows = _rows; \
243 Index depth = IsLower ? _depth : diagSize; \
244 Index cols = IsLower ? diagSize : _cols; \
245 \
246 using MatrixLhs = Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder>; \
247 using MatrixRhs = Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder>; \
248 \
249 /* Non-square case - doesn't fit to BLAS ?TRMM. Fall to default triangular product or call BLAS ?GEMM*/ \
250 if (cols != depth) { \
251 int nthr = 1 /*mkl_domain_get_max_threads(EIGEN_BLAS_DOMAIN_BLAS)*/; \
252 \
253 if ((nthr == 1) && (((std::max)(cols, depth) - diagSize) / (double)diagSize < 0.5)) { \
254 /* Most likely no benefit to call TRMM or GEMM from BLAS*/ \
255 product_triangular_matrix_matrix<EIGTYPE, Index, Mode, false, LhsStorageOrder, ConjugateLhs, \
256 RhsStorageOrder, ConjugateRhs, ColMajor, 1, BuiltIn>::run(_rows, _cols, \
257 _depth, _lhs, \
258 lhsStride, _rhs, \
259 rhsStride, res, \
260 1, resStride, \
261 alpha, blocking); \
262 /*std::cout << "TRMM_R: A is not square! Go to Eigen TRMM implementation!\n";*/ \
263 } else { \
264 /* Make sense to call GEMM */ \
265 Map<const MatrixRhs, 0, OuterStride<> > rhsMap(_rhs, depth, cols, OuterStride<>(rhsStride)); \
266 MatrixRhs aa_tmp = rhsMap.template triangularView<Mode>(); \
267 BlasIndex aStride = convert_index<BlasIndex>(aa_tmp.outerStride()); \
268 gemm_blocking_space<ColMajor, EIGTYPE, EIGTYPE, Dynamic, Dynamic, Dynamic> gemm_blocking(_rows, _cols, \
269 _depth, 1, true); \
270 general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
271 ConjugateRhs, ColMajor, 1>::run(rows, cols, depth, _lhs, lhsStride, \
272 aa_tmp.data(), aStride, res, 1, resStride, \
273 alpha, gemm_blocking, 0); \
274 \
275 /*std::cout << "TRMM_R: A is not square! Go to BLAS GEMM implementation! " << nthr<<" \n";*/ \
276 } \
277 return; \
278 } \
279 char side = 'R', transa, uplo, diag = 'N'; \
280 EIGTYPE* b; \
281 const EIGTYPE* a; \
282 BlasIndex m, n, lda, ldb; \
283 \
284 /* Set m, n */ \
285 m = convert_index<BlasIndex>(rows); \
286 n = convert_index<BlasIndex>(diagSize); \
287 \
288 /* Set trans */ \
289 transa = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N'; \
290 \
291 /* Set b, ldb */ \
292 Map<const MatrixLhs, 0, OuterStride<> > lhs(_lhs, rows, depth, OuterStride<>(lhsStride)); \
293 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
294 ColMajorMatrix b_tmp; \
295 \
296 EIGEN_IF_CONSTEXPR (ConjugateLhs) \
297 b_tmp = lhs.conjugate(); \
298 else \
299 b_tmp = lhs; \
300 b = b_tmp.data(); \
301 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
302 \
303 /* Set uplo */ \
304 uplo = IsLower ? 'L' : 'U'; \
305 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
306 uplo = (uplo == 'L') ? 'U' : 'L'; \
307 } \
308 /* Set a, lda */ \
309 Map<const MatrixRhs, 0, OuterStride<> > rhs(_rhs, depth, cols, OuterStride<>(rhsStride)); \
310 MatrixRhs a_tmp; \
311 \
312 EIGEN_IF_CONSTEXPR ((conjA != 0) || (SetDiag == 0)) { \
313 EIGEN_IF_CONSTEXPR (conjA) \
314 a_tmp = rhs.conjugate(); \
315 else \
316 a_tmp = rhs; \
317 EIGEN_IF_CONSTEXPR (IsZeroDiag) \
318 a_tmp.diagonal().setZero(); \
319 else EIGEN_IF_CONSTEXPR (IsUnitDiag) \
320 a_tmp.diagonal().setOnes(); \
321 a = a_tmp.data(); \
322 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
323 } else { \
324 a = _rhs; \
325 lda = blas_leading_dim<RhsStorageOrder>(depth, cols, rhsStride); \
326 } \
327 /*std::cout << "TRMM_R: A is square! Go to BLAS TRMM implementation! \n";*/ \
328 /* call ?trmm*/ \
329 BLASFUNC(&side, &uplo, &transa, &diag, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, \
330 &lda, (BLASTYPE*)b, &ldb); \
331 \
332 /* Add op(a_triangular)*b into res*/ \
333 Map<ColMajorMatrix, 0, OuterStride<> > res_tmp(res, rows, cols, OuterStride<>(resStride)); \
334 res_tmp = res_tmp + b_tmp; \
335 } \
336 };
337
338#ifdef EIGEN_USE_MKL
339EIGEN_BLAS_TRMM_R(double, double, d, dtrmm)
340EIGEN_BLAS_TRMM_R(dcomplex, MKL_Complex16, cd, ztrmm)
341EIGEN_BLAS_TRMM_R(float, float, f, strmm)
342EIGEN_BLAS_TRMM_R(scomplex, MKL_Complex8, cf, ctrmm)
343#else
344EIGEN_BLAS_TRMM_R(double, double, d, EIGEN_BLAS_SYM(dtrmm))
345EIGEN_BLAS_TRMM_R(dcomplex, double, cd, EIGEN_BLAS_SYM(ztrmm))
346EIGEN_BLAS_TRMM_R(float, float, f, EIGEN_BLAS_SYM(strmm))
347EIGEN_BLAS_TRMM_R(scomplex, float, cf, EIGEN_BLAS_SYM(ctrmm))
348#endif
349
350#undef EIGEN_BLAS_TRMM_SPECIALIZE
351#undef EIGEN_BLAS_TRMM_L
352#undef EIGEN_BLAS_TRMM_R
353} // end namespace internal
354
355} // end namespace Eigen
356
357#endif // EIGEN_TRIANGULAR_MATRIX_MATRIX_BLAS_H