Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
SelfadjointMatrixMatrix_BLAS.h
1/*
2 Copyright (c) 2011, Intel Corporation. All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without modification,
5 are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice, this
8 list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright notice,
10 this list of conditions and the following disclaimer in the documentation
11 and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors may
13 be used to endorse or promote products derived from this software without
14 specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
23 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 ********************************************************************************
28 * Content : Eigen bindings to BLAS F77
29 * Self adjoint matrix * matrix product functionality based on ?SYMM/?HEMM.
30 ********************************************************************************
31*/
32// SPDX-License-Identifier: BSD-3-Clause
33
34#ifndef EIGEN_SELFADJOINT_MATRIX_MATRIX_BLAS_H
35#define EIGEN_SELFADJOINT_MATRIX_MATRIX_BLAS_H
36
37// IWYU pragma: private
38#include "../InternalHeaderCheck.h"
39
40namespace Eigen {
41
42namespace internal {
43
44/* Optimized selfadjoint matrix * matrix (?SYMM/?HEMM) product */
45
46// Temporaries are ColMajor, also under EIGEN_DEFAULT_TO_ROW_MAJOR: BLAS takes column-major storage.
47#define EIGEN_BLAS_SYMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
48 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
49 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
50 ConjugateRhs, ColMajor, 1> { \
51 using Scalar = EIGTYPE; \
52 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
53 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
54 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
55 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
56 if (!blas_compatible_stride<LhsStorageOrder>(rows, rows, lhsStride) || \
57 !blas_compatible_stride<RhsStorageOrder>(rows, cols, rhsStride) || \
58 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
59 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
60 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
61 res, resIncr, resStride, alpha, blocking); \
62 return; \
63 } \
64 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
65 eigen_assert(resIncr == 1); \
66 char side = 'L', uplo = 'L'; \
67 BlasIndex m, n, lda, ldb, ldc; \
68 const EIGTYPE *a, *b; \
69 EIGTYPE beta(1); \
70 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
71 ColMajorMatrix b_tmp; \
72 \
73 /* Set transpose options */ \
74 /* Set m, n, k */ \
75 m = convert_index<BlasIndex>(rows); \
76 n = convert_index<BlasIndex>(cols); \
77 \
78 /* Set lda, ldb, ldc */ \
79 lda = blas_leading_dim<LhsStorageOrder>(rows, rows, lhsStride); \
80 ldb = blas_leading_dim<RhsStorageOrder>(rows, cols, rhsStride); \
81 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
82 \
83 /* Set a, b, c */ \
84 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
85 uplo = 'U'; \
86 } \
87 a = _lhs; \
88 \
89 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
90 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, n, m, OuterStride<>(rhsStride)); \
91 b_tmp = rhs.adjoint(); \
92 b = b_tmp.data(); \
93 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
94 } else \
95 b = _rhs; \
96 \
97 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
98 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
99 } \
100 };
101
102#define EIGEN_BLAS_HEMM_L(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
103 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
104 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
105 ConjugateRhs, ColMajor, 1> { \
106 using Scalar = EIGTYPE; \
107 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
108 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
109 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
110 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
111 if (!blas_compatible_stride<LhsStorageOrder>(rows, rows, lhsStride) || \
112 !blas_compatible_stride<RhsStorageOrder>(rows, cols, rhsStride) || \
113 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
114 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, true, ConjugateLhs, RhsStorageOrder, false, \
115 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
116 res, resIncr, resStride, alpha, blocking); \
117 return; \
118 } \
119 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
120 eigen_assert(resIncr == 1); \
121 char side = 'L', uplo = 'L'; \
122 BlasIndex m, n, lda, ldb, ldc; \
123 const EIGTYPE *a, *b; \
124 EIGTYPE beta(1); \
125 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
126 ColMajorMatrix b_tmp; \
127 Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder> a_tmp; \
128 \
129 /* Set transpose options */ \
130 /* Set m, n, k */ \
131 m = convert_index<BlasIndex>(rows); \
132 n = convert_index<BlasIndex>(cols); \
133 \
134 /* Set lda, ldb, ldc */ \
135 lda = blas_leading_dim<LhsStorageOrder>(rows, rows, lhsStride); \
136 ldb = blas_leading_dim<RhsStorageOrder>(rows, cols, rhsStride); \
137 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
138 \
139 /* Set a, b, c */ \
140 EIGEN_IF_CONSTEXPR (((LhsStorageOrder == ColMajor) && ConjugateLhs) || \
141 ((LhsStorageOrder == RowMajor) && (!ConjugateLhs))) { \
142 Map<const Matrix<EIGTYPE, Dynamic, Dynamic, LhsStorageOrder>, 0, OuterStride<> > lhs( \
143 _lhs, m, m, OuterStride<>(lhsStride)); \
144 a_tmp = lhs.conjugate(); \
145 a = a_tmp.data(); \
146 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
147 } else \
148 a = _lhs; \
149 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
150 uplo = 'U'; \
151 } \
152 \
153 EIGEN_IF_CONSTEXPR (RhsStorageOrder == ColMajor && (!ConjugateRhs)) { \
154 b = _rhs; \
155 } else { \
156 EIGEN_IF_CONSTEXPR (RhsStorageOrder == ColMajor && ConjugateRhs) { \
157 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, m, n, OuterStride<>(rhsStride)); \
158 b_tmp = rhs.conjugate(); \
159 } else EIGEN_IF_CONSTEXPR (ConjugateRhs) { \
160 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, n, m, OuterStride<>(rhsStride)); \
161 b_tmp = rhs.adjoint(); \
162 } else { \
163 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(_rhs, n, m, OuterStride<>(rhsStride)); \
164 b_tmp = rhs.transpose(); \
165 } \
166 b = b_tmp.data(); \
167 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
168 } \
169 \
170 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
171 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
172 } \
173 };
174
175#ifdef EIGEN_USE_MKL
176EIGEN_BLAS_SYMM_L(double, double, d, dsymm)
177EIGEN_BLAS_SYMM_L(float, float, f, ssymm)
178EIGEN_BLAS_HEMM_L(dcomplex, MKL_Complex16, cd, zhemm)
179EIGEN_BLAS_HEMM_L(scomplex, MKL_Complex8, cf, chemm)
180#else
181EIGEN_BLAS_SYMM_L(double, double, d, EIGEN_BLAS_SYM(dsymm))
182EIGEN_BLAS_SYMM_L(float, float, f, EIGEN_BLAS_SYM(ssymm))
183EIGEN_BLAS_HEMM_L(dcomplex, double, cd, EIGEN_BLAS_SYM(zhemm))
184EIGEN_BLAS_HEMM_L(scomplex, float, cf, EIGEN_BLAS_SYM(chemm))
185#endif
186
187/* Optimized matrix * selfadjoint matrix (?SYMM/?HEMM) product */
188
189#define EIGEN_BLAS_SYMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
190 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
191 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
192 ConjugateRhs, ColMajor, 1> { \
193 using Scalar = EIGTYPE; \
194 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
195 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
196 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
197 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
198 if (!blas_compatible_stride<LhsStorageOrder>(rows, cols, lhsStride) || \
199 !blas_compatible_stride<RhsStorageOrder>(cols, cols, rhsStride) || \
200 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
201 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
202 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
203 res, resIncr, resStride, alpha, blocking); \
204 return; \
205 } \
206 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
207 eigen_assert(resIncr == 1); \
208 char side = 'R', uplo = 'L'; \
209 BlasIndex m, n, lda, ldb, ldc; \
210 const EIGTYPE *a, *b; \
211 EIGTYPE beta(1); \
212 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
213 ColMajorMatrix b_tmp; \
214 \
215 /* Set m, n, k */ \
216 m = convert_index<BlasIndex>(rows); \
217 n = convert_index<BlasIndex>(cols); \
218 \
219 /* Set lda, ldb, ldc */ \
220 lda = blas_leading_dim<RhsStorageOrder>(cols, cols, rhsStride); \
221 ldb = blas_leading_dim<LhsStorageOrder>(rows, cols, lhsStride); \
222 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
223 \
224 /* Set a, b, c */ \
225 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
226 uplo = 'U'; \
227 } \
228 a = _rhs; \
229 \
230 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
231 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, n, m, OuterStride<>(lhsStride)); \
232 b_tmp = lhs.adjoint(); \
233 b = b_tmp.data(); \
234 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
235 } else \
236 b = _lhs; \
237 \
238 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
239 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
240 } \
241 };
242
243#define EIGEN_BLAS_HEMM_R(EIGTYPE, BLASTYPE, EIGPREFIX, BLASFUNC) \
244 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
245 struct product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
246 ConjugateRhs, ColMajor, 1> { \
247 using Scalar = EIGTYPE; \
248 static void run(Index rows, Index cols, const EIGTYPE* _lhs, Index lhsStride, const EIGTYPE* _rhs, \
249 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
250 level3_blocking<EIGTYPE, EIGTYPE>& blocking) { \
251 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
252 if (!blas_compatible_stride<LhsStorageOrder>(rows, cols, lhsStride) || \
253 !blas_compatible_stride<RhsStorageOrder>(cols, cols, rhsStride) || \
254 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
255 product_selfadjoint_matrix<EIGTYPE, Index, LhsStorageOrder, false, ConjugateLhs, RhsStorageOrder, true, \
256 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, _lhs, lhsStride, _rhs, rhsStride, \
257 res, resIncr, resStride, alpha, blocking); \
258 return; \
259 } \
260 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
261 eigen_assert(resIncr == 1); \
262 char side = 'R', uplo = 'L'; \
263 BlasIndex m, n, lda, ldb, ldc; \
264 const EIGTYPE *a, *b; \
265 EIGTYPE beta(1); \
266 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
267 ColMajorMatrix b_tmp; \
268 Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder> a_tmp; \
269 \
270 /* Set m, n, k */ \
271 m = convert_index<BlasIndex>(rows); \
272 n = convert_index<BlasIndex>(cols); \
273 \
274 /* Set lda, ldb, ldc */ \
275 lda = blas_leading_dim<RhsStorageOrder>(cols, cols, rhsStride); \
276 ldb = blas_leading_dim<LhsStorageOrder>(rows, cols, lhsStride); \
277 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
278 \
279 /* Set a, b, c */ \
280 EIGEN_IF_CONSTEXPR (((RhsStorageOrder == ColMajor) && ConjugateRhs) || \
281 ((RhsStorageOrder == RowMajor) && (!ConjugateRhs))) { \
282 Map<const Matrix<EIGTYPE, Dynamic, Dynamic, RhsStorageOrder>, 0, OuterStride<> > rhs( \
283 _rhs, n, n, OuterStride<>(rhsStride)); \
284 a_tmp = rhs.conjugate(); \
285 a = a_tmp.data(); \
286 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
287 } else \
288 a = _rhs; \
289 EIGEN_IF_CONSTEXPR (RhsStorageOrder == RowMajor) { \
290 uplo = 'U'; \
291 } \
292 \
293 EIGEN_IF_CONSTEXPR (LhsStorageOrder == ColMajor && (!ConjugateLhs)) { \
294 b = _lhs; \
295 } else { \
296 EIGEN_IF_CONSTEXPR (LhsStorageOrder == ColMajor && ConjugateLhs) { \
297 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, m, n, OuterStride<>(lhsStride)); \
298 b_tmp = lhs.conjugate(); \
299 } else EIGEN_IF_CONSTEXPR (ConjugateLhs) { \
300 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, n, m, OuterStride<>(lhsStride)); \
301 b_tmp = lhs.adjoint(); \
302 } else { \
303 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(_lhs, n, m, OuterStride<>(lhsStride)); \
304 b_tmp = lhs.transpose(); \
305 } \
306 b = b_tmp.data(); \
307 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
308 } \
309 \
310 BLASFUNC(&side, &uplo, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
311 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
312 } \
313 };
314
315#ifdef EIGEN_USE_MKL
316EIGEN_BLAS_SYMM_R(double, double, d, dsymm)
317EIGEN_BLAS_SYMM_R(float, float, f, ssymm)
318EIGEN_BLAS_HEMM_R(dcomplex, MKL_Complex16, cd, zhemm)
319EIGEN_BLAS_HEMM_R(scomplex, MKL_Complex8, cf, chemm)
320#else
321EIGEN_BLAS_SYMM_R(double, double, d, EIGEN_BLAS_SYM(dsymm))
322EIGEN_BLAS_SYMM_R(float, float, f, EIGEN_BLAS_SYM(ssymm))
323EIGEN_BLAS_HEMM_R(dcomplex, double, cd, EIGEN_BLAS_SYM(zhemm))
324EIGEN_BLAS_HEMM_R(scomplex, float, cf, EIGEN_BLAS_SYM(chemm))
325#endif
326
327#undef EIGEN_BLAS_SYMM_L
328#undef EIGEN_BLAS_SYMM_R
329#undef EIGEN_BLAS_HEMM_L
330#undef EIGEN_BLAS_HEMM_R
331} // end namespace internal
332
333} // end namespace Eigen
334
335#endif // EIGEN_SELFADJOINT_MATRIX_MATRIX_BLAS_H