Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TriangularMatrixVector_BLAS.h
1/*
2 Copyright (c) 2011, Intel Corporation. All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without modification,
5 are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice, this
8 list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright notice,
10 this list of conditions and the following disclaimer in the documentation
11 and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors may
13 be used to endorse or promote products derived from this software without
14 specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
23 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 ********************************************************************************
28 * Content : Eigen bindings to BLAS F77
29 * Triangular matrix-vector product functionality based on ?TRMV.
30 ********************************************************************************
31*/
32// SPDX-License-Identifier: BSD-3-Clause
33
34#ifndef EIGEN_TRIANGULAR_MATRIX_VECTOR_BLAS_H
35#define EIGEN_TRIANGULAR_MATRIX_VECTOR_BLAS_H
36
37// IWYU pragma: private
38#include "../InternalHeaderCheck.h"
39
40namespace Eigen {
41
42namespace internal {
43
44/**********************************************************************
45 * This file implements triangular matrix-vector multiplication using BLAS
46 **********************************************************************/
47
48// trmv/hemv specialization
49
50template <typename Index, int Mode, typename LhsScalar, bool ConjLhs, typename RhsScalar, bool ConjRhs,
51 int StorageOrder>
52struct triangular_matrix_vector_product_trmv
53 : triangular_matrix_vector_product<Index, Mode, LhsScalar, ConjLhs, RhsScalar, ConjRhs, StorageOrder, BuiltIn> {};
54
55template <typename Index, int Mode, typename Scalar, bool ConjLhs, bool ConjRhs, int Order>
56void trmv_builtin(Index rows, Index cols, const Scalar* lhs, Index lhsStride, const Scalar* rhs, Index rhsIncr,
57 Scalar* res, Index resIncr, Scalar alpha) {
58 using Vector = Matrix<Scalar, Dynamic, 1>;
59 Vector rhs_copy, result;
60 Scalar* output = res;
61 Index output_stride = resIncr;
62 const Index size = (std::min)(rows, cols);
63 const Index output_size = (Mode & Lower) ? rows : size;
64 // The row-major kernel needs contiguous input; the column-major kernel needs contiguous output.
65 EIGEN_IF_CONSTEXPR (Order == RowMajor) {
66 if (rhsIncr != 1) {
67 rhs_copy = Map<const Vector, 0, InnerStride<>>(rhs, (Mode & Lower) ? size : cols, InnerStride<>(rhsIncr));
68 rhs = rhs_copy.data();
69 rhsIncr = 1;
70 }
71 } else {
72 if (resIncr != 1) {
73 result = Vector::Zero(output_size);
74 output = result.data();
75 output_stride = 1;
76 }
77 }
78 triangular_matrix_vector_product<Index, Mode, Scalar, ConjLhs, Scalar, ConjRhs, Order, BuiltIn>::run(
79 rows, cols, lhs, lhsStride, rhs, rhsIncr, output, output_stride, alpha);
80 if (result.size() != 0) Map<Vector, 0, InnerStride<>>(res, output_size, InnerStride<>(resIncr)) += result;
81}
82
83#define EIGEN_BLAS_TRMV_SPECIALIZE(Scalar) \
84 template <typename Index, int Mode, bool ConjLhs, bool ConjRhs> \
85 struct triangular_matrix_vector_product<Index, Mode, Scalar, ConjLhs, Scalar, ConjRhs, ColMajor, Specialized> { \
86 static void run(Index rows_, Index cols_, const Scalar* lhs_, Index lhsStride, const Scalar* rhs_, Index rhsIncr, \
87 Scalar* res_, Index resIncr, Scalar alpha) { \
88 triangular_matrix_vector_product_trmv<Index, Mode, Scalar, ConjLhs, Scalar, ConjRhs, ColMajor>::run( \
89 rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, res_, resIncr, alpha); \
90 } \
91 }; \
92 template <typename Index, int Mode, bool ConjLhs, bool ConjRhs> \
93 struct triangular_matrix_vector_product<Index, Mode, Scalar, ConjLhs, Scalar, ConjRhs, RowMajor, Specialized> { \
94 static void run(Index rows_, Index cols_, const Scalar* lhs_, Index lhsStride, const Scalar* rhs_, Index rhsIncr, \
95 Scalar* res_, Index resIncr, Scalar alpha) { \
96 triangular_matrix_vector_product_trmv<Index, Mode, Scalar, ConjLhs, Scalar, ConjRhs, RowMajor>::run( \
97 rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, res_, resIncr, alpha); \
98 } \
99 };
100
101EIGEN_BLAS_TRMV_SPECIALIZE(double)
102EIGEN_BLAS_TRMV_SPECIALIZE(float)
103EIGEN_BLAS_TRMV_SPECIALIZE(dcomplex)
104EIGEN_BLAS_TRMV_SPECIALIZE(scomplex)
105
106// implements col-major: res += alpha * op(triangular) * vector
107#define EIGEN_BLAS_TRMV_CM(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX, BLASPOSTFIX) \
108 template <typename Index, int Mode, bool ConjLhs, bool ConjRhs> \
109 struct triangular_matrix_vector_product_trmv<Index, Mode, EIGTYPE, ConjLhs, EIGTYPE, ConjRhs, ColMajor> { \
110 using Scalar = EIGTYPE; \
111 enum { \
112 IsLower = (Mode & Lower) == Lower, \
113 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1, \
114 IsUnitDiag = (Mode & UnitDiag) ? 1 : 0, \
115 IsZeroDiag = (Mode & ZeroDiag) ? 1 : 0, \
116 LowUp = IsLower ? Lower : Upper \
117 }; \
118 static void run(Index rows_, Index cols_, const EIGTYPE* lhs_, Index lhsStride, const EIGTYPE* rhs_, \
119 Index rhsIncr, Scalar* res_, Index resIncr, EIGTYPE alpha) { \
120 if (rows_ == 0 || cols_ == 0 || numext::is_exactly_zero(alpha)) return; \
121 if (!blas_compatible_stride<ColMajor>(rows_, cols_, lhsStride) || (rows_ > 1 && resIncr == 0)) { \
122 trmv_builtin<Index, Mode, EIGTYPE, ConjLhs, ConjRhs, ColMajor>(rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, \
123 res_, resIncr, alpha); \
124 return; \
125 } \
126 EIGEN_IF_CONSTEXPR (ConjLhs || IsZeroDiag) { \
127 trmv_builtin<Index, Mode, EIGTYPE, ConjLhs, ConjRhs, ColMajor>(rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, \
128 res_, resIncr, alpha); \
129 return; \
130 } \
131 Index size = (std::min)(rows_, cols_); \
132 Index rows = IsLower ? rows_ : size; \
133 Index cols = IsLower ? size : cols_; \
134 \
135 using VectorRhs = VectorX##EIGPREFIX; \
136 Scalar *x, *y; \
137 \
138 /* Set x*/ \
139 Map<const VectorRhs, 0, InnerStride<>> rhs(rhs_, cols, InnerStride<>(rhsIncr)); \
140 VectorRhs x_tmp; \
141 EIGEN_IF_CONSTEXPR (ConjRhs) \
142 x_tmp = rhs.conjugate(); \
143 else \
144 x_tmp = rhs; \
145 x = x_tmp.data(); \
146 \
147 /* Square part handling */ \
148 \
149 char trans, uplo, diag; \
150 BlasIndex m, n, lda, incx, incy; \
151 EIGTYPE const* a; \
152 EIGTYPE beta(1); \
153 \
154 /* Set m, n */ \
155 n = convert_index<BlasIndex>(size); \
156 lda = blas_leading_dim<ColMajor>(rows, cols, lhsStride); \
157 incx = 1; \
158 incy = blas_vector_increment(rows, resIncr); \
159 \
160 /* Set uplo, trans and diag*/ \
161 trans = 'N'; \
162 uplo = IsLower ? 'L' : 'U'; \
163 diag = IsUnitDiag ? 'U' : 'N'; \
164 \
165 /* call ?TRMV*/ \
166 EIGEN_CAT(EIGEN_CAT(BLASPREFIX, trmv), BLASPOSTFIX) \
167 (&uplo, &trans, &diag, &n, (const BLASTYPE*)lhs_, &lda, (BLASTYPE*)x, &incx); \
168 \
169 /* Add op(a_tr)rhs into res*/ \
170 /* BLAS addresses a vector with a negative increment from its lowest address, Eigen from entry 0. */ \
171 EIGEN_CAT(EIGEN_CAT(BLASPREFIX, axpy), BLASPOSTFIX) \
172 (&n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)x, &incx, \
173 (BLASTYPE*)(resIncr < 0 ? res_ + (size - 1) * resIncr : res_), &incy); \
174 /* Non-square case - doesn't fit to BLAS ?TRMV. Fall to default triangular product*/ \
175 if (size < (std::max)(rows, cols)) { \
176 EIGEN_IF_CONSTEXPR (ConjRhs) \
177 x_tmp = rhs.conjugate(); \
178 else \
179 x_tmp = rhs; \
180 x = x_tmp.data(); \
181 if (size < rows) { \
182 y = res_ + size * resIncr; \
183 a = lhs_ + size; \
184 m = convert_index<BlasIndex>(rows - size); \
185 n = convert_index<BlasIndex>(size); \
186 } else { \
187 x += size; \
188 y = res_; \
189 a = lhs_ + size * lda; \
190 m = convert_index<BlasIndex>(size); \
191 n = convert_index<BlasIndex>(cols - size); \
192 } \
193 if (resIncr < 0) y += (m - 1) * resIncr; \
194 EIGEN_CAT(EIGEN_CAT(BLASPREFIX, gemv), BLASPOSTFIX) \
195 (&trans, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)x, \
196 &incx, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)y, &incy); \
197 } \
198 } \
199 };
200
201#ifdef EIGEN_USE_MKL
202EIGEN_BLAS_TRMV_CM(double, double, d, d, )
203EIGEN_BLAS_TRMV_CM(dcomplex, MKL_Complex16, cd, z, )
204EIGEN_BLAS_TRMV_CM(float, float, f, s, )
205EIGEN_BLAS_TRMV_CM(scomplex, MKL_Complex8, cf, c, )
206#else
207EIGEN_BLAS_TRMV_CM(double, double, d, d, EIGEN_BLAS_POSTFIX)
208EIGEN_BLAS_TRMV_CM(dcomplex, double, cd, z, EIGEN_BLAS_POSTFIX)
209EIGEN_BLAS_TRMV_CM(float, float, f, s, EIGEN_BLAS_POSTFIX)
210EIGEN_BLAS_TRMV_CM(scomplex, float, cf, c, EIGEN_BLAS_POSTFIX)
211#endif
212
213// implements row-major: res += alpha * op(triangular) * vector
214#define EIGEN_BLAS_TRMV_RM(EIGTYPE, BLASTYPE, EIGPREFIX, BLASPREFIX, BLASPOSTFIX) \
215 template <typename Index, int Mode, bool ConjLhs, bool ConjRhs> \
216 struct triangular_matrix_vector_product_trmv<Index, Mode, EIGTYPE, ConjLhs, EIGTYPE, ConjRhs, RowMajor> { \
217 using Scalar = EIGTYPE; \
218 enum { \
219 IsLower = (Mode & Lower) == Lower, \
220 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1, \
221 IsUnitDiag = (Mode & UnitDiag) ? 1 : 0, \
222 IsZeroDiag = (Mode & ZeroDiag) ? 1 : 0, \
223 LowUp = IsLower ? Lower : Upper \
224 }; \
225 static void run(Index rows_, Index cols_, const EIGTYPE* lhs_, Index lhsStride, const EIGTYPE* rhs_, \
226 Index rhsIncr, Scalar* res_, Index resIncr, EIGTYPE alpha) { \
227 if (rows_ == 0 || cols_ == 0 || numext::is_exactly_zero(alpha)) return; \
228 if (!blas_compatible_stride<RowMajor>(rows_, cols_, lhsStride) || (rows_ > 1 && resIncr == 0)) { \
229 trmv_builtin<Index, Mode, EIGTYPE, ConjLhs, ConjRhs, RowMajor>(rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, \
230 res_, resIncr, alpha); \
231 return; \
232 } \
233 EIGEN_IF_CONSTEXPR (IsZeroDiag) { \
234 trmv_builtin<Index, Mode, EIGTYPE, ConjLhs, ConjRhs, RowMajor>(rows_, cols_, lhs_, lhsStride, rhs_, rhsIncr, \
235 res_, resIncr, alpha); \
236 return; \
237 } \
238 Index size = (std::min)(rows_, cols_); \
239 Index rows = IsLower ? rows_ : size; \
240 Index cols = IsLower ? size : cols_; \
241 \
242 using VectorRhs = VectorX##EIGPREFIX; \
243 Scalar *x, *y; \
244 \
245 /* Set x*/ \
246 Map<const VectorRhs, 0, InnerStride<>> rhs(rhs_, cols, InnerStride<>(rhsIncr)); \
247 VectorRhs x_tmp; \
248 EIGEN_IF_CONSTEXPR (ConjRhs) \
249 x_tmp = rhs.conjugate(); \
250 else \
251 x_tmp = rhs; \
252 x = x_tmp.data(); \
253 \
254 /* Square part handling */ \
255 \
256 char trans, uplo, diag; \
257 BlasIndex m, n, lda, incx, incy; \
258 EIGTYPE const* a; \
259 EIGTYPE beta(1); \
260 \
261 /* Set m, n */ \
262 n = convert_index<BlasIndex>(size); \
263 lda = blas_leading_dim<RowMajor>(rows, cols, lhsStride); \
264 incx = 1; \
265 incy = blas_vector_increment(rows, resIncr); \
266 \
267 /* Set uplo, trans and diag*/ \
268 trans = ConjLhs ? 'C' : 'T'; \
269 uplo = IsLower ? 'U' : 'L'; \
270 diag = IsUnitDiag ? 'U' : 'N'; \
271 \
272 /* call ?TRMV*/ \
273 EIGEN_CAT(EIGEN_CAT(BLASPREFIX, trmv), BLASPOSTFIX) \
274 (&uplo, &trans, &diag, &n, (const BLASTYPE*)lhs_, &lda, (BLASTYPE*)x, &incx); \
275 \
276 /* Add op(a_tr)rhs into res*/ \
277 /* BLAS addresses a vector with a negative increment from its lowest address, Eigen from entry 0. */ \
278 EIGEN_CAT(EIGEN_CAT(BLASPREFIX, axpy), BLASPOSTFIX) \
279 (&n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)x, &incx, \
280 (BLASTYPE*)(resIncr < 0 ? res_ + (size - 1) * resIncr : res_), &incy); \
281 /* Non-square case - doesn't fit to BLAS ?TRMV. Fall to default triangular product*/ \
282 if (size < (std::max)(rows, cols)) { \
283 EIGEN_IF_CONSTEXPR (ConjRhs) \
284 x_tmp = rhs.conjugate(); \
285 else \
286 x_tmp = rhs; \
287 x = x_tmp.data(); \
288 if (size < rows) { \
289 y = res_ + size * resIncr; \
290 a = lhs_ + size * lda; \
291 m = convert_index<BlasIndex>(rows - size); \
292 n = convert_index<BlasIndex>(size); \
293 } else { \
294 x += size; \
295 y = res_; \
296 a = lhs_ + size; \
297 m = convert_index<BlasIndex>(size); \
298 n = convert_index<BlasIndex>(cols - size); \
299 } \
300 if (resIncr < 0) y += (m - 1) * resIncr; \
301 EIGEN_CAT(EIGEN_CAT(BLASPREFIX, gemv), BLASPOSTFIX) \
302 (&trans, &n, &m, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, (const BLASTYPE*)x, \
303 &incx, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)y, &incy); \
304 } \
305 } \
306 };
307
308#ifdef EIGEN_USE_MKL
309EIGEN_BLAS_TRMV_RM(double, double, d, d, )
310EIGEN_BLAS_TRMV_RM(dcomplex, MKL_Complex16, cd, z, )
311EIGEN_BLAS_TRMV_RM(float, float, f, s, )
312EIGEN_BLAS_TRMV_RM(scomplex, MKL_Complex8, cf, c, )
313#else
314EIGEN_BLAS_TRMV_RM(double, double, d, d, EIGEN_BLAS_POSTFIX)
315EIGEN_BLAS_TRMV_RM(dcomplex, double, cd, z, EIGEN_BLAS_POSTFIX)
316EIGEN_BLAS_TRMV_RM(float, float, f, s, EIGEN_BLAS_POSTFIX)
317EIGEN_BLAS_TRMV_RM(scomplex, float, cf, c, EIGEN_BLAS_POSTFIX)
318#endif
319
320#undef EIGEN_BLAS_TRMV_RM
321#undef EIGEN_BLAS_TRMV_SPECIALIZE
322#undef EIGEN_BLAS_TRMV_CM
323} // namespace internal
324
325} // end namespace Eigen
326
327#endif // EIGEN_TRIANGULAR_MATRIX_VECTOR_BLAS_H
@ Lower
Definition Constants.h:212
@ RowMajor
Definition Constants.h:321
Matrix< Type, Size, 1 > Vector
SizeƗ1 vector of type Type.
Definition Matrix.h:532