34#ifndef EIGEN_GENERAL_MATRIX_VECTOR_BLAS_H
35#define EIGEN_GENERAL_MATRIX_VECTOR_BLAS_H
38#include "../InternalHeaderCheck.h"
53template <
typename Index,
typename LhsScalar,
int StorageOrder,
bool ConjugateLhs,
typename RhsScalar,
55struct general_matrix_vector_product_gemv;
57#define EIGEN_BLAS_GEMV_SPECIALIZE(Scalar) \
58 template <typename Index, bool ConjugateLhs, bool ConjugateRhs> \
59 struct general_matrix_vector_product<Index, Scalar, const_blas_data_mapper<Scalar, Index, ColMajor>, ColMajor, \
60 ConjugateLhs, Scalar, const_blas_data_mapper<Scalar, Index, RowMajor>, \
61 ConjugateRhs, Specialized> { \
62 static void run(Index rows, Index cols, const const_blas_data_mapper<Scalar, Index, ColMajor>& lhs, \
63 const const_blas_data_mapper<Scalar, Index, RowMajor>& rhs, Scalar* res, Index resIncr, \
65 EIGEN_IF_CONSTEXPR (ConjugateLhs) { \
66 general_matrix_vector_product<Index, Scalar, const_blas_data_mapper<Scalar, Index, ColMajor>, ColMajor, \
67 ConjugateLhs, Scalar, const_blas_data_mapper<Scalar, Index, RowMajor>, \
68 ConjugateRhs, BuiltIn>::run(rows, cols, lhs, rhs, res, resIncr, alpha); \
70 general_matrix_vector_product_gemv<Index, Scalar, ColMajor, ConjugateLhs, Scalar, ConjugateRhs>::run( \
71 rows, cols, lhs.data(), lhs.stride(), rhs.data(), rhs.stride(), res, resIncr, alpha); \
75 template <typename Index, bool ConjugateLhs, bool ConjugateRhs> \
76 struct general_matrix_vector_product<Index, Scalar, const_blas_data_mapper<Scalar, Index, RowMajor>, RowMajor, \
77 ConjugateLhs, Scalar, const_blas_data_mapper<Scalar, Index, ColMajor>, \
78 ConjugateRhs, Specialized> { \
79 static void run(Index rows, Index cols, const const_blas_data_mapper<Scalar, Index, RowMajor>& lhs, \
80 const const_blas_data_mapper<Scalar, Index, ColMajor>& rhs, Scalar* res, Index resIncr, \
82 general_matrix_vector_product_gemv<Index, Scalar, RowMajor, ConjugateLhs, Scalar, ConjugateRhs>::run( \
83 rows, cols, lhs.data(), lhs.stride(), rhs.data(), rhs.stride(), res, resIncr, alpha); \
87EIGEN_BLAS_GEMV_SPECIALIZE(
double)
88EIGEN_BLAS_GEMV_SPECIALIZE(
float)
89EIGEN_BLAS_GEMV_SPECIALIZE(dcomplex)
90EIGEN_BLAS_GEMV_SPECIALIZE(scomplex)
92#define EIGEN_BLAS_GEMV_SPECIALIZATION(EIGTYPE, BLASTYPE, BLASFUNC) \
93 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, bool ConjugateRhs> \
94 struct general_matrix_vector_product_gemv<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, ConjugateRhs> { \
95 using Scalar = EIGTYPE; \
96 using GEMVVector = Matrix<EIGTYPE, Dynamic, 1, ColMajor>; \
98 static void run(Index rows, Index cols, const EIGTYPE* lhs, Index lhsStride, const EIGTYPE* rhs, Index rhsIncr, \
99 Scalar* res, Index resIncr, EIGTYPE alpha) { \
100 if (rows == 0 || cols == 0 || numext::is_exactly_zero(alpha)) return; \
101 if (!blas_compatible_stride<LhsStorageOrder>(rows, cols, lhsStride) || (cols > 1 && rhsIncr == 0) || \
102 (rows > 1 && resIncr == 0)) { \
103 using LhsMapper = const_blas_data_mapper<EIGTYPE, Index, LhsStorageOrder>; \
104 using RhsMapper = const_blas_data_mapper<EIGTYPE, Index, LhsStorageOrder == ColMajor ? RowMajor : ColMajor>; \
105 const GEMVVector rhs_copy = Map<const GEMVVector, 0, InnerStride<>>(rhs, cols, InnerStride<>(rhsIncr)); \
106 GEMVVector result = GEMVVector::Zero(rows); \
107 general_matrix_vector_product<Index, EIGTYPE, LhsMapper, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsMapper, \
108 ConjugateRhs, BuiltIn>::run(rows, cols, LhsMapper(lhs, lhsStride), \
109 RhsMapper(rhs_copy.data(), Index(1)), result.data(), \
111 Map<GEMVVector, 0, InnerStride<>>(res, rows, InnerStride<>(resIncr)) += result; \
114 BlasIndex m = convert_index<BlasIndex>(rows), n = convert_index<BlasIndex>(cols); \
115 const EIGTYPE beta(1); \
116 const EIGTYPE* x_ptr; \
117 char trans = (LhsStorageOrder == ColMajor) ? 'N' : (ConjugateLhs) ? 'C' : 'T'; \
118 EIGEN_IF_CONSTEXPR (LhsStorageOrder == RowMajor) { \
119 m = convert_index<BlasIndex>(cols); \
120 n = convert_index<BlasIndex>(rows); \
122 const BlasIndex lda = blas_leading_dim<LhsStorageOrder>(rows, cols, lhsStride); \
123 BlasIndex incx = blas_vector_increment(cols, rhsIncr); \
124 const BlasIndex incy = blas_vector_increment(rows, resIncr); \
126 EIGEN_IF_CONSTEXPR (ConjugateRhs) { \
127 Map<const GEMVVector, 0, InnerStride<>> map_x(rhs, cols, 1, InnerStride<>(incx)); \
128 x_tmp = map_x.conjugate(); \
129 x_ptr = x_tmp.data(); \
133 x_ptr = rhsIncr < 0 ? rhs + (cols - 1) * rhsIncr : rhs; \
135 auto* y_ptr = resIncr < 0 ? res + (rows - 1) * resIncr : res; \
136 BLASFUNC(&trans, &m, &n, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)lhs, &lda, \
137 (const BLASTYPE*)x_ptr, &incx, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)y_ptr, &incy); \
142EIGEN_BLAS_GEMV_SPECIALIZATION(
double,
double, dgemv)
143EIGEN_BLAS_GEMV_SPECIALIZATION(
float,
float, sgemv)
144EIGEN_BLAS_GEMV_SPECIALIZATION(dcomplex, MKL_Complex16, zgemv)
145EIGEN_BLAS_GEMV_SPECIALIZATION(scomplex, MKL_Complex8, cgemv)
147EIGEN_BLAS_GEMV_SPECIALIZATION(
double,
double, EIGEN_BLAS_SYM(dgemv))
148EIGEN_BLAS_GEMV_SPECIALIZATION(
float,
float, EIGEN_BLAS_SYM(sgemv))
149EIGEN_BLAS_GEMV_SPECIALIZATION(dcomplex,
double, EIGEN_BLAS_SYM(zgemv))
150EIGEN_BLAS_GEMV_SPECIALIZATION(scomplex,
float, EIGEN_BLAS_SYM(cgemv))
153#undef EIGEN_BLAS_GEMV_SPECIALIZE
154#undef EIGEN_BLAS_GEMV_SPECIALIZATION