163 static constexpr int value = FixedSizeCoeffBasedProduct ? CoeffBasedProductMode : ProductType;
164#ifdef EIGEN_DEBUG_PRODUCT
165 static void debug() {
166 const int rows_select = product_size_category<Rows, MaxRows>::value;
167 const int cols_select = product_size_category<Cols, MaxCols>::value;
168 const int depth_select = product_size_category<Depth, MaxDepth>::value;
169 EIGEN_DEBUG_VAR(Rows);
170 EIGEN_DEBUG_VAR(Cols);
171 EIGEN_DEBUG_VAR(Depth);
172 EIGEN_DEBUG_VAR(rows_select);
173 EIGEN_DEBUG_VAR(cols_select);
174 EIGEN_DEBUG_VAR(depth_select);
175 EIGEN_DEBUG_VAR(ProductType);
176 EIGEN_DEBUG_VAR(FixedSizeCoeffBasedProduct);
177 EIGEN_DEBUG_VAR(
value);
186template <
int M,
int N>
187struct product_type_selector<M, N, 1> : std::integral_constant<int, OuterProduct> {};
189struct product_type_selector<M, 1, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
191struct product_type_selector<1, N, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
193struct product_type_selector<1, 1, Depth> : std::integral_constant<int, InnerProduct> {};
195struct product_type_selector<1, 1, 1> : std::integral_constant<int, InnerProduct> {};
197struct product_type_selector<Small, 1, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
199struct product_type_selector<1, Small, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
201struct product_type_selector<Small, Small, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
203struct product_type_selector<Small, Small, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
205struct product_type_selector<Small, Large, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
207struct product_type_selector<Large, Small, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
209struct product_type_selector<1, Large, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
211struct product_type_selector<1, Large, Large> : std::integral_constant<int, GemvProduct> {};
213struct product_type_selector<1, Small, Large> : std::integral_constant<int, CoeffBasedProductMode> {};
215struct product_type_selector<Large, 1, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
217struct product_type_selector<Large, 1, Large> : std::integral_constant<int, GemvProduct> {};
219struct product_type_selector<Small, 1, Large> : std::integral_constant<int, CoeffBasedProductMode> {};
221struct product_type_selector<Small, Small, Large> : std::integral_constant<int, GemmProduct> {};
223struct product_type_selector<Large, Small, Large> : std::integral_constant<int, GemmProduct> {};
225struct product_type_selector<Small, Large, Large> : std::integral_constant<int, GemmProduct> {};
227struct product_type_selector<Large, Large, Large> : std::integral_constant<int, GemmProduct> {};
229struct product_type_selector<Large, Small, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
231struct product_type_selector<Small, Large, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
233struct product_type_selector<Large, Large, Small> : std::integral_constant<int, GemmProduct> {};
264template <
int S
ide,
int StorageOrder,
bool BlasCompatible>
265struct gemv_dense_selector;
271template <
typename Scalar,
int Size,
int MaxSize,
bool Cond>
272struct gemv_static_vector_if;
274template <
typename Scalar,
int Size,
int MaxSize>
275struct gemv_static_vector_if<Scalar, Size, MaxSize, false> {
276 EIGEN_DEVICE_FUNC
constexpr Scalar* data() {
277 eigen_internal_assert(
false &&
"should never be called");
282template <
typename Scalar,
int Size>
283struct gemv_static_vector_if<Scalar, Size, Dynamic, true> {
284 EIGEN_DEVICE_FUNC
constexpr Scalar* data() {
return 0; }
287template <
typename Scalar,
int Size>
288struct gemv_static_vector_if<Scalar, Size, 0, true> {
289 EIGEN_DEVICE_FUNC
constexpr Scalar* data() {
return nullptr; }
292template <
typename Scalar,
int Size,
int MaxSize>
293struct gemv_static_vector_if<Scalar, Size, MaxSize, true> {
294#if EIGEN_MAX_STATIC_ALIGN_BYTES != 0
295 internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize), 0, AlignedMax> m_data;
296 constexpr Scalar* data() {
return m_data.array; }
300 internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize) + EIGEN_MAX_ALIGN_BYTES, 0> m_data;
301 constexpr Scalar* data() {
302 return reinterpret_cast<Scalar*
>((std::uintptr_t(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES - 1))) +
303 EIGEN_MAX_ALIGN_BYTES);
308template <
typename ResScalar>
309using gemv_mapped_destination =
310 Map<Matrix<ResScalar, Dynamic, 1>, plain_enum_min(AlignedMax, internal::packet_traits<ResScalar>::size)>;
312template <
typename Dest>
313EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE gemv_mapped_destination<typename Dest::Scalar> gemv_construct_mapped_destination(
314 Dest& dest,
typename Dest::Scalar* actual_dest_ptr) {
315#ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
316 constexpr int Size = Dest::SizeAtCompileTime;
317 Index size = dest.size();
318 EIGEN_DENSE_STORAGE_CTOR_PLUGIN
320 return gemv_mapped_destination<typename Dest::Scalar>(actual_dest_ptr, dest.size());
324template <
bool EvalToDest,
typename Dest>
325EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void gemv_prepare_destination(Dest& dest,
326 typename Dest::Scalar* actual_dest_ptr) {
327 EIGEN_IF_CONSTEXPR (!EvalToDest) gemv_construct_mapped_destination(dest, actual_dest_ptr) = dest;
330template <
bool EvalToDest,
typename Dest>
331EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void gemv_prepare_destination(Dest& dest,
typename Dest::Scalar* actual_dest_ptr,
332 bool initialize_to_zero) {
333 if (initialize_to_zero) {
334 gemv_construct_mapped_destination(dest, actual_dest_ptr).setZero();
336 gemv_prepare_destination<EvalToDest>(dest, actual_dest_ptr);
340template <
bool EvalToDest,
typename Dest>
341EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void gemv_copy_destination(Dest& dest,
typename Dest::Scalar* actual_dest_ptr) {
342 EIGEN_IF_CONSTEXPR (!EvalToDest) {
343 dest = gemv_mapped_destination<typename Dest::Scalar>(actual_dest_ptr, dest.size());
348template <
typename RhsScalar,
typename ResScalar,
bool EvalToDestAtCompileTime,
bool ComplexByReal>
349class gemv_destination_policy {
351 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
explicit gemv_destination_policy(
const ResScalar& alpha) : alpha_(alpha) {}
353 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool eval_to_dest()
const {
354 return EvalToDestAtCompileTime && alpha_is_compatible();
357 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RhsScalar compatible_alpha()
const {
358 return alpha_is_compatible() ? get_factor<ResScalar, RhsScalar>::run(alpha_) : RhsScalar(1);
361 template <
typename Dest>
362 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void prepare(Dest& dest, ResScalar* actual_dest_ptr)
const {
363 gemv_prepare_destination<EvalToDestAtCompileTime>(dest, actual_dest_ptr, !alpha_is_compatible());
366 template <
typename Dest>
367 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void copy_back(Dest& dest, ResScalar* actual_dest_ptr)
const {
368 if (!alpha_is_compatible()) {
369 dest.matrix() += alpha_ * gemv_mapped_destination<ResScalar>(actual_dest_ptr, dest.size());
371 gemv_copy_destination<EvalToDestAtCompileTime>(dest, actual_dest_ptr);
376 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool alpha_is_compatible()
const {
377 return !ComplexByReal || numext::is_exactly_zero(numext::imag(alpha_));
384template <
bool DirectlyUseRhs,
typename ActualRhsType>
385EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void gemv_prepare_rhs(
386 const ActualRhsType& rhs,
typename remove_all_t<ActualRhsType>::Scalar* actual_rhs_ptr) {
387 using ActualRhsTypeCleaned = remove_all_t<ActualRhsType>;
388 EIGEN_IF_CONSTEXPR (!DirectlyUseRhs) {
389#ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
390 constexpr int Size = ActualRhsTypeCleaned::SizeAtCompileTime;
391 Index size = rhs.size();
392 EIGEN_DENSE_STORAGE_CTOR_PLUGIN
395 Map<typename ActualRhsTypeCleaned::PlainObject, AlignedMax>(actual_rhs_ptr, rhs.size()) = rhs;
400template <
int StorageOrder,
bool BlasCompatible>
401struct gemv_dense_selector<
OnTheLeft, StorageOrder, BlasCompatible> {
402 template <
typename Lhs,
typename Rhs,
typename Dest>
403 static void run(
const Lhs& lhs,
const Rhs& rhs, Dest& dest,
const typename Dest::Scalar& alpha) {
404 Transpose<Dest> destT(dest);
406 gemv_dense_selector<OnTheRight, OtherStorageOrder, BlasCompatible>::run(rhs.transpose(), lhs.transpose(), destT,
412template <
bool ConjugateLhs,
typename LhsScalar,
typename RhsScalar,
typename Dest>
413EIGEN_DONT_INLINE
void gemv_single_column_update(
const LhsScalar* data, Index rows, Index stride, RhsScalar rhsValue,
414 typename Dest::Scalar alpha, Dest& dest) {
415 using Column = Matrix<LhsScalar, Dynamic, 1>;
419 Map<const Column> column(data, rows);
420 dest.col(0).matrix() += alpha * (column.template conjugateIf<ConjugateLhs>() * rhsValue);
422 Map<const Column, Unaligned, InnerStride<>> column(data, rows, InnerStride<>(stride));
423 dest.col(0).matrix() += alpha * (column.template conjugateIf<ConjugateLhs>() * rhsValue);
427template <
bool Enable,
bool ConjugateLhs,
bool ConjugateRhs>
428struct gemv_single_column {
429 template <
typename Lhs,
typename Rhs,
typename Scalar,
typename Dest>
430 static EIGEN_STRONG_INLINE
bool run(
const Lhs&,
const Rhs&,
const Scalar&, Dest&,
bool) {
435template <
bool ConjugateLhs,
bool ConjugateRhs>
436struct gemv_single_column<true, ConjugateLhs, ConjugateRhs> {
437 template <
typename Lhs,
typename Rhs,
typename Scalar,
typename Dest>
438 static EIGEN_STRONG_INLINE
bool run(
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha, Dest& dest,
bool eligible) {
439 if (!eligible || lhs.cols() != 1)
return false;
440 if (!numext::is_exactly_zero(alpha)) {
441 gemv_single_column_update<ConjugateLhs>(lhs.data(), lhs.rows(), lhs.col(0).innerStride(),
442 conj_if<ConjugateRhs>()(rhs.coeff(0, 0)), alpha, dest);
450 template <
typename Lhs,
typename Rhs,
typename Dest>
451 static inline void run(
const Lhs& lhs,
const Rhs& rhs, Dest& dest,
const typename Dest::Scalar& alpha) {
452 using LhsScalar =
typename Lhs::Scalar;
453 using RhsScalar =
typename Rhs::Scalar;
454 using ResScalar =
typename Dest::Scalar;
456 using LhsBlasTraits = internal::blas_traits<Lhs>;
457 using ActualLhsType =
typename LhsBlasTraits::DirectLinearAccessType;
458 using RhsBlasTraits = internal::blas_traits<Rhs>;
459 using ActualRhsType =
typename RhsBlasTraits::DirectLinearAccessType;
461 ActualLhsType actualLhs = LhsBlasTraits::extract(lhs);
462 ActualRhsType actualRhs = RhsBlasTraits::extract(rhs);
464 ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
466 constexpr bool DoubleScalars = std::is_same<LhsScalar, RhsScalar>::value &&
467 std::is_same<LhsScalar, ResScalar>::value && std::is_same<ResScalar, double>::value;
469 constexpr bool EnableSingleColumn =
470 Lhs::ColsAtCompileTime == Dynamic && DoubleScalars && Dest::InnerStrideAtCompileTime != 1;
471 if (gemv_single_column<EnableSingleColumn, LhsBlasTraits::NeedToConjugate, RhsBlasTraits::NeedToConjugate>::run(
472 actualLhs, actualRhs, actualAlpha, dest,
473 Dest::InnerStrideAtCompileTime != Dynamic || dest.innerStride() != 1))
477 using ActualDest = std::conditional_t<Dest::IsVectorAtCompileTime, Dest, typename Dest::ColXpr>;
482 EvalToDestAtCompileTime = (ActualDest::InnerStrideAtCompileTime == 1),
483 ComplexByReal = (NumTraits<LhsScalar>::IsComplex) && (!NumTraits<RhsScalar>::IsComplex),
484 MightCannotUseDest = ((!EvalToDestAtCompileTime) || ComplexByReal) && (ActualDest::MaxSizeAtCompileTime != 0)
487 using LhsMapper = const_blas_data_mapper<LhsScalar, Index, ColMajor>;
488 using RhsMapper = const_blas_data_mapper<RhsScalar, Index, RowMajor>;
489 EIGEN_IF_CONSTEXPR (!MightCannotUseDest) {
492 general_matrix_vector_product<
493 Index, LhsScalar, LhsMapper,
ColMajor, LhsBlasTraits::NeedToConjugate, RhsScalar, RhsMapper,
494 RhsBlasTraits::NeedToConjugate>::run(actualLhs.rows(), actualLhs.cols(),
495 LhsMapper(actualLhs.data(), actualLhs.outerStride()),
496 RhsMapper(actualRhs.data(), actualRhs.innerStride()), dest.data(), 1,
497 get_factor<ResScalar, RhsScalar>::run(actualAlpha));
499 gemv_static_vector_if<ResScalar, ActualDest::SizeAtCompileTime, ActualDest::MaxSizeAtCompileTime,
503 gemv_destination_policy<RhsScalar, ResScalar, EvalToDestAtCompileTime, ComplexByReal> destPolicy(actualAlpha);
505 ei_declare_aligned_stack_constructed_variable(ResScalar, actualDestPtr, dest.size(),
506 destPolicy.eval_to_dest() ? dest.data() : static_dest.data());
508 destPolicy.prepare(dest, actualDestPtr);
510 general_matrix_vector_product<Index, LhsScalar, LhsMapper,
ColMajor, LhsBlasTraits::NeedToConjugate, RhsScalar,
511 RhsMapper, RhsBlasTraits::NeedToConjugate>::run(actualLhs.rows(), actualLhs.cols(),
512 LhsMapper(actualLhs.data(),
513 actualLhs.outerStride()),
514 RhsMapper(actualRhs.data(),
515 actualRhs.innerStride()),
517 destPolicy.compatible_alpha());
519 destPolicy.copy_back(dest, actualDestPtr);
526 template <
typename Lhs,
typename Rhs,
typename Dest>
527 static void run(
const Lhs& lhs,
const Rhs& rhs, Dest& dest,
const typename Dest::Scalar& alpha) {
528 using LhsScalar =
typename Lhs::Scalar;
529 using RhsScalar =
typename Rhs::Scalar;
530 using ResScalar =
typename Dest::Scalar;
532 using LhsBlasTraits = internal::blas_traits<Lhs>;
533 using ActualLhsType =
typename LhsBlasTraits::DirectLinearAccessType;
534 using RhsBlasTraits = internal::blas_traits<Rhs>;
535 using ActualRhsType =
typename RhsBlasTraits::DirectLinearAccessType;
536 using ActualRhsTypeCleaned = internal::remove_all_t<ActualRhsType>;
538 std::add_const_t<ActualLhsType> actualLhs = LhsBlasTraits::extract(lhs);
539 std::add_const_t<ActualRhsType> actualRhs = RhsBlasTraits::extract(rhs);
541 ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
544 constexpr bool EnableSingleColumn = Lhs::ColsAtCompileTime == Dynamic && !NumTraits<ResScalar>::IsComplex &&
545 std::is_trivially_copyable<LhsScalar>::value &&
546 std::is_trivially_copyable<RhsScalar>::value &&
547 std::is_trivially_copyable<ResScalar>::value;
548 if (gemv_single_column<EnableSingleColumn, LhsBlasTraits::NeedToConjugate, RhsBlasTraits::NeedToConjugate>::run(
549 actualLhs, actualRhs, actualAlpha, dest,
true))
556 ActualRhsTypeCleaned::InnerStrideAtCompileTime == 1 || ActualRhsTypeCleaned::MaxSizeAtCompileTime == 0
559 gemv_static_vector_if<RhsScalar, ActualRhsTypeCleaned::SizeAtCompileTime,
560 ActualRhsTypeCleaned::MaxSizeAtCompileTime, !DirectlyUseRhs>
563 ei_declare_aligned_stack_constructed_variable(
564 RhsScalar, actualRhsPtr, actualRhs.size(),
565 DirectlyUseRhs ?
const_cast<RhsScalar*
>(actualRhs.data()) : static_rhs.data());
567 gemv_prepare_rhs<DirectlyUseRhs>(actualRhs, actualRhsPtr);
569 using LhsMapper = const_blas_data_mapper<LhsScalar, Index, RowMajor>;
570 using RhsMapper = const_blas_data_mapper<RhsScalar, Index, ColMajor>;
571 general_matrix_vector_product<Index, LhsScalar, LhsMapper,
RowMajor, LhsBlasTraits::NeedToConjugate, RhsScalar,
572 RhsMapper, RhsBlasTraits::NeedToConjugate>::
573 run(actualLhs.rows(), actualLhs.cols(), LhsMapper(actualLhs.data(), actualLhs.outerStride()),
574 RhsMapper(actualRhsPtr, 1), dest.data(),
575 dest.col(0).innerStride(),
583 template <
typename Lhs,
typename Rhs,
typename Dest>
584 static void run(
const Lhs& lhs,
const Rhs& rhs, Dest& dest,
const typename Dest::Scalar& alpha) {
585 EIGEN_STATIC_ASSERT((!nested_eval<Lhs, 1>::Evaluate),
586 EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
589 typename nested_eval<Rhs, 1>::type actual_rhs(rhs);
590 const Index size = rhs.rows();
591 for (Index k = 0; k < size; ++k) dest.matrix() += (alpha * actual_rhs.coeff(k)) * lhs.col(k);
597 template <
typename Lhs,
typename Rhs,
typename Dest>
598 static void run(
const Lhs& lhs,
const Rhs& rhs, Dest& dest,
const typename Dest::Scalar& alpha) {
599 EIGEN_STATIC_ASSERT((!nested_eval<Lhs, 1>::Evaluate),
600 EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
601 typename nested_eval<Rhs, Lhs::RowsAtCompileTime>::type actual_rhs(rhs);
602 const Index rows = dest.rows();
603 for (Index i = 0; i < rows; ++i)
604 dest.coeffRef(i) += alpha * (lhs.row(i).cwiseProduct(actual_rhs.transpose())).sum();
620template <
typename Derived>
621template <
typename OtherDerived>
623 const MatrixBase<OtherDerived>& other)
const {
629 ProductIsValid = Derived::ColsAtCompileTime == Dynamic || OtherDerived::RowsAtCompileTime == Dynamic ||
630 int(Derived::ColsAtCompileTime) == int(OtherDerived::RowsAtCompileTime),
631 AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
632 SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived, OtherDerived)
638 ProductIsValid || !(AreVectors && SameSizes),
639 INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
640 EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
641 INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
642 EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)
643#ifdef EIGEN_DEBUG_PRODUCT
644 internal::product_type<Derived, OtherDerived>::debug();
661template <
typename Derived>
662template <
typename OtherDerived>
666 ProductIsValid = Derived::ColsAtCompileTime == Dynamic || OtherDerived::RowsAtCompileTime == Dynamic ||
667 int(Derived::ColsAtCompileTime) == int(OtherDerived::RowsAtCompileTime),
668 AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
669 SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived, OtherDerived)
675 ProductIsValid || !(AreVectors && SameSizes),
676 INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
677 EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
678 INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
679 EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)