Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GeneralProduct.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
5// Copyright (C) 2008-2011 Gael Guennebaud <gael.guennebaud@inria.fr>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_GENERAL_PRODUCT_H
13#define EIGEN_GENERAL_PRODUCT_H
14
15// IWYU pragma: private
16#include "./InternalHeaderCheck.h"
17
18namespace Eigen {
19
20enum { Large = 2, Small = 3 };
21
22// Runtime-size threshold for falling back from the generic matrix-matrix product
23// implementation (heavy) to the lightweight coeff-based product one. See
24// generic_product_impl<Lhs,Rhs,DenseShape,DenseShape,GemmProduct> in
25// products/GeneralMatrixMatrix.h for more details.
26// The crossover belongs to the kernel the GEMM path would select, not to the
27// build: a scalar type that falls back to the generic gebp kernel keeps this
28// value even in an SME build. The SME kernels have their own crossovers
29// -- sme_gemm_to_coeffbased_threshold and
30// EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD below.
31#ifndef EIGEN_GEMM_TO_COEFFBASED_THRESHOLD
32// This default value has been obtained on a Haswell architecture.
33#define EIGEN_GEMM_TO_COEFFBASED_THRESHOLD 20
34#endif
35
36// Fixed-size products can reach the GEMM product path even when the
37// coeff-based evaluator is still faster. Keep this threshold separate so
38// runtime-size dispatch preserves the historical heuristic above. This default
39// was tuned on the same Haswell system as the runtime threshold. It does not
40// follow a user-defined EIGEN_GEMM_TO_COEFFBASED_THRESHOLD, which still applies
41// at run time, so a fixed-size product's crossover is the larger of the two.
42#ifndef EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD
43#define EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD 40
44#endif
45
46#ifdef EIGEN_VECTORIZE_SME
47// Read only for SME-kernel scalar pairs; fitted on Apple M4 Pro to today's NEON small-block bounds (re-measure: #3194).
48#ifndef EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD
49#define EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD 61
50#endif
51#endif
52
53// The dimension-sum bound in sme_gemm_to_coeffbased_threshold below grows with
54// the depth, so it cannot reach a small output over a long depth: a 2x2 result
55// at k=1024 sums to 1028 and takes the SME path with nearly the whole ZA grid
56// predicated off. The crossover for that shape family is an output *area* rather
57// than a dimension sum, and it tracks the scalar width.
58//
59// Measured on Apple M4 at SVL=512, every m x n with m,n in [2,12] against the
60// coeff-based path: the largest area at which *no* shape loses is 27, 14, 15 and
61// 6 for float, complex<float>, double and complex<double>. 96/sizeof (24, 12,
62// 12, 6) sits just inside all four, and holds out to k=32768 -- the worst cell
63// there is float 4x6 at 1.03x, i.e. neutral, while 2x2 is 13x and 3x3 2.0x.
64// A flat constant does not work: the value that is safe for complex<double>
65// admits only 2x2 for float, and float's would cost complex<double> up to 2x.
66#ifndef EIGEN_SME_GEMM_TO_COEFFBASED_OUTPUT_AREA_THRESHOLD
67#define EIGEN_SME_GEMM_TO_COEFFBASED_OUTPUT_AREA_THRESHOLD(Scalar) (96 / int(sizeof(Scalar)))
68#endif
69
70namespace internal {
71
72#ifdef EIGEN_VECTORIZE_SME
73// Defined in products/GeneralBlockPanelKernel.h, which Core includes after this
74// header; only the declaration is needed to form the dependent type below.
75template <typename LhsScalar, typename RhsScalar>
76struct sme_has_gebp_kernel;
77// Dimension sum below which the coeff-based product beats the SME GEMM kernel.
78// mr x nr is sized in ZA tiles, so a product too small to fill the grid wastes
79// most of the outer products the kernel issues, and the block holds fewer
80// scalars as the scalar widens: the crossover sits well above the Haswell-tuned
81// EIGEN_GEMM_TO_COEFFBASED_THRESHOLD, and it moves with the scalar type.
82//
83// First cube (m=n=k) at which the SME kernel wins, from two sweeps that each
84// built the same source twice with only this threshold changed, so that the real
85// dispatch selected the path in both:
86//
87// scalar M4 M4 Pro M4 Pro, result 64-byte aligned
88// complex<double> 14 14 15
89// double 16 18 16
90// complex<float> 20 19 19
91// float 26 22 23
92//
93// The entries below are 3n for the largest of each row, so neither host loses at
94// the crossover it gets. Over the sizes where the two disagree the paths are
95// within 1.2x on both, while at n=14 a too-low bound costs 1.7x to 2.9x.
96//
97// Defining EIGEN_SME_GEMM_TO_COEFFBASED_THRESHOLD replaces every entry with one
98// value, which is what such a sweep wants. The primary template answers with the
99// generic threshold: sme_has_gebp_kernel gates the dispatch, so a scalar pair
100// without an SME kernel never reads the table.
101// With the NEON small-block path (arch/SME/GeneralBlockPanelKernel.h) the SME
102// kernels take the generic crossover: below it the small products run the NEON
103// packers and kernel, so the SME build matches a NEON build there.
104#ifdef EIGEN_SME_GEMM_TO_COEFFBASED_THRESHOLD
105template <typename Scalar>
106struct sme_gemm_to_coeffbased_threshold : std::integral_constant<int, EIGEN_SME_GEMM_TO_COEFFBASED_THRESHOLD> {};
107#else
108template <typename Scalar>
109struct sme_gemm_to_coeffbased_threshold : std::integral_constant<int, EIGEN_GEMM_TO_COEFFBASED_THRESHOLD> {};
110#endif
111#endif
112
113template <int Rows, int Cols, int Depth>
114struct product_type_selector;
115
116template <int Size, int MaxSize>
117struct product_size_category {
118 enum {
119#ifndef EIGEN_GPU_COMPILE_PHASE
120 is_large = MaxSize == Dynamic || Size >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD ||
121 (Size == Dynamic && MaxSize >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD),
122#else
123 is_large = 0,
124#endif
125 value = is_large ? Large
126 : Size == 1 ? 1
127 : Small
128 };
129};
130
131template <typename Lhs, typename Rhs>
132struct product_type {
133 using Lhs_ = remove_all_t<Lhs>;
134 using Rhs_ = remove_all_t<Rhs>;
135 enum {
136 MaxRows = traits<Lhs_>::MaxRowsAtCompileTime,
137 Rows = traits<Lhs_>::RowsAtCompileTime,
138 MaxCols = traits<Rhs_>::MaxColsAtCompileTime,
139 Cols = traits<Rhs_>::ColsAtCompileTime,
140 MaxDepth = min_size_prefer_fixed(traits<Lhs_>::MaxColsAtCompileTime, traits<Rhs_>::MaxRowsAtCompileTime),
141 Depth = min_size_prefer_fixed(traits<Lhs_>::ColsAtCompileTime, traits<Rhs_>::RowsAtCompileTime)
142 };
143
144 static constexpr int ProductType =
145 product_type_selector<product_size_category<Rows, MaxRows>::value, product_size_category<Cols, MaxCols>::value,
146 product_size_category<Depth, MaxDepth>::value>::value;
147
148 // Same rule as the runtime dispatch in products/GeneralMatrixMatrix.h: the
149 // threshold follows the kernel the GEMM path would select, so a scalar pair
150 // without an SME kernel keeps the generic crossover in an SME build.
151 static constexpr int FixedSizeThreshold =
152#ifdef EIGEN_VECTORIZE_SME
153 sme_has_gebp_kernel<typename traits<Lhs_>::Scalar, typename traits<Rhs_>::Scalar>::value
154 ? EIGEN_SME_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD
155 :
156#endif
157 EIGEN_FIXED_SIZE_GEMM_TO_COEFFBASED_THRESHOLD;
158
159 static constexpr bool FixedSizeCoeffBasedProduct = ProductType == GemmProduct && Rows != Dynamic && Cols != Dynamic &&
160 Depth != Dynamic &&
161 (int(Rows) + int(Cols) + int(Depth) < FixedSizeThreshold);
163 static constexpr int value = FixedSizeCoeffBasedProduct ? CoeffBasedProductMode : ProductType;
164#ifdef EIGEN_DEBUG_PRODUCT
165 static void debug() {
166 const int rows_select = product_size_category<Rows, MaxRows>::value;
167 const int cols_select = product_size_category<Cols, MaxCols>::value;
168 const int depth_select = product_size_category<Depth, MaxDepth>::value;
169 EIGEN_DEBUG_VAR(Rows);
170 EIGEN_DEBUG_VAR(Cols);
171 EIGEN_DEBUG_VAR(Depth);
172 EIGEN_DEBUG_VAR(rows_select);
173 EIGEN_DEBUG_VAR(cols_select);
174 EIGEN_DEBUG_VAR(depth_select);
175 EIGEN_DEBUG_VAR(ProductType);
176 EIGEN_DEBUG_VAR(FixedSizeCoeffBasedProduct);
177 EIGEN_DEBUG_VAR(value);
178 }
179#endif
180};
181
182/* The following allows to select the kind of product at compile time
183 * based on the three dimensions of the product.
184 * This is a compile time mapping from {1,Small,Large}^3 -> {product types} */
185// FIXME: the current compile-time product-type mapping may not be optimal.
186template <int M, int N>
187struct product_type_selector<M, N, 1> : std::integral_constant<int, OuterProduct> {};
188template <int M>
189struct product_type_selector<M, 1, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
190template <int N>
191struct product_type_selector<1, N, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
192template <int Depth>
193struct product_type_selector<1, 1, Depth> : std::integral_constant<int, InnerProduct> {};
194template <>
195struct product_type_selector<1, 1, 1> : std::integral_constant<int, InnerProduct> {};
196template <>
197struct product_type_selector<Small, 1, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
198template <>
199struct product_type_selector<1, Small, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
200template <>
201struct product_type_selector<Small, Small, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
202template <>
203struct product_type_selector<Small, Small, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
204template <>
205struct product_type_selector<Small, Large, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
206template <>
207struct product_type_selector<Large, Small, 1> : std::integral_constant<int, LazyCoeffBasedProductMode> {};
208template <>
209struct product_type_selector<1, Large, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
210template <>
211struct product_type_selector<1, Large, Large> : std::integral_constant<int, GemvProduct> {};
212template <>
213struct product_type_selector<1, Small, Large> : std::integral_constant<int, CoeffBasedProductMode> {};
214template <>
215struct product_type_selector<Large, 1, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
216template <>
217struct product_type_selector<Large, 1, Large> : std::integral_constant<int, GemvProduct> {};
218template <>
219struct product_type_selector<Small, 1, Large> : std::integral_constant<int, CoeffBasedProductMode> {};
220template <>
221struct product_type_selector<Small, Small, Large> : std::integral_constant<int, GemmProduct> {};
222template <>
223struct product_type_selector<Large, Small, Large> : std::integral_constant<int, GemmProduct> {};
224template <>
225struct product_type_selector<Small, Large, Large> : std::integral_constant<int, GemmProduct> {};
226template <>
227struct product_type_selector<Large, Large, Large> : std::integral_constant<int, GemmProduct> {};
228template <>
229struct product_type_selector<Large, Small, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
230template <>
231struct product_type_selector<Small, Large, Small> : std::integral_constant<int, CoeffBasedProductMode> {};
232template <>
233struct product_type_selector<Large, Large, Small> : std::integral_constant<int, GemmProduct> {};
234
235} // end namespace internal
236
237/***********************************************************************
238 * Implementation of Inner Vector Vector Product
239 ***********************************************************************/
240
241// FIXME: consider returning a Scalar instead of a 1x1 matrix for inner products.
242// Pro: more natural for the user.
243// Con: in a meta-unrolled algorithm a matrix-matrix product may reduce to a
244// row-vector times column-vector product. To handle this, we could specialize
245// Block<MatrixType,1,1> with operator=(Scalar x).
246
247/***********************************************************************
248 * Implementation of Outer Vector Vector Product
249 ***********************************************************************/
250
251/***********************************************************************
252 * Implementation of General Matrix Vector Product
253 ***********************************************************************/
254
255/* According to the shape/flags of the matrix we have to distinguish 3 different cases:
256 * 1 - the matrix is col-major, BLAS compatible and M is large => call fast BLAS-like colmajor routine
257 * 2 - the matrix is row-major, BLAS compatible and N is large => call fast BLAS-like rowmajor routine
258 * 3 - all other cases are handled using a simple loop along the outer-storage direction.
259 * Therefore we need a lower level meta selector.
260 * Furthermore, if the matrix is the rhs, then the product has to be transposed.
261 */
262namespace internal {
263
264template <int Side, int StorageOrder, bool BlasCompatible>
265struct gemv_dense_selector;
266
267} // end namespace internal
268
269namespace internal {
270
271template <typename Scalar, int Size, int MaxSize, bool Cond>
272struct gemv_static_vector_if;
273
274template <typename Scalar, int Size, int MaxSize>
275struct gemv_static_vector_if<Scalar, Size, MaxSize, false> {
276 EIGEN_DEVICE_FUNC constexpr Scalar* data() {
277 eigen_internal_assert(false && "should never be called");
278 return 0;
279 }
280};
281
282template <typename Scalar, int Size>
283struct gemv_static_vector_if<Scalar, Size, Dynamic, true> {
284 EIGEN_DEVICE_FUNC constexpr Scalar* data() { return 0; }
285};
286
287template <typename Scalar, int Size>
288struct gemv_static_vector_if<Scalar, Size, 0, true> {
289 EIGEN_DEVICE_FUNC constexpr Scalar* data() { return nullptr; }
290};
291
292template <typename Scalar, int Size, int MaxSize>
293struct gemv_static_vector_if<Scalar, Size, MaxSize, true> {
294#if EIGEN_MAX_STATIC_ALIGN_BYTES != 0
295 internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize), 0, AlignedMax> m_data;
296 constexpr Scalar* data() { return m_data.array; }
297#else
298 // Some architectures cannot align on the stack,
299 // => let's manually enforce alignment by allocating more data and return the address of the first aligned element.
300 internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize) + EIGEN_MAX_ALIGN_BYTES, 0> m_data;
301 constexpr Scalar* data() {
302 return reinterpret_cast<Scalar*>((std::uintptr_t(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES - 1))) +
303 EIGEN_MAX_ALIGN_BYTES);
304 }
305#endif
306};
307
308template <typename ResScalar>
309using gemv_mapped_destination =
310 Map<Matrix<ResScalar, Dynamic, 1>, plain_enum_min(AlignedMax, internal::packet_traits<ResScalar>::size)>;
311
312template <typename Dest>
313EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE gemv_mapped_destination<typename Dest::Scalar> gemv_construct_mapped_destination(
314 Dest& dest, typename Dest::Scalar* actual_dest_ptr) {
315#ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
316 constexpr int Size = Dest::SizeAtCompileTime;
317 Index size = dest.size();
318 EIGEN_DENSE_STORAGE_CTOR_PLUGIN
319#endif
320 return gemv_mapped_destination<typename Dest::Scalar>(actual_dest_ptr, dest.size());
321}
322
323// Prepares the temporary destination shared by general, triangular, and selfadjoint GEMV kernels.
324template <bool EvalToDest, typename Dest>
325EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void gemv_prepare_destination(Dest& dest,
326 typename Dest::Scalar* actual_dest_ptr) {
327 EIGEN_IF_CONSTEXPR (!EvalToDest) gemv_construct_mapped_destination(dest, actual_dest_ptr) = dest;
328}
329
330template <bool EvalToDest, typename Dest>
331EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void gemv_prepare_destination(Dest& dest, typename Dest::Scalar* actual_dest_ptr,
332 bool initialize_to_zero) {
333 if (initialize_to_zero) {
334 gemv_construct_mapped_destination(dest, actual_dest_ptr).setZero();
335 } else {
336 gemv_prepare_destination<EvalToDest>(dest, actual_dest_ptr);
337 }
338}
339
340template <bool EvalToDest, typename Dest>
341EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void gemv_copy_destination(Dest& dest, typename Dest::Scalar* actual_dest_ptr) {
342 EIGEN_IF_CONSTEXPR (!EvalToDest) {
343 dest = gemv_mapped_destination<typename Dest::Scalar>(actual_dest_ptr, dest.size());
344 }
345}
346
347// Adds complex-by-real scalar adaptation to the shared destination handling when necessary.
348template <typename RhsScalar, typename ResScalar, bool EvalToDestAtCompileTime, bool ComplexByReal>
349class gemv_destination_policy {
350 public:
351 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit gemv_destination_policy(const ResScalar& alpha) : alpha_(alpha) {}
352
353 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool eval_to_dest() const {
354 return EvalToDestAtCompileTime && alpha_is_compatible();
355 }
356
357 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RhsScalar compatible_alpha() const {
358 return alpha_is_compatible() ? get_factor<ResScalar, RhsScalar>::run(alpha_) : RhsScalar(1);
359 }
360
361 template <typename Dest>
362 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void prepare(Dest& dest, ResScalar* actual_dest_ptr) const {
363 gemv_prepare_destination<EvalToDestAtCompileTime>(dest, actual_dest_ptr, !alpha_is_compatible());
364 }
365
366 template <typename Dest>
367 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void copy_back(Dest& dest, ResScalar* actual_dest_ptr) const {
368 if (!alpha_is_compatible()) {
369 dest.matrix() += alpha_ * gemv_mapped_destination<ResScalar>(actual_dest_ptr, dest.size());
370 } else {
371 gemv_copy_destination<EvalToDestAtCompileTime>(dest, actual_dest_ptr);
372 }
373 }
374
375 private:
376 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool alpha_is_compatible() const {
377 return !ComplexByReal || numext::is_exactly_zero(numext::imag(alpha_));
378 }
379
380 ResScalar alpha_;
381};
382
383// Copies a non-contiguous GEMV right-hand side into the aligned buffer allocated by the caller.
384template <bool DirectlyUseRhs, typename ActualRhsType>
385EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void gemv_prepare_rhs(
386 const ActualRhsType& rhs, typename remove_all_t<ActualRhsType>::Scalar* actual_rhs_ptr) {
387 using ActualRhsTypeCleaned = remove_all_t<ActualRhsType>;
388 EIGEN_IF_CONSTEXPR (!DirectlyUseRhs) {
389#ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
390 constexpr int Size = ActualRhsTypeCleaned::SizeAtCompileTime;
391 Index size = rhs.size();
392 EIGEN_DENSE_STORAGE_CTOR_PLUGIN
393#endif
394 // The caller obtains this temporary from ei_declare_aligned_stack_constructed_variable.
395 Map<typename ActualRhsTypeCleaned::PlainObject, AlignedMax>(actual_rhs_ptr, rhs.size()) = rhs;
396 }
397}
398
399// The caller has already established commutative coefficient multiplication.
400template <int StorageOrder, bool BlasCompatible>
401struct gemv_dense_selector<OnTheLeft, StorageOrder, BlasCompatible> {
402 template <typename Lhs, typename Rhs, typename Dest>
403 static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
404 Transpose<Dest> destT(dest);
405 enum { OtherStorageOrder = StorageOrder == RowMajor ? ColMajor : RowMajor };
406 gemv_dense_selector<OnTheRight, OtherStorageOrder, BlasCompatible>::run(rhs.transpose(), lhs.transpose(), destT,
407 alpha);
408 }
409};
410
411// Keep the shortcut's expression out of the general GEMV caller's inlining budget.
412template <bool ConjugateLhs, typename LhsScalar, typename RhsScalar, typename Dest>
413EIGEN_DONT_INLINE void gemv_single_column_update(const LhsScalar* data, Index rows, Index stride, RhsScalar rhsValue,
414 typename Dest::Scalar alpha, Dest& dest) {
415 using Column = Matrix<LhsScalar, Dynamic, 1>;
416 // A runtime stride of one must be visible to the assignment evaluator to enable packets.
417 // Preserve alpha * (lhs * rhs): folding alpha into rhs can overflow before the product.
418 if (stride == 1) {
419 Map<const Column> column(data, rows);
420 dest.col(0).matrix() += alpha * (column.template conjugateIf<ConjugateLhs>() * rhsValue);
421 } else {
422 Map<const Column, Unaligned, InnerStride<>> column(data, rows, InnerStride<>(stride));
423 dest.col(0).matrix() += alpha * (column.template conjugateIf<ConjugateLhs>() * rhsValue);
424 }
425}
426
427template <bool Enable, bool ConjugateLhs, bool ConjugateRhs>
428struct gemv_single_column {
429 template <typename Lhs, typename Rhs, typename Scalar, typename Dest>
430 static EIGEN_STRONG_INLINE bool run(const Lhs&, const Rhs&, const Scalar&, Dest&, bool) {
431 return false;
432 }
433};
434
435template <bool ConjugateLhs, bool ConjugateRhs>
436struct gemv_single_column<true, ConjugateLhs, ConjugateRhs> {
437 template <typename Lhs, typename Rhs, typename Scalar, typename Dest>
438 static EIGEN_STRONG_INLINE bool run(const Lhs& lhs, const Rhs& rhs, const Scalar& alpha, Dest& dest, bool eligible) {
439 if (!eligible || lhs.cols() != 1) return false;
440 if (!numext::is_exactly_zero(alpha)) {
441 gemv_single_column_update<ConjugateLhs>(lhs.data(), lhs.rows(), lhs.col(0).innerStride(),
442 conj_if<ConjugateRhs>()(rhs.coeff(0, 0)), alpha, dest);
443 }
444 return true;
445 }
446};
447
448template <>
449struct gemv_dense_selector<OnTheRight, ColMajor, true> {
450 template <typename Lhs, typename Rhs, typename Dest>
451 static inline void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
452 using LhsScalar = typename Lhs::Scalar;
453 using RhsScalar = typename Rhs::Scalar;
454 using ResScalar = typename Dest::Scalar;
455
456 using LhsBlasTraits = internal::blas_traits<Lhs>;
457 using ActualLhsType = typename LhsBlasTraits::DirectLinearAccessType;
458 using RhsBlasTraits = internal::blas_traits<Rhs>;
459 using ActualRhsType = typename RhsBlasTraits::DirectLinearAccessType;
460
461 ActualLhsType actualLhs = LhsBlasTraits::extract(lhs);
462 ActualRhsType actualRhs = RhsBlasTraits::extract(rhs);
463
464 ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
465
466 constexpr bool DoubleScalars = std::is_same<LhsScalar, RhsScalar>::value &&
467 std::is_same<LhsScalar, ResScalar>::value && std::is_same<ResScalar, double>::value;
468 // Float updates can be slower than the buffered packet kernel; bypass it only for matching double scalars.
469 constexpr bool EnableSingleColumn =
470 Lhs::ColsAtCompileTime == Dynamic && DoubleScalars && Dest::InnerStrideAtCompileTime != 1;
471 if (gemv_single_column<EnableSingleColumn, LhsBlasTraits::NeedToConjugate, RhsBlasTraits::NeedToConjugate>::run(
472 actualLhs, actualRhs, actualAlpha, dest,
473 Dest::InnerStrideAtCompileTime != Dynamic || dest.innerStride() != 1))
474 return;
475
476 // make sure Dest is a compile-time vector type (bug 1166)
477 using ActualDest = std::conditional_t<Dest::IsVectorAtCompileTime, Dest, typename Dest::ColXpr>;
478
479 enum {
480 // FIXME: find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1
481 // on the other hand it is good for the cache to pack the vector anyways...
482 EvalToDestAtCompileTime = (ActualDest::InnerStrideAtCompileTime == 1),
483 ComplexByReal = (NumTraits<LhsScalar>::IsComplex) && (!NumTraits<RhsScalar>::IsComplex),
484 MightCannotUseDest = ((!EvalToDestAtCompileTime) || ComplexByReal) && (ActualDest::MaxSizeAtCompileTime != 0)
485 };
486
487 using LhsMapper = const_blas_data_mapper<LhsScalar, Index, ColMajor>;
488 using RhsMapper = const_blas_data_mapper<RhsScalar, Index, RowMajor>;
489 EIGEN_IF_CONSTEXPR (!MightCannotUseDest) {
490 // shortcut if we are sure to be able to use dest directly,
491 // this eases the compiler to generate cleaner and more optimized code for most common cases
492 general_matrix_vector_product<
493 Index, LhsScalar, LhsMapper, ColMajor, LhsBlasTraits::NeedToConjugate, RhsScalar, RhsMapper,
494 RhsBlasTraits::NeedToConjugate>::run(actualLhs.rows(), actualLhs.cols(),
495 LhsMapper(actualLhs.data(), actualLhs.outerStride()),
496 RhsMapper(actualRhs.data(), actualRhs.innerStride()), dest.data(), 1,
497 get_factor<ResScalar, RhsScalar>::run(actualAlpha));
498 } else {
499 gemv_static_vector_if<ResScalar, ActualDest::SizeAtCompileTime, ActualDest::MaxSizeAtCompileTime,
500 MightCannotUseDest>
501 static_dest;
502
503 gemv_destination_policy<RhsScalar, ResScalar, EvalToDestAtCompileTime, ComplexByReal> destPolicy(actualAlpha);
504
505 ei_declare_aligned_stack_constructed_variable(ResScalar, actualDestPtr, dest.size(),
506 destPolicy.eval_to_dest() ? dest.data() : static_dest.data());
507
508 destPolicy.prepare(dest, actualDestPtr);
509
510 general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, LhsBlasTraits::NeedToConjugate, RhsScalar,
511 RhsMapper, RhsBlasTraits::NeedToConjugate>::run(actualLhs.rows(), actualLhs.cols(),
512 LhsMapper(actualLhs.data(),
513 actualLhs.outerStride()),
514 RhsMapper(actualRhs.data(),
515 actualRhs.innerStride()),
516 actualDestPtr, 1,
517 destPolicy.compatible_alpha());
518
519 destPolicy.copy_back(dest, actualDestPtr);
520 }
521 }
522};
523
524template <>
525struct gemv_dense_selector<OnTheRight, RowMajor, true> {
526 template <typename Lhs, typename Rhs, typename Dest>
527 static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
528 using LhsScalar = typename Lhs::Scalar;
529 using RhsScalar = typename Rhs::Scalar;
530 using ResScalar = typename Dest::Scalar;
531
532 using LhsBlasTraits = internal::blas_traits<Lhs>;
533 using ActualLhsType = typename LhsBlasTraits::DirectLinearAccessType;
534 using RhsBlasTraits = internal::blas_traits<Rhs>;
535 using ActualRhsType = typename RhsBlasTraits::DirectLinearAccessType;
536 using ActualRhsTypeCleaned = internal::remove_all_t<ActualRhsType>;
537
538 std::add_const_t<ActualLhsType> actualLhs = LhsBlasTraits::extract(lhs);
539 std::add_const_t<ActualRhsType> actualRhs = RhsBlasTraits::extract(rhs);
540
541 ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
542
543 // Cost heuristic: broadcasts may allocate for owning scalars. Complex shortcuts can worsen GCC's kernel spilling.
544 constexpr bool EnableSingleColumn = Lhs::ColsAtCompileTime == Dynamic && !NumTraits<ResScalar>::IsComplex &&
545 std::is_trivially_copyable<LhsScalar>::value &&
546 std::is_trivially_copyable<RhsScalar>::value &&
547 std::is_trivially_copyable<ResScalar>::value;
548 if (gemv_single_column<EnableSingleColumn, LhsBlasTraits::NeedToConjugate, RhsBlasTraits::NeedToConjugate>::run(
549 actualLhs, actualRhs, actualAlpha, dest, true))
550 return;
551
552 enum {
553 // FIXME: find a way to allow an inner stride on the result if packet_traits<Scalar>::size==1
554 // on the other hand it is good for the cache to pack the vector anyways...
555 DirectlyUseRhs =
556 ActualRhsTypeCleaned::InnerStrideAtCompileTime == 1 || ActualRhsTypeCleaned::MaxSizeAtCompileTime == 0
557 };
558
559 gemv_static_vector_if<RhsScalar, ActualRhsTypeCleaned::SizeAtCompileTime,
560 ActualRhsTypeCleaned::MaxSizeAtCompileTime, !DirectlyUseRhs>
561 static_rhs;
562
563 ei_declare_aligned_stack_constructed_variable(
564 RhsScalar, actualRhsPtr, actualRhs.size(),
565 DirectlyUseRhs ? const_cast<RhsScalar*>(actualRhs.data()) : static_rhs.data());
566
567 gemv_prepare_rhs<DirectlyUseRhs>(actualRhs, actualRhsPtr);
568
569 using LhsMapper = const_blas_data_mapper<LhsScalar, Index, RowMajor>;
570 using RhsMapper = const_blas_data_mapper<RhsScalar, Index, ColMajor>;
571 general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, LhsBlasTraits::NeedToConjugate, RhsScalar,
572 RhsMapper, RhsBlasTraits::NeedToConjugate>::
573 run(actualLhs.rows(), actualLhs.cols(), LhsMapper(actualLhs.data(), actualLhs.outerStride()),
574 RhsMapper(actualRhsPtr, 1), dest.data(),
575 dest.col(0).innerStride(), // NOTE if dest is not a vector at compile-time, then dest.innerStride() might
576 // be wrong. (bug 1166)
577 actualAlpha);
578 }
579};
580
581template <>
582struct gemv_dense_selector<OnTheRight, ColMajor, false> {
583 template <typename Lhs, typename Rhs, typename Dest>
584 static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
585 EIGEN_STATIC_ASSERT((!nested_eval<Lhs, 1>::Evaluate),
586 EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
587 // TODO: if rhs is large enough it might be beneficial to make sure that dest is sequentially stored in memory,
588 // otherwise use a temp
589 typename nested_eval<Rhs, 1>::type actual_rhs(rhs);
590 const Index size = rhs.rows();
591 for (Index k = 0; k < size; ++k) dest.matrix() += (alpha * actual_rhs.coeff(k)) * lhs.col(k);
592 }
593};
594
595template <>
596struct gemv_dense_selector<OnTheRight, RowMajor, false> {
597 template <typename Lhs, typename Rhs, typename Dest>
598 static void run(const Lhs& lhs, const Rhs& rhs, Dest& dest, const typename Dest::Scalar& alpha) {
599 EIGEN_STATIC_ASSERT((!nested_eval<Lhs, 1>::Evaluate),
600 EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
601 typename nested_eval<Rhs, Lhs::RowsAtCompileTime>::type actual_rhs(rhs);
602 const Index rows = dest.rows();
603 for (Index i = 0; i < rows; ++i)
604 dest.coeffRef(i) += alpha * (lhs.row(i).cwiseProduct(actual_rhs.transpose())).sum();
605 }
606};
607
608} // end namespace internal
609
610/***************************************************************************
611 * Implementation of matrix base methods
612 ***************************************************************************/
613
620template <typename Derived>
621template <typename OtherDerived>
622EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Product<Derived, OtherDerived> MatrixBase<Derived>::operator*(
623 const MatrixBase<OtherDerived>& other) const {
624 // A note regarding the function declaration: In MSVC, this function will sometimes
625 // not be inlined since DenseStorage is an unwindable object for dynamic
626 // matrices and product types are holding a member to store the result.
627 // Thus it does not help tagging this function with EIGEN_STRONG_INLINE.
628 enum {
629 ProductIsValid = Derived::ColsAtCompileTime == Dynamic || OtherDerived::RowsAtCompileTime == Dynamic ||
630 int(Derived::ColsAtCompileTime) == int(OtherDerived::RowsAtCompileTime),
631 AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
632 SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived, OtherDerived)
633 };
634 // note to the lost user:
635 // * for a dot product use: v1.dot(v2)
636 // * for a coeff-wise product use: v1.cwiseProduct(v2)
637 EIGEN_STATIC_ASSERT(
638 ProductIsValid || !(AreVectors && SameSizes),
639 INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
640 EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
641 INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
642 EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)
643#ifdef EIGEN_DEBUG_PRODUCT
644 internal::product_type<Derived, OtherDerived>::debug();
645#endif
646
647 return Product<Derived, OtherDerived>(derived(), other.derived());
648}
649
661template <typename Derived>
662template <typename OtherDerived>
663EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Product<Derived, OtherDerived, LazyProduct>
664MatrixBase<Derived>::lazyProduct(const MatrixBase<OtherDerived>& other) const {
665 enum {
666 ProductIsValid = Derived::ColsAtCompileTime == Dynamic || OtherDerived::RowsAtCompileTime == Dynamic ||
667 int(Derived::ColsAtCompileTime) == int(OtherDerived::RowsAtCompileTime),
668 AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
669 SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived, OtherDerived)
670 };
671 // note to the lost user:
672 // * for a dot product use: v1.dot(v2)
673 // * for a coeff-wise product use: v1.cwiseProduct(v2)
674 EIGEN_STATIC_ASSERT(
675 ProductIsValid || !(AreVectors && SameSizes),
676 INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
677 EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
678 INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
679 EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)
680
681 return Product<Derived, OtherDerived, LazyProduct>(derived(), other.derived());
682}
683
684} // end namespace Eigen
685
686#endif // EIGEN_GENERAL_PRODUCT_H
CoeffReturnType value() const
Definition DenseBase.h:454
const Product< Derived, OtherDerived, LazyProduct > lazyProduct(const MatrixBase< OtherDerived > &other) const
Definition GeneralProduct.h:664
const Product< Derived, OtherDerived > operator*(const MatrixBase< OtherDerived > &other) const
Definition GeneralProduct.h:622
Expression of the product of two arbitrary matrices or vectors.
Definition Product.h:211
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
@ OnTheLeft
Definition Constants.h:332
@ OnTheRight
Definition Constants.h:334