Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
ProductEvaluators.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
5// Copyright (C) 2008-2010 Gael Guennebaud <gael.guennebaud@inria.fr>
6// Copyright (C) 2011 Jitse Niesen <jitse@maths.leeds.ac.uk>
7//
8// This Source Code Form is subject to the terms of the Mozilla
9// Public License v. 2.0. If a copy of the MPL was not distributed
10// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11// SPDX-License-Identifier: MPL-2.0
12
13#ifndef EIGEN_PRODUCTEVALUATORS_H
14#define EIGEN_PRODUCTEVALUATORS_H
15
16// IWYU pragma: private
17#include "./InternalHeaderCheck.h"
18
19// C4804: unsafe use of type 'bool' in operation. Unavoidable in generic code
20// instantiated with bool scalars (e.g. += and * on bool).
21#if EIGEN_COMP_MSVC
22#pragma warning(push)
23#pragma warning(disable : 4804)
24#endif
25
26namespace Eigen {
27
28namespace internal {
29
38template <typename Lhs, typename Rhs, int Options>
39struct evaluator<Product<Lhs, Rhs, Options>> : public product_evaluator<Product<Lhs, Rhs, Options>> {
40 using XprType = Product<Lhs, Rhs, Options>;
41 using Base = product_evaluator<XprType>;
42
43 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr) : Base(xpr) {}
44};
45
46// A scalar factor cannot be folded into a unit diagonal or a permutation.
47template <typename Lhs, typename Shape = typename evaluator_traits<Lhs>::Shape>
48struct product_can_fold_scalar
49 : bool_constant<std::is_same<Shape, DenseShape>::value || std::is_same<Shape, SparseShape>::value ||
50 std::is_same<Shape, DiagonalShape>::value || std::is_same<Shape, SelfAdjointShape>::value> {};
51
52template <typename Lhs>
53struct product_can_fold_scalar<Lhs, TriangularShape> : bool_constant<(int(Lhs::Mode) & int(UnitDiag)) == 0> {};
54
55template <typename Lhs, typename Rhs, typename LhsShape = typename evaluator_traits<Lhs>::Shape,
56 typename RhsShape = typename evaluator_traits<Rhs>::Shape>
57struct product_can_scale_unit_triangle : std::false_type {};
58
59// The kernel must accept a commuting outer scale without combining it with extracted operand factors.
60template <typename Lhs, typename Rhs>
61struct product_can_scale_unit_triangle<Lhs, Rhs, TriangularShape, DenseShape>
62 : bool_constant<
63 (int(Lhs::Mode) & int(UnitDiag)) != 0 && std::is_same<typename Lhs::Scalar, typename Rhs::Scalar>::value &&
64 (std::is_floating_point<typename Lhs::Scalar>::value ||
65 (std::is_same<typename Lhs::Scalar, std::complex<typename NumTraits<typename Lhs::Scalar>::Real>>::value &&
66 std::is_floating_point<typename NumTraits<typename Lhs::Scalar>::Real>::value)) &&
67 !blas_traits<typename Lhs::MatrixType>::HasScalarFactor && !blas_traits<Rhs>::HasScalarFactor> {};
68
69template <typename Lhs, typename Rhs>
70struct product_can_scale_unit_triangle<Lhs, Rhs, TriangularShape, DiagonalShape>
71 : product_can_scale_unit_triangle<Lhs, Rhs, TriangularShape, DenseShape> {};
72
73// The lazy selfadjoint/diagonal evaluator would conjugate a folded complex factor.
74template <typename Lhs, typename Rhs>
75struct product_evaluator_can_fold_scalar
76 : bool_constant<product_can_fold_scalar<Lhs>::value &&
77 !(std::is_same<typename evaluator_traits<Lhs>::Shape, SelfAdjointShape>::value &&
78 std::is_same<typename evaluator_traits<Rhs>::Shape, DiagonalShape>::value)> {};
79
80template <typename Xpr,
81 bool Fold = product_evaluator_can_fold_scalar<typename Xpr::Rhs::Lhs, typename Xpr::Rhs::Rhs>::value>
82struct scaled_product_evaluator_type {
83 // The assignment kernel extracts selfadjoint factors; materializing also protects nested aliases.
84 using type = std::conditional_t<product_can_fold_scalar<typename Xpr::Rhs::Lhs>::value, evaluator<EvalToTemp<Xpr>>,
85 binary_evaluator<Xpr>>;
86};
87
88template <typename Xpr>
89struct scaled_product_evaluator_type<Xpr, true> {
90 using type =
91 evaluator<remove_all_t<decltype((std::declval<Xpr>().lhs().functor().m_other * std::declval<Xpr>().rhs().lhs()) *
92 std::declval<Xpr>().rhs().rhs())>>;
93};
94
95// Catch "scalar * ( A * B )" and transform it to "(scalar*A) * B"
96// TODO: we should apply that rule only if that's really helpful
97template <typename Lhs, typename Rhs, typename Scalar1, typename Scalar2, typename Plain1>
98struct evaluator_assume_aliasing<CwiseBinaryOp<internal::scalar_product_op<Scalar1, Scalar2>,
99 const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>,
100 const Product<Lhs, Rhs, DefaultProduct>>> : std::true_type {};
101template <typename Lhs, typename Rhs, typename Scalar1, typename Scalar2, typename Plain1>
102struct evaluator<CwiseBinaryOp<internal::scalar_product_op<Scalar1, Scalar2>,
103 const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>,
104 const Product<Lhs, Rhs, DefaultProduct>>>
105 : scaled_product_evaluator_type<CwiseBinaryOp<internal::scalar_product_op<Scalar1, Scalar2>,
106 const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>,
107 const Product<Lhs, Rhs, DefaultProduct>>>::type {
108 using XprType = CwiseBinaryOp<internal::scalar_product_op<Scalar1, Scalar2>,
109 const CwiseNullaryOp<internal::scalar_constant_op<Scalar1>, Plain1>,
110 const Product<Lhs, Rhs, DefaultProduct>>;
111 using Base = typename scaled_product_evaluator_type<XprType>::type;
112
113 template <bool Fold = product_evaluator_can_fold_scalar<Lhs, Rhs>::value, std::enable_if_t<Fold, int> = 0>
114 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr)
115 : Base(xpr.lhs().functor().m_other * xpr.rhs().lhs() * xpr.rhs().rhs()) {}
116
117 template <bool Fold = product_evaluator_can_fold_scalar<Lhs, Rhs>::value, std::enable_if_t<!Fold, int> = 0>
118 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr) : Base(xpr) {}
119};
120
121template <typename Lhs, typename Rhs, int DiagIndex>
122struct evaluator<Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex>>
123 : public evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex>> {
124 using XprType = Diagonal<const Product<Lhs, Rhs, DefaultProduct>, DiagIndex>;
125 using Base = evaluator<Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex>>;
126
127 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit evaluator(const XprType& xpr)
128 : Base(Diagonal<const Product<Lhs, Rhs, LazyProduct>, DiagIndex>(
129 Product<Lhs, Rhs, LazyProduct>(xpr.nestedExpression().lhs(), xpr.nestedExpression().rhs()), xpr.index())) {}
130};
131
132// Helper class to perform a matrix product with the destination at hand.
133// Depending on the sizes of the factors, there are different evaluation strategies
134// as controlled by internal::product_type.
135template <typename Lhs, typename Rhs, typename LhsShape = typename evaluator_traits<Lhs>::Shape,
136 typename RhsShape = typename evaluator_traits<Rhs>::Shape,
137 int ProductType = internal::product_type<Lhs, Rhs>::value>
138struct generic_product_impl;
139
140template <typename Lhs, typename Rhs>
141struct evaluator_assume_aliasing<Product<Lhs, Rhs, DefaultProduct>> : std::true_type {};
142
143// This is the default evaluator implementation for products:
144// It creates a temporary and call generic_product_impl
145template <typename Lhs, typename Rhs, int Options, int ProductTag, typename LhsShape, typename RhsShape>
146struct product_evaluator<Product<Lhs, Rhs, Options>, ProductTag, LhsShape, RhsShape>
147 : public evaluator<typename Product<Lhs, Rhs, Options>::PlainObject> {
148 using XprType = Product<Lhs, Rhs, Options>;
149 using PlainObject = typename XprType::PlainObject;
150 using Base = evaluator<PlainObject>;
151 enum { Flags = Base::Flags | EvalBeforeNestingBit };
152
153 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit product_evaluator(const XprType& xpr)
154 : m_result(xpr.rows(), xpr.cols()) {
155 internal::construct_at<Base>(this, m_result);
156
157 // FIXME shall we handle nested_eval here?,
158 // if so, then we must take care at removing the call to nested_eval in the specializations (e.g., in
159 // permutation_matrix_product, transposition_matrix_product, etc.)
160 // typedef typename internal::nested_eval<Lhs,Rhs::ColsAtCompileTime>::type LhsNested;
161 // typedef typename internal::nested_eval<Rhs,Lhs::RowsAtCompileTime>::type RhsNested;
162 // typedef internal::remove_all_t<LhsNested> LhsNestedCleaned;
163 // typedef internal::remove_all_t<RhsNested> RhsNestedCleaned;
164 //
165 // const LhsNested lhs(xpr.lhs());
166 // const RhsNested rhs(xpr.rhs());
167 //
168 // generic_product_impl<LhsNestedCleaned, RhsNestedCleaned>::evalTo(m_result, lhs, rhs);
169
170 generic_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag>::evalTo(m_result, xpr.lhs(), xpr.rhs());
171 }
172
173 protected:
174 PlainObject m_result;
175};
176
177// The following three shortcuts are enabled only if the scalar types match exactly.
178// TODO: we could enable them for different scalar types when the product is not vectorized.
179
180// Dense = Product
181template <typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar>
182struct Assignment<DstXprType, Product<Lhs, Rhs, Options>, internal::assign_op<Scalar, Scalar>, Dense2Dense,
183 std::enable_if_t<(Options == DefaultProduct || Options == AliasFreeProduct)>> {
184 using SrcXprType = Product<Lhs, Rhs, Options>;
185 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
186 const internal::assign_op<Scalar, Scalar>& func) {
187 resize_if_allowed(dst, src, func);
188 // FIXME shall we handle nested_eval here?
189 generic_product_impl<Lhs, Rhs>::evalTo(dst, src.lhs(), src.rhs());
190 }
191};
192
193// Dense += Product
194template <typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar>
195struct Assignment<DstXprType, Product<Lhs, Rhs, Options>, internal::add_assign_op<Scalar, Scalar>, Dense2Dense,
196 std::enable_if_t<(Options == DefaultProduct || Options == AliasFreeProduct)>> {
197 using SrcXprType = Product<Lhs, Rhs, Options>;
198 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
199 const internal::add_assign_op<Scalar, Scalar>&) {
200 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
201 // FIXME shall we handle nested_eval here?
202 generic_product_impl<Lhs, Rhs>::addTo(dst, src.lhs(), src.rhs());
203 }
204};
205
206// Dense -= Product
207template <typename DstXprType, typename Lhs, typename Rhs, int Options, typename Scalar>
208struct Assignment<DstXprType, Product<Lhs, Rhs, Options>, internal::sub_assign_op<Scalar, Scalar>, Dense2Dense,
209 std::enable_if_t<(Options == DefaultProduct || Options == AliasFreeProduct)>> {
210 using SrcXprType = Product<Lhs, Rhs, Options>;
211 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
212 const internal::sub_assign_op<Scalar, Scalar>&) {
213 EIGEN_INTERNAL_ASSERT_SAME_SHAPE(dst, src);
214 // FIXME shall we handle nested_eval here?
215 generic_product_impl<Lhs, Rhs>::subTo(dst, src.lhs(), src.rhs());
216 }
217};
218
219// Dense ?= scalar * Product
220// TODO: we should apply that rule if that's really helpful
221// for instance, this is not good for inner products
222template <typename DstXprType, typename Lhs, typename Rhs, typename AssignFunc, typename Scalar, typename ScalarBis,
223 typename Plain>
224struct Assignment<DstXprType,
225 CwiseBinaryOp<internal::scalar_product_op<ScalarBis, Scalar>,
226 const CwiseNullaryOp<internal::scalar_constant_op<ScalarBis>, Plain>,
227 const Product<Lhs, Rhs, DefaultProduct>>,
228 AssignFunc, Dense2Dense, std::enable_if_t<product_can_fold_scalar<Lhs>::value>> {
229 using SrcXprType = CwiseBinaryOp<internal::scalar_product_op<ScalarBis, Scalar>,
230 const CwiseNullaryOp<internal::scalar_constant_op<ScalarBis>, Plain>,
231 const Product<Lhs, Rhs, DefaultProduct>>;
232 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
233 const AssignFunc& func) {
234 call_assignment_no_alias(dst, (src.lhs().functor().m_other * src.rhs().lhs()) * src.rhs().rhs(), func);
235 }
236};
237
238template <typename Dst, typename Lhs, typename Rhs, typename Func, typename Scalar, typename Scale, typename Plain>
239struct Assignment<
240 Dst,
241 CwiseBinaryOp<scalar_product_op<Scale, Scalar>, const CwiseNullaryOp<scalar_constant_op<Scale>, Plain>,
242 const Product<Lhs, Rhs, DefaultProduct>>,
243 Func, Dense2Dense,
244 std::enable_if_t<
245 product_can_scale_unit_triangle<Lhs, Rhs>::value && std::is_same<Scale, Scalar>::value &&
246 std::is_same<typename Lhs::Scalar, Scalar>::value && std::is_same<typename Dst::Scalar, Scalar>::value &&
247 std::is_same<typename traits<Dst>::XprKind, MatrixXpr>::value && ((int(Dst::Flags) & DirectAccessBit) != 0) &&
248 (Rhs::ColsAtCompileTime != 1 || Dst::IsVectorAtCompileTime) &&
249 (std::is_same<Func, assign_op<Scalar, Scalar>>::value ||
250 std::is_same<Func, add_assign_op<Scalar, Scalar>>::value ||
251 std::is_same<Func, sub_assign_op<Scalar, Scalar>>::value)>> {
252 using Src = CwiseBinaryOp<scalar_product_op<Scale, Scalar>, const CwiseNullaryOp<scalar_constant_op<Scale>, Plain>,
253 const Product<Lhs, Rhs, DefaultProduct>>;
254 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dst& dst, const Src& src, const Func& func) {
255 Scalar alpha = src.lhs().functor().m_other;
256 // Packed complex accumulation need not retain the sign of a zero scale component.
257 const bool signedZeroScale =
258 NumTraits<Scalar>::IsComplex &&
259 ((numext::real(alpha) == 0 && !numext::is_exactly_zero(numext::signbit(numext::real(alpha)))) ||
260 (numext::imag(alpha) == 0 && !numext::is_exactly_zero(numext::signbit(numext::imag(alpha)))));
261 // Preserve coefficient-wise 0*Inf and Inf*0 evaluation instead of taking the kernel's zero/empty shortcuts.
262 if (signedZeroScale || numext::is_exactly_zero(alpha) || !(numext::isfinite)(alpha) ||
263 src.rhs().lhs().cols() == 0) {
264 call_dense_assignment_loop(dst, src, func);
265 return;
266 }
267 resize_if_allowed(dst, src, func);
268 // The unscaled overwrite starts at +0; retain the sign of an outer scale applied to that zero.
269 EIGEN_IF_CONSTEXPR ((std::is_same<Func, assign_op<Scalar, Scalar>>::value)) dst.setConstant(alpha * Scalar(0));
270 EIGEN_IF_CONSTEXPR ((std::is_same<Func, sub_assign_op<Scalar, Scalar>>::value)) alpha = -alpha;
271 generic_product_impl<Lhs, Rhs>::scaleAndAddTo(dst, src.rhs().lhs(), src.rhs().rhs(), alpha);
272 }
273};
274
275//----------------------------------------
276// Catch "Dense ?= xpr + Product<>" expression to save one temporary
277// FIXME: consider enabling these rules for all product types, not only Dense and DefaultProduct.
278
279template <typename OtherXpr, typename Lhs, typename Rhs>
280struct evaluator_assume_aliasing<
281 CwiseBinaryOp<
282 internal::scalar_sum_op<typename OtherXpr::Scalar, typename Product<Lhs, Rhs, DefaultProduct>::Scalar>,
283 const OtherXpr, const Product<Lhs, Rhs, DefaultProduct>>,
284 DenseShape> : std::true_type {};
285
286template <typename OtherXpr, typename Lhs, typename Rhs>
287struct evaluator_assume_aliasing<
288 CwiseBinaryOp<
289 internal::scalar_difference_op<typename OtherXpr::Scalar, typename Product<Lhs, Rhs, DefaultProduct>::Scalar>,
290 const OtherXpr, const Product<Lhs, Rhs, DefaultProduct>>,
291 DenseShape> : std::true_type {};
292
293template <typename DstXprType, typename OtherXpr, typename ProductType, typename Func1, typename Func2>
294struct assignment_from_xpr_op_product {
295 template <typename SrcXprType, typename InitialFunc>
296 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src,
297 const InitialFunc& /*func*/) {
298 call_assignment_no_alias(dst, src.lhs(), Func1());
299 call_assignment_no_alias(dst, src.rhs(), Func2());
300 }
301};
302
303#define EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(ASSIGN_OP, BINOP, ASSIGN_OP2) \
304 template <typename DstXprType, typename OtherXpr, typename Lhs, typename Rhs, typename DstScalar, \
305 typename SrcScalar, typename OtherScalar, typename ProdScalar> \
306 struct Assignment<DstXprType, \
307 CwiseBinaryOp<internal::BINOP<OtherScalar, ProdScalar>, const OtherXpr, \
308 const Product<Lhs, Rhs, DefaultProduct>>, \
309 internal::ASSIGN_OP<DstScalar, SrcScalar>, Dense2Dense> \
310 : assignment_from_xpr_op_product<DstXprType, OtherXpr, Product<Lhs, Rhs, DefaultProduct>, \
311 internal::ASSIGN_OP<DstScalar, OtherScalar>, \
312 internal::ASSIGN_OP2<DstScalar, ProdScalar>> {}
313
314EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(assign_op, scalar_sum_op, add_assign_op);
315EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(add_assign_op, scalar_sum_op, add_assign_op);
316EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(sub_assign_op, scalar_sum_op, sub_assign_op);
317
318EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(assign_op, scalar_difference_op, sub_assign_op);
319EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(add_assign_op, scalar_difference_op, sub_assign_op);
320EIGEN_CATCH_ASSIGN_XPR_OP_PRODUCT(sub_assign_op, scalar_difference_op, add_assign_op);
321
322//----------------------------------------
323
324template <typename Lhs, typename Rhs>
325struct generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, InnerProduct> {
326 using impl = inner_product_dispatch<Lhs, Rhs, false>;
327 template <typename Dst>
328 static EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
329 dst.coeffRef(0, 0) = impl::run(lhs, rhs);
330 }
331
332 template <typename Dst>
333 static EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
334 dst.coeffRef(0, 0) += impl::run(lhs, rhs);
335 }
336
337 template <typename Dst>
338 static EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
339 dst.coeffRef(0, 0) -= impl::run(lhs, rhs);
340 }
341};
342
343/***********************************************************************
344 * Implementation of outer dense * dense vector product
345 ***********************************************************************/
346
347// Column major result
348template <typename Dst, typename Lhs, typename Rhs, typename Func>
349void EIGEN_DEVICE_FUNC outer_product_selector_run(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Func& func,
350 const std::false_type&) {
351 evaluator<Rhs> rhsEval(rhs);
352 ei_declare_local_nested_eval(Lhs, lhs, Rhs::SizeAtCompileTime, actual_lhs);
353 // FIXME if cols is large enough, then it might be useful to make sure that lhs is sequentially stored
354 // FIXME not very good if rhs is real and lhs complex while alpha is real too
355 const Index cols = dst.cols();
356 for (Index j = 0; j < cols; ++j) func(dst.col(j), actual_lhs * rhsEval.coeff(Index(0), j));
357}
358
359// Row major result
360template <typename Dst, typename Lhs, typename Rhs, typename Func>
361void EIGEN_DEVICE_FUNC outer_product_selector_run(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Func& func,
362 const std::true_type&) {
363 evaluator<Lhs> lhsEval(lhs);
364 ei_declare_local_nested_eval(Rhs, rhs, Lhs::SizeAtCompileTime, actual_rhs);
365 // FIXME if rows is large enough, then it might be useful to make sure that rhs is sequentially stored
366 // FIXME not very good if lhs is real and rhs complex while alpha is real too
367 const Index rows = dst.rows();
368 for (Index i = 0; i < rows; ++i) func(dst.row(i), lhsEval.coeff(i, Index(0)) * actual_rhs);
369}
370
371template <typename Dst>
372EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool outer_product_use_small_assignment(const Dst& dst) {
373 return dst.rows() <= 16 && dst.cols() <= 16;
374}
375
376template <bool ApplyScale = true, typename Dst, typename Lhs, typename Rhs, typename Func, typename Scalar>
377void EIGEN_DEVICE_FUNC outer_product_selector_run_small(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Func& func,
378 const Scalar& alpha, const std::false_type&) {
379 evaluator<Rhs> rhsEval(rhs);
380 ei_declare_local_nested_eval(Lhs, lhs, Rhs::SizeAtCompileTime, actual_lhs);
381 const Index rows = dst.rows();
382 const Index cols = dst.cols();
383 for (Index j = 0; j < cols; ++j) {
384 const typename Rhs::Scalar rhs_j = rhsEval.coeff(Index(0), j);
385 for (Index i = 0; i < rows; ++i) {
386 EIGEN_IF_CONSTEXPR (ApplyScale) {
387 func.assignCoeff(dst.coeffRef(i, j), internal::mul(alpha, internal::mul(actual_lhs.coeff(i, Index(0)), rhs_j)));
388 } else {
389 func.assignCoeff(dst.coeffRef(i, j), internal::mul(actual_lhs.coeff(i, Index(0)), rhs_j));
390 }
391 }
392 }
393}
394
395template <bool ApplyScale = true, typename Dst, typename Lhs, typename Rhs, typename Func, typename Scalar>
396void EIGEN_DEVICE_FUNC outer_product_selector_run_small(Dst& dst, const Lhs& lhs, const Rhs& rhs, const Func& func,
397 const Scalar& alpha, const std::true_type&) {
398 evaluator<Lhs> lhsEval(lhs);
399 ei_declare_local_nested_eval(Rhs, rhs, Lhs::SizeAtCompileTime, actual_rhs);
400 const Index rows = dst.rows();
401 const Index cols = dst.cols();
402 for (Index i = 0; i < rows; ++i) {
403 const typename Lhs::Scalar lhs_i = lhsEval.coeff(i, Index(0));
404 for (Index j = 0; j < cols; ++j) {
405 EIGEN_IF_CONSTEXPR (ApplyScale) {
406 func.assignCoeff(dst.coeffRef(i, j), internal::mul(alpha, internal::mul(lhs_i, actual_rhs.coeff(Index(0), j))));
407 } else {
408 func.assignCoeff(dst.coeffRef(i, j), internal::mul(lhs_i, actual_rhs.coeff(Index(0), j)));
409 }
410 }
411 }
412}
413
414template <typename Lhs, typename Rhs>
415struct generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, OuterProduct> {
416 template <typename T>
417 struct is_row_major : bool_constant<(int(T::Flags) & RowMajorBit)> {};
418 using Scalar = typename Product<Lhs, Rhs>::Scalar;
419
420 // TODO: it would be nice to be able to exploit our *_assign_op functors for that purpose
421 struct set {
422 template <typename Dst, typename Src>
423 EIGEN_DEVICE_FUNC void operator()(const Dst& dst, const Src& src) const {
424 dst.const_cast_derived() = src;
425 }
426 };
427 struct add {
429 template <typename Dst, typename Src>
430 EIGEN_DEVICE_FUNC void operator()(const Dst& dst, const Src& src) const {
431 dst.const_cast_derived() += src;
432 }
433 };
434 struct sub {
435 template <typename Dst, typename Src>
436 EIGEN_DEVICE_FUNC void operator()(const Dst& dst, const Src& src) const {
437 dst.const_cast_derived() -= src;
438 }
439 };
441 struct adds {
442 Scalar m_scale;
444 EIGEN_DEVICE_FUNC explicit adds(const Scalar& s) : m_scale(s) {}
446 template <typename Dst, typename Src>
447 void EIGEN_DEVICE_FUNC operator()(const Dst& dst, const Src& src) const {
448 dst.const_cast_derived() += m_scale * src;
449 }
450 };
451
452 template <typename Dst>
453 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
454 if (internal::outer_product_use_small_assignment(dst)) {
455 internal::outer_product_selector_run_small<false>(
456 dst, lhs, rhs, internal::assign_op<typename Dst::Scalar, Scalar>(), Scalar(1), is_row_major<Dst>());
457 } else {
458 internal::outer_product_selector_run(dst, lhs, rhs, set(), is_row_major<Dst>());
459 }
460 }
461
462 template <typename Dst>
463 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
464 if (internal::outer_product_use_small_assignment(dst)) {
465 internal::outer_product_selector_run_small<false>(
466 dst, lhs, rhs, internal::add_assign_op<typename Dst::Scalar, Scalar>(), Scalar(1), is_row_major<Dst>());
467 } else {
468 internal::outer_product_selector_run(dst, lhs, rhs, add(), is_row_major<Dst>());
469 }
470 }
471
472 template <typename Dst>
473 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
474 if (internal::outer_product_use_small_assignment(dst)) {
475 internal::outer_product_selector_run_small<false>(
476 dst, lhs, rhs, internal::sub_assign_op<typename Dst::Scalar, Scalar>(), Scalar(1), is_row_major<Dst>());
477 } else {
478 internal::outer_product_selector_run(dst, lhs, rhs, sub(), is_row_major<Dst>());
479 }
480 }
481
482 template <typename Dst>
483 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs,
484 const Scalar& alpha) {
485 if (internal::outer_product_use_small_assignment(dst)) {
486 internal::outer_product_selector_run_small(dst, lhs, rhs, internal::add_assign_op<typename Dst::Scalar, Scalar>(),
487 alpha, is_row_major<Dst>());
488 } else {
489 internal::outer_product_selector_run(dst, lhs, rhs, adds(alpha), is_row_major<Dst>());
490 }
491 }
492};
493
494// This base class provides default implementations for evalTo, addTo, subTo, in terms of scaleAndAddTo
495template <typename Lhs, typename Rhs, typename Derived>
496struct generic_product_impl_base {
497 using Scalar = typename Product<Lhs, Rhs>::Scalar;
498
499 template <typename Dst>
500 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
501 dst.setZero();
502 scaleAndAddTo(dst, lhs, rhs, Scalar(1));
503 }
504
505 template <typename Dst>
506 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
507 scaleAndAddTo(dst, lhs, rhs, Scalar(1));
508 }
509
510 template <typename Dst>
511 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
512 scaleAndAddTo(dst, lhs, rhs, Scalar(-1));
513 }
514
515 template <typename Dst>
516 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dst& dst, const Lhs& lhs, const Rhs& rhs,
517 const Scalar& alpha) {
518 Derived::scaleAndAddTo(dst, lhs, rhs, alpha);
519 }
520};
521
522template <typename Lhs, typename Rhs, typename Scalar>
523struct gemv_leading_constant : std::false_type {};
524
525template <typename Plain, typename Nested, typename Rhs, typename Scalar>
526struct gemv_leading_constant<
527 CwiseBinaryOp<scalar_product_op<Scalar>, const CwiseNullaryOp<scalar_constant_op<Scalar>, Plain>, Nested>, Rhs,
528 Scalar> : std::is_same<typename Rhs::Scalar, Scalar> {
529 using Inner = remove_all_t<Nested>;
530};
531
532// Seed all sums from their first term without requiring a default-constructible scalar.
533// Inline initialization so the compiler can keep the accumulator array in registers.
534template <typename Scalar, int Count>
535struct gemv_ordered_accumulators {
536 Scalar values[Count];
537 template <typename Term, int... Indices>
538 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit gemv_ordered_accumulators(const Term& term, Index offset,
539 std::integer_sequence<int, Indices...>)
540 : values{term(offset + Indices, 0)...} {}
541};
542
543template <typename Lhs, typename Rhs>
544struct generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemvProduct>
545 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemvProduct>> {
546 using LhsNested = typename nested_eval<Lhs, 1>::type;
547 using RhsNested = typename nested_eval<Rhs, 1>::type;
548 using Scalar = typename Product<Lhs, Rhs>::Scalar;
549 enum { Side = Lhs::IsVectorAtCompileTime ? OnTheLeft : OnTheRight };
550 using MatrixType = internal::remove_all_t<std::conditional_t<int(Side) == OnTheRight, LhsNested, RhsNested>>;
551
552 template <typename Dest>
553 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs,
554 const Scalar& alpha) {
555 run(dst, lhs, rhs, alpha, blas_product_is_commutative<typename Lhs::Scalar, typename Rhs::Scalar>());
556 }
557
558 private:
559 template <typename Dest>
560 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha,
561 std::false_type) {
562 if (lhs.cols() == 0 || numext::is_exactly_zero(alpha) || blas_traits<Lhs>::hasZeroScalarFactor(lhs) ||
563 blas_traits<Rhs>::hasZeroScalarFactor(rhs))
564 return;
565 run_ordered(dst, lhs, rhs, alpha, gemv_leading_constant<Lhs, Rhs, Scalar>());
566 }
567
568 template <typename Dest>
569 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_ordered(Dest& dst, const Lhs& lhs, const Rhs& rhs,
570 const Scalar& alpha, std::true_type) {
571 const Scalar scale = lhs.lhs().functor().m_other;
572 // A zero that is part of a constant expression must still multiply its coefficients and propagate NaNs.
573 if (numext::is_exactly_zero(scale)) return run_ordered(dst, lhs, rhs, alpha, std::false_type());
574 using Inner = typename gemv_leading_constant<Lhs, Rhs, Scalar>::Inner;
575 using Impl = generic_product_impl<Inner, Rhs, DenseShape, DenseShape, GemvProduct>;
576 if (numext::is_exactly_one(alpha)) return Impl::scaleAndAddTo(dst, lhs.rhs(), rhs, scale);
577 // Keep alpha * (scale * sum) separate: forming alpha * scale can underflow or overflow.
578 ei_declare_aligned_stack_constructed_variable(Scalar, buffer, dst.size(), nullptr);
579 Map<typename Product<Lhs, Rhs>::PlainObject> product(buffer, dst.rows(), dst.cols());
580 product.setZero();
581 Impl::scaleAndAddTo(product, lhs.rhs(), rhs, scale);
582 for (Index j = 0; j < dst.cols(); ++j)
583 for (Index i = 0; i < dst.rows(); ++i) dst.coeffRef(i, j) += alpha * product.coeff(i, j);
584 }
585
586 template <typename Dest>
587 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_ordered(Dest& dst, const Lhs& lhs, const Rhs& rhs,
588 const Scalar& alpha, std::false_type) {
589 // Factor extraction and alpha folding need not preserve the written scalar order.
590 LhsNested actual_lhs(lhs);
591 RhsNested actual_rhs(rhs);
592 const bool unit_alpha = numext::is_exactly_one(alpha);
593 constexpr bool OnRight = int(Side) == int(OnTheRight);
594 const Index size = OnRight ? lhs.rows() : rhs.cols();
595 const Index depth = lhs.cols();
596 const auto out = [&](Index i) -> decltype(auto) { return OnRight ? dst.coeffRef(i, 0) : dst.coeffRef(0, i); };
597 EIGEN_IF_CONSTEXPR (OnRight == !(MatrixType::Flags & RowMajorBit)) {
598 const auto accumulate = [&](const auto& at) {
599 for (Index k = 0; k < depth; ++k) {
600 EIGEN_IF_CONSTEXPR (OnRight) {
601 const typename Rhs::Scalar value = actual_rhs.coeff(k, 0);
602 for (Index i = 0; i < size; ++i) at(i) += actual_lhs.coeff(i, k) * value;
603 } else {
604 const typename Lhs::Scalar value = actual_lhs.coeff(0, k);
605 for (Index i = 0; i < size; ++i) at(i) += value * actual_rhs.coeff(k, i);
606 }
607 }
608 };
609 if (unit_alpha) {
610 accumulate(out);
611 } else {
612 ei_declare_aligned_stack_constructed_variable(Scalar, buffer, size, nullptr);
613 Map<Matrix<Scalar, Dynamic, 1>> product(buffer, size);
614 product.setZero();
615 accumulate([&](Index i) -> Scalar& { return product.coeffRef(i); });
616 for (Index i = 0; i < size; ++i) out(i) += alpha * product.coeff(i);
617 }
618 return;
619 }
620 const auto term = [&](Index i, Index k) -> Scalar {
621 return OnRight ? Scalar(actual_lhs.coeff(i, k) * actual_rhs.coeff(k, 0))
622 : Scalar(actual_lhs.coeff(0, k) * actual_rhs.coeff(k, i));
623 };
624 // Keep independent ordered sums in flight, sharing each vector coefficient across the block.
625 constexpr int Block = sizeof(Scalar) <= 16 ? 8 : 4;
626 Index i = 0;
627 for (; size - i >= Block; i += Block) {
628 gemv_ordered_accumulators<Scalar, Block> sums(term, i, std::make_integer_sequence<int, Block>{});
629 for (Index k = 1; k < depth; ++k) {
630 EIGEN_IF_CONSTEXPR (OnRight) {
631 const typename Rhs::Scalar value = actual_rhs.coeff(k, 0);
632 EIGEN_UNROLL_LOOP
633 for (int r = 0; r < Block; ++r) sums.values[r] += actual_lhs.coeff(i + r, k) * value;
634 } else {
635 const typename Lhs::Scalar value = actual_lhs.coeff(0, k);
636 EIGEN_UNROLL_LOOP
637 for (int r = 0; r < Block; ++r) sums.values[r] += value * actual_rhs.coeff(k, i + r);
638 }
639 }
640 EIGEN_UNROLL_LOOP
641 for (int r = 0; r < Block; ++r) out(i + r) += unit_alpha ? sums.values[r] : alpha * sums.values[r];
642 }
643 constexpr int OutputSize = OnRight ? int(traits<Lhs>::RowsAtCompileTime) : int(traits<Rhs>::ColsAtCompileTime);
644 EIGEN_IF_CONSTEXPR (OutputSize == Dynamic || OutputSize % Block != 0) {
645 for (; i < size; ++i) {
646 Scalar sum = term(i, 0);
647 for (Index k = 1; k < depth; ++k) sum += term(i, k);
648 out(i) += unit_alpha ? sum : alpha * sum;
649 }
650 }
651 }
652
653 template <typename Dest>
654 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha,
655 std::true_type) {
656 // Fallback to inner product if both the lhs and rhs is a runtime vector.
657 if (lhs.rows() == 1 && rhs.cols() == 1) {
658 dst.coeffRef(0, 0) += alpha * lhs.row(0).conjugate().dot(rhs.col(0));
659 return;
660 }
661 LhsNested actual_lhs(lhs);
662 RhsNested actual_rhs(rhs);
663 internal::gemv_dense_selector<Side, (int(MatrixType::Flags) & RowMajorBit) ? RowMajor : ColMajor,
664 bool(internal::blas_traits<MatrixType>::HasUsableDirectAccess)>::run(actual_lhs,
665 actual_rhs, dst,
666 alpha);
667 }
668};
669
670// Packet-cascade assignment for small fixed-size coefficient-based products
671// (dst = / += / -= lhs * rhs). The generic assignment path types the product's
672// packet via find_best_packet -- the widest packet that *exactly divides* the
673// vectorized extent, leaving no tail. For odd/non-power-of-two extents that
674// overshoots: with no exact divisor it falls back to a narrow packet, or all
675// the way to scalars (e.g. 3x3 double becomes scalar, ~3x slower than it needs
676// to be). find_largest_packet instead picks the widest packet that *fits*
677// (size <= extent) and we emit an explicit cascade -- full packet(s), then the
678// half packet, ... then a scalar tail -- so e.g. 3x3 double does one Packet2d
679// plus one scalar per column, and 6x6 double does one Packet4d plus one Packet2d
680// instead of three Packet2d. See find_largest_packet in XprHelper.h.
681
682// Emit the inner segment [begin, end) for a single outer index, cascading from
683// `Packet` down through its half packets to a scalar tail. `ColMajor` selects
684// whether the contiguous inner index is the row (true) or the column (false).
685// The bool `Terminal` partial specialization stops the recursion at the
686// narrowest packet without requiring C++17 `if constexpr`.
687template <typename Packet, bool ColMajor,
688 bool Terminal = std::is_same<typename unpacket_traits<Packet>::half, Packet>::value>
689struct product_packet_cascade {
690 template <typename Func, typename Dst, typename ProdEval>
691 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Func& func, Dst& dst, const ProdEval& prod, Index outer,
692 Index begin, Index end) {
693 constexpr int PacketSize = unpacket_traits<Packet>::size;
694 Index pktEnd = begin + numext::round_down(end - begin, PacketSize);
695 Index i = begin;
696 EIGEN_IF_CONSTEXPR (ProdEval::MixedRealComplex) {
697 if (pktEnd - begin == PacketSize && pktEnd < end) {
698 // Read the tail before storing the full packet so their real-factor broadcasts can be shared.
699 Index r = ColMajor ? begin : outer, c = ColMajor ? outer : begin;
700 const Packet result = prod.template packet<Unaligned, Packet>(r, c);
701 product_packet_cascade<typename unpacket_traits<Packet>::half, ColMajor>::run(func, dst, prod, outer, pktEnd,
702 end);
703 func.template assignPacket<Unaligned, Packet>(&dst.coeffRef(r, c), result);
704 return;
705 }
706 }
707 for (; i < pktEnd; i += PacketSize) {
708 Index r = ColMajor ? i : outer, c = ColMajor ? outer : i;
709 func.template assignPacket<Unaligned, Packet>(&dst.coeffRef(r, c), prod.template packet<Unaligned, Packet>(r, c));
710 }
711 // Fall back to the half packet for the remainder [i, end).
712 product_packet_cascade<typename unpacket_traits<Packet>::half, ColMajor>::run(func, dst, prod, outer, i, end);
713 }
714};
715
716// Narrowest packet in the chain: lay down its full packets, then a scalar tail.
717template <typename Packet, bool ColMajor>
718struct product_packet_cascade<Packet, ColMajor, /*Terminal=*/true> {
719 template <typename Func, typename Dst, typename ProdEval>
720 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Func& func, Dst& dst, const ProdEval& prod, Index outer,
721 Index begin, Index end) {
722 constexpr int PacketSize = unpacket_traits<Packet>::size;
723 Index pktEnd = begin + numext::round_down(end - begin, PacketSize);
724 Index i = begin;
725 for (; i < pktEnd; i += PacketSize) {
726 Index r = ColMajor ? i : outer, c = ColMajor ? outer : i;
727 func.template assignPacket<Unaligned, Packet>(&dst.coeffRef(r, c), prod.template packet<Unaligned, Packet>(r, c));
728 }
729 EIGEN_IF_CONSTEXPR (ProdEval::MixedRealComplex &&
730 (ColMajor ? bool(NumTraits<typename ProdEval::LhsScalar>::IsComplex)
731 : bool(NumTraits<typename ProdEval::RhsScalar>::IsComplex))) {
732 if (i > begin && i < end) {
733 // Overlap complex loads to avoid scalar tail reductions; duplicating real lanes again is more expensive.
734 // Store only new lanes so compound assignments visit each coefficient once.
735 const Index start = end - PacketSize;
736 const Index r = ColMajor ? start : outer, c = ColMajor ? outer : start;
737 const Packet result = prod.template packet<Unaligned, Packet>(r, c);
738 func.template assignPacketSegment<Unaligned, Packet>(&dst.coeffRef(r, c), result, i - start, end - i);
739 return;
740 }
741 }
742 for (; i < end; ++i) {
743 Index r = ColMajor ? i : outer, c = ColMajor ? outer : i;
744 func.assignCoeff(dst.coeffRef(r, c), prod.coeff(r, c));
745 }
746 }
747};
748
749// Largest packet size in the (halving) chain that *exactly divides* Size, or 1
750// if none does. This is what the generic InnerVectorized assignment path can
751// actually use -- unlike find_best_packet, whose stop condition returns the
752// terminal (non-dividing) packet when the chain bottoms out above size 1 (on
753// x86 the chain ends at Packet2d / Packet4f, not a size-1 packet), so e.g.
754// find_best_packet<double,3> reports Packet2d even though 3 % 2 != 0.
755template <typename Packet, int Size,
756 bool Terminal = std::is_same<typename unpacket_traits<Packet>::half, Packet>::value>
757struct largest_exact_divisor_size { // recurse into the narrower half packet
758 static constexpr int s = unpacket_traits<Packet>::size;
759 static constexpr int value =
760 (Size % s == 0) ? s : largest_exact_divisor_size<typename unpacket_traits<Packet>::half, Size>::value;
761};
762template <typename Packet, int Size>
763struct largest_exact_divisor_size<Packet, Size, /*Terminal=*/true> { // narrowest packet in the chain
764 static constexpr int s = unpacket_traits<Packet>::size;
765 static constexpr int value = (Size % s == 0) ? s : 1;
766};
767
768// When is the cascade worth diverging from the generic assignment path, and
769// along which axis? Only for fixed-size products where the widest packet that
770// fits the vectorized extent (find_largest_packet) is strictly wider than the
771// widest that exactly divides it -- i.e. the extent is not a clean packet
772// multiple, so the generic path is forced down to a narrower packet or scalars.
773// Exact-fit sizes (2, 4, 8, ...) keep the proven generic path, as do dynamic
774// sizes (GEMM / generic kernels) and extents too small to hold even one packet
775// (e.g. 3-row float). Complex is included: the generic path scalarizes odd
776// extents (find_best<complex<double>,3> == Packet1cd), so the cascade's
777// Packet2cd/Packet4cf tail is a sizeable win there too.
778//
779// The destination must also store the vectorized axis contiguously
780// (DirectAccessBit and InnerStrideAtCompileTime == 1): the cascade writes whole
781// packets with a single unaligned store, so a strided destination such as
782// `m.row(i)` of a column-major matrix would clobber neighbouring coefficients.
783// The generic assignment path makes this check along with MightVectorize and
784// EIGEN_UNALIGNED_VECTORIZE; here we must replicate them since we bypass that path.
785template <typename ProdEval, typename Dst, typename Func>
786struct product_packet_cascade_traits {
787 using Scalar = typename ProdEval::Scalar;
788 using DstEval = evaluator<Dst>;
789 static constexpr int Rows = ProdEval::RowsAtCompileTime;
790 static constexpr int Cols = ProdEval::ColsAtCompileTime;
791 static constexpr bool DstRowMajor = bool(Dst::IsRowMajor);
792 static constexpr bool DstContiguous =
793 (int(DstEval::Flags) & DirectAccessBit) != 0 && int(Dst::InnerStrideAtCompileTime) == 1;
794 // The product's packets run along its storage order. The cascade axis must match
795 // it -- and so must the destination's storage order, so the contiguous packet
796 // store lands on contiguous coefficients.
797 static constexpr bool ProdRowMajor = (int(ProdEval::Flags) & RowMajorBit) != 0;
798 static constexpr bool VecLhs = // column-major product & dst: vectorize down each column
799 bool(ProdEval::Vectorize) && !ProdRowMajor && !DstRowMajor && Rows != Dynamic;
800 static constexpr bool VecRhs = // row-major product & dst: vectorize along each row
801 bool(ProdEval::Vectorize) && ProdRowMajor && DstRowMajor && Cols != Dynamic;
802 static constexpr int Extent = VecLhs ? Rows : VecRhs ? Cols : 1;
803 using Packet = typename find_largest_packet<Scalar, Extent>::type;
804 static constexpr int LargestSize = unpacket_traits<Packet>::size;
805 static constexpr int GenericSize = largest_exact_divisor_size<typename packet_traits<Scalar>::type, Extent>::value;
806 static constexpr bool MightVectorize =
807 bool(EIGEN_UNALIGNED_VECTORIZE) &&
808 (int(DstEval::Flags) & int(ProdEval::Flags) & int(ActualPacketAccessBit)) != 0 &&
809 bool(functor_traits<Func>::PacketAccess);
810 static constexpr bool Enable = (ProdEval::SameType || ProdEval::MixedRealComplex) && (VecLhs || VecRhs) &&
811 DstContiguous && MightVectorize && (LargestSize <= Extent) &&
812 (LargestSize > GenericSize);
813};
814
815// Run the cascade over every outer index of dst. Traits::VecLhs is a compile-time
816// constant, so it folds the bounds away and selects the cascade's ColMajor axis,
817// instantiating only the active orientation (no dead branch on either axis).
818template <typename Traits, typename Func, typename Dst, typename ProdEval>
819EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void product_run_packet_cascade(const Func& func, Dst& dst,
820 const ProdEval& prod) {
821 using Packet = typename Traits::Packet;
822 Index rows = dst.rows(), cols = dst.cols();
823 Index outerCount = Traits::VecLhs ? cols : rows;
824 Index innerEnd = Traits::VecLhs ? rows : cols;
825 for (Index outer = 0; outer < outerCount; ++outer)
826 product_packet_cascade<Packet, Traits::VecLhs>::run(func, dst, prod, outer, 0, innerEnd);
827}
828
829// Product assignment: take the packet cascade when the traits enable
830// it, otherwise the generic assignment path. Dispatch on a bool tag (rather than
831// a plain `if`) so the cascade is only *instantiated* for the vectorizable
832// fixed-size case -- a runtime branch would instantiate it for every product,
833// including scalar types with no packet support.
834template <typename Lhs, typename Rhs, bool EvalToRowMajor>
835struct coeff_based_product_evaluator;
836
837// The coefficient-based evaluator of lhs * rhs with packets along dst's storage order.
838template <typename Lhs, typename Rhs, typename Dst>
839using dst_ordered_product_evaluator = coeff_based_product_evaluator<Lhs, Rhs, bool(Dst::IsRowMajor)>;
840
841template <typename Func, typename Dst, typename Lhs, typename Rhs>
842EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void product_packet_assign(std::true_type, const Func& func, Dst& dst,
843 const Lhs& lhs, const Rhs& rhs) {
844 using ProdEval = dst_ordered_product_evaluator<Lhs, Rhs, Dst>;
845 using Traits = product_packet_cascade_traits<ProdEval, Dst, Func>;
846 const ProdEval prodEval(lhs, rhs);
847 product_run_packet_cascade<Traits>(func, dst, prodEval);
848}
849template <typename Func, typename Dst, typename Lhs, typename Rhs>
850EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void product_packet_assign(std::false_type, const Func& func, Dst& dst,
851 const Lhs& lhs, const Rhs& rhs) {
852 call_assignment_no_alias(dst, lhs.lazyProduct(rhs), func);
853}
854
855template <typename Lhs, typename Rhs>
856struct generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode> {
857 using Scalar = typename Product<Lhs, Rhs>::Scalar;
858
859 template <typename Dst>
860 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
861 // Same as: dst.noalias() = lhs.lazyProduct(rhs);
862 // but easier on the compiler side.
863 // NB: same-scalar plain assignment stays on the generic path.
864 // The packet cascade is a measured win for the compound (+=, -=) case, but
865 // for plain assignment the generic InnerVectorized path is already well
866 // tuned -- the cascade was at best neutral there and could regress float
867 // (whose packet half-chain bottoms out at Packet4f, so odd-size tails fall
868 // to scalars). Mixed products also use the cascade when it widens their packets.
869 using ProdEval = dst_ordered_product_evaluator<Lhs, Rhs, Dst>;
870 using Func = internal::assign_op<typename Dst::Scalar, Scalar>;
871 using Traits = product_packet_cascade_traits<ProdEval, Dst, Func>;
872 product_packet_assign(bool_constant<(ProdEval::MixedRealComplex && Traits::Enable)>(), Func(), dst, lhs, rhs);
873 }
874
875 template <typename Dst>
876 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
877 // dst.noalias() += lhs.lazyProduct(rhs);
878 using ProdEval = dst_ordered_product_evaluator<Lhs, Rhs, Dst>;
879 using Func = internal::add_assign_op<typename Dst::Scalar, Scalar>;
880 using Traits = product_packet_cascade_traits<ProdEval, Dst, Func>;
881 product_packet_assign(bool_constant<Traits::Enable>(), Func(), dst, lhs, rhs);
882 }
883
884 template <typename Dst>
885 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
886 // dst.noalias() -= lhs.lazyProduct(rhs);
887 using ProdEval = dst_ordered_product_evaluator<Lhs, Rhs, Dst>;
888 using Func = internal::sub_assign_op<typename Dst::Scalar, Scalar>;
889 using Traits = product_packet_cascade_traits<ProdEval, Dst, Func>;
890 product_packet_assign(bool_constant<Traits::Enable>(), Func(), dst, lhs, rhs);
891 }
892
893 // This is a special evaluation path called from generic_product_impl<...,GemmProduct> in file GeneralMatrixMatrix.h
894 // This variant tries to extract scalar multiples from both the LHS and RHS and factor them out. For instance:
895 // dst {,+,-}= (s1*A)*(B*s2)
896 // will be rewritten as:
897 // dst {,+,-}= (s1*s2) * (A.lazyProduct(B))
898 // There are at least four benefits of doing so:
899 // 1 - huge performance gain for heap-allocated matrix types as it save costly allocations.
900 // 2 - it is faster than simply by-passing the heap allocation through stack allocation.
901 // 3 - it makes this fallback consistent with the heavy GEMM routine.
902 // 4 - it fully by-passes huge stack allocation attempts when multiplying huge fixed-size matrices.
903 // (see https://stackoverflow.com/questions/54738495)
904 // For small fixed sizes matrices, however, the gains are less obvious, it is sometimes x2 faster, but sometimes x3
905 // slower, and the behavior depends also a lot on the compiler... This is why this re-writing strategy is currently
906 // enabled only when falling back from the main GEMM.
907 template <typename Dst, typename Func>
908 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void eval_dynamic(Dst& dst, const Lhs& lhs, const Rhs& rhs,
909 const Func& func) {
910 enum {
911 HasScalarFactor = blas_traits<Lhs>::HasScalarFactor || blas_traits<Rhs>::HasScalarFactor,
912 ConjLhs = blas_traits<Lhs>::NeedToConjugate,
913 ConjRhs = blas_traits<Rhs>::NeedToConjugate
914 };
915 // FIXME: this should be auto, and extractScalarFactor should also return auto
916 // this is important for real*complex_mat
917 Scalar actualAlpha = combine_scalar_factors<Scalar>(lhs, rhs);
918
919 eval_dynamic_impl(dst, blas_traits<Lhs>::extract(lhs).template conjugateIf<ConjLhs>(),
920 blas_traits<Rhs>::extract(rhs).template conjugateIf<ConjRhs>(), func, actualAlpha,
921 bool_constant<HasScalarFactor>());
922 }
923
924 protected:
925 template <typename Dst, typename LhsT, typename RhsT, typename Func, typename Scalar>
926 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void eval_dynamic_impl(Dst& dst, const LhsT& lhs, const RhsT& rhs,
927 const Func& func, const Scalar& s /* == 1 */,
928 std::false_type) {
929 EIGEN_UNUSED_VARIABLE(s);
930 eigen_internal_assert(numext::is_exactly_one(s));
931 call_restricted_packet_assignment_no_alias(dst, lhs.lazyProduct(rhs), func);
932 }
933
934 template <typename Dst, typename LhsT, typename RhsT, typename Func, typename Scalar>
935 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void eval_dynamic_impl(Dst& dst, const LhsT& lhs, const RhsT& rhs,
936 const Func& func, const Scalar& s,
937 std::true_type) {
938 call_restricted_packet_assignment_no_alias(dst, s * lhs.lazyProduct(rhs), func);
939 }
940};
941
942// This specialization enforces the use of a coefficient-based evaluation strategy
943template <typename Lhs, typename Rhs>
944struct generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, LazyCoeffBasedProductMode>
945 : generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode> {};
946
947template <typename Packet, typename Scalar, typename ResultScalar = typename unpacket_traits<Packet>::type,
948 bool Real = std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value>
949struct product_operand_packet {
950 using type = Packet;
951};
952
953// Preserve real operands: promoting them to complex would introduce artificial 0*inf terms.
954template <typename Packet, typename Real>
955struct product_operand_packet<Packet, Real, std::complex<Real>, true> {
956 using type = typename unpacket_traits<Packet>::as_real;
957};
958
959template <typename Packet, typename Scalar>
960using product_operand_packet_t = typename product_operand_packet<Packet, Scalar>::type;
961
962// ploaddup reads half a real packet. Only contiguous, addressable coefficients may use it.
963template <typename Xpr>
964struct product_real_packet_access {
965 using CoeffReturnType = decltype(std::declval<const evaluator<Xpr>&>().coeff(Index(0), Index(0)));
966 static constexpr bool value =
967 (evaluator<Xpr>::Flags & DirectAccessBit) != 0 && Xpr::InnerStrideAtCompileTime == 1 &&
968 std::is_same<std::remove_const_t<std::remove_reference_t<CoeffReturnType>>, typename Xpr::Scalar>::value &&
969 std::is_lvalue_reference<CoeffReturnType>::value;
970};
971
972template <typename Xpr>
973struct product_real_packet_evaluator : evaluator<Xpr> {
974 using Base = evaluator<Xpr>;
975 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit product_real_packet_evaluator(const Xpr& xpr) : Base(xpr) {}
976
977 template <int LoadMode, typename Packet>
978 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename unpacket_traits<Packet>::as_real packet(Index row, Index col) const {
979 return ploaddup<typename unpacket_traits<Packet>::as_real>(&this->coeff(row, col));
980 }
981
982 template <int LoadMode, typename Packet>
983 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename unpacket_traits<Packet>::as_real packetSegment(Index row, Index col,
984 Index begin,
985 Index count) const {
986 // The product evaluator handles partial packets before loading either operand.
987 EIGEN_UNUSED_VARIABLE(begin);
988 EIGEN_UNUSED_VARIABLE(count);
989 eigen_internal_assert(begin == 0 && count == unpacket_traits<Packet>::size);
990 return packet<LoadMode, Packet>(row, col);
991 }
992};
993
994template <int StorageOrder, int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode>
995struct etor_product_packet_impl;
996
997struct product_empty_packet_evaluator {
998 template <typename Xpr>
999 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit product_empty_packet_evaluator(const Xpr&) {}
1000};
1001
1002// Coefficient-based evaluator of lhs * rhs whose packets run along rows if EvalToRowMajor_, else along columns.
1003template <typename Lhs, typename Rhs, bool EvalToRowMajor_>
1004struct coeff_based_product_evaluator : evaluator_base<Product<Lhs, Rhs, LazyProduct>> {
1005 using XprType = Product<Lhs, Rhs, LazyProduct>;
1006 using Scalar = typename XprType::Scalar;
1007 using CoeffReturnType = typename XprType::CoeffReturnType;
1008
1009 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit coeff_based_product_evaluator(const XprType& xpr)
1010 : coeff_based_product_evaluator(xpr.lhs(), xpr.rhs()) {}
1011
1012 // Also construct directly from the product operands so packet-cascade callers
1013 // do not have to materialize a temporary Product expression first.
1014 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE coeff_based_product_evaluator(const Lhs& lhs, const Rhs& rhs)
1015 : m_lhs(lhs),
1016 m_rhs(rhs),
1017 m_lhsImpl(m_lhs), // Real evaluator objects for packet products; empty placeholders for scalar-only products.
1018 m_rhsImpl(m_rhs),
1019 m_innerDim(lhs.cols()) {
1020 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::MulCost);
1021 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::AddCost);
1022 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
1023 }
1024
1025 // Everything below here is taken from CoeffBasedProduct.h
1026
1027 using LhsNested = typename internal::nested_eval<Lhs, Rhs::ColsAtCompileTime>::type;
1028 using RhsNested = typename internal::nested_eval<Rhs, Lhs::RowsAtCompileTime>::type;
1029
1030 using LhsNestedCleaned = internal::remove_all_t<LhsNested>;
1031 using RhsNestedCleaned = internal::remove_all_t<RhsNested>;
1032
1033 using LhsScalar = typename LhsNestedCleaned::Scalar;
1034 using RhsScalar = typename RhsNestedCleaned::Scalar;
1035
1036 using LhsEtorType = evaluator<LhsNestedCleaned>;
1037 using RhsEtorType = evaluator<RhsNestedCleaned>;
1038
1039 enum {
1040 RowsAtCompileTime = LhsNestedCleaned::RowsAtCompileTime,
1041 ColsAtCompileTime = RhsNestedCleaned::ColsAtCompileTime,
1042 InnerSize = min_size_prefer_fixed(LhsNestedCleaned::ColsAtCompileTime, RhsNestedCleaned::RowsAtCompileTime),
1043 MaxRowsAtCompileTime = LhsNestedCleaned::MaxRowsAtCompileTime,
1044 MaxColsAtCompileTime = RhsNestedCleaned::MaxColsAtCompileTime
1045 };
1046
1047 using LhsVecPacketType = typename find_best_packet<Scalar, RowsAtCompileTime>::type;
1048 using RhsVecPacketType = typename find_best_packet<Scalar, ColsAtCompileTime>::type;
1049
1050 static constexpr int LhsCoeffReadCost = LhsEtorType::CoeffReadCost;
1051 static constexpr int RhsCoeffReadCost = RhsEtorType::CoeffReadCost;
1052 static constexpr int CoeffReadCost =
1053 InnerSize == 0 ? NumTraits<Scalar>::ReadCost
1054 : InnerSize == Dynamic
1055 ? HugeCost
1056 : InnerSize * (NumTraits<Scalar>::MulCost + int(LhsCoeffReadCost) + int(RhsCoeffReadCost)) +
1057 (InnerSize - 1) * NumTraits<Scalar>::AddCost;
1058
1059 static constexpr bool Unroll = CoeffReadCost <= EIGEN_UNROLLING_LIMIT;
1060
1061 static constexpr int LhsFlags = LhsEtorType::Flags;
1062 static constexpr int RhsFlags = RhsEtorType::Flags;
1063
1064 static constexpr int LhsRowMajor = LhsFlags & RowMajorBit;
1065 static constexpr int RhsRowMajor = RhsFlags & RowMajorBit;
1066
1067 static constexpr int LhsVecPacketSize = unpacket_traits<LhsVecPacketType>::size;
1068 static constexpr int RhsVecPacketSize = unpacket_traits<RhsVecPacketType>::size;
1069
1070 // Here, we don't care about alignment larger than the usable packet size.
1071 static constexpr int LhsAlignment =
1072 plain_enum_min(LhsEtorType::Alignment, LhsVecPacketSize* int(sizeof(typename LhsNestedCleaned::Scalar)));
1073 static constexpr int RhsAlignment =
1074 plain_enum_min(RhsEtorType::Alignment, RhsVecPacketSize* int(sizeof(typename RhsNestedCleaned::Scalar)));
1075
1076 static constexpr bool SameType =
1077 std::is_same<typename LhsNestedCleaned::Scalar, typename RhsNestedCleaned::Scalar>::value;
1078
1079 static constexpr bool CanVectorizeRhs = bool(RhsRowMajor) && (RhsFlags & PacketAccessBit) && (ColsAtCompileTime != 1);
1080 static constexpr bool CanVectorizeLhs = (!LhsRowMajor) && (LhsFlags & PacketAccessBit) && (RowsAtCompileTime != 1);
1081
1082 // Only the operand that is contiguous along the packets' axis can vectorize the product.
1083 static constexpr bool EvalToRowMajor = EvalToRowMajor_;
1084 using RealScalar = typename NumTraits<Scalar>::Real;
1085 static constexpr bool LhsIsReal = std::is_same<LhsScalar, RealScalar>::value;
1086 static constexpr bool RhsIsReal = std::is_same<RhsScalar, RealScalar>::value;
1087 static constexpr bool MixedScalarTypes =
1088 std::is_same<Scalar, std::complex<RealScalar>>::value &&
1089 (std::is_same<RealScalar, float>::value || std::is_same<RealScalar, double>::value) &&
1090 ((LhsIsReal && std::is_same<RhsScalar, Scalar>::value) || (RhsIsReal && std::is_same<LhsScalar, Scalar>::value));
1091 static constexpr bool RealPacketLhs = LhsIsReal && product_real_packet_access<LhsNestedCleaned>::value;
1092 static constexpr bool RealPacketRhs = RhsIsReal && product_real_packet_access<RhsNestedCleaned>::value;
1093 static constexpr bool PacketOperandVectorizable = EvalToRowMajor ? CanVectorizeRhs : CanVectorizeLhs;
1094 // Expression and evaluator storage orders must agree for consumers that index product packets.
1095 static constexpr bool MixedRealComplex =
1096 MixedScalarTypes && packet_traits<Scalar>::Vectorizable && PacketOperandVectorizable &&
1097 (EvalToRowMajor ? (!RhsIsReal || RealPacketRhs) : (!LhsIsReal || RealPacketLhs)) &&
1098 (bool(int(traits<XprType>::Flags) & RowMajorBit) == bool(EvalToRowMajor));
1099
1100 static constexpr bool Vectorize = (SameType && PacketOperandVectorizable) || MixedRealComplex;
1101 static constexpr int Flags = ((int(LhsFlags) | int(RhsFlags)) & HereditaryBits & ~RowMajorBit) |
1102 (EvalToRowMajor ? RowMajorBit : 0) | (Vectorize ? PacketAccessBit : 0) |
1103 (XprType::IsVectorAtCompileTime ? LinearAccessBit : 0);
1104 using LhsPacketEtorType =
1105 std::conditional_t<bool(int(Flags) & PacketAccessBit),
1106 std::conditional_t<MixedRealComplex && RealPacketLhs && !EvalToRowMajor,
1107 product_real_packet_evaluator<LhsNestedCleaned>, LhsEtorType>,
1108 product_empty_packet_evaluator>;
1109 using RhsPacketEtorType =
1110 std::conditional_t<bool(int(Flags) & PacketAccessBit),
1111 std::conditional_t<MixedRealComplex && RealPacketRhs && EvalToRowMajor,
1112 product_real_packet_evaluator<RhsNestedCleaned>, RhsEtorType>,
1113 product_empty_packet_evaluator>;
1114
1115 static constexpr bool SingleTermCoeff = InnerSize == 1 && Vectorize;
1116
1117 static constexpr int LhsOuterStrideBytes =
1118 int(LhsNestedCleaned::OuterStrideAtCompileTime) * int(sizeof(typename LhsNestedCleaned::Scalar));
1119 static constexpr int RhsOuterStrideBytes =
1120 int(RhsNestedCleaned::OuterStrideAtCompileTime) * int(sizeof(typename RhsNestedCleaned::Scalar));
1121
1122 // Packets come from the operand that vectorizes the product, at its outer index k.
1123 static constexpr int VecOuterStrideBytes = EvalToRowMajor ? RhsOuterStrideBytes : LhsOuterStrideBytes;
1124 static constexpr int VecAlignment = EvalToRowMajor ? RhsAlignment : LhsAlignment;
1125 static constexpr int Alignment =
1126 (Vectorize && VecOuterStrideBytes > 0 && VecOuterStrideBytes % plain_enum_max(1, VecAlignment) == 0)
1127 ? VecAlignment
1128 : 0;
1129
1130 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index row, Index col) const {
1131 return coeff_impl(row, col, bool_constant<SingleTermCoeff>());
1132 }
1133
1134 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE CoeffReturnType coeff_impl(Index row, Index col,
1135 std::true_type) const {
1136 return fast_mult_op<LhsScalar, RhsScalar>()(m_lhsImpl.coeff(row, Index(0)), m_rhsImpl.coeff(Index(0), col));
1137 }
1138
1139 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE CoeffReturnType coeff_impl(Index row, Index col,
1140 std::false_type) const {
1141 return coeff_reduced(row, col, bool_constant<MixedRealComplex>());
1142 }
1143
1144 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff_reduced(Index row, Index col, std::true_type) const {
1145 using SinglePacket = typename find_best_packet<Scalar, 1>::type;
1146 EIGEN_IF_CONSTEXPR (unpacket_traits<SinglePacket>::size == 1) {
1147 return pfirst(packetSegment<Unaligned, SinglePacket>(row, col, 0, 1));
1148 }
1149 return coeff_reduced(row, col, std::false_type());
1150 }
1151
1152 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE CoeffReturnType coeff_reduced(Index row, Index col,
1153 std::false_type) const {
1154 // fast_mult_op is cwiseProduct's scalar_product_op with a pmul-based scalar path, so the
1155 // reduction (and its precision) is unchanged but complex scalars avoid std::complex::operator*
1156 // (the slow libgcc __mul?c3). See fast_mult_op.
1157 return m_lhs.row(row).transpose().binaryExpr(m_rhs.col(col), fast_mult_op<LhsScalar, RhsScalar>()).sum();
1158 }
1159
1160 /* Allow index-based non-packet access. It is impossible though to allow index-based packed access,
1161 * which is why we don't set the LinearAccessBit.
1162 * TODO: this seems possible when the result is a vector
1163 */
1164 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE const CoeffReturnType coeff(Index index) const {
1165 const Index row = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? 0 : index;
1166 const Index col = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? index : 0;
1167 return coeff(row, col);
1168 }
1169
1170 template <int LoadMode, typename PacketType>
1171 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packet(Index row, Index col) const {
1172 PacketType res;
1173 using PacketImpl = etor_product_packet_impl<bool(int(Flags) & RowMajorBit) ? RowMajor : ColMajor,
1174 Unroll ? int(InnerSize) : Dynamic, LhsPacketEtorType, RhsPacketEtorType,
1175 PacketType, LoadMode>;
1176 PacketImpl::run(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res);
1177 return res;
1178 }
1179
1180 template <int LoadMode, typename PacketType>
1181 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packet(Index index) const {
1182 const Index row = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? 0 : index;
1183 const Index col = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? index : 0;
1184 return packet<LoadMode, PacketType>(row, col);
1185 }
1186
1187 template <int LoadMode, typename PacketType>
1188 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packetSegment(Index row, Index col, Index begin,
1189 Index count) const {
1190 EIGEN_IF_CONSTEXPR (MixedRealComplex) {
1191 if (count == 0) return pset1<PacketType>(Scalar(0));
1192 }
1193 EIGEN_IF_CONSTEXPR (MixedRealComplex && (EvalToRowMajor ? RealPacketRhs : RealPacketLhs) &&
1194 unpacket_traits<PacketType>::size > 1) {
1195 if (count != unpacket_traits<PacketType>::size) {
1196 // The real-packet adapter's ploaddup requires a full source half-packet; a partial load could overread.
1197 Scalar values[unpacket_traits<PacketType>::size] = {};
1198 for (Index i = begin; i < begin + count; ++i) {
1199 values[i] = coeff_reduced(row + (EvalToRowMajor ? 0 : i), col + (EvalToRowMajor ? i : 0), std::false_type());
1200 }
1201 return ploadu<PacketType>(values);
1202 }
1203 }
1204 PacketType res;
1205 using PacketImpl = etor_product_packet_impl<bool(int(Flags) & RowMajorBit) ? RowMajor : ColMajor,
1206 Unroll ? int(InnerSize) : Dynamic, LhsPacketEtorType, RhsPacketEtorType,
1207 PacketType, LoadMode>;
1208 PacketImpl::run_segment(row, col, m_lhsImpl, m_rhsImpl, m_innerDim, res, begin, count);
1209 return res;
1210 }
1211
1212 template <int LoadMode, typename PacketType>
1213 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const PacketType packetSegment(Index index, Index begin, Index count) const {
1214 const Index row = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? 0 : index;
1215 const Index col = (RowsAtCompileTime == 1 || MaxRowsAtCompileTime == 1) ? index : 0;
1216 return packetSegment<LoadMode, PacketType>(row, col, begin, count);
1217 }
1218
1219 protected:
1220 add_const_on_value_type_t<LhsNested> m_lhs;
1221 add_const_on_value_type_t<RhsNested> m_rhs;
1222
1223 LhsPacketEtorType m_lhsImpl;
1224 RhsPacketEtorType m_rhsImpl;
1225
1226 variable_if_dynamic<Index, InnerSize> m_innerDim;
1227};
1228
1229// Evaluate in the storage order the expression reports, which consumers such as Reverse, Redux, Block and FindCoeff
1230// use to index packets and size their loops.
1231template <typename Lhs, typename Rhs, int ProductTag>
1232struct product_evaluator<Product<Lhs, Rhs, LazyProduct>, ProductTag, DenseShape, DenseShape>
1233 : coeff_based_product_evaluator<Lhs, Rhs, (int(traits<Product<Lhs, Rhs, LazyProduct>>::Flags) & RowMajorBit) != 0> {
1234 using XprType = Product<Lhs, Rhs, LazyProduct>;
1235 using Base =
1236 coeff_based_product_evaluator<Lhs, Rhs, (int(traits<Product<Lhs, Rhs, LazyProduct>>::Flags) & RowMajorBit) != 0>;
1237
1238 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit product_evaluator(const XprType& xpr) : Base(xpr) {}
1239 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE product_evaluator(const Lhs& lhs, const Rhs& rhs) : Base(lhs, rhs) {}
1240};
1241
1242// A dense assignment reads its source only through the evaluator, so it may evaluate a coefficient-based product
1243// along the destination's storage order, which the destination's packets need. It does so only where that order
1244// vectorizes; otherwise the assignment is scalar in either order, and the expression's evaluator is kept.
1245template <typename DstEvaluatorType, typename Lhs, typename Rhs>
1246struct dense_assignment_source_evaluator<
1247 DstEvaluatorType, Product<Lhs, Rhs, LazyProduct>,
1248 std::enable_if_t<std::is_same<typename evaluator_traits<Lhs>::Shape, DenseShape>::value &&
1249 std::is_same<typename evaluator_traits<Rhs>::Shape, DenseShape>::value>> {
1250 using DstOrdered = coeff_based_product_evaluator<Lhs, Rhs, (int(DstEvaluatorType::Flags) & RowMajorBit) != 0>;
1251 using type = std::conditional_t<DstOrdered::Vectorize, DstOrdered, evaluator<Product<Lhs, Rhs, LazyProduct>>>;
1252};
1253
1254template <typename Lhs, typename Rhs>
1255struct product_evaluator<Product<Lhs, Rhs, DefaultProduct>, LazyCoeffBasedProductMode, DenseShape, DenseShape>
1256 : product_evaluator<Product<Lhs, Rhs, LazyProduct>, CoeffBasedProductMode, DenseShape, DenseShape> {
1257 using XprType = Product<Lhs, Rhs, DefaultProduct>;
1258 using BaseProduct = Product<Lhs, Rhs, LazyProduct>;
1259 using Base = product_evaluator<BaseProduct, CoeffBasedProductMode, DenseShape, DenseShape>;
1260 static_assert((int(traits<XprType>::Flags) & RowMajorBit) == (int(Base::Flags) & RowMajorBit),
1261 "the evaluator must use the storage order its expression reports");
1262 enum { Flags = Base::Flags | EvalBeforeNestingBit };
1263 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit product_evaluator(const XprType& xpr)
1264 : Base(BaseProduct(xpr.lhs(), xpr.rhs())) {}
1265};
1266
1267/****************************************
1268*** Coeff based product, Packet path ***
1269****************************************/
1270
1271template <int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode>
1272struct etor_product_packet_impl<RowMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode> {
1273 using FactorPacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Lhs&>().coeff(0, 0))>>;
1274 using ValuePacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Rhs&>().coeff(0, 0))>>;
1275 using Op = conj_helper<FactorPacket, ValuePacket, false, false>;
1276 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1277 Index innerDim, Packet& res) {
1278 etor_product_packet_impl<RowMajor, UnrollingIndex - 1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs,
1279 innerDim, res);
1280 res = Op().pmadd(pset1<FactorPacket>(lhs.coeff(row, Index(UnrollingIndex - 1))),
1281 rhs.template packet<LoadMode, Packet>(Index(UnrollingIndex - 1), col), res);
1282 }
1283 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1284 Index innerDim, Packet& res, Index begin, Index count) {
1285 etor_product_packet_impl<RowMajor, UnrollingIndex - 1, Lhs, Rhs, Packet, LoadMode>::run_segment(
1286 row, col, lhs, rhs, innerDim, res, begin, count);
1287 res = Op().pmadd(pset1<FactorPacket>(lhs.coeff(row, Index(UnrollingIndex - 1))),
1288 rhs.template packetSegment<LoadMode, Packet>(Index(UnrollingIndex - 1), col, begin, count), res);
1289 }
1290};
1291
1292template <int UnrollingIndex, typename Lhs, typename Rhs, typename Packet, int LoadMode>
1293struct etor_product_packet_impl<ColMajor, UnrollingIndex, Lhs, Rhs, Packet, LoadMode> {
1294 using FactorPacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Rhs&>().coeff(0, 0))>>;
1295 using ValuePacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Lhs&>().coeff(0, 0))>>;
1296 using Op = conj_helper<ValuePacket, FactorPacket, false, false>;
1297 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1298 Index innerDim, Packet& res) {
1299 etor_product_packet_impl<ColMajor, UnrollingIndex - 1, Lhs, Rhs, Packet, LoadMode>::run(row, col, lhs, rhs,
1300 innerDim, res);
1301 res = Op().pmadd(lhs.template packet<LoadMode, Packet>(row, Index(UnrollingIndex - 1)),
1302 pset1<FactorPacket>(rhs.coeff(Index(UnrollingIndex - 1), col)), res);
1303 }
1304 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1305 Index innerDim, Packet& res, Index begin, Index count) {
1306 etor_product_packet_impl<ColMajor, UnrollingIndex - 1, Lhs, Rhs, Packet, LoadMode>::run_segment(
1307 row, col, lhs, rhs, innerDim, res, begin, count);
1308 res = Op().pmadd(lhs.template packetSegment<LoadMode, Packet>(row, Index(UnrollingIndex - 1), begin, count),
1309 pset1<FactorPacket>(rhs.coeff(Index(UnrollingIndex - 1), col)), res);
1310 }
1311};
1312
1313template <typename Lhs, typename Rhs, typename Packet, int LoadMode>
1314struct etor_product_packet_impl<RowMajor, 1, Lhs, Rhs, Packet, LoadMode> {
1315 using FactorPacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Lhs&>().coeff(0, 0))>>;
1316 using ValuePacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Rhs&>().coeff(0, 0))>>;
1317 using Op = conj_helper<FactorPacket, ValuePacket, false, false>;
1318 static EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1319 Index /*innerDim*/, Packet& res) {
1320 res =
1321 Op().pmul(pset1<FactorPacket>(lhs.coeff(row, Index(0))), rhs.template packet<LoadMode, Packet>(Index(0), col));
1322 }
1323 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1324 Index /*innerDim*/, Packet& res, Index begin,
1325 Index count) {
1326 res = Op().pmul(pset1<FactorPacket>(lhs.coeff(row, Index(0))),
1327 rhs.template packetSegment<LoadMode, Packet>(Index(0), col, begin, count));
1328 }
1329};
1330
1331template <typename Lhs, typename Rhs, typename Packet, int LoadMode>
1332struct etor_product_packet_impl<ColMajor, 1, Lhs, Rhs, Packet, LoadMode> {
1333 using FactorPacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Rhs&>().coeff(0, 0))>>;
1334 using ValuePacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Lhs&>().coeff(0, 0))>>;
1335 using Op = conj_helper<ValuePacket, FactorPacket, false, false>;
1336 static EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1337 Index /*innerDim*/, Packet& res) {
1338 res =
1339 Op().pmul(lhs.template packet<LoadMode, Packet>(row, Index(0)), pset1<FactorPacket>(rhs.coeff(Index(0), col)));
1340 }
1341 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1342 Index /*innerDim*/, Packet& res, Index begin,
1343 Index count) {
1344 res = Op().pmul(lhs.template packetSegment<LoadMode, Packet>(row, Index(0), begin, count),
1345 pset1<FactorPacket>(rhs.coeff(Index(0), col)));
1346 }
1347};
1348
1349template <typename Lhs, typename Rhs, typename Packet, int LoadMode>
1350struct etor_product_packet_impl<RowMajor, 0, Lhs, Rhs, Packet, LoadMode> {
1351 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/,
1352 const Rhs& /*rhs*/, Index /*innerDim*/, Packet& res) {
1353 res = pset1<Packet>(typename unpacket_traits<Packet>::type(0));
1354 }
1355 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/,
1356 const Rhs& /*rhs*/, Index /*innerDim*/, Packet& res,
1357 Index /*begin*/, Index /*count*/) {
1358 res = pset1<Packet>(typename unpacket_traits<Packet>::type(0));
1359 }
1360};
1361
1362template <typename Lhs, typename Rhs, typename Packet, int LoadMode>
1363struct etor_product_packet_impl<ColMajor, 0, Lhs, Rhs, Packet, LoadMode> {
1364 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/,
1365 const Rhs& /*rhs*/, Index /*innerDim*/, Packet& res) {
1366 res = pset1<Packet>(typename unpacket_traits<Packet>::type(0));
1367 }
1368 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index /*row*/, Index /*col*/, const Lhs& /*lhs*/,
1369 const Rhs& /*rhs*/, Index /*innerDim*/, Packet& res,
1370 Index /*begin*/, Index /*count*/) {
1371 res = pset1<Packet>(typename unpacket_traits<Packet>::type(0));
1372 }
1373};
1374
1375template <typename Packet, typename ReadLhs, typename ReadRhs>
1376EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet product_packet_accumulate(Index innerDim, const ReadLhs& lhs,
1377 const ReadRhs& rhs) {
1378 using LhsPacket = decltype(lhs(Index(0)));
1379 using RhsPacket = decltype(rhs(Index(0)));
1380 const conj_helper<LhsPacket, RhsPacket, false, false> op;
1381 Packet res = pset1<Packet>(typename unpacket_traits<Packet>::type(0));
1382 Index i = 0;
1383 // Seeding mixed products from their first term preserves single-term signed zeros.
1384 EIGEN_IF_CONSTEXPR ((!std::is_same<LhsPacket, RhsPacket>::value)) {
1385 if (innerDim > 0) {
1386 res = op.pmul(lhs(0), rhs(0));
1387 i = 1;
1388 }
1389 }
1390 for (; i < innerDim; ++i) res = op.pmadd(lhs(i), rhs(i), res);
1391 return res;
1392}
1393
1394template <typename Lhs, typename Rhs, typename Packet, int LoadMode>
1395struct etor_product_packet_impl<RowMajor, Dynamic, Lhs, Rhs, Packet, LoadMode> {
1396 using FactorPacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Lhs&>().coeff(0, 0))>>;
1397 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1398 Index innerDim, Packet& res) {
1399 res = product_packet_accumulate<Packet>(
1400 innerDim, [&](Index i) { return pset1<FactorPacket>(lhs.coeff(row, i)); },
1401 [&](Index i) { return rhs.template packet<LoadMode, Packet>(i, col); });
1402 }
1403 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1404 Index innerDim, Packet& res, Index begin, Index count) {
1405 res = product_packet_accumulate<Packet>(
1406 innerDim, [&](Index i) { return pset1<FactorPacket>(lhs.coeff(row, i)); },
1407 [&](Index i) { return rhs.template packetSegment<LoadMode, Packet>(i, col, begin, count); });
1408 }
1409};
1410
1411template <typename Lhs, typename Rhs, typename Packet, int LoadMode>
1412struct etor_product_packet_impl<ColMajor, Dynamic, Lhs, Rhs, Packet, LoadMode> {
1413 using FactorPacket = product_operand_packet_t<Packet, remove_all_t<decltype(std::declval<const Rhs&>().coeff(0, 0))>>;
1414 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1415 Index innerDim, Packet& res) {
1416 res = product_packet_accumulate<Packet>(
1417 innerDim, [&](Index i) { return lhs.template packet<LoadMode, Packet>(row, i); },
1418 [&](Index i) { return pset1<FactorPacket>(rhs.coeff(i, col)); });
1419 }
1420 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run_segment(Index row, Index col, const Lhs& lhs, const Rhs& rhs,
1421 Index innerDim, Packet& res, Index begin, Index count) {
1422 res = product_packet_accumulate<Packet>(
1423 innerDim, [&](Index i) { return lhs.template packetSegment<LoadMode, Packet>(row, i, begin, count); },
1424 [&](Index i) { return pset1<FactorPacket>(rhs.coeff(i, col)); });
1425 }
1426};
1427
1428/***************************************************************************
1429 * Triangular products
1430 ***************************************************************************/
1431template <int Mode, bool LhsIsTriangular, typename Lhs, bool LhsIsVector, typename Rhs, bool RhsIsVector>
1432struct triangular_product_impl;
1433
1434template <int ProductOrder>
1435struct diagonal_product_segment_impl;
1436
1437template <>
1438struct diagonal_product_segment_impl<OnTheLeft> {
1439 template <typename DstSegment, typename Coeffs, typename DiagonalType, typename Alpha>
1440 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstSegment& dst, const Coeffs& coeffs,
1441 const DiagonalType& diagonal, Index begin, Index /*col*/,
1442 const Alpha& alpha) {
1443 dst += alpha * diagonal.segment(begin, dst.size()).cwiseProduct(coeffs);
1444 }
1445 template <typename DstSegment, typename Coeffs, typename DiagonalType>
1446 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void runOverwrite(DstSegment& dst, const Coeffs& coeffs,
1447 const DiagonalType& diagonal, Index begin,
1448 Index /*col*/) {
1449 dst = diagonal.segment(begin, dst.size()).cwiseProduct(coeffs);
1450 }
1451};
1452
1453template <>
1454struct diagonal_product_segment_impl<OnTheRight> {
1455 template <typename DstSegment, typename Coeffs, typename DiagonalType, typename Alpha>
1456 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstSegment& dst, const Coeffs& coeffs,
1457 const DiagonalType& diagonal, Index /*begin*/, Index col,
1458 const Alpha& alpha) {
1459 dst += alpha * (coeffs * diagonal.coeff(col));
1460 }
1461 template <typename DstSegment, typename Coeffs, typename DiagonalType>
1462 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void runOverwrite(DstSegment& dst, const Coeffs& coeffs,
1463 const DiagonalType& diagonal, Index /*begin*/,
1464 Index col) {
1465 dst = coeffs * diagonal.coeff(col);
1466 }
1467};
1468
1469template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType>
1470struct triangular_diagonal_product_impl {
1471 using MatrixScalar = typename MatrixType::Scalar;
1472
1473 template <typename Dest, typename Alpha>
1474 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const MatrixType& matrix,
1475 const DiagonalType& diagonal, const Alpha& alpha) {
1476 EIGEN_INTERNAL_ASSERT_SIZE_MATCHES_AXIS_MSG(diagonal, matrix, ProductOrder == OnTheLeft, "invalid matrix product");
1477
1478 const Index rows = matrix.rows();
1479 const Index cols = matrix.cols();
1480 for (Index col = 0; col < cols; ++col) {
1481 EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
1482 const Index end = (std::min)(rows, ((Mode & (UnitDiag | ZeroDiag)) ? col : col + 1));
1483 addStoredSegment(dst, matrix, diagonal, 0, end, col, alpha);
1484 } else {
1485 const Index begin = ((Mode & (UnitDiag | ZeroDiag)) ? col + 1 : col);
1486 addStoredSegment(dst, matrix, diagonal, begin, rows - begin, col, alpha);
1487 }
1488
1489 EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
1490 if (col < rows) addUnitCoeff(dst, diagonal, col, alpha);
1491 }
1492 }
1493 }
1494
1495 private:
1496 template <typename Dest, typename Alpha>
1497 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addStoredSegment(Dest& dst, const MatrixType& matrix,
1498 const DiagonalType& diagonal, Index begin,
1499 Index size, Index col, const Alpha& alpha) {
1500 if (size <= 0) return;
1501 auto dstSegment = dst.col(col).segment(begin, size);
1502 diagonal_product_segment_impl<ProductOrder>::run(dstSegment, matrix.col(col).segment(begin, size), diagonal, begin,
1503 col, alpha);
1504 }
1505
1506 template <typename Dest, typename Alpha>
1507 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addUnitCoeff(Dest& dst, const DiagonalType& diagonal, Index index,
1508 const Alpha& alpha) {
1509 EIGEN_IF_CONSTEXPR (ProductOrder == OnTheLeft) {
1510 dst.coeffRef(index, index) += alpha * (diagonal.coeff(index) * MatrixScalar(1));
1511 } else {
1512 dst.coeffRef(index, index) += alpha * (MatrixScalar(1) * diagonal.coeff(index));
1513 }
1514 }
1515};
1516
1517template <typename Lhs, typename Rhs, int ProductTag>
1518struct generic_product_impl<Lhs, Rhs, TriangularShape, DenseShape, ProductTag>
1519 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, TriangularShape, DenseShape, ProductTag>> {
1520 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1521
1522 template <typename Dest>
1523 static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
1524 triangular_product_impl<Lhs::Mode, true, typename Lhs::MatrixType, false, Rhs, Rhs::ColsAtCompileTime == 1>::run(
1525 dst, lhs.nestedExpression(), rhs, alpha);
1526 }
1527};
1528
1529template <typename Lhs, typename Rhs, int ProductTag>
1530struct generic_product_impl<Lhs, Rhs, DenseShape, TriangularShape, ProductTag>
1531 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DenseShape, TriangularShape, ProductTag>> {
1532 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1533
1534 template <typename Dest>
1535 static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
1536 triangular_product_impl<Rhs::Mode, false, Lhs, Lhs::RowsAtCompileTime == 1, typename Rhs::MatrixType, false>::run(
1537 dst, lhs, rhs.nestedExpression(), alpha);
1538 }
1539};
1540
1541template <typename Lhs, typename Rhs, int ProductTag>
1542struct generic_product_impl<Lhs, Rhs, TriangularShape, DiagonalShape, ProductTag>
1543 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, TriangularShape, DiagonalShape, ProductTag>> {
1544 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1545
1546 template <typename Dest>
1547 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs,
1548 const Scalar& alpha) {
1549 triangular_diagonal_product_impl<Lhs::Mode, OnTheRight, typename Lhs::MatrixType,
1550 typename Rhs::DiagonalVectorType>::run(dst, lhs.nestedExpression(), rhs.diagonal(),
1551 alpha);
1552 }
1553};
1554
1555template <typename Lhs, typename Rhs, int ProductTag>
1556struct generic_product_impl<Lhs, Rhs, DiagonalShape, TriangularShape, ProductTag>
1557 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DiagonalShape, TriangularShape, ProductTag>> {
1558 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1559
1560 template <typename Dest>
1561 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs,
1562 const Scalar& alpha) {
1563 triangular_diagonal_product_impl<Rhs::Mode, OnTheLeft, typename Rhs::MatrixType,
1564 typename Lhs::DiagonalVectorType>::run(dst, rhs.nestedExpression(), lhs.diagonal(),
1565 alpha);
1566 }
1567};
1568
1569/***************************************************************************
1570 * SelfAdjoint products
1571 ***************************************************************************/
1572template <typename Lhs, int LhsMode, bool LhsIsVector, typename Rhs, int RhsMode, bool RhsIsVector>
1573struct selfadjoint_product_impl;
1574
1575template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType>
1576struct selfadjoint_diagonal_product_impl {
1577 // Accumulating: dst += alpha * (matrix.selfadjointView<Mode>() * diagonal[asDiagonal])
1578 // (with the diagonal on the right or left as ProductOrder dictates).
1579 template <typename Dest, typename Alpha>
1580 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const MatrixType& matrix,
1581 const DiagonalType& diagonal, const Alpha& alpha) {
1582 runImpl<true>(dst, matrix, diagonal, alpha);
1583 }
1584
1585 // Overwriting: dst = matrix.selfadjointView<Mode>() * diagonal[asDiagonal].
1586 // Each output entry is written exactly once, so the caller can skip the
1587 // dst.setZero() pass that generic_product_impl_base::evalTo would do.
1588 template <typename Dest>
1589 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void runOverwrite(Dest& dst, const MatrixType& matrix,
1590 const DiagonalType& diagonal) {
1591 using Scalar = typename traits<MatrixType>::Scalar;
1592 runImpl<false>(dst, matrix, diagonal, Scalar(1));
1593 }
1594
1595 private:
1596 // Tile size for the blocked mirror pass. Tuned so that one BlockSize x
1597 // BlockSize source tile fits comfortably in L1 across common scalar / SIMD
1598 // combinations: 32x32 of double = 8 KB, of complex<double> = 16 KB, both
1599 // well under typical 32 KB L1. Smaller tiles leave SIMD work on the table
1600 // for AVX/AVX-512; larger tiles spill out of L1 on machines with smaller
1601 // caches.
1602 static constexpr Index BlockSize = 32;
1603
1604 // The mirror half writes the strict-other-triangle of dst. The naive
1605 // per-column form reads matrix.row(col).segment(...) which has stride =
1606 // leading dimension, so on an N x N source it streams cold cache lines.
1607 // We walk the mirror in BlockSize x BlockSize tiles instead: off-diagonal
1608 // tiles use a blocked conjugate-transpose, and the small diagonal tile
1609 // falls back to the per-column row-strided loop where the working set is
1610 // L1-hot.
1611 template <bool Accumulate, typename Dest, typename Alpha>
1612 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void runImpl(Dest& dst, const MatrixType& matrix,
1613 const DiagonalType& diagonal, const Alpha& alpha) {
1614 EIGEN_INTERNAL_ASSERT_SQUARE_MSG(matrix, "SelfAdjointView is only for squared matrices");
1615 EIGEN_INTERNAL_ASSERT_SIZE_MATCHES_ROWS_MSG(diagonal, matrix, "invalid matrix product");
1616
1617 const Index size = matrix.rows();
1618
1619 // Stored half: one column-strided segment per output column.
1620 for (Index col = 0; col < size; ++col) {
1621 EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
1622 storedSegment<Accumulate>(dst, matrix, diagonal, 0, col + 1, col, alpha);
1623 } else {
1624 storedSegment<Accumulate>(dst, matrix, diagonal, col, size - col, col, alpha);
1625 }
1626 }
1627
1628 // Mirror half.
1629 for (Index ib = 0; ib < size; ib += BlockSize) {
1630 const Index ib_end = numext::mini(size, ib + BlockSize);
1631 const Index br = ib_end - ib;
1632 EIGEN_IF_CONSTEXPR ((Mode & Upper) == Upper) {
1633 // Off-diagonal: write strict-lower of dst from strict-upper of source.
1634 for (Index jb = 0; jb < ib; jb += BlockSize) {
1635 const Index bc = numext::mini(jb + BlockSize, ib) - jb;
1636 mirrorBlock<Accumulate>(dst, matrix, diagonal, ib, jb, br, bc, alpha);
1637 }
1638 // Diagonal tile: in-tile strict-lower mirror.
1639 for (Index col = ib; col < ib_end; ++col)
1640 conjugateSegment<Accumulate>(dst, matrix, diagonal, col + 1, ib_end - col - 1, col, alpha);
1641 } else {
1642 // Off-diagonal: write strict-upper of dst from strict-lower of source.
1643 for (Index jb = ib_end; jb < size; jb += BlockSize) {
1644 const Index bc = numext::mini(size, jb + BlockSize) - jb;
1645 mirrorBlock<Accumulate>(dst, matrix, diagonal, ib, jb, br, bc, alpha);
1646 }
1647 // Diagonal tile: in-tile strict-upper mirror.
1648 for (Index col = ib; col < ib_end; ++col)
1649 conjugateSegment<Accumulate>(dst, matrix, diagonal, ib, col - ib, col, alpha);
1650 }
1651 }
1652 }
1653
1654 template <bool Accumulate, typename Dest, typename Alpha>
1655 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void storedSegment(Dest& dst, const MatrixType& matrix,
1656 const DiagonalType& diagonal, Index begin, Index size,
1657 Index col, const Alpha& alpha) {
1658 if (size <= 0) return;
1659 auto dstSegment = dst.col(col).segment(begin, size);
1660 auto srcSegment = matrix.col(col).segment(begin, size);
1661 EIGEN_IF_CONSTEXPR (Accumulate) {
1662 diagonal_product_segment_impl<ProductOrder>::run(dstSegment, srcSegment, diagonal, begin, col, alpha);
1663 } else {
1664 diagonal_product_segment_impl<ProductOrder>::runOverwrite(dstSegment, srcSegment, diagonal, begin, col);
1665 }
1666 }
1667
1668 template <bool Accumulate, typename Dest, typename Alpha>
1669 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void conjugateSegment(Dest& dst, const MatrixType& matrix,
1670 const DiagonalType& diagonal, Index begin,
1671 Index size, Index col, const Alpha& alpha) {
1672 if (size <= 0) return;
1673 auto dstSegment = dst.col(col).segment(begin, size);
1674 auto srcSegment = matrix.row(col).segment(begin, size).conjugate().transpose();
1675 EIGEN_IF_CONSTEXPR (Accumulate) {
1676 diagonal_product_segment_impl<ProductOrder>::run(dstSegment, srcSegment, diagonal, begin, col, alpha);
1677 } else {
1678 diagonal_product_segment_impl<ProductOrder>::runOverwrite(dstSegment, srcSegment, diagonal, begin, col);
1679 }
1680 }
1681
1682 // dst.block(ib, jb, br, bc) [+= alpha *] matrix.block(jb, ib, bc, br).adjoint() * <diag>,
1683 // where <diag> scales each output column (OnTheRight) or row (OnTheLeft).
1684 // Loop bounds in runImpl guarantee br > 0 && bc > 0.
1685 template <bool Accumulate, typename Dest, typename Alpha>
1686 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void mirrorBlock(Dest& dst, const MatrixType& matrix,
1687 const DiagonalType& diagonal, Index ib, Index jb,
1688 Index br, Index bc, const Alpha& alpha) {
1689 auto dstBlock = dst.block(ib, jb, br, bc);
1690 auto srcAdjoint = matrix.block(jb, ib, bc, br).adjoint();
1691 EIGEN_IF_CONSTEXPR (ProductOrder == OnTheRight) {
1692 auto scaled = srcAdjoint * diagonal.segment(jb, bc).asDiagonal();
1693 EIGEN_IF_CONSTEXPR (Accumulate) {
1694 dstBlock += alpha * scaled;
1695 } else {
1696 dstBlock = scaled;
1697 }
1698 } else {
1699 auto scaled = diagonal.segment(ib, br).asDiagonal() * srcAdjoint;
1700 EIGEN_IF_CONSTEXPR (Accumulate) {
1701 dstBlock += alpha * scaled;
1702 } else {
1703 dstBlock = scaled;
1704 }
1705 }
1706 }
1707};
1708
1709template <typename Lhs, typename Rhs, int ProductTag>
1710struct generic_product_impl<Lhs, Rhs, SelfAdjointShape, DenseShape, ProductTag>
1711 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, SelfAdjointShape, DenseShape, ProductTag>> {
1712 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1713
1714 template <typename Dest>
1715 static EIGEN_DEVICE_FUNC void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
1716 selfadjoint_product_impl<typename Lhs::MatrixType, Lhs::Mode, false, Rhs, 0, Rhs::ColsAtCompileTime == 1>::run(
1717 dst, lhs.nestedExpression(), rhs, alpha);
1718 }
1719};
1720
1721template <typename Lhs, typename Rhs, int ProductTag>
1722struct generic_product_impl<Lhs, Rhs, DenseShape, SelfAdjointShape, ProductTag>
1723 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DenseShape, SelfAdjointShape, ProductTag>> {
1724 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1725
1726 template <typename Dest>
1727 static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
1728 selfadjoint_product_impl<Lhs, 0, Lhs::RowsAtCompileTime == 1, typename Rhs::MatrixType, Rhs::Mode, false>::run(
1729 dst, lhs, rhs.nestedExpression(), alpha);
1730 }
1731};
1732
1733template <typename Lhs, typename Rhs, int ProductTag>
1734struct generic_product_impl<Lhs, Rhs, SelfAdjointShape, DiagonalShape, ProductTag>
1735 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, SelfAdjointShape, DiagonalShape, ProductTag>> {
1736 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1737 // The "Dense ?= scalar * Product" rewriting rule folds an outer alpha into the
1738 // SelfAdjointView via SelfAdjointView::operator*(scalar), whose nested
1739 // expression becomes (matrix * alpha). For complex alpha this is no longer
1740 // Hermitian — the mirror half of our kernel would produce conj(alpha) on the
1741 // off-triangle. Strip the scalar factor with blas_traits and re-fold it into
1742 // the kernel's alpha so the same scalar multiplies every output entry.
1743 using LhsBlasTraits = blas_traits<typename Lhs::MatrixType>;
1744 // A named constant, not bool(...): nvcc's front end re-emits that cast as a function type MSVC rejects.
1745 static constexpr bool ConjLhs = LhsBlasTraits::NeedToConjugate;
1746 // conjugateIf<false> returns a reference to its argument, which dangles when extract returns by value.
1747 using ActualLhsMatrixXpr =
1748 decltype(LhsBlasTraits::extract(std::declval<const typename Lhs::MatrixType&>()).template conjugateIf<ConjLhs>());
1749 using ActualLhsMatrix = std::conditional_t<std::is_reference<typename LhsBlasTraits::ExtractType>::value,
1750 ActualLhsMatrixXpr, remove_all_t<ActualLhsMatrixXpr>>;
1751 using ActualLhsMatrixType = remove_all_t<ActualLhsMatrix>;
1752 using Kernel =
1753 selfadjoint_diagonal_product_impl<Lhs::Mode, OnTheRight, ActualLhsMatrixType, typename Rhs::DiagonalVectorType>;
1754
1755 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ActualLhsMatrix actualLhsMatrix(const typename Lhs::MatrixType& matrix) {
1756 return LhsBlasTraits::extract(matrix).template conjugateIf<ConjLhs>();
1757 }
1758
1759 template <typename Dest>
1760 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
1761 EIGEN_IF_CONSTEXPR (LhsBlasTraits::HasScalarFactor) {
1762 // Folded scalar factor present: zero dst then accumulate at the extracted alpha.
1763 Scalar factor = LhsBlasTraits::extractScalarFactor(lhs.nestedExpression());
1764 dst.setZero();
1765 Kernel::run(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal(), factor);
1766 } else {
1767 // No scalar factor: kernel writes every entry exactly once, skip setZero.
1768 Kernel::runOverwrite(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal());
1769 }
1770 }
1771
1772 template <typename Dest>
1773 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs,
1774 const Scalar& alpha) {
1775 Scalar combinedAlpha = alpha * LhsBlasTraits::extractScalarFactor(lhs.nestedExpression());
1776 Kernel::run(dst, actualLhsMatrix(lhs.nestedExpression()), rhs.diagonal(), combinedAlpha);
1777 }
1778};
1779
1780template <typename Lhs, typename Rhs, int ProductTag>
1781struct generic_product_impl<Lhs, Rhs, DiagonalShape, SelfAdjointShape, ProductTag>
1782 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DiagonalShape, SelfAdjointShape, ProductTag>> {
1783 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1784 // See note on the SelfAdjointShape, DiagonalShape specialization above for why
1785 // we extract the scalar factor with blas_traits.
1786 using RhsBlasTraits = blas_traits<typename Rhs::MatrixType>;
1787 static constexpr bool ConjRhs = RhsBlasTraits::NeedToConjugate;
1788 // conjugateIf<false> returns a reference to its argument, which dangles when extract returns by value.
1789 using ActualRhsMatrixXpr =
1790 decltype(RhsBlasTraits::extract(std::declval<const typename Rhs::MatrixType&>()).template conjugateIf<ConjRhs>());
1791 using ActualRhsMatrix = std::conditional_t<std::is_reference<typename RhsBlasTraits::ExtractType>::value,
1792 ActualRhsMatrixXpr, remove_all_t<ActualRhsMatrixXpr>>;
1793 using ActualRhsMatrixType = remove_all_t<ActualRhsMatrix>;
1794 using Kernel =
1795 selfadjoint_diagonal_product_impl<Rhs::Mode, OnTheLeft, ActualRhsMatrixType, typename Lhs::DiagonalVectorType>;
1796
1797 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ActualRhsMatrix actualRhsMatrix(const typename Rhs::MatrixType& matrix) {
1798 return RhsBlasTraits::extract(matrix).template conjugateIf<ConjRhs>();
1799 }
1800
1801 template <typename Dest>
1802 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
1803 EIGEN_IF_CONSTEXPR (RhsBlasTraits::HasScalarFactor) {
1804 Scalar factor = RhsBlasTraits::extractScalarFactor(rhs.nestedExpression());
1805 dst.setZero();
1806 Kernel::run(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal(), factor);
1807 } else {
1808 Kernel::runOverwrite(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal());
1809 }
1810 }
1811
1812 template <typename Dest>
1813 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs,
1814 const Scalar& alpha) {
1815 Scalar combinedAlpha = alpha * RhsBlasTraits::extractScalarFactor(rhs.nestedExpression());
1816 Kernel::run(dst, actualRhsMatrix(rhs.nestedExpression()), lhs.diagonal(), combinedAlpha);
1817 }
1818};
1819
1820/***************************************************************************
1821 * Products of two triangular or self-adjoint views
1822 ***************************************************************************/
1823
1830template <typename View, typename Shape = typename evaluator_traits<View>::Shape>
1831struct densified_view_operand {
1832 using Scalar = typename View::Scalar;
1833 template <typename Dense>
1834 static Scalar run(Dense& dense, const View& view) {
1835 dense = view;
1836 return Scalar(1);
1837 }
1838};
1839
1840template <typename View>
1841struct densified_view_operand<View, SelfAdjointShape> {
1842 using Scalar = typename View::Scalar;
1843 using BlasTraits = blas_traits<typename View::MatrixType>;
1844 static constexpr unsigned int UpLo = unsigned(int(View::Mode) & int(Upper | Lower));
1845 template <typename Dense>
1846 static Scalar run(Dense& dense, const View& view) {
1847 dense = BlasTraits::extract(view.nestedExpression())
1848 .template conjugateIf<bool(BlasTraits::NeedToConjugate)>()
1849 .template selfadjointView<UpLo>();
1850 return BlasTraits::extractScalarFactor(view.nestedExpression());
1851 }
1852};
1853
1861template <typename Lhs, typename Rhs, typename LhsShape, typename RhsShape, int ProductTag, bool DensifyRhs>
1862struct structured_view_product_impl;
1863
1864template <typename Lhs, typename Rhs, typename LhsShape, typename RhsShape, int ProductTag>
1865struct structured_view_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag, true>
1866 : generic_product_impl_base<Lhs, Rhs,
1867 structured_view_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag, true>> {
1868 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1869 using RhsDenseType = typename Rhs::DenseMatrixType;
1870
1871 template <typename Dest>
1872 static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
1873 RhsDenseType rhsDense(rhs.rows(), rhs.cols());
1874 const Scalar factor = densified_view_operand<Rhs>::run(rhsDense, rhs);
1875 generic_product_impl<Lhs, RhsDenseType, LhsShape, DenseShape, ProductTag>::scaleAndAddTo(dst, lhs, rhsDense,
1876 alpha * factor);
1877 }
1878};
1879
1880template <typename Lhs, typename Rhs, typename LhsShape, typename RhsShape, int ProductTag>
1881struct structured_view_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag, false>
1882 : generic_product_impl_base<Lhs, Rhs,
1883 structured_view_product_impl<Lhs, Rhs, LhsShape, RhsShape, ProductTag, false>> {
1884 using Scalar = typename Product<Lhs, Rhs>::Scalar;
1885 using LhsDenseType = typename Lhs::DenseMatrixType;
1886
1887 template <typename Dest>
1888 static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
1889 LhsDenseType lhsDense(lhs.rows(), lhs.cols());
1890 const Scalar factor = densified_view_operand<Lhs>::run(lhsDense, lhs);
1891 generic_product_impl<LhsDenseType, Rhs, DenseShape, RhsShape, ProductTag>::scaleAndAddTo(dst, lhsDense, rhs,
1892 alpha * factor);
1893 }
1894};
1895
1896template <typename Lhs, typename Rhs, int ProductTag>
1897struct generic_product_impl<Lhs, Rhs, TriangularShape, TriangularShape, ProductTag>
1898 : structured_view_product_impl<Lhs, Rhs, TriangularShape, TriangularShape, ProductTag, true> {};
1899
1900template <typename Lhs, typename Rhs, int ProductTag>
1901struct generic_product_impl<Lhs, Rhs, TriangularShape, SelfAdjointShape, ProductTag>
1902 : structured_view_product_impl<Lhs, Rhs, TriangularShape, SelfAdjointShape, ProductTag, true> {};
1903
1904template <typename Lhs, typename Rhs, int ProductTag>
1905struct generic_product_impl<Lhs, Rhs, SelfAdjointShape, TriangularShape, ProductTag>
1906 : structured_view_product_impl<Lhs, Rhs, SelfAdjointShape, TriangularShape, ProductTag, false> {};
1907
1908template <typename Lhs, typename Rhs, int ProductTag>
1909struct generic_product_impl<Lhs, Rhs, SelfAdjointShape, SelfAdjointShape, ProductTag>
1910 : structured_view_product_impl<Lhs, Rhs, SelfAdjointShape, SelfAdjointShape, ProductTag, true> {};
1911
1912/***************************************************************************
1913 * Diagonal products
1914 ***************************************************************************/
1915
1916template <typename MatrixType, typename DiagonalType, typename Derived, int ProductOrder>
1917struct diagonal_product_evaluator_base : evaluator_base<Derived> {
1918 using Scalar = typename Derived::Scalar;
1919
1920 public:
1921 enum {
1922 CoeffReadCost = int(NumTraits<Scalar>::MulCost) + int(evaluator<MatrixType>::CoeffReadCost) +
1923 int(evaluator<DiagonalType>::CoeffReadCost),
1924
1925 MatrixFlags = evaluator<MatrixType>::Flags,
1926 DiagFlags = evaluator<DiagonalType>::Flags,
1927
1928 StorageOrder_ = (Derived::MaxRowsAtCompileTime == 1 && Derived::MaxColsAtCompileTime != 1) ? RowMajor
1929 : (Derived::MaxColsAtCompileTime == 1 && Derived::MaxRowsAtCompileTime != 1) ? ColMajor
1930 : MatrixFlags & RowMajorBit ? RowMajor
1931 : ColMajor,
1932 SameStorageOrder_ = int(StorageOrder_) == ((MatrixFlags & RowMajorBit) ? RowMajor : ColMajor),
1933
1934 ScalarAccessOnDiag_ = !((int(StorageOrder_) == ColMajor && int(ProductOrder) == OnTheLeft) ||
1935 (int(StorageOrder_) == RowMajor && int(ProductOrder) == OnTheRight)),
1936 SameTypes_ = std::is_same<typename MatrixType::Scalar, typename DiagonalType::Scalar>::value,
1937 // FIXME currently we need same types, but in the future the next rule should be the one
1938 // Vectorizable_ = bool(int(MatrixFlags)&PacketAccessBit) && ((!_PacketOnDiag) || (SameTypes_ &&
1939 // bool(int(DiagFlags)&PacketAccessBit))),
1940 Vectorizable_ = bool(int(MatrixFlags) & PacketAccessBit) && SameTypes_ &&
1941 (SameStorageOrder_ || (MatrixFlags & LinearAccessBit) == LinearAccessBit) &&
1942 (ScalarAccessOnDiag_ || (bool(int(DiagFlags) & PacketAccessBit))),
1943 LinearAccessMask_ =
1944 (MatrixType::RowsAtCompileTime == 1 || MatrixType::ColsAtCompileTime == 1) ? LinearAccessBit : 0,
1945 Flags = ((HereditaryBits | LinearAccessMask_) & static_cast<unsigned int>(MatrixFlags)) |
1946 (Vectorizable_ ? PacketAccessBit : 0),
1947 Alignment = evaluator<MatrixType>::Alignment,
1948
1949 AsScalarProduct =
1950 (DiagonalType::SizeAtCompileTime == 1) ||
1951 (DiagonalType::SizeAtCompileTime == Dynamic && MatrixType::RowsAtCompileTime == 1 &&
1952 ProductOrder == OnTheLeft) ||
1953 (DiagonalType::SizeAtCompileTime == Dynamic && MatrixType::ColsAtCompileTime == 1 && ProductOrder == OnTheRight)
1954 };
1955
1956 EIGEN_DEVICE_FUNC diagonal_product_evaluator_base(const MatrixType& mat, const DiagonalType& diag)
1957 : m_diagImpl(diag), m_matImpl(mat) {
1958 EIGEN_INTERNAL_CHECK_COST_VALUE(NumTraits<Scalar>::MulCost);
1959 EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
1960 }
1961
1962 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE const Scalar coeff(Index idx) const {
1963 using DiagonalFirst = bool_constant<
1964 ProductOrder == OnTheLeft ||
1965 functor_is_commutative<scalar_product_op<typename MatrixType::Scalar, typename DiagonalType::Scalar>>::value>;
1966 return coeff_impl(idx, DiagonalFirst());
1967 }
1968
1969 protected:
1970 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Scalar coeff_impl(Index idx, std::true_type) const {
1971 EIGEN_IF_CONSTEXPR (AsScalarProduct)
1972 return m_diagImpl.coeff(0) * m_matImpl.coeff(idx);
1973 else
1974 return m_diagImpl.coeff(idx) * m_matImpl.coeff(idx);
1975 }
1976
1977 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE Scalar coeff_impl(Index idx, std::false_type) const {
1978 EIGEN_IF_CONSTEXPR (AsScalarProduct)
1979 return m_matImpl.coeff(idx) * m_diagImpl.coeff(0);
1980 else
1981 return m_matImpl.coeff(idx) * m_diagImpl.coeff(idx);
1982 }
1983
1984 template <int LoadMode, typename PacketType>
1985 EIGEN_STRONG_INLINE PacketType packet_impl(Index row, Index col, Index id, std::true_type) const {
1986 return internal::pmul(m_matImpl.template packet<LoadMode, PacketType>(row, col),
1987 internal::pset1<PacketType>(m_diagImpl.coeff(id)));
1988 }
1989
1990 template <int LoadMode, typename PacketType>
1991 EIGEN_STRONG_INLINE PacketType packet_impl(Index row, Index col, Index id, std::false_type) const {
1992 enum { DiagonalPacketLoadMode = plain_enum_min(LoadMode, int(evaluator<DiagonalType>::Alignment)) };
1993 return internal::pmul(m_matImpl.template packet<LoadMode, PacketType>(row, col),
1994 m_diagImpl.template packet<DiagonalPacketLoadMode, PacketType>(id));
1995 }
1996
1997 template <int LoadMode, typename PacketType>
1998 EIGEN_STRONG_INLINE PacketType packet_segment_impl(Index row, Index col, Index id, Index begin, Index count,
1999 std::true_type) const {
2000 return internal::pmul(m_matImpl.template packetSegment<LoadMode, PacketType>(row, col, begin, count),
2001 internal::pset1<PacketType>(m_diagImpl.coeff(id)));
2002 }
2003
2004 template <int LoadMode, typename PacketType>
2005 EIGEN_STRONG_INLINE PacketType packet_segment_impl(Index row, Index col, Index id, Index begin, Index count,
2006 std::false_type) const {
2007 enum { DiagonalPacketLoadMode = plain_enum_min(LoadMode, int(evaluator<DiagonalType>::Alignment)) };
2008 return internal::pmul(m_matImpl.template packetSegment<LoadMode, PacketType>(row, col, begin, count),
2009 m_diagImpl.template packetSegment<DiagonalPacketLoadMode, PacketType>(id, begin, count));
2010 }
2011
2012 evaluator<DiagonalType> m_diagImpl;
2013 evaluator<MatrixType> m_matImpl;
2014};
2015
2016// diagonal * dense
2017template <typename Lhs, typename Rhs, int ProductKind, int ProductTag>
2018struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, DenseShape>
2019 : diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
2020 OnTheLeft> {
2021 using Base =
2022 diagonal_product_evaluator_base<Rhs, typename Lhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>, OnTheLeft>;
2023 using Base::coeff;
2024 using Base::m_diagImpl;
2025 using Base::m_matImpl;
2026 using Scalar = typename Base::Scalar;
2027
2028 using XprType = Product<Lhs, Rhs, ProductKind>;
2029 using PlainObject = typename XprType::PlainObject;
2030
2031 static constexpr int StorageOrder = Base::StorageOrder_;
2032 using IsRowMajor_t = bool_constant<StorageOrder == RowMajor>;
2033
2034 EIGEN_DEVICE_FUNC constexpr explicit product_evaluator(const XprType& xpr) : Base(xpr.rhs(), xpr.lhs().diagonal()) {}
2035
2036 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE const Scalar coeff(Index row, Index col) const {
2037 return m_diagImpl.coeff(row) * m_matImpl.coeff(row, col);
2038 }
2039
2040#ifndef EIGEN_GPUCC
2041 template <int LoadMode, typename PacketType>
2042 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
2043 // FIXME: NVCC used to complain about the template keyword, but we have to check whether this is still the case.
2044 // See also similar calls below.
2045 return this->template packet_impl<LoadMode, PacketType>(row, col, row, IsRowMajor_t());
2046 }
2047
2048 template <int LoadMode, typename PacketType>
2049 EIGEN_STRONG_INLINE PacketType packet(Index idx) const {
2050 return packet<LoadMode, PacketType>(int(StorageOrder) == ColMajor ? idx : 0,
2051 int(StorageOrder) == ColMajor ? 0 : idx);
2052 }
2053
2054 template <int LoadMode, typename PacketType>
2055 EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const {
2056 // FIXME: NVCC used to complain about the template keyword, but we have to check whether this is still the case.
2057 // See also similar calls below.
2058 return this->template packet_segment_impl<LoadMode, PacketType>(row, col, row, begin, count, IsRowMajor_t());
2059 }
2060
2061 template <int LoadMode, typename PacketType>
2062 EIGEN_STRONG_INLINE PacketType packetSegment(Index idx, Index begin, Index count) const {
2063 return packetSegment<LoadMode, PacketType>(StorageOrder == ColMajor ? idx : 0, StorageOrder == ColMajor ? 0 : idx,
2064 begin, count);
2065 }
2066#endif
2067};
2068
2069// dense * diagonal
2070template <typename Lhs, typename Rhs, int ProductKind, int ProductTag>
2071struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DenseShape, DiagonalShape>
2072 : diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
2073 OnTheRight> {
2074 using Base = diagonal_product_evaluator_base<Lhs, typename Rhs::DiagonalVectorType, Product<Lhs, Rhs, LazyProduct>,
2075 OnTheRight>;
2076 using Base::coeff;
2077 using Base::m_diagImpl;
2078 using Base::m_matImpl;
2079 using Scalar = typename Base::Scalar;
2080
2081 using XprType = Product<Lhs, Rhs, ProductKind>;
2082 using PlainObject = typename XprType::PlainObject;
2083
2084 static constexpr int StorageOrder = Base::StorageOrder_;
2085 using IsColMajor_t = bool_constant<StorageOrder == ColMajor>;
2086
2087 EIGEN_DEVICE_FUNC constexpr explicit product_evaluator(const XprType& xpr) : Base(xpr.lhs(), xpr.rhs().diagonal()) {}
2088
2089 EIGEN_DEVICE_FUNC constexpr EIGEN_STRONG_INLINE const Scalar coeff(Index row, Index col) const {
2090 return m_matImpl.coeff(row, col) * m_diagImpl.coeff(col);
2091 }
2092
2093#ifndef EIGEN_GPUCC
2094 template <int LoadMode, typename PacketType>
2095 EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const {
2096 return this->template packet_impl<LoadMode, PacketType>(row, col, col, IsColMajor_t());
2097 }
2098
2099 template <int LoadMode, typename PacketType>
2100 EIGEN_STRONG_INLINE PacketType packet(Index idx) const {
2101 return packet<LoadMode, PacketType>(StorageOrder == ColMajor ? idx : 0, StorageOrder == ColMajor ? 0 : idx);
2102 }
2103
2104 template <int LoadMode, typename PacketType>
2105 EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const {
2106 return this->template packet_segment_impl<LoadMode, PacketType>(row, col, col, begin, count, IsColMajor_t());
2107 }
2108
2109 template <int LoadMode, typename PacketType>
2110 EIGEN_STRONG_INLINE PacketType packetSegment(Index idx, Index begin, Index count) const {
2111 return packetSegment<LoadMode, PacketType>(StorageOrder == ColMajor ? idx : 0, StorageOrder == ColMajor ? 0 : idx,
2112 begin, count);
2113 }
2114#endif
2115};
2116
2117template <typename MatrixScalar, typename DiagonalScalar>
2118EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto diagonal_product_coeff(const MatrixScalar& matrix,
2119 const DiagonalScalar& diagonal, true_type)
2120 -> decltype(diagonal * matrix) {
2121 return diagonal * matrix;
2122}
2123
2124template <typename MatrixScalar, typename DiagonalScalar>
2125EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto diagonal_product_coeff(const MatrixScalar& matrix,
2126 const DiagonalScalar& diagonal, false_type)
2127 -> decltype(matrix * diagonal) {
2128 return matrix * diagonal;
2129}
2130
2131// Lazy evaluators for triangular/selfadjoint × diagonal products.
2132//
2133// The triangular-assignment dispatcher in TriangularMatrix.h routes
2134// `triangularView() ?= structured * diagonal` through call_triangular_assignment_loop,
2135// which constructs an evaluator over the Product expression. Without these
2136// specializations, evaluator<Product<TriangularView, Diagonal>> falls back to the
2137// default product_evaluator that materializes a full PlainObject — defeating the
2138// triangular-aware dispatcher. The kernel reads only the destination's active
2139// triangle one coefficient at a time, so a coeff()-only evaluator avoids the
2140// temporary at no loss in functionality.
2141
2142template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType, typename ProductXpr>
2143struct triangular_diagonal_product_lazy_evaluator_base : evaluator_base<ProductXpr> {
2144 using Scalar = typename ProductXpr::Scalar;
2145 using DiagonalOnLeft = bool_constant<ProductOrder == OnTheLeft>;
2146 using MatrixScalar = typename MatrixType::Scalar;
2147
2148 enum {
2149 CoeffReadCost = int(NumTraits<Scalar>::MulCost) + int(evaluator<MatrixType>::CoeffReadCost) +
2150 int(evaluator<DiagonalType>::CoeffReadCost),
2151 Flags = HereditaryBits & static_cast<unsigned int>(evaluator<MatrixType>::Flags),
2152 Alignment = 0
2153 };
2154
2155 EIGEN_DEVICE_FUNC triangular_diagonal_product_lazy_evaluator_base(const MatrixType& mat, const DiagonalType& diag)
2156 : m_diagImpl(diag), m_matImpl(mat) {}
2157
2158 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const {
2159 const bool inActive = ((Mode & Upper) == Upper) ? (row <= col) : (row >= col);
2160 if (!inActive) return Scalar(0);
2161 if (row == col) {
2162 EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
2163 return Scalar(diagonal_product_coeff(MatrixScalar(1), m_diagImpl.coeff(row), DiagonalOnLeft()));
2164 }
2165 EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag) return Scalar(0);
2166 }
2167 return Scalar(diagonal_product_coeff(m_matImpl.coeff(row, col),
2168 m_diagImpl.coeff(ProductOrder == OnTheLeft ? row : col), DiagonalOnLeft()));
2169 }
2170
2171 protected:
2172 evaluator<DiagonalType> m_diagImpl;
2173 evaluator<MatrixType> m_matImpl;
2174};
2175
2176// Triangular × Diagonal
2177template <typename Lhs, typename Rhs, int ProductKind, int ProductTag>
2178struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, TriangularShape, DiagonalShape>
2179 : triangular_diagonal_product_lazy_evaluator_base<Lhs::Mode, OnTheRight, typename Lhs::MatrixType,
2180 typename Rhs::DiagonalVectorType,
2181 Product<Lhs, Rhs, ProductKind>> {
2182 using XprType = Product<Lhs, Rhs, ProductKind>;
2183 using Base = triangular_diagonal_product_lazy_evaluator_base<Lhs::Mode, OnTheRight, typename Lhs::MatrixType,
2184 typename Rhs::DiagonalVectorType, XprType>;
2185
2186 EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr)
2187 : Base(xpr.lhs().nestedExpression(), xpr.rhs().diagonal()) {}
2188};
2189
2190// Diagonal × Triangular
2191template <typename Lhs, typename Rhs, int ProductKind, int ProductTag>
2192struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, TriangularShape>
2193 : triangular_diagonal_product_lazy_evaluator_base<Rhs::Mode, OnTheLeft, typename Rhs::MatrixType,
2194 typename Lhs::DiagonalVectorType,
2195 Product<Lhs, Rhs, ProductKind>> {
2196 using XprType = Product<Lhs, Rhs, ProductKind>;
2197 using Base = triangular_diagonal_product_lazy_evaluator_base<Rhs::Mode, OnTheLeft, typename Rhs::MatrixType,
2198 typename Lhs::DiagonalVectorType, XprType>;
2199
2200 EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr)
2201 : Base(xpr.rhs().nestedExpression(), xpr.lhs().diagonal()) {}
2202};
2203
2204// Dense SelfAdjointView statically rejects the Upper|Lower mode (only one half is stored), so the
2205// off-stored coefficient is always reconstructed by conjugating its mirror.
2206template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType, typename ProductXpr,
2207 bool Materialize =
2208 NumTraits<typename MatrixType::Scalar>::IsComplex && blas_traits<MatrixType>::HasScalarFactor>
2209struct selfadjoint_diagonal_product_lazy_evaluator_base : evaluator_base<ProductXpr> {
2210 using Scalar = typename ProductXpr::Scalar;
2211 using MatrixScalar = typename MatrixType::Scalar;
2212 using DiagonalOnLeft = bool_constant<ProductOrder == OnTheLeft>;
2213
2214 enum {
2215 CoeffReadCost = int(NumTraits<Scalar>::MulCost) + int(evaluator<MatrixType>::CoeffReadCost) +
2216 int(evaluator<DiagonalType>::CoeffReadCost),
2217 Flags = HereditaryBits & static_cast<unsigned int>(evaluator<MatrixType>::Flags),
2218 Alignment = 0
2219 };
2220
2221 EIGEN_DEVICE_FUNC selfadjoint_diagonal_product_lazy_evaluator_base(const ProductXpr&, const MatrixType& mat,
2222 const DiagonalType& diag)
2223 : m_diagImpl(diag), m_matImpl(mat) {}
2224
2225 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const {
2226 const bool storedHere = ((Mode & Upper) == Upper) ? (row <= col) : (row >= col);
2227 const MatrixScalar matCoeff =
2228 storedHere ? MatrixScalar(m_matImpl.coeff(row, col)) : MatrixScalar(numext::conj(m_matImpl.coeff(col, row)));
2229 return Scalar(
2230 diagonal_product_coeff(matCoeff, m_diagImpl.coeff(ProductOrder == OnTheLeft ? row : col), DiagonalOnLeft()));
2231 }
2232
2233 protected:
2234 evaluator<DiagonalType> m_diagImpl;
2235 evaluator<MatrixType> m_matImpl;
2236};
2237
2238// Extract complex factors before conjugating mirrored entries; the temporary also protects nested aliases.
2239// The parameter is not named XprType: MSVC lets the dependent base's XprType (an EvalToTemp) hide it.
2240template <int Mode, int ProductOrder, typename MatrixType, typename DiagonalType, typename ProductXpr>
2241struct selfadjoint_diagonal_product_lazy_evaluator_base<Mode, ProductOrder, MatrixType, DiagonalType, ProductXpr, true>
2242 : evaluator<EvalToTemp<Product<typename ProductXpr::Lhs, typename ProductXpr::Rhs, DefaultProduct>>> {
2243 using DefaultProductType = Product<typename ProductXpr::Lhs, typename ProductXpr::Rhs, DefaultProduct>;
2244 using Base = evaluator<EvalToTemp<DefaultProductType>>;
2245 EIGEN_DEVICE_FUNC selfadjoint_diagonal_product_lazy_evaluator_base(const ProductXpr& xpr, const MatrixType&,
2246 const DiagonalType&)
2247 : Base(DefaultProductType(xpr.lhs(), xpr.rhs())) {}
2248};
2249
2250// SelfAdjoint × Diagonal
2251template <typename Lhs, typename Rhs, int ProductKind, int ProductTag>
2252struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, SelfAdjointShape, DiagonalShape>
2253 : selfadjoint_diagonal_product_lazy_evaluator_base<Lhs::Mode, OnTheRight, typename Lhs::MatrixType,
2254 typename Rhs::DiagonalVectorType,
2255 Product<Lhs, Rhs, ProductKind>> {
2256 using XprType = Product<Lhs, Rhs, ProductKind>;
2257 using Base = selfadjoint_diagonal_product_lazy_evaluator_base<Lhs::Mode, OnTheRight, typename Lhs::MatrixType,
2258 typename Rhs::DiagonalVectorType, XprType>;
2259
2260 EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr)
2261 : Base(xpr, xpr.lhs().nestedExpression(), xpr.rhs().diagonal()) {}
2262};
2263
2264// Diagonal × SelfAdjoint
2265template <typename Lhs, typename Rhs, int ProductKind, int ProductTag>
2266struct product_evaluator<Product<Lhs, Rhs, ProductKind>, ProductTag, DiagonalShape, SelfAdjointShape>
2267 : selfadjoint_diagonal_product_lazy_evaluator_base<Rhs::Mode, OnTheLeft, typename Rhs::MatrixType,
2268 typename Lhs::DiagonalVectorType,
2269 Product<Lhs, Rhs, ProductKind>> {
2270 using XprType = Product<Lhs, Rhs, ProductKind>;
2271 using Base = selfadjoint_diagonal_product_lazy_evaluator_base<Rhs::Mode, OnTheLeft, typename Rhs::MatrixType,
2272 typename Lhs::DiagonalVectorType, XprType>;
2273
2274 EIGEN_DEVICE_FUNC explicit product_evaluator(const XprType& xpr)
2275 : Base(xpr, xpr.rhs().nestedExpression(), xpr.lhs().diagonal()) {}
2276};
2277
2278// Fallback for indexed products whose operand shape has no direct accumulator.
2279template <typename Lhs, typename Rhs, typename Derived>
2280struct product_accumulate_through_temporary {
2281 template <typename Dest>
2282 static EIGEN_DEVICE_FUNC void addTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2283 typename Product<Lhs, Rhs>::PlainObject tmp(lhs.rows(), rhs.cols());
2284 Derived::evalTo(tmp, lhs, rhs);
2285 dst += tmp;
2286 }
2287
2288 template <typename Dest>
2289 static EIGEN_DEVICE_FUNC void subTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2290 typename Product<Lhs, Rhs>::PlainObject tmp(lhs.rows(), rhs.cols());
2291 Derived::evalTo(tmp, lhs, rhs);
2292 dst -= tmp;
2293 }
2294};
2295
2296template <typename Lhs, typename Rhs, typename Derived, typename MatrixShape>
2297struct indexed_product_accumulation : product_accumulate_through_temporary<Lhs, Rhs, Derived> {};
2298
2299template <typename Lhs, typename Rhs, typename Derived>
2300struct indexed_product_accumulation<Lhs, Rhs, Derived, DenseShape> {
2301 template <typename Dest>
2302 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void addTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2303 Derived::accumulate(dst, lhs, rhs, add_assign_op<typename Dest::Scalar, typename Product<Lhs, Rhs>::Scalar>());
2304 }
2305
2306 template <typename Dest>
2307 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void subTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2308 Derived::accumulate(dst, lhs, rhs, sub_assign_op<typename Dest::Scalar, typename Product<Lhs, Rhs>::Scalar>());
2309 }
2310};
2311
2312template <typename Lhs, typename Rhs, typename Derived>
2313struct indexed_product_accumulation<Lhs, Rhs, Derived, TriangularShape>
2314 : indexed_product_accumulation<Lhs, Rhs, Derived, DenseShape> {};
2315
2316template <typename Lhs, typename Rhs, typename Derived>
2317struct indexed_product_accumulation<Lhs, Rhs, Derived, SelfAdjointShape>
2318 : indexed_product_accumulation<Lhs, Rhs, Derived, DenseShape> {};
2319
2320template <int Side, bool Transposed, typename Dest, typename Indices, typename Expression, typename Func>
2321EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void indexed_matrix_update(Dest& dst, const Indices& indices,
2322 const Expression& xpr, const Func& func) {
2323 using MatrixType = typename nested_eval<Expression, 1>::type;
2324 using MatrixTypeCleaned = remove_all_t<MatrixType>;
2325 MatrixType mat(xpr);
2326 // An inner-dimension permutation reuses its index expression for every outer vector.
2327 typename nested_eval<Indices, Dynamic>::type actualIndices(indices);
2328 const Index n = Side == OnTheLeft ? mat.rows() : mat.cols();
2329 EIGEN_IF_CONSTEXPR (bool(Dest::IsRowMajor) == bool(MatrixTypeCleaned::IsRowMajor) &&
2330 bool(Dest::IsRowMajor) == (Side == OnTheRight)) {
2331 for (Index j = 0; j < (Side == OnTheLeft ? mat.cols() : mat.rows()); ++j) {
2332 auto target = dst.template subVector < Side == OnTheLeft ? Vertical : Horizontal > (j);
2333 const auto source = mat.template subVector < Side == OnTheLeft ? Vertical : Horizontal > (j);
2334 for (Index i = 0; i < n; ++i) {
2335 const Index p = actualIndices.coeff(i);
2336 func.assignCoeff(target.coeffRef(((Side == OnTheLeft) ^ Transposed) ? p : i),
2337 source.coeff(((Side == OnTheRight) ^ Transposed) ? p : i));
2338 }
2339 }
2340 } else {
2341 for (Index i = 0; i < n; ++i) {
2342 const Index p = actualIndices.coeff(i);
2343 auto target = dst.template subVector < Side == OnTheLeft
2344 ? Horizontal
2345 : Vertical > (((Side == OnTheLeft) ^ Transposed) ? p : i);
2346 const auto source = mat.template subVector < Side == OnTheLeft
2347 ? Horizontal
2348 : Vertical > (((Side == OnTheRight) ^ Transposed) ? p : i);
2349 call_assignment_no_alias(target, source, func);
2350 }
2351 }
2352}
2353
2354/***************************************************************************
2355 * Products with permutation matrices
2356 ***************************************************************************/
2357
2366template <typename ExpressionType, int Side, bool Transposed, typename ExpressionShape>
2367struct permutation_matrix_product {
2368 template <typename Dest, typename PermutationType>
2369 static EIGEN_DEVICE_FUNC void run(Dest&, const PermutationType&, const ExpressionType&) {
2370 static_assert(std::is_same<ExpressionShape, DenseShape>::value,
2371 "PERMUTATION_PRODUCTS_ARE_NOT_IMPLEMENTED_FOR_THIS_OPERAND_SHAPE");
2372 }
2373};
2374
2375template <typename ExpressionType, int Side, bool Transposed>
2376struct permutation_matrix_product<ExpressionType, Side, Transposed, DenseShape> {
2377 using MatrixType = typename nested_eval<ExpressionType, 1>::type;
2378 using MatrixTypeCleaned = remove_all_t<MatrixType>;
2379
2380 template <typename Dest, typename Indices, typename Func>
2381 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Indices& indices,
2382 const ExpressionType& xpr, const Func& func) {
2383 indexed_matrix_update<Side, Transposed>(dst, indices, xpr, func);
2384 }
2385
2386 template <typename Dest, typename PermutationType>
2387 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const PermutationType& perm,
2388 const ExpressionType& xpr) {
2389 MatrixType mat(xpr);
2390 const Index n = Side == OnTheLeft ? mat.rows() : mat.cols();
2391 // FIXME we need an is_same for expression that is not sensitive to constness. For instance
2392 // is_same_xpr<Block<const Matrix>, Block<Matrix> >::value should be true.
2393 // if(std::is_same<MatrixTypeCleaned,Dest>::value && extract_data(dst) == extract_data(mat))
2394 if (is_same_dense(dst, mat)) {
2395 // apply the permutation inplace
2396 Matrix<bool, PermutationType::RowsAtCompileTime, 1, 0, PermutationType::MaxRowsAtCompileTime> mask(perm.size());
2397 mask.fill(false);
2398 Index r = 0;
2399 while (r < perm.size()) {
2400 // search for the next seed
2401 while (r < perm.size() && mask[r]) r++;
2402 if (r >= perm.size()) break;
2403 // we got one, let's follow it until we are back to the seed
2404 Index k0 = r++;
2405 Index kPrev = k0;
2406 mask.coeffRef(k0) = true;
2407 for (Index k = perm.indices().coeff(k0); k != k0; k = perm.indices().coeff(k)) {
2408 Block<Dest, Side == OnTheLeft ? 1 : Dest::RowsAtCompileTime,
2409 Side == OnTheRight ? 1 : Dest::ColsAtCompileTime>(dst, k)
2410 .swap(Block < Dest, Side == OnTheLeft ? 1 : Dest::RowsAtCompileTime,
2411 Side == OnTheRight
2412 ? 1
2413 : Dest::ColsAtCompileTime > (dst, ((Side == OnTheLeft) ^ Transposed) ? k0 : kPrev));
2414
2415 mask.coeffRef(k) = true;
2416 kPrev = k;
2417 }
2418 }
2419 } else {
2420 for (Index i = 0; i < n; ++i) {
2421 Block<Dest, Side == OnTheLeft ? 1 : Dest::RowsAtCompileTime, Side == OnTheRight ? 1 : Dest::ColsAtCompileTime>(
2422 dst, ((Side == OnTheLeft) ^ Transposed) ? perm.indices().coeff(i) : i)
2423
2424 =
2425
2426 Block < const MatrixTypeCleaned,
2427 Side == OnTheLeft ? 1 : MatrixTypeCleaned::RowsAtCompileTime,
2428 Side == OnTheRight ? 1
2429 : MatrixTypeCleaned::ColsAtCompileTime >
2430 (mat, ((Side == OnTheRight) ^ Transposed) ? perm.indices().coeff(i) : i);
2431 }
2432 }
2433 }
2434};
2435
2441template <typename ExpressionType, int Side, bool Transposed>
2442struct permutation_densified_matrix_product {
2443 using DenseType = typename ExpressionType::DenseMatrixType;
2444
2445 template <typename Dest, typename Indices, typename Func>
2446 static EIGEN_DEVICE_FUNC void accumulate(Dest& dst, const Indices& indices, const ExpressionType& xpr,
2447 const Func& func) {
2448 const DenseType dense(xpr);
2449 indexed_matrix_update<Side, Transposed>(dst, indices, dense, func);
2450 }
2451
2452 template <typename Dest, typename PermutationType>
2453 static EIGEN_DEVICE_FUNC void run(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) {
2454 const DenseType dense(xpr);
2455 permutation_matrix_product<DenseType, Side, Transposed, DenseShape>::run(dst, perm, dense);
2456 }
2457};
2458
2459template <typename ExpressionType, int Side, bool Transposed>
2460struct permutation_matrix_product<ExpressionType, Side, Transposed, TriangularShape>
2461 : permutation_densified_matrix_product<ExpressionType, Side, Transposed> {};
2462
2463// The scalar factor of a self-adjoint view, folded in from s * (A.selfadjointView() * P), scales the whole view as
2464// in the other self-adjoint products, not only its stored triangle.
2465template <typename ExpressionType, int Side, bool Transposed>
2466struct permutation_matrix_product<ExpressionType, Side, Transposed, SelfAdjointShape> {
2467 using DenseType = typename ExpressionType::DenseMatrixType;
2468
2469 template <typename Dest, typename Indices, typename Func>
2470 static void accumulate(Dest& dst, const Indices& indices, const ExpressionType& xpr, const Func& func) {
2471 DenseType dense(xpr.rows(), xpr.cols());
2472 const typename ExpressionType::Scalar factor = densified_view_operand<ExpressionType>::run(dense, xpr);
2473 EIGEN_IF_CONSTEXPR (blas_traits<typename ExpressionType::MatrixType>::HasScalarFactor) dense *= factor;
2474 indexed_matrix_update<Side, Transposed>(dst, indices, dense, func);
2475 }
2476
2477 template <typename Dest, typename PermutationType>
2478 static void run(Dest& dst, const PermutationType& perm, const ExpressionType& xpr) {
2479 DenseType dense(xpr.rows(), xpr.cols());
2480 const typename ExpressionType::Scalar factor = densified_view_operand<ExpressionType>::run(dense, xpr);
2481 EIGEN_IF_CONSTEXPR (blas_traits<typename ExpressionType::MatrixType>::HasScalarFactor) dense *= factor;
2482 permutation_matrix_product<DenseType, Side, Transposed, DenseShape>::run(dst, perm, dense);
2483 }
2484};
2485
2486template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2487struct generic_product_impl<Lhs, Rhs, PermutationShape, MatrixShape, ProductTag>
2488 : indexed_product_accumulation<Lhs, Rhs, generic_product_impl<Lhs, Rhs, PermutationShape, MatrixShape, ProductTag>,
2489 MatrixShape> {
2490 template <typename Dest, typename Func>
2491 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Lhs& lhs, const Rhs& rhs,
2492 const Func& func) {
2493 permutation_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::accumulate(dst, lhs.indices(), rhs, func);
2494 }
2495 template <typename Dest>
2496 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2497 permutation_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::run(dst, lhs, rhs);
2498 }
2499};
2500
2501template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2502struct generic_product_impl<Lhs, Rhs, MatrixShape, PermutationShape, ProductTag>
2503 : indexed_product_accumulation<Lhs, Rhs, generic_product_impl<Lhs, Rhs, MatrixShape, PermutationShape, ProductTag>,
2504 MatrixShape> {
2505 template <typename Dest, typename Func>
2506 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Lhs& lhs, const Rhs& rhs,
2507 const Func& func) {
2508 permutation_matrix_product<Lhs, OnTheRight, false, MatrixShape>::accumulate(dst, rhs.indices(), lhs, func);
2509 }
2510 template <typename Dest>
2511 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2512 permutation_matrix_product<Lhs, OnTheRight, false, MatrixShape>::run(dst, rhs, lhs);
2513 }
2514};
2515
2516template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2517struct generic_product_impl<Inverse<Lhs>, Rhs, PermutationShape, MatrixShape, ProductTag>
2518 : indexed_product_accumulation<Inverse<Lhs>, Rhs,
2519 generic_product_impl<Inverse<Lhs>, Rhs, PermutationShape, MatrixShape, ProductTag>,
2520 MatrixShape> {
2521 template <typename Dest, typename Func>
2522 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Inverse<Lhs>& lhs, const Rhs& rhs,
2523 const Func& func) {
2524 permutation_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::accumulate(dst, lhs.nestedExpression().indices(),
2525 rhs, func);
2526 }
2527 template <typename Dest>
2528 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Inverse<Lhs>& lhs, const Rhs& rhs) {
2529 permutation_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::run(dst, lhs.nestedExpression(), rhs);
2530 }
2531};
2532
2533template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2534struct generic_product_impl<Lhs, Inverse<Rhs>, MatrixShape, PermutationShape, ProductTag>
2535 : indexed_product_accumulation<Lhs, Inverse<Rhs>,
2536 generic_product_impl<Lhs, Inverse<Rhs>, MatrixShape, PermutationShape, ProductTag>,
2537 MatrixShape> {
2538 template <typename Dest, typename Func>
2539 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Lhs& lhs, const Inverse<Rhs>& rhs,
2540 const Func& func) {
2541 permutation_matrix_product<Lhs, OnTheRight, true, MatrixShape>::accumulate(dst, rhs.nestedExpression().indices(),
2542 lhs, func);
2543 }
2544 template <typename Dest>
2545 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Inverse<Rhs>& rhs) {
2546 permutation_matrix_product<Lhs, OnTheRight, true, MatrixShape>::run(dst, rhs.nestedExpression(), lhs);
2547 }
2548};
2549
2550/***************************************************************************
2551 * Products with transpositions matrices
2552 ***************************************************************************/
2553
2554// Overwrite can apply swaps in place; compound updates must preserve the original source.
2555template <typename Lhs, typename Rhs, typename DstScalar, typename SrcScalar>
2556struct assignment_assume_aliasing<Product<Lhs, Rhs, AliasFreeProduct>, add_assign_op<DstScalar, SrcScalar>>
2557 : bool_constant<evaluator_assume_aliasing<Product<Lhs, Rhs, AliasFreeProduct>>::value ||
2558 std::is_same<typename evaluator_traits<Lhs>::Shape, TranspositionsShape>::value ||
2559 std::is_same<typename evaluator_traits<Rhs>::Shape, TranspositionsShape>::value> {};
2560
2561template <typename Lhs, typename Rhs, typename DstScalar, typename SrcScalar>
2562struct assignment_assume_aliasing<Product<Lhs, Rhs, AliasFreeProduct>, sub_assign_op<DstScalar, SrcScalar>>
2563 : assignment_assume_aliasing<Product<Lhs, Rhs, AliasFreeProduct>, add_assign_op<DstScalar, SrcScalar>> {};
2564
2565// FIXME: consider unifying Transpositions and Permutation into a single shape.
2566
2571template <typename ExpressionType, int Side, bool Transposed, typename ExpressionShape>
2572struct transposition_matrix_product {
2573 using MatrixType = typename nested_eval<ExpressionType, 1>::type;
2574 using MatrixTypeCleaned = remove_all_t<MatrixType>;
2575
2576 template <typename Dest, typename TranspositionType, typename Func>
2577 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const TranspositionType& tr,
2578 const ExpressionType& xpr, const Func& func) {
2579 using SourceIndices = typename TranspositionType::IndicesType;
2580 // Swap partners can fit a narrow StorageIndex even when composed positions do not.
2581 constexpr bool FitsStack = SourceIndices::MaxSizeAtCompileTime != Dynamic &&
2582 (EIGEN_STACK_ALLOCATION_LIMIT == 0 || std::size_t(SourceIndices::MaxSizeAtCompileTime) <=
2583 EIGEN_STACK_ALLOCATION_LIMIT / sizeof(Index));
2584 using Indices = Matrix<Index, FitsStack ? SourceIndices::SizeAtCompileTime : Dynamic, 1, 0,
2585 FitsStack ? SourceIndices::MaxSizeAtCompileTime : Dynamic, 1>;
2586 Indices indices(tr.size());
2587 for (Index i = 0; i < tr.size(); ++i) indices.coeffRef(i) = i;
2588 for (Index k = tr.size(); k > 0; --k) numext::swap(indices.coeffRef(k - 1), indices.coeffRef(tr.coeff(k - 1)));
2589 // Right products apply the swap sequence in the opposite matrix-product order.
2590 indexed_matrix_update<Side, Transposed ^ (Side == OnTheRight)>(dst, indices, xpr, func);
2591 }
2592
2593 template <typename Dest, typename TranspositionType>
2594 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Dest& dst, const TranspositionType& tr,
2595 const ExpressionType& xpr) {
2596 // The in-place row/column swaps below need a dense operand.
2597 static_assert(std::is_same<ExpressionShape, DenseShape>::value,
2598 "TRANSPOSITIONS_PRODUCTS_ARE_ONLY_IMPLEMENTED_FOR_DENSE_OPERANDS");
2599 MatrixType mat(xpr);
2600 using StorageIndex = typename TranspositionType::StorageIndex;
2601 const Index size = tr.size();
2602
2603 if (!is_same_dense(dst, mat)) dst = mat;
2604
2605 for (Index k = (Transposed ? size - 1 : 0); Transposed ? k >= 0 : k < size; Transposed ? --k : ++k) {
2606 const StorageIndex j = tr.coeff(k);
2607 if (Index(j) != k) {
2608 EIGEN_IF_CONSTEXPR (Side == OnTheLeft) {
2609 dst.row(k).swap(dst.row(j));
2610 } else EIGEN_IF_CONSTEXPR (Side == OnTheRight) {
2611 dst.col(k).swap(dst.col(j));
2612 }
2613 }
2614 }
2615 }
2616};
2617
2618template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2619struct generic_product_impl<Lhs, Rhs, TranspositionsShape, MatrixShape, ProductTag>
2620 : indexed_product_accumulation<
2621 Lhs, Rhs, generic_product_impl<Lhs, Rhs, TranspositionsShape, MatrixShape, ProductTag>, MatrixShape> {
2622 template <typename Dest, typename Func>
2623 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Lhs& lhs, const Rhs& rhs,
2624 const Func& func) {
2625 transposition_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::accumulate(dst, lhs, rhs, func);
2626 }
2627 template <typename Dest>
2628 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2629 transposition_matrix_product<Rhs, OnTheLeft, false, MatrixShape>::run(dst, lhs, rhs);
2630 }
2631};
2632
2633template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2634struct generic_product_impl<Lhs, Rhs, MatrixShape, TranspositionsShape, ProductTag>
2635 : indexed_product_accumulation<
2636 Lhs, Rhs, generic_product_impl<Lhs, Rhs, MatrixShape, TranspositionsShape, ProductTag>, MatrixShape> {
2637 template <typename Dest, typename Func>
2638 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Lhs& lhs, const Rhs& rhs,
2639 const Func& func) {
2640 transposition_matrix_product<Lhs, OnTheRight, false, MatrixShape>::accumulate(dst, rhs, lhs, func);
2641 }
2642 template <typename Dest>
2643 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2644 transposition_matrix_product<Lhs, OnTheRight, false, MatrixShape>::run(dst, rhs, lhs);
2645 }
2646};
2647
2648template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2649struct generic_product_impl<Transpose<Lhs>, Rhs, TranspositionsShape, MatrixShape, ProductTag>
2650 : indexed_product_accumulation<
2651 Transpose<Lhs>, Rhs, generic_product_impl<Transpose<Lhs>, Rhs, TranspositionsShape, MatrixShape, ProductTag>,
2652 MatrixShape> {
2653 template <typename Dest, typename Func>
2654 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Transpose<Lhs>& lhs, const Rhs& rhs,
2655 const Func& func) {
2656 transposition_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::accumulate(dst, lhs.nestedExpression(), rhs, func);
2657 }
2658 template <typename Dest>
2659 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Transpose<Lhs>& lhs, const Rhs& rhs) {
2660 transposition_matrix_product<Rhs, OnTheLeft, true, MatrixShape>::run(dst, lhs.nestedExpression(), rhs);
2661 }
2662};
2663
2664template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2665struct generic_product_impl<Lhs, Transpose<Rhs>, MatrixShape, TranspositionsShape, ProductTag>
2666 : indexed_product_accumulation<
2667 Lhs, Transpose<Rhs>, generic_product_impl<Lhs, Transpose<Rhs>, MatrixShape, TranspositionsShape, ProductTag>,
2668 MatrixShape> {
2669 template <typename Dest, typename Func>
2670 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void accumulate(Dest& dst, const Lhs& lhs, const Transpose<Rhs>& rhs,
2671 const Func& func) {
2672 transposition_matrix_product<Lhs, OnTheRight, true, MatrixShape>::accumulate(dst, rhs.nestedExpression(), lhs,
2673 func);
2674 }
2675 template <typename Dest>
2676 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Transpose<Rhs>& rhs) {
2677 transposition_matrix_product<Lhs, OnTheRight, true, MatrixShape>::run(dst, rhs.nestedExpression(), lhs);
2678 }
2679};
2680
2681/***************************************************************************
2682 * skew symmetric products
2683 * The 3x3 skew-symmetric operand is densified and the product is re-dispatched on the other
2684 * operand's shape, so shapes whose products with a dense matrix are evaluator-only (DiagonalShape)
2685 * resolve too.
2686 ***************************************************************************/
2687template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2688struct generic_product_impl<Lhs, Rhs, SkewSymmetricShape, MatrixShape, ProductTag> {
2689 template <typename Dest>
2690 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2691 call_assignment_no_alias(dst, typename Lhs::DenseMatrixType(lhs) * rhs);
2692 }
2693};
2694
2695template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2696struct generic_product_impl<Lhs, Rhs, MatrixShape, SkewSymmetricShape, ProductTag> {
2697 template <typename Dest>
2698 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2699 call_assignment_no_alias(dst, lhs * typename Rhs::DenseMatrixType(rhs));
2700 }
2701};
2702
2703template <typename Lhs, typename Rhs, int ProductTag>
2704struct generic_product_impl<Lhs, Rhs, SkewSymmetricShape, SkewSymmetricShape, ProductTag> {
2705 template <typename Dest>
2706 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
2707 generic_product_impl<typename Lhs::DenseMatrixType, typename Rhs::DenseMatrixType, DenseShape, DenseShape,
2708 ProductTag>::evalTo(dst, lhs, rhs);
2709 }
2710};
2711
2712template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2713struct generic_product_impl<Lhs, Rhs, MatrixShape, HomogeneousShape, ProductTag>
2714 : generic_product_impl<Lhs, typename Rhs::PlainObject, MatrixShape, DenseShape, ProductTag> {};
2715
2716template <typename Lhs, typename Rhs, int ProductTag, typename MatrixShape>
2717struct generic_product_impl<Lhs, Rhs, HomogeneousShape, MatrixShape, ProductTag>
2718 : generic_product_impl<typename Lhs::PlainObject, Rhs, DenseShape, MatrixShape, ProductTag> {};
2719
2720template <typename Lhs, typename Rhs, int ProductTag>
2721struct generic_product_impl<Lhs, Rhs, PermutationShape, HomogeneousShape, ProductTag>
2722 : generic_product_impl<Lhs, Rhs, PermutationShape, DenseShape, ProductTag> {};
2723
2724template <typename Lhs, typename Rhs, int ProductTag>
2725struct generic_product_impl<Lhs, Rhs, HomogeneousShape, PermutationShape, ProductTag>
2726 : generic_product_impl<Lhs, Rhs, DenseShape, PermutationShape, ProductTag> {};
2727
2728} // end namespace internal
2729
2730} // end namespace Eigen
2731
2732#if EIGEN_COMP_MSVC
2733#pragma warning(pop)
2734#endif
2735
2736#endif // EIGEN_PRODUCTEVALUATORS_H
@ UnitDiag
Definition Constants.h:216
@ ZeroDiag
Definition Constants.h:218
@ Lower
Definition Constants.h:212
@ Upper
Definition Constants.h:214
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
@ OnTheLeft
Definition Constants.h:332
@ OnTheRight
Definition Constants.h:334
@ Horizontal
Definition Constants.h:270
@ Vertical
Definition Constants.h:267
constexpr unsigned int ActualPacketAccessBit
Definition Constants.h:109
constexpr unsigned int PacketAccessBit
Definition Constants.h:98
constexpr unsigned int DirectAccessBit
Definition Constants.h:160
constexpr unsigned int EvalBeforeNestingBit
Definition Constants.h:75
constexpr unsigned int LinearAccessBit
Definition Constants.h:134
constexpr unsigned int RowMajorBit
Definition Constants.h:71
void operator()(const Dst &dst, const Src &src) const
Definition ProductEvaluators.h:447