11#ifndef EIGEN_INNER_PRODUCT_EVAL_H
12#define EIGEN_INNER_PRODUCT_EVAL_H
15#include "./InternalHeaderCheck.h"
22template <
typename Lhs,
typename Rhs>
23struct inner_product_assert {
24 EIGEN_STATIC_ASSERT_VECTOR_ONLY(Lhs)
25 EIGEN_STATIC_ASSERT_VECTOR_ONLY(Rhs)
26 static EIGEN_DEVICE_FUNC
void run(
const Lhs& lhs,
const Rhs& rhs) {
27 EIGEN_INTERNAL_ASSERT_SAME_SIZE_MSG(lhs, rhs,
"Inner product: lhs and rhs vectors must have same size");
31template <
typename Func,
typename Lhs,
typename Rhs>
32struct inner_product_evaluator {
33 static constexpr int LhsFlags = evaluator<Lhs>::Flags;
34 static constexpr int RhsFlags = evaluator<Rhs>::Flags;
35 static constexpr int SizeAtCompileTime = size_prefer_fixed(Lhs::SizeAtCompileTime, Rhs::SizeAtCompileTime);
36 static constexpr int MaxSizeAtCompileTime =
37 min_size_prefer_fixed(Lhs::MaxSizeAtCompileTime, Rhs::MaxSizeAtCompileTime);
38 static constexpr int LhsAlignment = evaluator<Lhs>::Alignment;
39 static constexpr int RhsAlignment = evaluator<Rhs>::Alignment;
41 using Scalar =
typename Func::result_type;
42 using Packet =
typename find_largest_packet<Scalar, SizeAtCompileTime>::type;
44 static constexpr bool Vectorize =
46 ((MaxSizeAtCompileTime == Dynamic) || (unpacket_traits<Packet>::size <= MaxSizeAtCompileTime));
48 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
explicit inner_product_evaluator(
const Lhs& lhs,
const Rhs& rhs,
50 : m_func(func), m_lhs(lhs), m_rhs(rhs), m_size(lhs.size()) {
51 inner_product_assert<Lhs, Rhs>::run(lhs, rhs);
54 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size()
const {
return m_size.value(); }
56 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index)
const {
57 return m_func.coeff(m_lhs.coeff(index), m_rhs.coeff(index));
60 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(
const Scalar& value, Index index)
const {
61 return m_func.coeff(value, m_lhs.coeff(index), m_rhs.coeff(index));
64 template <
typename PacketType,
int LhsMode = LhsAlignment,
int RhsMode = RhsAlignment>
65 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index)
const {
66 return m_func.packet(m_lhs.template packet<LhsMode, PacketType>(index),
67 m_rhs.template packet<RhsMode, PacketType>(index));
70 template <
typename PacketType,
int LhsMode = LhsAlignment,
int RhsMode = RhsAlignment>
71 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(
const PacketType& value, Index index)
const {
72 return m_func.packet(value, m_lhs.template packet<LhsMode, PacketType>(index),
73 m_rhs.template packet<RhsMode, PacketType>(index));
77 const evaluator<Lhs> m_lhs;
78 const evaluator<Rhs> m_rhs;
79 const variable_if_dynamic<Index, SizeAtCompileTime> m_size;
82template <
typename Evaluator,
bool Vectorize = Evaluator::Vectorize>
83struct inner_product_impl;
86template <
typename Evaluator>
87struct inner_product_impl<Evaluator, false> {
88 using Scalar =
typename Evaluator::Scalar;
89 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(
const Evaluator& eval) {
90 const Index size = eval.size();
91 if (size == 0)
return Scalar(0);
93 Scalar result = eval.coeff(0);
94 for (Index k = 1; k < size; k++) {
95 result = eval.coeff(result, k);
103template <
typename Evaluator>
104struct inner_product_impl<Evaluator, true> {
105 using UnsignedIndex = std::make_unsigned_t<Index>;
106 using Scalar =
typename Evaluator::Scalar;
107 using Packet =
typename Evaluator::Packet;
108 static constexpr int PacketSize = unpacket_traits<Packet>::size;
109 static constexpr int MaxSize = Evaluator::MaxSizeAtCompileTime;
110 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(
const Evaluator& eval) {
111 const UnsignedIndex size =
static_cast<UnsignedIndex
>(eval.size());
112 if (size < PacketSize)
return inner_product_impl<Evaluator, false>::run(eval);
114 const UnsignedIndex packetEnd = numext::round_down(size, PacketSize);
115 const UnsignedIndex quadEnd = numext::round_down(size, 4 * PacketSize);
116 const UnsignedIndex numPackets = size / PacketSize;
117 const UnsignedIndex numRemPackets = (packetEnd - quadEnd) / PacketSize;
119 Packet presult0 = eval.template packet<Packet>(0 * PacketSize);
121 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 2) {
122 if (numPackets >= 2) {
123 Packet presult1 = eval.template packet<Packet>(1 * PacketSize);
124 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 3) {
125 if (numPackets >= 3) {
126 Packet presult2 = eval.template packet<Packet>(2 * PacketSize);
127 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 4) {
128 if (numPackets >= 4) {
129 Packet presult3 = eval.template packet<Packet>(3 * PacketSize);
131 for (UnsignedIndex k = 4 * PacketSize; k < quadEnd; k += 4 * PacketSize) {
132 presult0 = eval.packet(presult0, k + 0 * PacketSize);
133 presult1 = eval.packet(presult1, k + 1 * PacketSize);
134 presult2 = eval.packet(presult2, k + 2 * PacketSize);
135 presult3 = eval.packet(presult3, k + 3 * PacketSize);
138 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 5) {
139 if (numRemPackets >= 1) {
140 presult0 = eval.packet(presult0, quadEnd + 0 * PacketSize);
141 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 6) {
142 if (numRemPackets >= 2) {
143 presult1 = eval.packet(presult1, quadEnd + 1 * PacketSize);
144 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 7) {
145 if (numRemPackets == 3) presult2 = eval.packet(presult2, quadEnd + 2 * PacketSize);
152 presult2 = padd(presult2, presult3);
155 presult1 = padd(presult1, presult2);
158 presult0 = padd(presult0, presult1);
162 Scalar result = predux(presult0);
163 for (UnsignedIndex k = packetEnd; k < size; k++) {
164 result = eval.coeff(result, k);
171template <
typename LhsScalar,
typename RhsScalar,
bool Conj>
172struct scalar_inner_product_op {
173 using result_type =
typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
174 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(
const LhsScalar& a,
const RhsScalar& b)
const {
175 return (conj_if<Conj>()(a) * b);
177 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(
const result_type& accum,
const LhsScalar& a,
178 const RhsScalar& b)
const {
179 return (conj_if<Conj>()(a) * b) + accum;
181 static constexpr bool PacketAccess =
false;
186template <
typename Scalar,
bool Conj>
187struct scalar_inner_product_op<
189 std::enable_if_t<std::is_same<typename ScalarBinaryOpTraits<Scalar, Scalar>::ReturnType, Scalar>::value, Scalar>,
191 using result_type = Scalar;
192 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(
const Scalar& a,
const Scalar& b)
const {
193 return pmul(conj_if<Conj>()(a), b);
195 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(
const Scalar& accum,
const Scalar& a,
const Scalar& b)
const {
196 return pmadd(conj_if<Conj>()(a), b, accum);
198 template <
typename Packet>
199 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(
const Packet& a,
const Packet& b)
const {
200 return pmul(conj_if<Conj>().pconj(a), b);
202 template <
typename Packet>
203 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(
const Packet& accum,
const Packet& a,
const Packet& b)
const {
204 return pmadd(conj_if<Conj>().pconj(a), b, accum);
206 static constexpr bool PacketAccess = packet_traits<Scalar>::HasMul && packet_traits<Scalar>::HasAdd;
210template <
typename Lhs,
typename Rhs,
bool Conj,
typename Enable =
void>
211struct default_inner_product_impl {
212 using LhsScalar =
typename traits<Lhs>::Scalar;
213 using RhsScalar =
typename traits<Rhs>::Scalar;
214 using Op = scalar_inner_product_op<LhsScalar, RhsScalar, Conj>;
215 using Evaluator = inner_product_evaluator<Op, Lhs, Rhs>;
216 using result_type =
typename Evaluator::Scalar;
217 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run(
const MatrixBase<Lhs>& a,
const MatrixBase<Rhs>& b) {
218 Evaluator eval(a.derived(), b.derived(), Op());
219 return inner_product_impl<Evaluator>::run(eval);
226 static constexpr bool HasDirectAccess = bool(traits<type>::Flags &
DirectAccessBit);
228 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
constexpr T
const& get(T
const& xpr) {
return xpr; }
232struct unwrap_unary<T const> : unwrap_unary<T> {};
234template <
typename Op,
typename Xpr>
235struct unwrap_unary<CwiseUnaryOp<Op, Xpr>> : unwrap_unary<Xpr> {
236 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
constexpr typename unwrap_unary<Xpr>::type
const& get(
237 CwiseUnaryOp<Op, Xpr>
const& xpr) {
238 return unwrap_unary<Xpr>::get(xpr.nestedExpression());
242template <
typename T,
typename Target>
246 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
constexpr Target
const& apply(T
const&, Target
const& target) {
251template <
typename T,
typename Target>
252struct rewrap_unary<T const, Target> : rewrap_unary<T, Target const> {};
254template <
typename Op,
typename Xpr,
typename Target>
255struct rewrap_unary<CwiseUnaryOp<Op, Xpr>, Target> {
257 using type = CwiseUnaryOp<Op, std::add_const_t<typename rewrap_unary<Xpr, Target>::type>>;
259 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
constexpr type apply(CwiseUnaryOp<Op, Xpr>
const& base,
260 Target
const& target) {
261 return rewrap_unary<Xpr, Target>::apply(base.nestedExpression(), target).unaryExpr(base.functor());
267template <
typename Lhs,
typename Rhs,
bool Conj,
268 bool MayMap = unwrap_unary<Lhs>::HasDirectAccess && unwrap_unary<Rhs>::HasDirectAccess &&
269 (inner_stride_at_compile_time<typename unwrap_unary<Lhs>::type>::value != 1 ||
270 inner_stride_at_compile_time<typename unwrap_unary<Rhs>::type>::value != 1)>
271struct inner_product_dispatch : default_inner_product_impl<Lhs, Rhs, Conj> {};
273template <
typename Lhs,
typename Rhs,
bool Conj>
274struct inner_product_dispatch<Lhs, Rhs, Conj, true> {
275 using Impl = default_inner_product_impl<Lhs, Rhs, Conj>;
276 using result_type =
typename Impl::result_type;
278 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run(
const MatrixBase<Lhs>& a,
const MatrixBase<Rhs>& b) {
279 EIGEN_IF_CONSTEXPR (Conj) {
280 return run_general(a, b);
284 typename Impl::Evaluator eval(a.derived(), b.derived());
285 if (eval.size() == 0)
return result_type(0);
286 result_type result = eval.coeff(0);
287 if (eval.size() > 1) result = eval.coeff(result, 1);
288 if (eval.size() > 2) result = eval.coeff(result, 2);
289 if (eval.size() > 3) result = eval.coeff(result, 3);
292 return run_large_product(a, b);
296 static EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE result_type run_large_product(
const MatrixBase<Lhs>& a,
297 const MatrixBase<Rhs>& b) {
298 return run_general(a, b);
301 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run_general(
const MatrixBase<Lhs>& a,
302 const MatrixBase<Rhs>& b) {
303 using LhsUnwrapper = unwrap_unary<Lhs>;
304 using RhsUnwrapper = unwrap_unary<Rhs>;
305 using LhsInner =
typename LhsUnwrapper::type;
306 using RhsInner =
typename RhsUnwrapper::type;
308 LhsInner
const& lhs_inner = LhsUnwrapper::get(a.derived());
309 RhsInner
const& rhs_inner = RhsUnwrapper::get(b.derived());
311 if (lhs_inner.innerStride() == 1 && rhs_inner.innerStride() == 1) {
312 using LhsMap = Map<Vector<typename LhsInner::Scalar, size_of_xpr_at_compile_time<LhsInner>::value>
const,
313 evaluator<LhsInner>::Alignment>;
314 using RhsMap = Map<Vector<typename RhsInner::Scalar, size_of_xpr_at_compile_time<RhsInner>::value>
const,
315 evaluator<RhsInner>::Alignment>;
317 LhsMap
const lhs_map(lhs_inner.data(), lhs_inner.size());
318 RhsMap
const rhs_map(rhs_inner.data(), rhs_inner.size());
320 using LhsRewrap = rewrap_unary<Lhs, LhsMap>;
321 using RhsRewrap = rewrap_unary<Rhs, RhsMap>;
323 return default_inner_product_impl<typename LhsRewrap::type, typename RhsRewrap::type, Conj>::run(
324 LhsRewrap::apply(a.derived(), lhs_map), RhsRewrap::apply(b.derived(), rhs_map));
327 return Impl::run(a, b);
constexpr unsigned int PacketAccessBit
Definition Constants.h:98
constexpr unsigned int DirectAccessBit
Definition Constants.h:160