Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
InnerProduct.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2024 Charlie Schlosser <cs.schlosser@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_INNER_PRODUCT_EVAL_H
12#define EIGEN_INNER_PRODUCT_EVAL_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21// Shared accumulation kernel for dot() (Conj = true) and vector products (Conj = false).
22template <typename Lhs, typename Rhs>
23struct inner_product_assert {
24 EIGEN_STATIC_ASSERT_VECTOR_ONLY(Lhs)
25 EIGEN_STATIC_ASSERT_VECTOR_ONLY(Rhs)
26 static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, const Rhs& rhs) {
27 EIGEN_INTERNAL_ASSERT_SAME_SIZE_MSG(lhs, rhs, "Inner product: lhs and rhs vectors must have same size");
28 }
29};
30
31template <typename Func, typename Lhs, typename Rhs>
32struct inner_product_evaluator {
33 static constexpr int LhsFlags = evaluator<Lhs>::Flags;
34 static constexpr int RhsFlags = evaluator<Rhs>::Flags;
35 static constexpr int SizeAtCompileTime = size_prefer_fixed(Lhs::SizeAtCompileTime, Rhs::SizeAtCompileTime);
36 static constexpr int MaxSizeAtCompileTime =
37 min_size_prefer_fixed(Lhs::MaxSizeAtCompileTime, Rhs::MaxSizeAtCompileTime);
38 static constexpr int LhsAlignment = evaluator<Lhs>::Alignment;
39 static constexpr int RhsAlignment = evaluator<Rhs>::Alignment;
40
41 using Scalar = typename Func::result_type;
42 using Packet = typename find_largest_packet<Scalar, SizeAtCompileTime>::type;
43
44 static constexpr bool Vectorize =
45 bool(LhsFlags & RhsFlags & PacketAccessBit) && Func::PacketAccess &&
46 ((MaxSizeAtCompileTime == Dynamic) || (unpacket_traits<Packet>::size <= MaxSizeAtCompileTime));
47
48 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit inner_product_evaluator(const Lhs& lhs, const Rhs& rhs,
49 Func func = Func())
50 : m_func(func), m_lhs(lhs), m_rhs(rhs), m_size(lhs.size()) {
51 inner_product_assert<Lhs, Rhs>::run(lhs, rhs);
52 }
53
54 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return m_size.value(); }
55
56 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const {
57 return m_func.coeff(m_lhs.coeff(index), m_rhs.coeff(index));
58 }
59
60 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& value, Index index) const {
61 return m_func.coeff(value, m_lhs.coeff(index), m_rhs.coeff(index));
62 }
63
64 template <typename PacketType, int LhsMode = LhsAlignment, int RhsMode = RhsAlignment>
65 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const {
66 return m_func.packet(m_lhs.template packet<LhsMode, PacketType>(index),
67 m_rhs.template packet<RhsMode, PacketType>(index));
68 }
69
70 template <typename PacketType, int LhsMode = LhsAlignment, int RhsMode = RhsAlignment>
71 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(const PacketType& value, Index index) const {
72 return m_func.packet(value, m_lhs.template packet<LhsMode, PacketType>(index),
73 m_rhs.template packet<RhsMode, PacketType>(index));
74 }
75
76 const Func m_func;
77 const evaluator<Lhs> m_lhs;
78 const evaluator<Rhs> m_rhs;
79 const variable_if_dynamic<Index, SizeAtCompileTime> m_size;
80};
81
82template <typename Evaluator, bool Vectorize = Evaluator::Vectorize>
83struct inner_product_impl;
84
85// scalar loop
86template <typename Evaluator>
87struct inner_product_impl<Evaluator, false> {
88 using Scalar = typename Evaluator::Scalar;
89 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval) {
90 const Index size = eval.size();
91 if (size == 0) return Scalar(0);
92
93 Scalar result = eval.coeff(0);
94 for (Index k = 1; k < size; k++) {
95 result = eval.coeff(result, k);
96 }
97
98 return result;
99 }
100};
101
102// vector loop
103template <typename Evaluator>
104struct inner_product_impl<Evaluator, true> {
105 using UnsignedIndex = std::make_unsigned_t<Index>;
106 using Scalar = typename Evaluator::Scalar;
107 using Packet = typename Evaluator::Packet;
108 static constexpr int PacketSize = unpacket_traits<Packet>::size;
109 static constexpr int MaxSize = Evaluator::MaxSizeAtCompileTime;
110 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval) {
111 const UnsignedIndex size = static_cast<UnsignedIndex>(eval.size());
112 if (size < PacketSize) return inner_product_impl<Evaluator, false>::run(eval);
113
114 const UnsignedIndex packetEnd = numext::round_down(size, PacketSize);
115 const UnsignedIndex quadEnd = numext::round_down(size, 4 * PacketSize);
116 const UnsignedIndex numPackets = size / PacketSize;
117 const UnsignedIndex numRemPackets = (packetEnd - quadEnd) / PacketSize;
118
119 Packet presult0 = eval.template packet<Packet>(0 * PacketSize);
120 // Exclude unreachable packet loads for bounded vectors (also avoids GCC bounds warnings).
121 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 2) {
122 if (numPackets >= 2) {
123 Packet presult1 = eval.template packet<Packet>(1 * PacketSize);
124 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 3) {
125 if (numPackets >= 3) {
126 Packet presult2 = eval.template packet<Packet>(2 * PacketSize);
127 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 4) {
128 if (numPackets >= 4) {
129 Packet presult3 = eval.template packet<Packet>(3 * PacketSize);
130
131 for (UnsignedIndex k = 4 * PacketSize; k < quadEnd; k += 4 * PacketSize) {
132 presult0 = eval.packet(presult0, k + 0 * PacketSize);
133 presult1 = eval.packet(presult1, k + 1 * PacketSize);
134 presult2 = eval.packet(presult2, k + 2 * PacketSize);
135 presult3 = eval.packet(presult3, k + 3 * PacketSize);
136 }
137
138 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 5) {
139 if (numRemPackets >= 1) {
140 presult0 = eval.packet(presult0, quadEnd + 0 * PacketSize);
141 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 6) {
142 if (numRemPackets >= 2) {
143 presult1 = eval.packet(presult1, quadEnd + 1 * PacketSize);
144 EIGEN_IF_CONSTEXPR (MaxSize == Dynamic || MaxSize / PacketSize >= 7) {
145 if (numRemPackets == 3) presult2 = eval.packet(presult2, quadEnd + 2 * PacketSize);
146 }
147 }
148 }
149 }
150 }
151
152 presult2 = padd(presult2, presult3);
153 }
154 }
155 presult1 = padd(presult1, presult2);
156 }
157 }
158 presult0 = padd(presult0, presult1);
159 }
160 }
161
162 Scalar result = predux(presult0);
163 for (UnsignedIndex k = packetEnd; k < size; k++) {
164 result = eval.coeff(result, k);
165 }
166
167 return result;
168 }
169};
170
171template <typename LhsScalar, typename RhsScalar, bool Conj>
172struct scalar_inner_product_op {
173 using result_type = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
174 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(const LhsScalar& a, const RhsScalar& b) const {
175 return (conj_if<Conj>()(a) * b);
176 }
177 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(const result_type& accum, const LhsScalar& a,
178 const RhsScalar& b) const {
179 return (conj_if<Conj>()(a) * b) + accum;
180 }
181 static constexpr bool PacketAccess = false;
182};
183
184// Partial specialization for packet access if and only if
185// LhsScalar == RhsScalar == ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType.
186template <typename Scalar, bool Conj>
187struct scalar_inner_product_op<
188 Scalar,
189 std::enable_if_t<std::is_same<typename ScalarBinaryOpTraits<Scalar, Scalar>::ReturnType, Scalar>::value, Scalar>,
190 Conj> {
191 using result_type = Scalar;
192 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& a, const Scalar& b) const {
193 return pmul(conj_if<Conj>()(a), b);
194 }
195 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& accum, const Scalar& a, const Scalar& b) const {
196 return pmadd(conj_if<Conj>()(a), b, accum);
197 }
198 template <typename Packet>
199 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& a, const Packet& b) const {
200 return pmul(conj_if<Conj>().pconj(a), b);
201 }
202 template <typename Packet>
203 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& accum, const Packet& a, const Packet& b) const {
204 return pmadd(conj_if<Conj>().pconj(a), b, accum);
205 }
206 static constexpr bool PacketAccess = packet_traits<Scalar>::HasMul && packet_traits<Scalar>::HasAdd;
207};
208
209// Backends specialize Enable = enable_if_t<...> (void); Enable = false_type always names this generic definition.
210template <typename Lhs, typename Rhs, bool Conj, typename Enable = void>
211struct default_inner_product_impl {
212 using LhsScalar = typename traits<Lhs>::Scalar;
213 using RhsScalar = typename traits<Rhs>::Scalar;
214 using Op = scalar_inner_product_op<LhsScalar, RhsScalar, Conj>;
215 using Evaluator = inner_product_evaluator<Op, Lhs, Rhs>;
216 using result_type = typename Evaluator::Scalar;
217 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run(const MatrixBase<Lhs>& a, const MatrixBase<Rhs>& b) {
218 Evaluator eval(a.derived(), b.derived(), Op());
219 return inner_product_impl<Evaluator>::run(eval);
220 }
221};
222
223template <typename T>
224struct unwrap_unary {
225 using type = T;
226 static constexpr bool HasDirectAccess = bool(traits<type>::Flags & DirectAccessBit);
227
228 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr T const& get(T const& xpr) { return xpr; }
229};
230
231template <typename T>
232struct unwrap_unary<T const> : unwrap_unary<T> {};
233
234template <typename Op, typename Xpr>
235struct unwrap_unary<CwiseUnaryOp<Op, Xpr>> : unwrap_unary<Xpr> {
236 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr typename unwrap_unary<Xpr>::type const& get(
237 CwiseUnaryOp<Op, Xpr> const& xpr) {
238 return unwrap_unary<Xpr>::get(xpr.nestedExpression());
239 }
240};
241
242template <typename T, typename Target>
243struct rewrap_unary {
244 using type = Target;
245
246 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr Target const& apply(T const&, Target const& target) {
247 return target;
248 }
249};
250
251template <typename T, typename Target>
252struct rewrap_unary<T const, Target> : rewrap_unary<T, Target const> {};
253
254template <typename Op, typename Xpr, typename Target>
255struct rewrap_unary<CwiseUnaryOp<Op, Xpr>, Target> {
256 // unaryExpr() nests its operand as const; Xpr itself need not be (adjoint() nests a non-const Transpose).
257 using type = CwiseUnaryOp<Op, std::add_const_t<typename rewrap_unary<Xpr, Target>::type>>;
258
259 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr type apply(CwiseUnaryOp<Op, Xpr> const& base,
260 Target const& target) {
261 return rewrap_unary<Xpr, Target>::apply(base.nestedExpression(), target).unaryExpr(base.functor());
262 }
263};
264
265// Rewrap coefficient-wise unary operations around contiguous maps only when both
266// operands expose storage and at least one lacks a compile-time unit inner stride.
267template <typename Lhs, typename Rhs, bool Conj,
268 bool MayMap = unwrap_unary<Lhs>::HasDirectAccess && unwrap_unary<Rhs>::HasDirectAccess &&
269 (inner_stride_at_compile_time<typename unwrap_unary<Lhs>::type>::value != 1 ||
270 inner_stride_at_compile_time<typename unwrap_unary<Rhs>::type>::value != 1)>
271struct inner_product_dispatch : default_inner_product_impl<Lhs, Rhs, Conj> {};
272
273template <typename Lhs, typename Rhs, bool Conj>
274struct inner_product_dispatch<Lhs, Rhs, Conj, true> {
275 using Impl = default_inner_product_impl<Lhs, Rhs, Conj>;
276 using result_type = typename Impl::result_type;
277
278 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run(const MatrixBase<Lhs>& a, const MatrixBase<Rhs>& b) {
279 EIGEN_IF_CONSTEXPR (Conj) {
280 return run_general(a, b);
281 }
282 // Keep tiny products inlined without the remapping and packet-loop setup.
283 if (a.size() <= 4) {
284 typename Impl::Evaluator eval(a.derived(), b.derived());
285 if (eval.size() == 0) return result_type(0);
286 result_type result = eval.coeff(0);
287 if (eval.size() > 1) result = eval.coeff(result, 1);
288 if (eval.size() > 2) result = eval.coeff(result, 2);
289 if (eval.size() > 3) result = eval.coeff(result, 3);
290 return result;
291 }
292 return run_large_product(a, b);
293 }
294
295 // Keep the larger product kernel out of tiny callers, while dot() retains its existing inlining.
296 static EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE result_type run_large_product(const MatrixBase<Lhs>& a,
297 const MatrixBase<Rhs>& b) {
298 return run_general(a, b);
299 }
300
301 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run_general(const MatrixBase<Lhs>& a,
302 const MatrixBase<Rhs>& b) {
303 using LhsUnwrapper = unwrap_unary<Lhs>;
304 using RhsUnwrapper = unwrap_unary<Rhs>;
305 using LhsInner = typename LhsUnwrapper::type;
306 using RhsInner = typename RhsUnwrapper::type;
307
308 LhsInner const& lhs_inner = LhsUnwrapper::get(a.derived());
309 RhsInner const& rhs_inner = RhsUnwrapper::get(b.derived());
310
311 if (lhs_inner.innerStride() == 1 && rhs_inner.innerStride() == 1) {
312 using LhsMap = Map<Vector<typename LhsInner::Scalar, size_of_xpr_at_compile_time<LhsInner>::value> const,
313 evaluator<LhsInner>::Alignment>;
314 using RhsMap = Map<Vector<typename RhsInner::Scalar, size_of_xpr_at_compile_time<RhsInner>::value> const,
315 evaluator<RhsInner>::Alignment>;
316
317 LhsMap const lhs_map(lhs_inner.data(), lhs_inner.size());
318 RhsMap const rhs_map(rhs_inner.data(), rhs_inner.size());
319
320 using LhsRewrap = rewrap_unary<Lhs, LhsMap>;
321 using RhsRewrap = rewrap_unary<Rhs, RhsMap>;
322
323 return default_inner_product_impl<typename LhsRewrap::type, typename RhsRewrap::type, Conj>::run(
324 LhsRewrap::apply(a.derived(), lhs_map), RhsRewrap::apply(b.derived(), rhs_map));
325 }
326
327 return Impl::run(a, b);
328 }
329};
330
331} // namespace internal
332} // namespace Eigen
333
334#endif // EIGEN_INNER_PRODUCT_EVAL_H
constexpr unsigned int PacketAccessBit
Definition Constants.h:98
constexpr unsigned int DirectAccessBit
Definition Constants.h:160