Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GeneralMatrixVector.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2016 Gael Guennebaud <gael.guennebaud@inria.fr>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_GENERAL_MATRIX_VECTOR_H
12#define EIGEN_GENERAL_MATRIX_VECTOR_H
13
14// IWYU pragma: private
15#include "../InternalHeaderCheck.h"
16
17// C4804: unsafe use of type 'bool' in operation. Unavoidable in generic code
18// instantiated with bool scalars (e.g. += and * on bool).
19#if EIGEN_COMP_MSVC
20#pragma warning(push)
21#pragma warning(disable : 4804)
22#endif
23
24namespace Eigen {
25
26namespace internal {
27
28template <typename Scalar, typename NativePacket, typename ResultPacket, bool Mixed>
29struct gemv_operand_packet {
30 using type = NativePacket;
31};
32
33template <typename Scalar, typename NativePacket, typename ResultPacket>
34struct gemv_operand_packet<Scalar, NativePacket, ResultPacket, true> {
35 using type = std::conditional_t<std::is_same<Scalar, typename unpacket_traits<ResultPacket>::type>::value,
36 ResultPacket, typename unpacket_traits<ResultPacket>::as_real>;
37};
38
39template <typename LhsScalar, typename RhsScalar, int PacketSize_ = GEBPPacketFull, bool EnableMixedPackets = true>
40class gemv_traits {
41 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
42 using RealScalar = typename NumTraits<ResScalar>::Real;
43
44#define PACKET_DECL_COND_POSTFIX(postfix, name, packet_size) \
45 using name##Packet##postfix = \
46 typename packet_conditional<packet_size, typename packet_traits<name##Scalar>::type, \
47 typename packet_traits<name##Scalar>::half, \
48 typename unpacket_traits<typename packet_traits<name##Scalar>::half>::half>::type
49
50 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
51 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
52 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
53#undef PACKET_DECL_COND_POSTFIX
54
55 public:
56 static constexpr bool MixedRealComplex =
57 EnableMixedPackets && (std::is_same<RealScalar, float>::value || std::is_same<RealScalar, double>::value) &&
58 std::is_same<ResScalar, std::complex<RealScalar>>::value &&
59 ((std::is_same<LhsScalar, ResScalar>::value && std::is_same<RhsScalar, RealScalar>::value) ||
60 (std::is_same<LhsScalar, RealScalar>::value && std::is_same<RhsScalar, ResScalar>::value)) &&
61 unpacket_traits<ResPacket_>::vectorizable;
62 enum {
63 Vectorizable =
64 MixedRealComplex || (unpacket_traits<LhsPacket_>::vectorizable && unpacket_traits<RhsPacket_>::vectorizable &&
65 int(unpacket_traits<LhsPacket_>::size) == int(unpacket_traits<RhsPacket_>::size)),
66 // Real component packets duplicate each coefficient, so both operands advance by the complex packet size.
67 LhsPacketSize = Vectorizable ? (MixedRealComplex ? int(unpacket_traits<ResPacket_>::size)
68 : int(unpacket_traits<LhsPacket_>::size))
69 : 1,
70 RhsPacketSize = Vectorizable ? (MixedRealComplex ? int(unpacket_traits<ResPacket_>::size)
71 : int(unpacket_traits<RhsPacket_>::size))
72 : 1,
73 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1
74 };
75
76 using LhsPacket = std::conditional_t<
77 Vectorizable, typename gemv_operand_packet<LhsScalar, LhsPacket_, ResPacket_, MixedRealComplex>::type, LhsScalar>;
78 using RhsPacket = std::conditional_t<
79 Vectorizable, typename gemv_operand_packet<RhsScalar, RhsPacket_, ResPacket_, MixedRealComplex>::type, RhsScalar>;
80 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
81};
82
83// Whether a mapper's coefficients are addressable and consecutive along the given storage order: the BLAS mappers
84// with unit inner stride. Other mappers (e.g. the Tensor contraction input mappers) return coefficients by value.
85template <typename Mapper, int Order>
86struct gemv_mapper_is_contiguous : std::false_type {};
87template <typename Scalar, typename Index, int Order, int Alignment>
88struct gemv_mapper_is_contiguous<blas_data_mapper<Scalar, Index, Order, Alignment, 1>, Order> : std::true_type {};
89template <typename Scalar, typename Index, int Order>
90struct gemv_mapper_is_contiguous<const_blas_data_mapper<Scalar, Index, Order>, Order> : std::true_type {};
91
92template <typename Packet, typename OtherPacket, int Order, int Alignment,
93 bool Duplicate = (std::is_same<typename unpacket_traits<Packet>::type, float>::value ||
94 std::is_same<typename unpacket_traits<Packet>::type, double>::value) &&
95 std::is_same<typename unpacket_traits<OtherPacket>::type,
96 std::complex<typename unpacket_traits<Packet>::type>>::value &&
97 (unpacket_traits<Packet>::size == 2 * unpacket_traits<OtherPacket>::size)>
98struct gemv_packet_loader {
99 template <typename Mapper, typename Index, typename ColIndex>
100 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE Packet run(const Mapper& mapper, Index i, ColIndex j) {
101 return mapper.template load<Packet, Alignment>(i, j);
102 }
103};
104
105template <typename Packet, typename OtherPacket, int Order, int Alignment>
106struct gemv_packet_loader<Packet, OtherPacket, Order, Alignment, true> {
107 template <typename Mapper, typename Index, typename ColIndex>
108 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Mapper& mapper, Index i, ColIndex j) {
109 return run(mapper, i, j, gemv_mapper_is_contiguous<Mapper, Order>());
110 }
111
112 private:
113 template <typename Mapper, typename Index, typename ColIndex>
114 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Mapper& mapper, Index i, ColIndex j, std::true_type) {
115 return ploaddup<Packet>(&mapper(i, j));
116 }
117
118 template <typename Mapper, typename Index, typename ColIndex>
119 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Packet run(const Mapper& mapper, Index i, ColIndex j, std::false_type) {
120 typename unpacket_traits<Packet>::type values[unpacket_traits<Packet>::size];
121 for (Index k = 0; k < unpacket_traits<OtherPacket>::size; ++k) {
122 values[2 * k] = values[2 * k + 1] = mapper(i + (Order == ColMajor ? k : 0), j + (Order == RowMajor ? k : 0));
123 }
124 return ploadu<Packet>(values);
125 }
126};
127
128// Whether the GEMV kernels finish a partial packet with masked segment loads. Segment loads zero the lanes outside
129// the segment, which the row-major kernel relies on before its horizontal reduction.
130template <typename LhsPacket, typename RhsPacket, typename ResPacket>
131using gemv_use_packet_segment =
132 bool_constant<has_packet_segment<ResPacket>::value && std::is_same<LhsPacket, ResPacket>::value &&
133 std::is_same<RhsPacket, ResPacket>::value>;
134
135// Loads count < packet-size consecutive coefficients starting at (i, j), along Order. Only contiguous mappers take
136// the masked load; the kernels never select segments for other mappers, whose branch merely has to compile.
137template <typename Packet, int Order, typename Mapper,
138 bool Contiguous = gemv_mapper_is_contiguous<Mapper, Order>::value>
139struct gemv_segment_loader {
140 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Mapper&, Index, Index, Index) {
141 eigen_internal_assert(false && "segment load from a non-contiguous mapper");
142 return pzero(Packet{});
143 }
144};
145template <typename Packet, int Order, typename Mapper>
146struct gemv_segment_loader<Packet, Order, Mapper, true> {
147 // i and j keep the kernel's index type, which is the mapper's.
148 template <typename RowIndex, typename ColIndex>
149 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Mapper& m, RowIndex i, ColIndex j, Index count) {
150 return ploaduSegment<Packet>(&m(i, j), 0, count);
151 }
152};
153
154/* Optimized col-major matrix * vector product:
155 * This algorithm processes the matrix per vertical panels,
156 * which are then processed horizontally per chunk of 8*PacketSize x 1 vertical segments.
157 *
158 * Mixing type logic: C += alpha * A * B
159 * | A | B |alpha| comments
160 * |real |cplx |cplx | real coefficients are duplicated into component packets
161 * |real |cplx |real | alpha is converted to a cplx when calling the run function
162 * |cplx |real |cplx | invalid, the caller has to do tmp: = A * B; C += alpha*tmp
163 * |cplx |real |real | optimal case, vectorization possible via real-cplx mul
164 *
165 * The same reasoning applies for the transposed case.
166 */
167template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
168 typename RhsMapper, bool ConjugateRhs, int Version>
169struct general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper,
170 ConjugateRhs, Version> {
171 using Traits = gemv_traits<LhsScalar, RhsScalar>;
172 using HalfTraits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketHalf>;
173 using QuarterTraits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketQuarter>;
174
175 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
176
177 using LhsPacket = typename Traits::LhsPacket;
178 using RhsPacket = typename Traits::RhsPacket;
179 using ResPacket = typename Traits::ResPacket;
180
181 using LhsPacketHalf = typename HalfTraits::LhsPacket;
182 using RhsPacketHalf = typename HalfTraits::RhsPacket;
183 using ResPacketHalf = typename HalfTraits::ResPacket;
184
185 using LhsPacketQuarter = typename QuarterTraits::LhsPacket;
186 using RhsPacketQuarter = typename QuarterTraits::RhsPacket;
187 using ResPacketQuarter = typename QuarterTraits::ResPacket;
188
189 EIGEN_DEVICE_FUNC inline static void run(Index rows, Index cols, const LhsMapper& lhs, const RhsMapper& rhs,
190 ResScalar* res, Index resIncr, RhsScalar alpha);
191
192 // Processes N full packets of rows starting at row i and, if Segment, the next count < ResPacketSize rows as one
193 // masked packet, all in a single pass over the columns [j2, jend).
194 template <int N, bool Segment = false>
195 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE void process_rows(
196 Index i, Index j2, Index jend, const LhsMapper& lhs, const RhsMapper& rhs, ResScalar* res,
197 const ResPacket& palpha, conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj, Index count = 0);
198
199 // Finishes the rows from i on, full_packets < 10 full packets followed by 0 < count < ResPacketSize rows, in one
200 // process_rows pass. Out of line: it runs at most once per column block, and only when rows is not a multiple of
201 // the packet size. lhs is taken by value: a reference would make run() keep its copy in memory, which clang fills
202 // with a load that cannot be forwarded from the caller's stores.
203 EIGEN_DEVICE_FUNC static EIGEN_DONT_INLINE void process_segment_tail(
204 std::true_type, Index full_packets, Index i, Index j2, Index jend, LhsMapper lhs, const RhsMapper& rhs,
205 ResScalar* res, const ResPacket& palpha, conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj,
206 Index count);
207 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void process_segment_tail(
208 std::false_type, Index, Index, Index, Index, const LhsMapper&, const RhsMapper&, ResScalar*, const ResPacket&,
209 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>&, Index) {}
210};
211
212// Integer-sequence helper for col-major GEMV full-packet row blocks.
213template <int N>
214struct gemv_colmajor_unroller {
215 template <typename Packet, int... K>
216 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void init_zero_impl(std::integer_sequence<int, K...>, Packet* c) {
217 int unused[] = {0, ((c[K] = pzero(Packet{})), 0)...};
218 EIGEN_UNUSED_VARIABLE(unused);
219 }
220
221 template <typename Packet>
222 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void init_zero(Packet* c) {
223 init_zero_impl(std::make_integer_sequence<int, N>{}, c);
224 }
225
226 template <typename LhsPacket, int LhsStride, int Alignment, typename AccPacket, typename RhsPacket,
227 typename ConjHelper, typename LhsMapper, typename Index, int... K>
228 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void madd_impl(std::integer_sequence<int, K...>, AccPacket* c,
229 const LhsMapper& lhs, Index i, Index j,
230 const RhsPacket& b0, ConjHelper& pcj) {
231 int unused[] = {
232 0,
233 ((c[K] = pcj.pmadd(
234 gemv_packet_loader<LhsPacket, RhsPacket, ColMajor, Alignment>::run(lhs, i + LhsStride * K, j), b0, c[K])),
235 0)...};
236 EIGEN_UNUSED_VARIABLE(unused);
237 }
238
239 template <typename LhsPacket, int LhsStride, int Alignment, typename AccPacket, typename RhsPacket,
240 typename ConjHelper, typename LhsMapper, typename Index>
241 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void madd(AccPacket* c, const LhsMapper& lhs, Index i, Index j,
242 const RhsPacket& b0, ConjHelper& pcj) {
243 madd_impl<LhsPacket, LhsStride, Alignment>(std::make_integer_sequence<int, N>{}, c, lhs, i, j, b0, pcj);
244 }
245
246 template <int K, typename ResPacket, int ResStride, typename ResScalar, typename Index>
247 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void store_one(const ResPacket* c, ResScalar* res, Index i,
248 const ResPacket& palpha) {
249 ResScalar* r = res + i + ResStride * K;
250 pstoreu(r, pmadd(c[K], palpha, ploadu<ResPacket>(r)));
251 }
252
253 template <typename ResPacket, int ResStride, typename ResScalar, typename Index, int... K>
254 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void store_impl(std::integer_sequence<int, K...>, const ResPacket* c,
255 ResScalar* res, Index i, const ResPacket& palpha) {
256 int unused[] = {0, (store_one<K, ResPacket, ResStride>(c, res, i, palpha), 0)...};
257 EIGEN_UNUSED_VARIABLE(unused);
258 }
259
260 template <typename ResPacket, int ResStride, typename ResScalar, typename Index>
261 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void store(const ResPacket* c, ResScalar* res, Index i,
262 const ResPacket& palpha) {
263 store_impl<ResPacket, ResStride>(std::make_integer_sequence<int, N>{}, c, res, i, palpha);
264 }
265};
266
267// No full packets: the final pass may hold only the masked partial packet.
268template <>
269struct gemv_colmajor_unroller<0> {
270 template <typename Packet>
271 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void init_zero(Packet*) {}
272
273 template <typename LhsPacket, int LhsStride, int Alignment, typename AccPacket, typename RhsPacket,
274 typename ConjHelper, typename LhsMapper, typename Index>
275 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void madd(AccPacket*, const LhsMapper&, Index, Index, const RhsPacket&,
276 ConjHelper&) {}
277
278 template <typename ResPacket, int ResStride, typename ResScalar, typename Index>
279 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void store(const ResPacket*, ResScalar*, Index, const ResPacket&) {}
280};
281
282template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
283 typename RhsMapper, bool ConjugateRhs, int Version>
284template <int N, bool Segment>
285EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void
286general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs,
287 Version>::process_rows(Index i, Index j2, Index jend, const LhsMapper& lhs,
288 const RhsMapper& rhs, ResScalar* res, const ResPacket& palpha,
289 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj,
290 Index count) {
291 enum { LhsAlignment = Unaligned, LhsPacketSize = Traits::LhsPacketSize, ResPacketSize = Traits::ResPacketSize };
292 using Unroller = gemv_colmajor_unroller<N>;
293 const Index iseg = i + N * ResPacketSize;
294
295 // c_seg accumulates the masked partial packet when Segment is set. It is not part of c: a wider array grows the
296 // stack frame GCC estimates for run() and stops it from being inlined.
297 ResPacket c[N > 0 ? N : 1];
298 Unroller::init_zero(c);
299 ResPacket c_seg = pzero(ResPacket{});
300 for (Index j = j2; j < jend; ++j) {
301 RhsPacket b0 = pset1<RhsPacket>(rhs(j, 0));
302 Unroller::template madd<LhsPacket, LhsPacketSize, LhsAlignment>(c, lhs, i, j, b0, pcj);
303 EIGEN_IF_CONSTEXPR (Segment) {
304 c_seg = pcj.pmadd(gemv_segment_loader<LhsPacket, ColMajor, LhsMapper>::run(lhs, iseg, j, count), b0, c_seg);
305 }
306 }
307 Unroller::template store<ResPacket, ResPacketSize>(c, res, i, palpha);
308 EIGEN_IF_CONSTEXPR (Segment) {
309 pstoreuSegment(res + iseg, pmadd(c_seg, palpha, ploaduSegment<ResPacket>(res + iseg, 0, count)), 0, count);
310 }
311}
312
313template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
314 typename RhsMapper, bool ConjugateRhs, int Version>
315EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE void general_matrix_vector_product<
316 Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs,
317 Version>::process_segment_tail(std::true_type, Index full_packets, Index i, Index j2, Index jend, LhsMapper lhs,
318 const RhsMapper& rhs, ResScalar* res, const ResPacket& palpha,
319 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>& pcj, Index count) {
320#define EIGEN_GEMV_PROCESS_ROW(n) \
321 case n: \
322 process_rows<n, true>(i, j2, jend, lhs, rhs, res, palpha, pcj, count); \
323 break
324 switch (full_packets) {
325 EIGEN_GEMV_PROCESS_ROW(0);
326 EIGEN_GEMV_PROCESS_ROW(1);
327 EIGEN_GEMV_PROCESS_ROW(2);
328 EIGEN_GEMV_PROCESS_ROW(3);
329 EIGEN_GEMV_PROCESS_ROW(4);
330 EIGEN_GEMV_PROCESS_ROW(5);
331 EIGEN_GEMV_PROCESS_ROW(6);
332 EIGEN_GEMV_PROCESS_ROW(7);
333 EIGEN_GEMV_PROCESS_ROW(8);
334 EIGEN_GEMV_PROCESS_ROW(9);
335 default:
336 eigen_internal_assert(false);
337 break;
338 }
339#undef EIGEN_GEMV_PROCESS_ROW
340}
341
342template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
343 typename RhsMapper, bool ConjugateRhs, int Version>
344EIGEN_DEVICE_FUNC inline void
345general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs,
346 Version>::run(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs,
347 ResScalar* res, Index resIncr, RhsScalar alpha) {
348 EIGEN_UNUSED_VARIABLE(resIncr);
349 eigen_internal_assert(resIncr == 1);
350
351 // BLAS contract: if alpha == 0, the result is unchanged (and lhs/rhs need not be read).
352 if (numext::is_exactly_zero(alpha)) return;
353
354 // The following copy tells the compiler that lhs's attributes are not modified outside this function
355 // This helps GCC to generate proper code.
356 LhsMapper lhs(alhs);
357
358 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
359 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs> pcj;
360 conj_helper<LhsPacketHalf, RhsPacketHalf, ConjugateLhs, ConjugateRhs> pcj_half;
361 conj_helper<LhsPacketQuarter, RhsPacketQuarter, ConjugateLhs, ConjugateRhs> pcj_quarter;
362
363 const Index lhsStride = lhs.stride();
364 // LhsAlignment stays Unaligned; enabling aligned reads would require
365 // propagating the Mapper's Alignment through the run() template, and on
366 // modern x86 aligned/unaligned packet loads are equivalent anyway.
367 enum {
368 LhsAlignment = Unaligned,
369 ResPacketSize = Traits::ResPacketSize,
370 ResPacketSizeHalf = HalfTraits::ResPacketSize,
371 ResPacketSizeQuarter = QuarterTraits::ResPacketSize,
372 LhsPacketSize = Traits::LhsPacketSize,
373 HasHalf = (int)ResPacketSizeHalf < (int)ResPacketSize,
374 HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf,
375 UseSegment = gemv_use_packet_segment<LhsPacket, RhsPacket, ResPacket>::value &&
376 gemv_mapper_is_contiguous<LhsMapper, ColMajor>::value
377 };
378
379 using UnsignedIndex = std::make_unsigned_t<Index>;
380 // With segments, the count trailing rows of a partial packet take one masked pass together with the full packets
381 // before them. The 8-packet blocks then stop while at least 2 full packets remain, so that pass is never a lone
382 // packet whose accumulator chain is bound by the FMA latency. Exact multiples of the packet size keep the 4, 3, 2
383 // and 1-packet passes.
384 const Index count = UseSegment ? Index(UnsignedIndex(rows) % ResPacketSize) : Index(0);
385 const Index n8 = rows - (count > 0 ? 10 : 8) * ResPacketSize + 1;
386 const Index n4 = rows - 4 * ResPacketSize + 1;
387 const Index n3 = rows - 3 * ResPacketSize + 1;
388 const Index n2 = rows - 2 * ResPacketSize + 1;
389 const Index n1 = rows - 1 * ResPacketSize + 1;
390 const Index n_half = rows - 1 * ResPacketSizeHalf + 1;
391 const Index n_quarter = rows - 1 * ResPacketSizeQuarter + 1;
392
393 // Choose block_cols so that one column slice of the LHS roughly fits in L1.
394 // When it does not, fall back to a smaller batch to keep cache pressure down.
395 std::ptrdiff_t l1, l2, l3;
396 manage_caching_sizes(GetAction, &l1, &l2, &l3);
397 const Index block_cols =
398 cols < 128 ? cols : (lhsStride * Index(sizeof(LhsScalar)) < Index(l1) ? Index(16) : Index(4));
399 ResPacket palpha = pset1<ResPacket>(alpha);
400 ResPacketHalf palpha_half = pset1<ResPacketHalf>(alpha);
401 ResPacketQuarter palpha_quarter = pset1<ResPacketQuarter>(alpha);
402
403 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
404 Index jend = numext::mini(j2 + block_cols, cols);
405 Index i = 0;
406 for (; i < n8; i += ResPacketSize * 8) process_rows<8>(i, j2, jend, lhs, rhs, res, palpha, pcj);
407 if (count > 0) {
408 process_segment_tail(bool_constant<UseSegment>(), Index(UnsignedIndex(rows - i) / ResPacketSize), i, j2, jend,
409 lhs, rhs, res, palpha, pcj, count);
410 } else {
411#define EIGEN_GEMV_PROCESS_ROW(k) \
412 if (i < n##k) { \
413 process_rows<k>(i, j2, jend, lhs, rhs, res, palpha, pcj); \
414 i += ResPacketSize * (k); \
415 } \
416 static_assert(true, "Trailing semicolon required")
417 EIGEN_GEMV_PROCESS_ROW(4);
418 EIGEN_GEMV_PROCESS_ROW(3);
419 EIGEN_GEMV_PROCESS_ROW(2);
420 EIGEN_GEMV_PROCESS_ROW(1);
421#undef EIGEN_GEMV_PROCESS_ROW
422 EIGEN_IF_CONSTEXPR (HasHalf) {
423 if (i < n_half) {
424 ResPacketHalf c0 = pzero(ResPacketHalf{});
425 for (Index j = j2; j < jend; j += 1) {
426 RhsPacketHalf b0 = pset1<RhsPacketHalf>(rhs(j, 0));
427 c0 = pcj_half.pmadd(
428 gemv_packet_loader<LhsPacketHalf, RhsPacketHalf, ColMajor, LhsAlignment>::run(lhs, i + 0, j), b0, c0);
429 }
430 pstoreu(res + i + ResPacketSizeHalf * 0,
431 pmadd(c0, palpha_half, ploadu<ResPacketHalf>(res + i + ResPacketSizeHalf * 0)));
432 i += ResPacketSizeHalf;
433 }
434 }
435 EIGEN_IF_CONSTEXPR (HasQuarter) {
436 if (i < n_quarter) {
437 ResPacketQuarter c0 = pzero(ResPacketQuarter{});
438 for (Index j = j2; j < jend; j += 1) {
439 RhsPacketQuarter b0 = pset1<RhsPacketQuarter>(rhs(j, 0));
440 c0 = pcj_quarter.pmadd(
441 gemv_packet_loader<LhsPacketQuarter, RhsPacketQuarter, ColMajor, LhsAlignment>::run(lhs, i + 0, j), b0,
442 c0);
443 }
444 pstoreu(res + i + ResPacketSizeQuarter * 0,
445 pmadd(c0, palpha_quarter, ploadu<ResPacketQuarter>(res + i + ResPacketSizeQuarter * 0)));
446 i += ResPacketSizeQuarter;
447 }
448 }
449 for (; i < rows; ++i) {
450 ResScalar c0(0);
451 for (Index j = j2; j < jend; j += 1) c0 += cj.pmul(lhs(i, j), rhs(j, 0));
452 res[i] += alpha * c0;
453 }
454 }
455 }
456}
457
458/* Optimized row-major matrix * vector product:
459 * This algorithm processes 4 rows at once that allows to both reduce
460 * the number of load/stores of the result by a factor 4 and to reduce
461 * the instruction dependency. Moreover, we know that all bands have the
462 * same alignment pattern.
463 *
464 * Mixing type logic:
465 * - alpha is always a complex (or converted to a complex)
466 * - mixed operands use matching complex/component packets
467 */
468template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
469 typename RhsMapper, bool ConjugateRhs, int Version>
470struct general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper,
471 ConjugateRhs, Version> {
472 using Traits = gemv_traits<LhsScalar, RhsScalar>;
473 using HalfTraits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketHalf>;
474 using QuarterTraits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketQuarter>;
475
476 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
477
478 using LhsPacket = typename Traits::LhsPacket;
479 using RhsPacket = typename Traits::RhsPacket;
480 using ResPacket = typename Traits::ResPacket;
481
482 using LhsPacketHalf = typename HalfTraits::LhsPacket;
483 using RhsPacketHalf = typename HalfTraits::RhsPacket;
484 using ResPacketHalf = typename HalfTraits::ResPacket;
485
486 using LhsPacketQuarter = typename QuarterTraits::LhsPacket;
487 using RhsPacketQuarter = typename QuarterTraits::RhsPacket;
488 using ResPacketQuarter = typename QuarterTraits::ResPacket;
489
490 EIGEN_DEVICE_FUNC static inline void run(Index rows, Index cols, const LhsMapper& lhs, const RhsMapper& rhs,
491 ResScalar* res, Index resIncr, ResScalar alpha);
492
493 // Specialized path for when cols < full packet size.
494 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static void run_small_cols(Index rows, Index cols, const LhsMapper& lhs,
495 const RhsMapper& rhs, ResScalar* res, Index resIncr,
496 ResScalar alpha);
497
498 // Templated helper that processes N rows in run_small_cols. N is a compile-time
499 // constant; row-dimension unrolling is done inside flat helper loops.
500 template <int N>
501 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE void process_rows_small_cols(Index i, Index cols, const LhsMapper& lhs,
502 const RhsMapper& rhs, ResScalar* res,
503 Index resIncr, ResScalar alpha,
504 Index halfColBlockEnd,
505 Index quarterColBlockEnd);
506};
507
508template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
509 typename RhsMapper, bool ConjugateRhs, int Version>
510EIGEN_DEVICE_FUNC inline void
511general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs,
512 Version>::run(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs,
513 ResScalar* res, Index resIncr, ResScalar alpha) {
514 // BLAS contract: if alpha == 0, the result is unchanged (and lhs/rhs need not be read).
515 if (numext::is_exactly_zero(alpha)) return;
516
517 // When cols < full packet size, the main vectorized loops are empty.
518 // Use the sub-packet helper only when half or quarter packets can do useful work;
519 // otherwise it would just duplicate the scalar cleanup.
520 enum {
521 LhsPacketSize_ = Traits::LhsPacketSize,
522 MinUsefulCols_ =
523 ((int)QuarterTraits::LhsPacketSize < (int)HalfTraits::LhsPacketSize)
524 ? (int)QuarterTraits::LhsPacketSize
525 : (((int)HalfTraits::LhsPacketSize < (int)Traits::LhsPacketSize) ? (int)HalfTraits::LhsPacketSize
526 : (int)Traits::LhsPacketSize),
527 HasSubPackets_ = (int)MinUsefulCols_ < (int)LhsPacketSize_,
528 UseSegment_ = gemv_use_packet_segment<LhsPacket, RhsPacket, ResPacket>::value &&
529 gemv_mapper_is_contiguous<LhsMapper, RowMajor>::value &&
530 gemv_mapper_is_contiguous<RhsMapper, ColMajor>::value,
531 // With segments, one masked full packet per row beats a half packet followed by two or more scalar columns.
532 SmallColsEnd_ = UseSegment_ ? (int)HalfTraits::LhsPacketSize + 2 : (int)LhsPacketSize_
533 };
534 EIGEN_IF_CONSTEXPR (HasSubPackets_) {
535 if (cols >= MinUsefulCols_) {
536 if (cols < SmallColsEnd_ && cols < LhsPacketSize_) {
537 run_small_cols(rows, cols, alhs, rhs, res, resIncr, alpha);
538 return;
539 }
540 }
541 }
542
543 // The following copy tells the compiler that lhs's attributes are not modified outside this function
544 // This helps GCC to generate proper code.
545 LhsMapper lhs(alhs);
546
547 eigen_internal_assert(rhs.stride() == 1);
548 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
549 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs> pcj;
550 conj_helper<LhsPacketHalf, RhsPacketHalf, ConjugateLhs, ConjugateRhs> pcj_half;
551 conj_helper<LhsPacketQuarter, RhsPacketQuarter, ConjugateLhs, ConjugateRhs> pcj_quarter;
552
553 // Disable the 8-row inner unroll once a single column slice no longer fits in L1; with very
554 // large LHS strides each unrolled iteration evicts the previously-loaded rows from cache.
555 std::ptrdiff_t l1, l2, l3;
556 manage_caching_sizes(GetAction, &l1, &l2, &l3);
557 const Index n8 = lhs.stride() * Index(sizeof(LhsScalar)) > Index(l1) ? 0 : rows - 7;
558 const Index n4 = rows - 3;
559 const Index n2 = rows - 1;
560
561 // LhsAlignment stays Unaligned; enabling aligned reads would require
562 // propagating the Mapper's Alignment through the run() template, and on
563 // modern x86 aligned/unaligned packet loads are equivalent anyway.
564 enum {
565 LhsAlignment = Unaligned,
566 ResPacketSize = Traits::ResPacketSize,
567 ResPacketSizeHalf = HalfTraits::ResPacketSize,
568 ResPacketSizeQuarter = QuarterTraits::ResPacketSize,
569 LhsPacketSize = Traits::LhsPacketSize,
570 LhsPacketSizeHalf = HalfTraits::LhsPacketSize,
571 LhsPacketSizeQuarter = QuarterTraits::LhsPacketSize,
572 HasHalf = (int)ResPacketSizeHalf < (int)ResPacketSize,
573 HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf,
574 UseSegment = UseSegment_
575 };
576
577 using UnsignedIndex = std::make_unsigned_t<Index>;
578 const Index fullColBlockEnd = LhsPacketSize * (UnsignedIndex(cols) / LhsPacketSize);
579 const Index halfColBlockEnd = LhsPacketSizeHalf * (UnsignedIndex(cols) / LhsPacketSizeHalf);
580 const Index quarterColBlockEnd = LhsPacketSizeQuarter * (UnsignedIndex(cols) / LhsPacketSizeQuarter);
581 // With segments, the last cols - fullColBlockEnd < LhsPacketSize columns take one masked packet per row and the
582 // scalar column loops below are empty.
583 const Index segmentCount = cols - fullColBlockEnd;
584 const Index scalarColStart = UseSegment ? cols : fullColBlockEnd;
585 using LhsSegmentLoader = gemv_segment_loader<LhsPacket, RowMajor, LhsMapper>;
586 using RhsSegmentLoader = gemv_segment_loader<RhsPacket, ColMajor, RhsMapper>;
587
588 Index i = 0;
589 for (; i < n8; i += 8) {
590 ResPacket c0 = pzero(ResPacket{}), c1 = pzero(ResPacket{}), c2 = pzero(ResPacket{}), c3 = pzero(ResPacket{}),
591 c4 = pzero(ResPacket{}), c5 = pzero(ResPacket{}), c6 = pzero(ResPacket{}), c7 = pzero(ResPacket{});
592
593 for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) {
594 RhsPacket b0 = gemv_packet_loader<RhsPacket, LhsPacket, ColMajor, Unaligned>::run(rhs, j, 0);
595
596 c0 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 0, j), b0, c0);
597 c1 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 1, j), b0, c1);
598 c2 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 2, j), b0, c2);
599 c3 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 3, j), b0, c3);
600 c4 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 4, j), b0, c4);
601 c5 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 5, j), b0, c5);
602 c6 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 6, j), b0, c6);
603 c7 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 7, j), b0, c7);
604 }
605 if (UseSegment && segmentCount > 0) {
606 RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount);
607 c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 0, fullColBlockEnd, segmentCount), b0, c0);
608 c1 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 1, fullColBlockEnd, segmentCount), b0, c1);
609 c2 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 2, fullColBlockEnd, segmentCount), b0, c2);
610 c3 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 3, fullColBlockEnd, segmentCount), b0, c3);
611 c4 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 4, fullColBlockEnd, segmentCount), b0, c4);
612 c5 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 5, fullColBlockEnd, segmentCount), b0, c5);
613 c6 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 6, fullColBlockEnd, segmentCount), b0, c6);
614 c7 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 7, fullColBlockEnd, segmentCount), b0, c7);
615 }
616 ResScalar cc0 = predux(c0);
617 ResScalar cc1 = predux(c1);
618 ResScalar cc2 = predux(c2);
619 ResScalar cc3 = predux(c3);
620 ResScalar cc4 = predux(c4);
621 ResScalar cc5 = predux(c5);
622 ResScalar cc6 = predux(c6);
623 ResScalar cc7 = predux(c7);
624
625 for (Index j = scalarColStart; j < cols; ++j) {
626 RhsScalar b0 = rhs(j, 0);
627
628 cc0 += cj.pmul(lhs(i + 0, j), b0);
629 cc1 += cj.pmul(lhs(i + 1, j), b0);
630 cc2 += cj.pmul(lhs(i + 2, j), b0);
631 cc3 += cj.pmul(lhs(i + 3, j), b0);
632 cc4 += cj.pmul(lhs(i + 4, j), b0);
633 cc5 += cj.pmul(lhs(i + 5, j), b0);
634 cc6 += cj.pmul(lhs(i + 6, j), b0);
635 cc7 += cj.pmul(lhs(i + 7, j), b0);
636 }
637 res[(i + 0) * resIncr] += alpha * cc0;
638 res[(i + 1) * resIncr] += alpha * cc1;
639 res[(i + 2) * resIncr] += alpha * cc2;
640 res[(i + 3) * resIncr] += alpha * cc3;
641 res[(i + 4) * resIncr] += alpha * cc4;
642 res[(i + 5) * resIncr] += alpha * cc5;
643 res[(i + 6) * resIncr] += alpha * cc6;
644 res[(i + 7) * resIncr] += alpha * cc7;
645 }
646 for (; i < n4; i += 4) {
647 ResPacket c0 = pzero(ResPacket{}), c1 = pzero(ResPacket{}), c2 = pzero(ResPacket{}), c3 = pzero(ResPacket{});
648
649 for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) {
650 RhsPacket b0 = gemv_packet_loader<RhsPacket, LhsPacket, ColMajor, Unaligned>::run(rhs, j, 0);
651
652 c0 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 0, j), b0, c0);
653 c1 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 1, j), b0, c1);
654 c2 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 2, j), b0, c2);
655 c3 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 3, j), b0, c3);
656 }
657 if (UseSegment && segmentCount > 0) {
658 RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount);
659 c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 0, fullColBlockEnd, segmentCount), b0, c0);
660 c1 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 1, fullColBlockEnd, segmentCount), b0, c1);
661 c2 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 2, fullColBlockEnd, segmentCount), b0, c2);
662 c3 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 3, fullColBlockEnd, segmentCount), b0, c3);
663 }
664 ResScalar cc0 = predux(c0);
665 ResScalar cc1 = predux(c1);
666 ResScalar cc2 = predux(c2);
667 ResScalar cc3 = predux(c3);
668
669 for (Index j = scalarColStart; j < cols; ++j) {
670 RhsScalar b0 = rhs(j, 0);
671
672 cc0 += cj.pmul(lhs(i + 0, j), b0);
673 cc1 += cj.pmul(lhs(i + 1, j), b0);
674 cc2 += cj.pmul(lhs(i + 2, j), b0);
675 cc3 += cj.pmul(lhs(i + 3, j), b0);
676 }
677 res[(i + 0) * resIncr] += alpha * cc0;
678 res[(i + 1) * resIncr] += alpha * cc1;
679 res[(i + 2) * resIncr] += alpha * cc2;
680 res[(i + 3) * resIncr] += alpha * cc3;
681 }
682 for (; i < n2; i += 2) {
683 ResPacket c0 = pzero(ResPacket{}), c1 = pzero(ResPacket{});
684
685 for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) {
686 RhsPacket b0 = gemv_packet_loader<RhsPacket, LhsPacket, ColMajor, Unaligned>::run(rhs, j, 0);
687
688 c0 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 0, j), b0, c0);
689 c1 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i + 1, j), b0, c1);
690 }
691 if (UseSegment && segmentCount > 0) {
692 RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount);
693 c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 0, fullColBlockEnd, segmentCount), b0, c0);
694 c1 = pcj.pmadd(LhsSegmentLoader::run(lhs, i + 1, fullColBlockEnd, segmentCount), b0, c1);
695 }
696 ResScalar cc0 = predux(c0);
697 ResScalar cc1 = predux(c1);
698
699 for (Index j = scalarColStart; j < cols; ++j) {
700 RhsScalar b0 = rhs(j, 0);
701
702 cc0 += cj.pmul(lhs(i + 0, j), b0);
703 cc1 += cj.pmul(lhs(i + 1, j), b0);
704 }
705 res[(i + 0) * resIncr] += alpha * cc0;
706 res[(i + 1) * resIncr] += alpha * cc1;
707 }
708 for (; i < rows; ++i) {
709 ResPacket c0 = pzero(ResPacket{});
710 ResPacketHalf c0_h = pzero(ResPacketHalf{});
711 ResPacketQuarter c0_q = pzero(ResPacketQuarter{});
712
713 for (Index j = 0; j < fullColBlockEnd; j += LhsPacketSize) {
714 RhsPacket b0 = gemv_packet_loader<RhsPacket, LhsPacket, ColMajor, Unaligned>::run(rhs, j, 0);
715 c0 = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsPacket, RowMajor, LhsAlignment>::run(lhs, i, j), b0, c0);
716 }
717 if (UseSegment && segmentCount > 0) {
718 RhsPacket b0 = RhsSegmentLoader::run(rhs, fullColBlockEnd, 0, segmentCount);
719 c0 = pcj.pmadd(LhsSegmentLoader::run(lhs, i, fullColBlockEnd, segmentCount), b0, c0);
720 }
721 ResScalar cc0 = predux(c0);
722 EIGEN_IF_CONSTEXPR (HasHalf && !UseSegment) {
723 for (Index j = fullColBlockEnd; j < halfColBlockEnd; j += LhsPacketSizeHalf) {
724 RhsPacketHalf b0 = gemv_packet_loader<RhsPacketHalf, LhsPacketHalf, ColMajor, Unaligned>::run(rhs, j, 0);
725 c0_h = pcj_half.pmadd(gemv_packet_loader<LhsPacketHalf, RhsPacketHalf, RowMajor, LhsAlignment>::run(lhs, i, j),
726 b0, c0_h);
727 }
728 cc0 += predux(c0_h);
729 }
730 EIGEN_IF_CONSTEXPR (HasQuarter && !UseSegment) {
731 for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) {
732 RhsPacketQuarter b0 =
733 gemv_packet_loader<RhsPacketQuarter, LhsPacketQuarter, ColMajor, Unaligned>::run(rhs, j, 0);
734 c0_q = pcj_quarter.pmadd(
735 gemv_packet_loader<LhsPacketQuarter, RhsPacketQuarter, RowMajor, LhsAlignment>::run(lhs, i, j), b0, c0_q);
736 }
737 cc0 += predux(c0_q);
738 }
739 for (Index j = UseSegment ? cols : quarterColBlockEnd; j < cols; ++j) {
740 cc0 += cj.pmul(lhs(i, j), rhs(j, 0));
741 }
742 res[i * resIncr] += alpha * cc0;
743 }
744}
745
746// Integer-sequence helper for process_rows_small_cols.
747template <int N>
748struct gemv_small_cols_unroller {
749 template <typename LhsPacket, typename AccPacket, int Alignment, typename RhsType, typename ConjHelper,
750 typename LhsMapper, typename Index, int... K>
751 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void madd_impl(std::integer_sequence<int, K...>, AccPacket* acc,
752 const LhsMapper& lhs, Index i, Index j, const RhsType& b0,
753 ConjHelper& pcj) {
754 int unused[] = {
755 0, ((acc[K] = pcj.pmadd(gemv_packet_loader<LhsPacket, RhsType, RowMajor, Alignment>::run(lhs, i + K, j), b0,
756 acc[K])),
757 0)...};
758 EIGEN_UNUSED_VARIABLE(unused);
759 }
760
761 template <typename LhsPacket, typename AccPacket, int Alignment, typename RhsType, typename ConjHelper,
762 typename LhsMapper, typename Index>
763 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void madd(AccPacket* acc, const LhsMapper& lhs, Index i, Index j,
764 const RhsType& b0, ConjHelper& pcj) {
765 madd_impl<LhsPacket, AccPacket, Alignment>(std::make_integer_sequence<int, N>{}, acc, lhs, i, j, b0, pcj);
766 }
767
768 template <typename ResScalar, typename RhsScalar, typename ConjHelper, typename LhsMapper, typename Index, int... K>
769 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void scalar_madd_impl(std::integer_sequence<int, K...>, ResScalar* cc,
770 const LhsMapper& lhs, Index i, Index j,
771 const RhsScalar& b0, ConjHelper& cj) {
772 int unused[] = {0, ((cc[K] += cj.pmul(lhs(i + K, j), b0)), 0)...};
773 EIGEN_UNUSED_VARIABLE(unused);
774 }
775
776 template <typename ResScalar, typename RhsScalar, typename ConjHelper, typename LhsMapper, typename Index>
777 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void scalar_madd(ResScalar* cc, const LhsMapper& lhs, Index i, Index j,
778 const RhsScalar& b0, ConjHelper& cj) {
779 scalar_madd_impl(std::make_integer_sequence<int, N>{}, cc, lhs, i, j, b0, cj);
780 }
781
782 template <typename Scalar, typename Packet, int... K>
783 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void predux_accum_impl(std::integer_sequence<int, K...>, Scalar* cc,
784 const Packet* acc) {
785 int unused[] = {0, ((cc[K] += predux(acc[K])), 0)...};
786 EIGEN_UNUSED_VARIABLE(unused);
787 }
788
789 template <typename Scalar, typename Packet>
790 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void predux_accum(Scalar* cc, const Packet* acc) {
791 predux_accum_impl(std::make_integer_sequence<int, N>{}, cc, acc);
792 }
793
794 template <typename Packet, int... K>
795 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void init_zero_impl(std::integer_sequence<int, K...>, Packet* acc) {
796 int unused[] = {0, ((acc[K] = pzero(Packet{})), 0)...};
797 EIGEN_UNUSED_VARIABLE(unused);
798 }
799
800 template <typename Packet>
801 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void init_zero(Packet* acc) {
802 init_zero_impl(std::make_integer_sequence<int, N>{}, acc);
803 }
804
805 template <typename Scalar, typename Index, int... K>
806 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void write_result_impl(std::integer_sequence<int, K...>, Scalar* res,
807 Index resIncr, Index i, Scalar alpha,
808 const Scalar* cc) {
809 int unused[] = {0, ((res[(i + K) * resIncr] += alpha * cc[K]), 0)...};
810 EIGEN_UNUSED_VARIABLE(unused);
811 }
812
813 template <typename Scalar, typename Index>
814 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void write_result(Scalar* res, Index resIncr, Index i, Scalar alpha,
815 const Scalar* cc) {
816 write_result_impl(std::make_integer_sequence<int, N>{}, res, resIncr, i, alpha, cc);
817 }
818};
819
820template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
821 typename RhsMapper, bool ConjugateRhs, int Version>
822template <int N>
823EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void
824general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs,
825 Version>::process_rows_small_cols(Index i, Index cols, const LhsMapper& lhs,
826 const RhsMapper& rhs, ResScalar* res, Index resIncr,
827 ResScalar alpha, Index halfColBlockEnd,
828 Index quarterColBlockEnd) {
829 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
830 conj_helper<LhsPacketHalf, RhsPacketHalf, ConjugateLhs, ConjugateRhs> pcj_half;
831 conj_helper<LhsPacketQuarter, RhsPacketQuarter, ConjugateLhs, ConjugateRhs> pcj_quarter;
832
833 enum {
834 LhsAlignment = Unaligned,
835 ResPacketSizeHalf = HalfTraits::ResPacketSize,
836 ResPacketSizeQuarter = QuarterTraits::ResPacketSize,
837 LhsPacketSizeHalf = HalfTraits::LhsPacketSize,
838 LhsPacketSizeQuarter = QuarterTraits::LhsPacketSize,
839 HasHalf = (int)ResPacketSizeHalf < (int)Traits::ResPacketSize,
840 HasQuarter = (int)ResPacketSizeQuarter < (int)ResPacketSizeHalf
841 };
842
843 using Unroll = gemv_small_cols_unroller<N>;
844
845 ResScalar cc[N] = {};
846 EIGEN_IF_CONSTEXPR (HasHalf) {
847 ResPacketHalf h[N];
848 Unroll::init_zero(h);
849 for (Index j = 0; j < halfColBlockEnd; j += LhsPacketSizeHalf) {
850 RhsPacketHalf b0 = gemv_packet_loader<RhsPacketHalf, LhsPacketHalf, ColMajor, Unaligned>::run(rhs, j, 0);
851 Unroll::template madd<LhsPacketHalf, ResPacketHalf, LhsAlignment>(h, lhs, i, j, b0, pcj_half);
852 }
853 Unroll::predux_accum(cc, h);
854 }
855 EIGEN_IF_CONSTEXPR (HasQuarter) {
856 ResPacketQuarter q[N];
857 Unroll::init_zero(q);
858 for (Index j = halfColBlockEnd; j < quarterColBlockEnd; j += LhsPacketSizeQuarter) {
859 RhsPacketQuarter b0 = gemv_packet_loader<RhsPacketQuarter, LhsPacketQuarter, ColMajor, Unaligned>::run(rhs, j, 0);
860 Unroll::template madd<LhsPacketQuarter, ResPacketQuarter, LhsAlignment>(q, lhs, i, j, b0, pcj_quarter);
861 }
862 Unroll::predux_accum(cc, q);
863 }
864 for (Index j = quarterColBlockEnd; j < cols; ++j) {
865 RhsScalar b0 = rhs(j, 0);
866 Unroll::scalar_madd(cc, lhs, i, j, b0, cj);
867 }
868 Unroll::write_result(res, resIncr, i, alpha, cc);
869}
870
871template <typename Index, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, typename RhsScalar,
872 typename RhsMapper, bool ConjugateRhs, int Version>
873EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void
874general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper, ConjugateRhs,
875 Version>::run_small_cols(Index rows, Index cols, const LhsMapper& alhs,
876 const RhsMapper& rhs, ResScalar* res, Index resIncr,
877 ResScalar alpha) {
878 LhsMapper lhs(alhs);
879 eigen_internal_assert(rhs.stride() == 1);
880
881 enum {
882 LhsPacketSizeHalf = HalfTraits::LhsPacketSize,
883 LhsPacketSizeQuarter = QuarterTraits::LhsPacketSize,
884 };
885
886 using UnsignedIndex = std::make_unsigned_t<Index>;
887 const Index halfColBlockEnd = LhsPacketSizeHalf * (UnsignedIndex(cols) / LhsPacketSizeHalf);
888 const Index quarterColBlockEnd = LhsPacketSizeQuarter * (UnsignedIndex(cols) / LhsPacketSizeQuarter);
889
890 // Disable the 8-row inner unroll once a single column slice no longer fits in L1; with very
891 // large LHS strides each unrolled iteration evicts the previously-loaded rows from cache.
892 std::ptrdiff_t l1, l2, l3;
893 manage_caching_sizes(GetAction, &l1, &l2, &l3);
894 const Index n8 = lhs.stride() * Index(sizeof(LhsScalar)) > Index(l1) ? 0 : rows - 7;
895 const Index n4 = rows - 3;
896 const Index n2 = rows - 1;
897
898 Index i = 0;
899 for (; i < n8; i += 8) {
900 process_rows_small_cols<8>(i, cols, lhs, rhs, res, resIncr, alpha, halfColBlockEnd, quarterColBlockEnd);
901 }
902 // Process remaining groups of 4 rows in case n8 was 0.
903 for (; i < n4; i += 4) {
904 process_rows_small_cols<4>(i, cols, lhs, rhs, res, resIncr, alpha, halfColBlockEnd, quarterColBlockEnd);
905 }
906 if (i < n2) {
907 process_rows_small_cols<2>(i, cols, lhs, rhs, res, resIncr, alpha, halfColBlockEnd, quarterColBlockEnd);
908 i += 2;
909 }
910 if (i < rows) {
911 process_rows_small_cols<1>(i, cols, lhs, rhs, res, resIncr, alpha, halfColBlockEnd, quarterColBlockEnd);
912 }
913}
914
915} // end namespace internal
916
917} // end namespace Eigen
918
919#if EIGEN_COMP_MSVC
920#pragma warning(pop)
921#endif
922
923#endif // EIGEN_GENERAL_MATRIX_VECTOR_H
@ Unaligned
Definition Constants.h:236
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321