Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GeneralBlockPanelKernel.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_GENERAL_BLOCK_PANEL_H
12#define EIGEN_GENERAL_BLOCK_PANEL_H
13
14// IWYU pragma: private
15#include "../InternalHeaderCheck.h"
16
17// C4804: unsafe use of type 'bool' in operation. Unavoidable in generic code
18// instantiated with bool scalars (e.g. += and * on bool).
19#if EIGEN_COMP_MSVC
20#pragma warning(push)
21#pragma warning(disable : 4804)
22#endif
23
24namespace Eigen {
25
26namespace internal {
27
28enum GEBPPacketSizeType { GEBPPacketFull = 0, GEBPPacketHalf, GEBPPacketQuarter };
29
30template <typename LhsScalar_, typename RhsScalar_, bool ConjLhs_ = false, bool ConjRhs_ = false,
31 int Arch = Architecture::Target, int PacketSize_ = GEBPPacketFull>
32class gebp_traits;
33
35inline std::ptrdiff_t manage_caching_sizes_helper(std::ptrdiff_t a, std::ptrdiff_t b) { return a <= 0 ? b : a; }
36
37#if defined(EIGEN_DEFAULT_L1_CACHE_SIZE)
38#define EIGEN_SET_DEFAULT_L1_CACHE_SIZE(val) EIGEN_DEFAULT_L1_CACHE_SIZE
39#else
40#define EIGEN_SET_DEFAULT_L1_CACHE_SIZE(val) val
41#endif // defined(EIGEN_DEFAULT_L1_CACHE_SIZE)
42
43#if defined(EIGEN_DEFAULT_L2_CACHE_SIZE)
44#define EIGEN_SET_DEFAULT_L2_CACHE_SIZE(val) EIGEN_DEFAULT_L2_CACHE_SIZE
45#else
46#define EIGEN_SET_DEFAULT_L2_CACHE_SIZE(val) val
47#endif // defined(EIGEN_DEFAULT_L2_CACHE_SIZE)
48
49#if defined(EIGEN_DEFAULT_L3_CACHE_SIZE)
50#define EIGEN_SET_DEFAULT_L3_CACHE_SIZE(val) EIGEN_DEFAULT_L3_CACHE_SIZE
51#else
52#define EIGEN_SET_DEFAULT_L3_CACHE_SIZE(val) val
53#endif // defined(EIGEN_DEFAULT_L3_CACHE_SIZE)
54
55#if EIGEN_ARCH_i386_OR_x86_64
56const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(32 * 1024);
57const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(256 * 1024);
58const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(2 * 1024 * 1024);
59#elif EIGEN_ARCH_PPC
60const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(64 * 1024);
61#ifdef _ARCH_PWR10
62const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(2 * 1024 * 1024);
63const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(8 * 1024 * 1024);
64#else
65const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512 * 1024);
66const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(4 * 1024 * 1024);
67#endif
68#elif EIGEN_ARCH_ARM_OR_ARM64
69const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(64 * 1024);
70const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(1024 * 1024);
71const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(4 * 1024 * 1024);
72#else
73const std::ptrdiff_t defaultL1CacheSize = EIGEN_SET_DEFAULT_L1_CACHE_SIZE(16 * 1024);
74const std::ptrdiff_t defaultL2CacheSize = EIGEN_SET_DEFAULT_L2_CACHE_SIZE(512 * 1024);
75const std::ptrdiff_t defaultL3CacheSize = EIGEN_SET_DEFAULT_L3_CACHE_SIZE(512 * 1024);
76#endif
77
78#undef EIGEN_SET_DEFAULT_L1_CACHE_SIZE
79#undef EIGEN_SET_DEFAULT_L2_CACHE_SIZE
80#undef EIGEN_SET_DEFAULT_L3_CACHE_SIZE
81
83struct CacheSizes {
84 CacheSizes() : m_l1(-1), m_l2(-1), m_l3(-1), m_l3_per_cpu(0) {
85 std::ptrdiff_t l1CacheSize, l2CacheSize, l3CacheSize;
86 queryCacheSizes(l1CacheSize, l2CacheSize, l3CacheSize, m_l3_per_cpu);
87 m_l1 = manage_caching_sizes_helper(l1CacheSize, defaultL1CacheSize);
88 m_l2 = manage_caching_sizes_helper(l2CacheSize, defaultL2CacheSize);
89 m_l3 = manage_caching_sizes_helper(l3CacheSize, defaultL3CacheSize);
90 }
91
92 std::ptrdiff_t m_l1;
93 std::ptrdiff_t m_l2;
94 std::ptrdiff_t m_l3;
95 // Bytes of L3 backing one CPU, or 0 when unknown. Cleared by setCpuCacheSizes so that an
96 // explicit override is never overruled by the detected geometry.
97 std::ptrdiff_t m_l3_per_cpu;
98};
99
101inline void manage_caching_sizes(Action action, std::ptrdiff_t* l1, std::ptrdiff_t* l2, std::ptrdiff_t* l3,
102 std::ptrdiff_t* l3_per_cpu = nullptr) {
103 static CacheSizes m_cacheSizes;
104
105 if (action == SetAction) {
106 // set the cpu cache size and cache all block sizes from a global cache size in byte
107 eigen_internal_assert(l1 != 0 && l2 != 0);
108 m_cacheSizes.m_l1 = *l1;
109 m_cacheSizes.m_l2 = *l2;
110 m_cacheSizes.m_l3 = *l3;
111 m_cacheSizes.m_l3_per_cpu = l3_per_cpu != nullptr ? *l3_per_cpu : 0;
112 } else if (action == GetAction) {
113 eigen_internal_assert(l1 != 0 && l2 != 0);
114 *l1 = m_cacheSizes.m_l1;
115 *l2 = m_cacheSizes.m_l2;
116 if (l3_per_cpu != nullptr) *l3_per_cpu = m_cacheSizes.m_l3_per_cpu;
117 *l3 = m_cacheSizes.m_l3;
118 } else {
119 eigen_internal_assert(false);
120 }
121}
122
123/* Helper for computeProductBlockingSizes.
124 *
125 * Given a m x k times k x n matrix product of scalar types \c LhsScalar and \c RhsScalar,
126 * this function computes the blocking size parameters along the respective dimensions
127 * for matrix products and related algorithms. The blocking sizes depends on various
128 * parameters:
129 * - the L1 and L2 cache sizes,
130 * - the register level blocking sizes defined by gebp_traits,
131 * - the number of scalars that fit into a packet (when vectorization is enabled).
132 *
133 * \sa setCpuCacheSizes */
134
135// GCC clones a function for constant arguments (IPA-CP, IPA-SRA), which would compile a runtime overwrite flag's
136// body twice again; clang 18 and Apple clang 21 did not in our builds.
137#if EIGEN_COMP_GNUC_STRICT
138#define EIGEN_GEMM_NO_CLONE __attribute__((noclone))
139#else
140#define EIGEN_GEMM_NO_CLONE
141#endif
142
143// Tag for the gebp_kernel call that assigns C = alpha * A' * B' over the block instead of accumulating into it,
144// independently of C's prior contents. A kernel offers it by declaring `static constexpr bool CanOverwrite = true`
145// next to that overload; every other kernel, including user specializations, keeps the accumulate-only contract and
146// the GEMM driver zeroes the result before calling it. A specialization inheriting a true CanOverwrite while hiding
147// the tagged overload must declare it false.
148struct gebp_overwrite_t {};
149template <typename Gebp, typename = void>
150struct gebp_kernel_overwrites : std::false_type {};
151template <typename Gebp>
152struct gebp_kernel_overwrites<Gebp, std::enable_if_t<Gebp::CanOverwrite>> : std::true_type {};
153
154// Whether an overwriting kernel should overwrite a rows x cols x depth product rather than have the driver zero the
155// result first: Gebp::overwrite_pays where the kernel declares it, true otherwise.
156template <typename Gebp, typename Index>
157EIGEN_ALWAYS_INLINE bool gebp_overwrite_pays_impl(int, Index rows, Index cols, Index depth,
158 decltype(Gebp::overwrite_pays(rows, cols, depth))* = nullptr) {
159 return Gebp::overwrite_pays(rows, cols, depth);
160}
161template <typename Gebp, typename Index>
162EIGEN_ALWAYS_INLINE bool gebp_overwrite_pays_impl(long, Index, Index, Index) {
163 return true;
164}
165template <typename Gebp, typename Index>
166EIGEN_ALWAYS_INLINE bool gebp_overwrite_pays(Index rows, Index cols, Index depth) {
167 return gebp_overwrite_pays_impl<Gebp>(0, rows, cols, depth);
168}
169
170#ifdef EIGEN_VECTORIZE_SME
171// True for the scalar pairs the SME gebp_kernel specializes (see
172// arch/SME/GeneralBlockPanelKernel.h, which static_asserts that it agrees with
173// this list); every other pair keeps Eigen's generic kernel, packers, cache
174// blocking and GEMM loop order.
175template <typename LhsScalar, typename RhsScalar>
176struct sme_has_gebp_kernel : std::false_type {};
177template <>
178struct sme_has_gebp_kernel<float, float> : std::true_type {};
179#ifdef EIGEN_VECTORIZE_SME_F64F64
180template <>
181struct sme_has_gebp_kernel<double, double> : std::true_type {};
182#endif
183// A complex accumulator is a pair of the corresponding real ZA tiles, so the
184// complex kernels exist exactly where the real ones do.
185template <typename RealScalar>
186struct sme_has_gebp_kernel<std::complex<RealScalar>, std::complex<RealScalar>>
187 : sme_has_gebp_kernel<RealScalar, RealScalar> {};
188
189// Overridable SME packed-panel budgets. The defaults are empirically tuned
190// fp32 working-set limits for Apple M4 — heuristic budgets, not generic ARM64
191// cache defaults; redefine them to retune for other SME implementations.
192#ifndef EIGEN_SME_MAX_KC
193#define EIGEN_SME_MAX_KC 2048
194#endif
195#ifndef EIGEN_SME_PACKED_RHS_BUDGET_BYTES
196#define EIGEN_SME_PACKED_RHS_BUDGET_BYTES (32 * 1024 * 1024)
197#endif
198#ifndef EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES
199#define EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES (7 * 1024 * 1024)
200#endif
201#ifndef EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES
202#define EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES (4 * 1024 * 1024)
203#endif
204
205template <typename LhsScalar, typename RhsScalar, typename Index>
206void evaluateProductBlockingSizesHeuristicForSme(Index& k, Index& m, Index& n) {
207 using Traits = gebp_traits<LhsScalar, RhsScalar>;
208
209 const Index mr = static_cast<Index>(Traits::mr);
210 const Index nr = static_cast<Index>(Traits::nr);
211
212#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
213 // Fixed scaled-down budgets so that test-sized products (see
214 // EIGEN_TEST_MAX_SIZE) exercise multi-pass blocking along all three
215 // dimensions. Like the l1/l2/l3 reduction applied to the generic heuristic
216 // below, this intentionally overrides any user-configured budgets.
217 constexpr Index sme_max_kc = static_cast<Index>(128);
218 constexpr Index sme_packed_rhs_budget_bytes = static_cast<Index>(128 * 1024);
219 constexpr Index sme_lhs_working_set_budget_bytes = static_cast<Index>(128 * 1024);
220 constexpr Index sme_single_pass_rhs_budget_bytes = static_cast<Index>(128 * 1024);
221#else
222 constexpr Index sme_max_kc = static_cast<Index>(EIGEN_SME_MAX_KC);
223 constexpr Index sme_packed_rhs_budget_bytes = static_cast<Index>(EIGEN_SME_PACKED_RHS_BUDGET_BYTES);
224 constexpr Index sme_lhs_working_set_budget_bytes = static_cast<Index>(EIGEN_SME_LHS_WORKING_SET_BUDGET_BYTES);
225 constexpr Index sme_single_pass_rhs_budget_bytes = static_cast<Index>(EIGEN_SME_SINGLE_PASS_RHS_BUDGET_BYTES);
226#endif
227
228 // Keep kc large enough to amortize SME setup and accumulation, but cap very
229 // deep products to avoid too many result store passes. The cap is a scalar
230 // count tuned for fp32; scale it by the scalar width so every element type
231 // gets the same packed-panel byte budget.
232 const Index max_kc = (numext::maxi)(Index(1), sme_max_kc * Index(sizeof(float)) / Index(sizeof(LhsScalar)));
233 k = (numext::mini)(k, max_kc);
234
235 const Index block_b_hot_bytes = k * nr * Index(sizeof(RhsScalar));
236 const Index min_lhs_bytes = mr * k * Index(sizeof(LhsScalar));
237 const Index block_a_bytes = sme_lhs_working_set_budget_bytes > block_b_hot_bytes
238 ? sme_lhs_working_set_budget_bytes - block_b_hot_bytes
239 : min_lhs_bytes;
240 Index mc = block_a_bytes / (k * Index(sizeof(LhsScalar)));
241 mc = (mc / mr) * mr;
242
243 // Bound the packed RHS strip so very wide matrices do not allocate an unbounded blockB panel. When all rows fit
244 // one LHS block, each packed RHS block is read once, so it stays L2-sized.
245 const Index rhs_budget = m <= mc ? (numext::mini)(sme_packed_rhs_budget_bytes, sme_single_pass_rhs_budget_bytes)
246 : sme_packed_rhs_budget_bytes;
247 Index nc = rhs_budget / (numext::maxi)(Index(1), k * Index(sizeof(RhsScalar)));
248 nc = (nc / nr) * nr;
249 n = (numext::mini)(n, (numext::maxi)(nr, nc));
250
251 m = (numext::mini)(m, (numext::maxi)(mr, mc));
252}
253#endif
254
255template <typename LhsScalar, typename RhsScalar, int KcFactor, typename Index>
256void evaluateProductBlockingSizesHeuristic(Index& k, Index& m, Index& n, Index num_threads = 1) {
257 using Traits = gebp_traits<LhsScalar, RhsScalar>;
258
259 // Explanations:
260 // Let's recall that the product algorithms form mc x kc vertical panels A' on the lhs and
261 // kc x nc blocks B' on the rhs. B' has to fit into L2/L3 cache. Moreover, A' is processed
262 // per mr x kc horizontal small panels where mr is the blocking size along the m dimension
263 // at the register level. This small horizontal panel has to stay within L1 cache.
264 std::ptrdiff_t l1, l2, l3, l3_per_cpu;
265 manage_caching_sizes(GetAction, &l1, &l2, &l3, &l3_per_cpu);
266#ifdef EIGEN_VECTORIZE_AVX512
267 const std::ptrdiff_t phys_l1 = l1;
268 // We need to find a rationale for that, but without this adjustment,
269 // performance with AVX512 is pretty bad, like -20% slower.
270 // One reason is that with increasing packet-size, the blocking size k
271 // has to become pretty small if we want that 1 lhs panel fit within L1.
272 // For instance, with the 3pX4 kernel and double, the size of the lhs+rhs panels are:
273 // k*(3*64 + 4*8) Bytes, with l1=32kBytes, and k%8=0, we have k=144.
274 // This is quite small for a good reuse of the accumulation registers.
275 l1 *= 4;
276#endif
277
278 if (num_threads > 1) {
279 using ResScalar = typename Traits::ResScalar;
280 enum {
281 kdiv = KcFactor * (Traits::mr * sizeof(LhsScalar) + Traits::nr * sizeof(RhsScalar)),
282 ksub = Traits::mr * (Traits::nr * sizeof(ResScalar)),
283 kr = 8,
284 mr = Traits::mr,
285 nr = Traits::nr
286 };
287 // Increasing k gives us more time to prefetch the content of the "C"
288 // registers. However once the latency is hidden there is no point in
289 // increasing the value of k, so we'll cap it at 320 (value determined
290 // experimentally).
291 // To avoid that k vanishes, we make k_cache at least as big as kr
292 const Index k_cache = numext::maxi<Index>(kr, (numext::mini<Index>)(static_cast<Index>((l1 - ksub) / kdiv), 320));
293 if (k_cache < k) {
294 k = k_cache - (k_cache % kr);
295 eigen_internal_assert(k > 0);
296 }
297
298 const Index n_cache = static_cast<Index>((l2 - l1) / (nr * sizeof(RhsScalar) * k));
299 const Index nr_index = static_cast<Index>(nr);
300 // If the cache model says that even one kernel-width RHS panel does not fit,
301 // fall back to the minimum useful panel width instead of producing a zero block.
302 const Index n_cache_aligned = n_cache >= nr_index ? n_cache - (n_cache % nr_index) : nr_index;
303 const Index n_per_thread = numext::div_ceil(n, num_threads);
304 if (n_cache <= n_per_thread) {
305 // Don't exceed the capacity of the l2 cache unless the minimum panel width
306 // is already larger than the modeled capacity.
307 n = (numext::mini<Index>)(n, n_cache_aligned);
308 eigen_internal_assert(n > 0);
309 } else {
310 n = (numext::mini<Index>)(n, (n_per_thread + nr - 1) - ((n_per_thread + nr - 1) % nr));
311 }
312
313 if (l3 > l2) {
314 // l3 is shared between all cores, so we'll give each thread its own chunk of l3.
315 const Index m_cache = static_cast<Index>((l3 - l2) / (sizeof(LhsScalar) * k * num_threads));
316 const Index m_per_thread = numext::div_ceil(m, num_threads);
317 if (m_cache < m_per_thread && m_cache >= static_cast<Index>(mr)) {
318 m = m_cache - (m_cache % mr);
319 eigen_internal_assert(m > 0);
320 } else {
321 m = (numext::mini<Index>)(m, (m_per_thread + mr - 1) - ((m_per_thread + mr - 1) % mr));
322 }
323 }
324 } else {
325 // In unit tests we do not want to use extra large matrices,
326 // so we reduce the cache size to check the blocking strategy is not flawed
327#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
328 l1 = 9 * 1024;
329 l2 = 32 * 1024;
330 l3 = 512 * 1024;
331 // The detected share would otherwise swamp these synthetic sizes and defeat the whole point.
332 l3_per_cpu = 0;
333#endif
334
335 // Early return for small problems because the computation below are time consuming for small problems.
336 // Perhaps it would make more sense to consider k*n*m?
337 // Note that for very tiny problem, this function should be bypassed anyway
338 // because we use the coefficient-based implementation for them.
339 if ((numext::maxi)(k, (numext::maxi)(m, n)) < 48) return;
340
341#ifdef EIGEN_VECTORIZE_SME
342 // Only the scalar pairs the SME kernel specializes use the SME budgets;
343 // the others run the generic kernel below and would thrash L1/L2 with them.
344 EIGEN_IF_CONSTEXPR ((sme_has_gebp_kernel<LhsScalar, RhsScalar>::value)) {
345 evaluateProductBlockingSizesHeuristicForSme<LhsScalar, RhsScalar>(k, m, n);
346 return;
347 }
348#endif
349
350 using ResScalar = typename Traits::ResScalar;
351 enum {
352 k_peeling = 8,
353 k_div = KcFactor * (Traits::mr * sizeof(LhsScalar) + Traits::nr * sizeof(RhsScalar)),
354 k_sub = Traits::mr * (Traits::nr * sizeof(ResScalar))
355 };
356
357 // ---- 1st level of blocking on L1, yields kc ----
358
359 // Blocking on the third dimension (i.e., k) is chosen so that an horizontal panel
360 // of size mr x kc of the lhs plus a vertical panel of kc x nr of the rhs both fits within L1 cache.
361 // We also include a register-level block of the result (mx x nr).
362 // (In an ideal world only the lhs panel would stay in L1)
363 // Moreover, kc has to be a multiple of 8 to be compatible with loop peeling, leading to a maximum blocking size of:
364 const Index max_kc = numext::maxi<Index>(static_cast<Index>(((l1 - k_sub) / k_div) & (~(k_peeling - 1))), 1);
365 const Index old_k = k;
366 if (k > max_kc) {
367 // We are really blocking on the third dimension:
368 // -> reduce blocking size to make sure the last block is as large as possible
369 // while keeping the same number of sweeps over the result.
370 k = (k % max_kc) == 0 ? max_kc
371 : max_kc - k_peeling * ((max_kc - 1 - (k % max_kc)) / (k_peeling * (k / max_kc + 1)));
372
373 eigen_internal_assert(((old_k / k) == (old_k / max_kc)) && "the number of sweeps has to remain the same");
374 }
375
376#ifdef EIGEN_VECTORIZE_AVX512
377 // The l1 *= 4 inflation above allows larger kc for better accumulator reuse,
378 // but can overfill the physical L1. Recompute max_kc using 85% of actual L1
379 // to leave headroom for RHS streaming, prefetch buffers, and stack.
380 {
381 const Index phys_l1_eff = convert_index<Index>(phys_l1 * 85 / 100);
382 const Index max_kc_phys = numext::maxi<Index>(((phys_l1_eff - k_sub) / k_div) & (~(k_peeling - 1)), k_peeling);
383 if (max_kc_phys < k) {
384 k = (old_k % max_kc_phys) == 0 ? max_kc_phys
385 : max_kc_phys - k_peeling * ((max_kc_phys - 1 - (old_k % max_kc_phys)) /
386 (k_peeling * (old_k / max_kc_phys + 1)));
387 }
388 }
389#endif
390
391// ---- 2nd level of blocking on max(L2,L3), yields nc ----
392
393// Estimate the effective per-core L2 capacity for 2nd-level blocking.
394// Use 1.5x the runtime-detected L2 size. The extra 50% accounts for data
395// that spills to L3 but remains accessible with low latency. This matches
396// the empirically-tuned constant (1.5MB) previously used when L2 was 1MB.
397#ifdef EIGEN_DEBUG_SMALL_PRODUCT_BLOCKS
398 const Index actual_l2 = static_cast<Index>(l3);
399#else
400 const Index actual_l2 = static_cast<Index>(l2 * 3 / 2);
401#endif
402
403 // Budget for the packed rhs panel. The 1.5x above stands in for an L3 whose geometry was
404 // unknown, and was calibrated against a 1MB placeholder L2, so it underestimates the reachable
405 // working set on a core whose real L2 is much smaller. Prefer this CPU's measured share of L3
406 // where the platform reports it -- a share rather than the whole cache, since sizing one CPU's
407 // panel to all of a server's L3 would evict every other CPU's working set. This deliberately
408 // does not feed actual_lm below: that governs the blockA allocation, whose L1/L2 tuning is
409 // separate.
410 const Index rhs_panel_budget = numext::maxi<Index>(actual_l2, static_cast<Index>(l3_per_cpu));
411
412 // Here, nc is chosen such that a block of kc x nc of the rhs fit within half of L2.
413 // The second half is implicitly reserved to access the result and lhs coefficients.
414 // When k<max_kc, then nc can grow without bound. In practice, it seems to be fruitful
415 // to limit this growth: we bound nc growth to a factor of 1.5x.
416 // However, if the entire lhs block fit within L1, then we are not going to block on the rows at all,
417 // and it becomes fruitful to keep the packed rhs blocks in L1 if there is enough remaining space.
418 Index max_nc;
419 const Index lhs_bytes = m * k * sizeof(LhsScalar);
420 const Index remaining_l1 = static_cast<Index>(l1 - k_sub - lhs_bytes);
421 if (remaining_l1 >= Index(Traits::nr * sizeof(RhsScalar)) * k) {
422 // L1 blocking
423 max_nc = remaining_l1 / (k * sizeof(RhsScalar));
424 } else {
425 // L2 blocking: use actual kc (k) rather than max_kc so that nc is not
426 // unnecessarily squeezed when k < max_kc (e.g. on CPUs with large L1).
427 max_nc = (3 * rhs_panel_budget) / (2 * 2 * k * sizeof(RhsScalar));
428 }
429 // WARNING Below, we assume that Traits::nr is a power of two.
430 Index nc = numext::mini<Index>(rhs_panel_budget / (2 * k * sizeof(RhsScalar)), max_nc) & (~(Traits::nr - 1));
431 // As in the threaded branch, a budget below one kernel-width panel still takes nr columns: a zero
432 // width would divide by zero below.
433 nc = numext::maxi<Index>(nc, Traits::nr);
434 if (n > nc) {
435 // We are really blocking over the columns:
436 // -> reduce blocking size to make sure the last block is as large as possible
437 // while keeping the same number of sweeps over the packed lhs.
438 // Here we allow one more sweep if this gives us a perfect match, thus the commented "-1"
439 n = (n % nc) == 0 ? nc : (nc - Traits::nr * ((nc /*-1*/ - (n % nc)) / (Traits::nr * (n / nc + 1))));
440 } else if (old_k == k) {
441 // No k- or n-blocking happened yet (kc==depth, nc>=n). gebp already
442 // strip-chunks the packed lhs via its own `actual_panel_rows` budget,
443 // so cache residency is honored whatever mc we pick here. What this
444 // branch actually governs is the size of the `mc * kc` packing buffer
445 // (blockA) that the caller allocates — capping mc keeps it bounded for
446 // tall-m / small-k shapes, where leaving mc=m would allocate up to
447 // `rows * depth * sizeof(LhsScalar)`. A budget-based alternative
448 // (e.g. cap blockA at ~L3/4) is no faster in benchmarks and increases
449 // heap use, so the original L1/L2-residency tuning is kept.
450 Index problem_size = k * n * sizeof(LhsScalar);
451 Index actual_lm = actual_l2;
452 Index max_mc = m;
453 if (problem_size <= 1024) {
454 // problem is small enough to keep in L1
455 // Let's choose m such that lhs's block fit in 1/3 of L1
456 actual_lm = static_cast<Index>(l1);
457 } else if (l3 != 0 && problem_size <= l1) {
458 // We have both L2 and L3, and the rhs panel still fits in L1. Choose mc so the
459 // lhs block fits in 1/3 of L2 and avoid spilling into the L2+50% fallback band.
460 // The 32768 byte threshold previously used here was a stand-in for typical x86
461 // L1 size; using the runtime-detected l1 generalizes this to current cache sizes.
462 actual_lm = static_cast<Index>(l2);
463 max_mc = (numext::mini<Index>)(576, max_mc);
464 }
465 Index mc = (numext::mini<Index>)(actual_lm / (3 * k * sizeof(LhsScalar)), max_mc);
466 if (mc > Traits::mr)
467 mc -= mc % Traits::mr;
468 else if (mc == 0)
469 return;
470 m = (m % mc) == 0 ? mc : (mc - Traits::mr * ((mc /*-1*/ - (m % mc)) / (Traits::mr * (m / mc + 1))));
471 }
472 }
473}
474
475template <typename Index>
476inline bool useSpecificBlockingSizes(Index& k, Index& m, Index& n) {
477#ifdef EIGEN_TEST_SPECIFIC_BLOCKING_SIZES
478 if (EIGEN_TEST_SPECIFIC_BLOCKING_SIZES) {
479 k = numext::mini<Index>(k, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_K);
480 m = numext::mini<Index>(m, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_M);
481 n = numext::mini<Index>(n, EIGEN_TEST_SPECIFIC_BLOCKING_SIZE_N);
482 return true;
483 }
484#else
485 EIGEN_UNUSED_VARIABLE(k);
486 EIGEN_UNUSED_VARIABLE(m);
487 EIGEN_UNUSED_VARIABLE(n);
488#endif
489 return false;
490}
491
509
510template <typename LhsScalar, typename RhsScalar, int KcFactor, typename Index>
511void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1) {
512 if (!useSpecificBlockingSizes(k, m, n)) {
513 evaluateProductBlockingSizesHeuristic<LhsScalar, RhsScalar, KcFactor, Index>(k, m, n, num_threads);
514 }
515}
516
517template <typename LhsScalar, typename RhsScalar, typename Index>
518inline void computeProductBlockingSizes(Index& k, Index& m, Index& n, Index num_threads = 1) {
519 computeProductBlockingSizes<LhsScalar, RhsScalar, 1, Index>(k, m, n, num_threads);
520}
521
522// rhs_registers is the number of registers one RhsPacket takes, so RhsPacketx4 takes four times that.
523template <typename RhsPacket, typename RhsPacketx4, int registers_taken, int rhs_registers = 1>
524struct RhsPanelHelper {
525 private:
526 static constexpr int remaining_registers =
527 (std::max)(int(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS) - registers_taken, 0);
528
529 public:
530 using type = std::conditional_t<remaining_registers >= 4 * rhs_registers, RhsPacketx4, RhsPacket>;
531};
532
533template <typename Packet>
534struct QuadPacket {
535 Packet B_0, B1, B2, B3;
536 const Packet& get(const FixedInt<0>&) const { return B_0; }
537 const Packet& get(const FixedInt<1>&) const { return B1; }
538 const Packet& get(const FixedInt<2>&) const { return B2; }
539 const Packet& get(const FixedInt<3>&) const { return B3; }
540};
541
542template <int N, typename T1, typename T2, typename T3>
543struct packet_conditional {
544 using type = T3;
545};
546
547template <typename T1, typename T2, typename T3>
548struct packet_conditional<GEBPPacketFull, T1, T2, T3> {
549 using type = T1;
550};
551
552template <typename T1, typename T2, typename T3>
553struct packet_conditional<GEBPPacketHalf, T1, T2, T3> {
554 using type = T2;
555};
556
557#define PACKET_DECL_COND_POSTFIX(postfix, name, packet_size) \
558 using name##Packet##postfix = \
559 typename packet_conditional<packet_size, typename packet_traits<name##Scalar>::type, \
560 typename packet_traits<name##Scalar>::half, \
561 typename unpacket_traits<typename packet_traits<name##Scalar>::half>::half>::type
562
563#define PACKET_DECL_COND(name, packet_size) \
564 using name##Packet = \
565 typename packet_conditional<packet_size, typename packet_traits<name##Scalar>::type, \
566 typename packet_traits<name##Scalar>::half, \
567 typename unpacket_traits<typename packet_traits<name##Scalar>::half>::half>::type
568
569#define PACKET_DECL_COND_SCALAR_POSTFIX(postfix, packet_size) \
570 using ScalarPacket##postfix = \
571 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
572 typename packet_traits<Scalar>::half, \
573 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
574
575#define PACKET_DECL_COND_SCALAR(packet_size) \
576 using ScalarPacket = \
577 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
578 typename packet_traits<Scalar>::half, \
579 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
580
581/* Vectorization logic
582 * real*real: unpack rhs to constant packets, ...
583 *
584 * cd*cd : unpack rhs to (b_r,b_r), (b_i,b_i), mul to get (a_r b_r,a_i b_r) (a_r b_i,a_i b_i),
585 * storing each res packet into two packets (2x2),
586 * at the end combine them: swap the second and addsub them
587 * cf*cf : same but with 2x4 blocks
588 * cplx*real : unpack rhs to constant packets, ...
589 * real*cplx : load lhs as (a0,a0,a1,a1), and mul as usual
590 */
591template <typename LhsScalar_, typename RhsScalar_, bool ConjLhs_, bool ConjRhs_, int Arch, int PacketSize_>
592class gebp_traits {
593 public:
594 using LhsScalar = LhsScalar_;
595 using RhsScalar = RhsScalar_;
596 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
597
598 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
599 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
600 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
601
602 enum {
603 ConjLhs = ConjLhs_,
604 ConjRhs = ConjRhs_,
605 Vectorizable = unpacket_traits<LhsPacket_>::vectorizable && unpacket_traits<RhsPacket_>::vectorizable,
606 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
607 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
608 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
609
610 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
611
612 // register block size along the N direction must be 1 or 4
613 nr = 4,
614
615 // register block size along the M direction (currently, this one cannot be modified)
616 default_mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * LhsPacketSize,
617#if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && \
618 !defined(EIGEN_VECTORIZE_VSX) && ((!EIGEN_COMP_MSVC) || (EIGEN_COMP_MSVC >= 1914))
619 // we assume 16 registers or more
620 // See bug 992, if the scalar type is not vectorizable but that EIGEN_HAS_SINGLE_INSTRUCTION_MADD is defined,
621 // then using 3*LhsPacketSize triggers non-implemented paths in syrk.
622 // Bug 1515: MSVC prior to v19.14 yields to register spilling.
623 mr = Vectorizable ? 3 * LhsPacketSize : default_mr,
624#else
625 mr = default_mr,
626#endif
627
628 LhsProgress = LhsPacketSize,
629 RhsProgress = 1
630 };
631
632 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
633 using RhsPacket = std::conditional_t<Vectorizable, RhsPacket_, RhsScalar>;
634 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
635 using LhsPacket4Packing = LhsPacket;
636
637 using RhsPacketx4 = QuadPacket<RhsPacket>;
638 using AccPacket = ResPacket;
639
640 EIGEN_STRONG_INLINE void initAcc(AccPacket& p) const { p = pset1<ResPacket>(ResScalar(0)); }
641
642 template <typename RhsPacketType>
643 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const {
644 dest = pset1<RhsPacketType>(*b);
645 }
646
647 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
648 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
649 }
650
651 template <typename RhsPacketType>
652 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const {
653 loadRhs(b, dest);
654 }
655
656 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
657
658 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = ploadquad<RhsPacket>(b); }
659
660 template <typename LhsPacketType>
661 EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacketType& dest) const {
662 dest = pload<LhsPacketType>(a);
663 }
664
665 template <typename LhsPacketType>
666 EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const {
667 dest = ploadu<LhsPacketType>(a);
668 }
669
670 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType>
671 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
672 const LaneIdType&) const {
673 conj_helper<LhsPacketType, RhsPacketType, ConjLhs, ConjRhs> cj;
674 // It would be a lot cleaner to call pmadd all the time. Unfortunately if we
675 // let gcc allocate the register in which to store the result of the pmul
676 // (in the case where there is no FMA) gcc fails to figure out how to avoid
677 // spilling register.
678#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
679 EIGEN_UNUSED_VARIABLE(tmp);
680 c = cj.pmadd(a, b, c);
681#else
682 madd_unfused(a, b, c, tmp, cj, bool_constant<std::is_same<RhsPacketType, decltype(cj.pmul(a, b))>::value>());
683#endif
684 }
685
686 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename Conjugator>
687 EIGEN_STRONG_INLINE void madd_unfused(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c,
688 RhsPacketType& tmp, const Conjugator& cj, std::true_type) const {
689 tmp = b;
690 tmp = cj.pmul(a, tmp);
691 c = padd(c, tmp);
692 }
693
694 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename Conjugator>
695 EIGEN_STRONG_INLINE void madd_unfused(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c,
696 RhsPacketType& tmp, const Conjugator& cj, std::false_type) const {
697 EIGEN_UNUSED_VARIABLE(tmp);
698 c = padd(c, cj.pmul(a, b));
699 }
700
701 template <typename LhsPacketType, typename AccPacketType, typename LaneIdType>
702 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
703 const LaneIdType& lane) const {
704 madd(a, b.get(lane), c, tmp, lane);
705 }
706
707 EIGEN_STRONG_INLINE void acc(const AccPacket& c, const ResPacket& alpha, ResPacket& r) const {
708 r = pmadd(c, alpha, r);
709 }
710
711 template <typename ResPacketHalf>
712 EIGEN_STRONG_INLINE void acc(const ResPacketHalf& c, const ResPacketHalf& alpha, ResPacketHalf& r) const {
713 r = pmadd(c, alpha, r);
714 }
715};
716
717template <typename RealScalar, bool ConjLhs_, int Arch, int PacketSize_>
718class gebp_traits<std::complex<RealScalar>, RealScalar, ConjLhs_, false, Arch, PacketSize_> {
719 public:
720 using LhsScalar = std::complex<RealScalar>;
721 using RhsScalar = RealScalar;
722 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
723
724 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
725 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
726 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
727
728 enum {
729 ConjLhs = ConjLhs_,
730 ConjRhs = false,
731 Vectorizable = unpacket_traits<LhsPacket_>::vectorizable && unpacket_traits<RhsPacket_>::vectorizable,
732 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
733 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
734 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
735
736 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
737 nr = 4,
738#if defined(EIGEN_HAS_SINGLE_INSTRUCTION_MADD) && !defined(EIGEN_VECTORIZE_ALTIVEC) && !defined(EIGEN_VECTORIZE_VSX)
739 // we assume 16 registers
740 mr = 3 * LhsPacketSize,
741#else
742 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * LhsPacketSize,
743#endif
744
745 LhsProgress = LhsPacketSize,
746 RhsProgress = 1
747 };
748
749 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
750 using RhsPacket = std::conditional_t<Vectorizable, RhsPacket_, RhsScalar>;
751 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
752 using LhsPacket4Packing = LhsPacket;
753
754 using RhsPacketx4 = QuadPacket<RhsPacket>;
755
756 using AccPacket = ResPacket;
757
758 EIGEN_STRONG_INLINE void initAcc(AccPacket& p) const { p = pset1<ResPacket>(ResScalar(0)); }
759
760 template <typename RhsPacketType>
761 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const {
762 dest = pset1<RhsPacketType>(*b);
763 }
764
765 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
766 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
767 }
768
769 template <typename RhsPacketType>
770 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const {
771 loadRhs(b, dest);
772 }
773
774 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
775
776 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const {
777 loadRhsQuad_impl(b, dest, bool_constant<RhsPacketSize == 16>());
778 }
779
780 EIGEN_STRONG_INLINE void loadRhsQuad_impl(const RhsScalar* b, RhsPacket& dest, const std::true_type&) const {
781 // FIXME: replace with a dedicated ploadheight operation for more efficient quad loading.
782 RhsScalar tmp[4] = {b[0], b[0], b[1], b[1]};
783 dest = ploadquad<RhsPacket>(tmp);
784 }
785
786 EIGEN_STRONG_INLINE void loadRhsQuad_impl(const RhsScalar* b, RhsPacket& dest, const std::false_type&) const {
787 eigen_internal_assert(RhsPacketSize <= 8);
788 dest = pset1<RhsPacket>(*b);
789 }
790
791 EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacket& dest) const { dest = pload<LhsPacket>(a); }
792
793 template <typename LhsPacketType>
794 EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const {
795 dest = ploadu<LhsPacketType>(a);
796 }
797
798 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType>
799 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
800 const LaneIdType&) const {
801 madd_impl(a, b, c, tmp, bool_constant<Vectorizable>());
802 }
803
804 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType>
805 EIGEN_STRONG_INLINE void madd_impl(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c,
806 RhsPacketType& tmp, const std::true_type&) const {
807#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
808 EIGEN_UNUSED_VARIABLE(tmp);
809 // Conjugation is deferred to acc().
810 conj_helper<LhsPacketType, RhsPacketType, false, false> cj;
811 c = cj.pmadd(a, b, c);
812#else
813 tmp = b;
814 tmp = pmul(a.v, tmp);
815 c.v = padd(c.v, tmp);
816#endif
817 }
818
819 EIGEN_STRONG_INLINE void madd_impl(const LhsScalar& a, const RhsScalar& b, ResScalar& c, RhsScalar& /*tmp*/,
820 const std::false_type&) const {
821 c += a * b;
822 }
823
824 template <typename LhsPacketType, typename AccPacketType, typename LaneIdType>
825 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
826 const LaneIdType& lane) const {
827 madd(a, b.get(lane), c, tmp, lane);
828 }
829
830 template <typename ResPacketType, typename AccPacketType>
831 EIGEN_STRONG_INLINE void acc(const AccPacketType& c, const ResPacketType& alpha, ResPacketType& r) const {
832 conj_helper<ResPacketType, ResPacketType, ConjLhs, false> cj;
833 r = cj.pmadd(c, alpha, r);
834 }
835};
836
837template <typename Packet>
838struct DoublePacket {
839 Packet first;
840 Packet second;
841};
842
843template <typename Packet>
844struct is_double_packet : std::false_type {};
845template <typename Packet>
846struct is_double_packet<DoublePacket<Packet>> : std::true_type {};
847
848template <typename Packet>
849DoublePacket<Packet> padd(const DoublePacket<Packet>& a, const DoublePacket<Packet>& b) {
850 DoublePacket<Packet> res;
851 res.first = padd(a.first, b.first);
852 res.second = padd(a.second, b.second);
853 return res;
854}
855
856template <typename Packet, std::enable_if_t<unpacket_traits<Packet>::size <= 8, int> = 0>
857const DoublePacket<Packet>& predux_half(const DoublePacket<Packet>& a) {
858 return a;
859}
860
861template <typename Packet, std::enable_if_t<unpacket_traits<Packet>::size >= 16 &&
862 !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex,
863 int> = 0>
864DoublePacket<typename unpacket_traits<Packet>::half> predux_half(const DoublePacket<Packet>& a) {
865 // Workaround: reduce real packets to half size by reinterpreting as complex.
866 DoublePacket<typename unpacket_traits<Packet>::half> res;
867 using Cplx = std::complex<typename unpacket_traits<Packet>::type>;
868 using CplxPacket = typename packet_traits<Cplx>::type;
869 res.first = predux_half(CplxPacket(a.first)).v;
870 res.second = predux_half(CplxPacket(a.second)).v;
871 return res;
872}
873
874// same here, "quad" actually means "8" in terms of real coefficients
875template <typename Scalar, typename RealPacket, std::enable_if_t<unpacket_traits<RealPacket>::size <= 8, int> = 0>
876void loadQuadToDoublePacket(const Scalar* b, DoublePacket<RealPacket>& dest) {
877 dest.first = pset1<RealPacket>(numext::real(*b));
878 dest.second = pset1<RealPacket>(numext::imag(*b));
879}
880
881// A real packet of N lanes carries N/8 complex values, each spread over eight
882// lanes. ploadquad repeats every element four times, so it needs each value
883// listed twice. Sized off N rather than a fixed 16: SVE reaches 32 real lanes
884// at VL=1024 and 64 at VL=2048, which no fixed overload covers.
885template <typename Scalar, typename RealPacket, std::enable_if_t<(unpacket_traits<RealPacket>::size > 8), int> = 0>
886void loadQuadToDoublePacket(const Scalar* b, DoublePacket<RealPacket>& dest) {
887 using RealScalar = typename NumTraits<Scalar>::Real;
888 constexpr int kQuads = unpacket_traits<RealPacket>::size / 4;
889 RealScalar r[kQuads], i[kQuads];
890 for (int j = 0; j < kQuads; ++j) {
891 r[j] = numext::real(b[j / 2]);
892 i[j] = numext::imag(b[j / 2]);
893 }
894 dest.first = ploadquad<RealPacket>(r);
895 dest.second = ploadquad<RealPacket>(i);
896}
897
898template <typename Packet>
899struct unpacket_traits<DoublePacket<Packet>> {
900 using half = DoublePacket<typename unpacket_traits<Packet>::half>;
901 enum { size = 2 * unpacket_traits<Packet>::size };
902};
903
904template <typename RealScalar, bool ConjLhs_, bool ConjRhs_, int Arch, int PacketSize_>
905class gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, ConjLhs_, ConjRhs_, Arch, PacketSize_> {
906 public:
907 using Scalar = std::complex<RealScalar>;
908 using LhsScalar = std::complex<RealScalar>;
909 using RhsScalar = std::complex<RealScalar>;
910 using ResScalar = std::complex<RealScalar>;
911
912 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
913 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
914 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
915 PACKET_DECL_COND(Real, PacketSize_);
916 PACKET_DECL_COND_SCALAR(PacketSize_);
917
918 enum {
919 ConjLhs = ConjLhs_,
920 ConjRhs = ConjRhs_,
921 Vectorizable = unpacket_traits<RealPacket>::vectorizable && unpacket_traits<ScalarPacket>::vectorizable,
922 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
923 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
924 RhsPacketSize = Vectorizable ? unpacket_traits<RhsScalar>::size : 1,
925 RealPacketSize = Vectorizable ? unpacket_traits<RealPacket>::size : 1,
926 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
927
928 nr = 4,
929#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
930 // Each DoublePacket accumulator takes two registers.
931 mr = plain_enum_max(1, NumberOfRegisters / 4 / nr) * ResPacketSize,
932#else
933 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * ResPacketSize,
934#endif
935
936 LhsProgress = ResPacketSize,
937 RhsProgress = 1
938 };
939
940 using DoublePacketType = DoublePacket<RealPacket>;
941
942 using LhsPacket4Packing = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
943 using LhsPacket = std::conditional_t<Vectorizable, RealPacket, Scalar>;
944 using RhsPacket = std::conditional_t<Vectorizable, DoublePacketType, Scalar>;
945 using ResPacket = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
946 using AccPacket = std::conditional_t<Vectorizable, DoublePacketType, Scalar>;
947
948 // this actually holds 8 packets!
949 using RhsPacketx4 = QuadPacket<RhsPacket>;
950
951 EIGEN_STRONG_INLINE void initAcc(Scalar& p) const { p = Scalar(0); }
952
953 EIGEN_STRONG_INLINE void initAcc(DoublePacketType& p) const {
954 p.first = pset1<RealPacket>(RealScalar(0));
955 p.second = pset1<RealPacket>(RealScalar(0));
956 }
957
958 // Scalar path
959 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, ScalarPacket& dest) const { dest = pset1<ScalarPacket>(*b); }
960
961 // Vectorized path
962 template <typename RealPacketType>
963 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, DoublePacket<RealPacketType>& dest) const {
964 dest.first = pset1<RealPacketType>(numext::real(*b));
965 dest.second = pset1<RealPacketType>(numext::imag(*b));
966 }
967
968 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
969 loadRhs(b, dest.B_0);
970 loadRhs(b + 1, dest.B1);
971 loadRhs(b + 2, dest.B2);
972 loadRhs(b + 3, dest.B3);
973 }
974
975 // Scalar path
976 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, ScalarPacket& dest) const { loadRhs(b, dest); }
977
978 // Vectorized path
979 template <typename RealPacketType>
980 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, DoublePacket<RealPacketType>& dest) const {
981 loadRhs(b, dest);
982 }
983
984 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
985
986 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, ResPacket& dest) const { loadRhs(b, dest); }
987 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, DoublePacketType& dest) const {
988 loadQuadToDoublePacket(b, dest);
989 }
990
991 // nothing special here
992 EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacket& dest) const {
993 dest = pload<LhsPacket>((const typename unpacket_traits<LhsPacket>::type*)(a));
994 }
995
996 template <typename LhsPacketType>
997 EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const {
998 dest = ploadu<LhsPacketType>((const typename unpacket_traits<LhsPacketType>::type*)(a));
999 }
1000
1001 template <typename LhsPacketType, typename RhsPacketType, typename ResPacketType, typename TmpType,
1002 typename LaneIdType>
1003 EIGEN_STRONG_INLINE std::enable_if_t<!std::is_same<RhsPacketType, RhsPacketx4>::value> madd(
1004 const LhsPacketType& a, const RhsPacketType& b, DoublePacket<ResPacketType>& c, TmpType& /*tmp*/,
1005 const LaneIdType&) const {
1006 c.first = pmadd(a, b.first, c.first);
1007 c.second = pmadd(a, b.second, c.second);
1008 }
1009
1010 template <typename LaneIdType>
1011 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, ResPacket& c, RhsPacket& /*tmp*/,
1012 const LaneIdType&) const {
1013 c = cj.pmadd(a, b, c);
1014 }
1015
1016 template <typename LhsPacketType, typename AccPacketType, typename LaneIdType>
1017 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
1018 const LaneIdType& lane) const {
1019 madd(a, b.get(lane), c, tmp, lane);
1020 }
1021
1022 EIGEN_STRONG_INLINE void acc(const Scalar& c, const Scalar& alpha, Scalar& r) const { r += alpha * c; }
1023
1024 template <typename RealPacketType, typename ResPacketType>
1025 EIGEN_STRONG_INLINE void acc(const DoublePacket<RealPacketType>& c, const ResPacketType& alpha,
1026 ResPacketType& r) const {
1027 // assemble c
1028 ResPacketType tmp;
1029 EIGEN_IF_CONSTEXPR ((!ConjLhs) && (!ConjRhs)) {
1030 tmp = pcplxflip(pconj(ResPacketType(c.second)));
1031 tmp = padd(ResPacketType(c.first), tmp);
1032 } else EIGEN_IF_CONSTEXPR ((!ConjLhs) && (ConjRhs)) {
1033 tmp = pconj(pcplxflip(ResPacketType(c.second)));
1034 tmp = padd(ResPacketType(c.first), tmp);
1035 } else EIGEN_IF_CONSTEXPR ((ConjLhs) && (!ConjRhs)) {
1036 tmp = pcplxflip(ResPacketType(c.second));
1037 tmp = padd(pconj(ResPacketType(c.first)), tmp);
1038 } else {
1039 tmp = pcplxflip(ResPacketType(c.second));
1040 tmp = psub(pconj(ResPacketType(c.first)), tmp);
1041 }
1042
1043 r = pmadd(tmp, alpha, r);
1044 }
1045
1046 protected:
1047 conj_helper<LhsScalar, RhsScalar, ConjLhs, ConjRhs> cj;
1048};
1049
1050template <typename RealScalar, bool ConjRhs_, int Arch, int PacketSize_>
1051class gebp_traits<RealScalar, std::complex<RealScalar>, false, ConjRhs_, Arch, PacketSize_> {
1052 public:
1053 using Scalar = std::complex<RealScalar>;
1054 using LhsScalar = RealScalar;
1055 using RhsScalar = Scalar;
1056 using ResScalar = Scalar;
1057
1058 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
1059 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
1060 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
1061 PACKET_DECL_COND_POSTFIX(_, Real, PacketSize_);
1062 PACKET_DECL_COND_SCALAR_POSTFIX(_, PacketSize_);
1063
1064#undef PACKET_DECL_COND_SCALAR_POSTFIX
1065#undef PACKET_DECL_COND_POSTFIX
1066#undef PACKET_DECL_COND_SCALAR
1067#undef PACKET_DECL_COND
1068
1069 enum {
1070 ConjLhs = false,
1071 ConjRhs = ConjRhs_,
1072 Vectorizable = unpacket_traits<RealPacket_>::vectorizable && unpacket_traits<ScalarPacket_>::vectorizable,
1073 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
1074 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
1075 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
1076
1077 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
1078 // FIXME: should depend on NumberOfRegisters
1079 nr = 4,
1080 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * ResPacketSize,
1081
1082 LhsProgress = ResPacketSize,
1083 RhsProgress = 1
1084 };
1085
1086 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
1087 using RhsPacket = std::conditional_t<Vectorizable, RhsPacket_, RhsScalar>;
1088 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
1089 using LhsPacket4Packing = LhsPacket;
1090 using RhsPacketx4 = QuadPacket<RhsPacket>;
1091 using AccPacket = ResPacket;
1092
1093 EIGEN_STRONG_INLINE void initAcc(AccPacket& p) const { p = pset1<ResPacket>(ResScalar(0)); }
1094
1095 template <typename RhsPacketType>
1096 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const {
1097 dest = pset1<RhsPacketType>(*b);
1098 }
1099
1100 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
1101 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
1102 }
1103
1104 template <typename RhsPacketType>
1105 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const {
1106 loadRhs(b, dest);
1107 }
1108
1109 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
1110
1111 EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacket& dest) const { dest = ploaddup<LhsPacket>(a); }
1112
1113 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = ploadquad<RhsPacket>(b); }
1114
1115 template <typename LhsPacketType>
1116 EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const {
1117 dest = ploaddup<LhsPacketType>(a);
1118 }
1119
1120 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType>
1121 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
1122 const LaneIdType&) const {
1123 madd_impl(a, b, c, tmp, bool_constant<Vectorizable>());
1124 }
1125
1126 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType>
1127 EIGEN_STRONG_INLINE void madd_impl(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c,
1128 RhsPacketType& tmp, const std::true_type&) const {
1129#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
1130 EIGEN_UNUSED_VARIABLE(tmp);
1131 // Conjugation is deferred to acc().
1132 conj_helper<LhsPacketType, RhsPacketType, false, false> cj;
1133 c = cj.pmadd(a, b, c);
1134#else
1135 tmp = b;
1136 tmp.v = pmul(a, tmp.v);
1137 c = padd(c, tmp);
1138#endif
1139 }
1140
1141 EIGEN_STRONG_INLINE void madd_impl(const LhsScalar& a, const RhsScalar& b, ResScalar& c, RhsScalar& /*tmp*/,
1142 const std::false_type&) const {
1143 c += a * b;
1144 }
1145
1146 template <typename LhsPacketType, typename AccPacketType, typename LaneIdType>
1147 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
1148 const LaneIdType& lane) const {
1149 madd(a, b.get(lane), c, tmp, lane);
1150 }
1151
1152 template <typename ResPacketType, typename AccPacketType>
1153 EIGEN_STRONG_INLINE void acc(const AccPacketType& c, const ResPacketType& alpha, ResPacketType& r) const {
1154 conj_helper<ResPacketType, ResPacketType, false, ConjRhs> cj;
1155 r = cj.pmadd(alpha, c, r);
1156 }
1157};
1158
1159/* optimized General packed Block * packed Panel product kernel
1160 *
1161 * Mixing type logic: C += A * B
1162 * | A | B | comments
1163 * |real |cplx | no vectorization yet, would require to pack A with duplication
1164 * |cplx |real | easy vectorization
1165 */
1166// The result scalars whose kernel stores may overwrite: assigning one is known to give what adding it onto a zero
1167// does. Other scalars, including custom ones over a built-in real type, keep accumulating.
1168template <typename Scalar>
1169struct gebp_overwritable_scalar : bool_constant<std::is_arithmetic<Scalar>::value> {};
1170template <typename Real>
1171struct gebp_overwritable_scalar<std::complex<Real>> : bool_constant<std::is_arithmetic<Real>::value> {};
1172
1173template <typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr,
1174 bool ConjugateLhs, bool ConjugateRhs>
1175struct gebp_kernel {
1176 using Traits = gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target>;
1177 using HalfTraits =
1178 gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target, GEBPPacketHalf>;
1179 using QuarterTraits =
1180 gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target, GEBPPacketQuarter>;
1181
1182 using ResScalar = typename Traits::ResScalar;
1183 using LhsPacket = typename Traits::LhsPacket;
1184 using RhsPacket = typename Traits::RhsPacket;
1185 using ResPacket = typename Traits::ResPacket;
1186 using AccPacket = typename Traits::AccPacket;
1187 using RhsPacketx4 = typename Traits::RhsPacketx4;
1188
1189 using SwappedTraits = gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>;
1190
1191 using SLhsPacket = typename SwappedTraits::LhsPacket;
1192 using SRhsPacket = typename SwappedTraits::RhsPacket;
1193 using SResPacket = typename SwappedTraits::ResPacket;
1194 using SAccPacket = typename SwappedTraits::AccPacket;
1195
1196 using LhsPacketHalf = typename HalfTraits::LhsPacket;
1197 using RhsPacketHalf = typename HalfTraits::RhsPacket;
1198 using ResPacketHalf = typename HalfTraits::ResPacket;
1199 using AccPacketHalf = typename HalfTraits::AccPacket;
1200
1201 using LhsPacketQuarter = typename QuarterTraits::LhsPacket;
1202 using RhsPacketQuarter = typename QuarterTraits::RhsPacket;
1203 using ResPacketQuarter = typename QuarterTraits::ResPacket;
1204 using AccPacketQuarter = typename QuarterTraits::AccPacket;
1205
1206 using LinearMapper = typename DataMapper::LinearMapper;
1207
1208 enum {
1209 Vectorizable = Traits::Vectorizable,
1210 LhsProgress = Traits::LhsProgress,
1211 LhsProgressHalf = HalfTraits::LhsProgress,
1212 LhsProgressQuarter = QuarterTraits::LhsProgress,
1213 RhsProgress = Traits::RhsProgress,
1214 RhsProgressHalf = HalfTraits::RhsProgress,
1215 RhsProgressQuarter = QuarterTraits::RhsProgress,
1216 ResPacketSize = Traits::ResPacketSize
1217 };
1218
1219 // Overwriting starts each result packet from zero instead of loading it, so it computes exactly what accumulating
1220 // into a zeroed result does (see gebp_overwritable_scalar for the scalars that holds for).
1221 static constexpr bool CanOverwrite = gebp_overwritable_scalar<ResScalar>::value;
1222
1223 EIGEN_STRONG_INLINE void operator()(const DataMapper& res, const LhsScalar* blockA, const RhsScalar* blockB,
1224 Index rows, Index depth, Index cols, ResScalar alpha, Index strideA = -1,
1225 Index strideB = -1, Index offsetA = 0, Index offsetB = 0) const {
1226 run(false, res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
1227 }
1228 // C = alpha * A' * B' over the block, whatever C held (see gebp_overwrite_t).
1229 EIGEN_STRONG_INLINE void operator()(gebp_overwrite_t, const DataMapper& res, const LhsScalar* blockA,
1230 const RhsScalar* blockB, Index rows, Index depth, Index cols, ResScalar alpha,
1231 Index strideA = -1, Index strideB = -1, Index offsetA = 0,
1232 Index offsetB = 0) const {
1233 run(true, res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
1234 }
1235
1236 // One body for both: `overwrite` is a runtime flag, so that assignment and accumulation share one
1237 // instantiation.
1238 EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE void run(bool overwrite, const DataMapper& res, const LhsScalar* blockA,
1239 const RhsScalar* blockB, Index rows, Index depth, Index cols,
1240 ResScalar alpha, Index strideA, Index strideB, Index offsetA,
1241 Index offsetB) const;
1242};
1243
1244// One scalar result, alpha * c: added to the destination, or, overwriting, assigned as the sum onto zero that
1245// accumulating into a zeroed destination computes. A scalar that cannot overwrite (CanOverwrite) takes an overload
1246// without the overwriting form, so that accumulation asks of it only the `dst += alpha * c` it always did.
1247template <typename ResScalar, typename Alpha, typename Acc>
1248EIGEN_ALWAYS_INLINE void gebp_store_scalar_impl(std::false_type, bool, ResScalar& dst, const Alpha& alpha,
1249 const Acc& c) {
1250 dst += alpha * c;
1251}
1252template <typename ResScalar, typename Alpha, typename Acc>
1253EIGEN_ALWAYS_INLINE void gebp_store_scalar_impl(std::true_type, bool overwrite, ResScalar& dst, const Alpha& alpha,
1254 const Acc& c) {
1255 if (overwrite)
1256 dst = ResScalar(0) + alpha * c;
1257 else
1258 dst += alpha * c;
1259}
1260template <bool CanOverwrite, typename ResScalar, typename Alpha, typename Acc>
1261EIGEN_ALWAYS_INLINE void gebp_store_scalar(bool overwrite, ResScalar& dst, const Alpha& alpha, const Acc& c) {
1262 gebp_store_scalar_impl(bool_constant<CanOverwrite>(), overwrite, dst, alpha, c);
1263}
1264
1265template <typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr,
1266 bool ConjugateLhs, bool ConjugateRhs,
1267 int SwappedLhsProgress =
1268 gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>::LhsProgress>
1269struct last_row_process_16_packets {
1270 using Traits = gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target>;
1271 using SwappedTraits = gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>;
1272
1273 using ResScalar = typename Traits::ResScalar;
1274 using SLhsPacket = typename SwappedTraits::LhsPacket;
1275 using SRhsPacket = typename SwappedTraits::RhsPacket;
1276 using SResPacket = typename SwappedTraits::ResPacket;
1277 using SAccPacket = typename SwappedTraits::AccPacket;
1278
1279 EIGEN_STRONG_INLINE void operator()(const DataMapper& res, SwappedTraits& straits, const LhsScalar* blA,
1280 const RhsScalar* blB, Index depth, const Index endk, Index i, Index j2,
1281 ResScalar alpha, SAccPacket& C0, bool overwrite = false) const {
1282 EIGEN_UNUSED_VARIABLE(overwrite);
1283 EIGEN_UNUSED_VARIABLE(res);
1284 EIGEN_UNUSED_VARIABLE(straits);
1285 EIGEN_UNUSED_VARIABLE(blA);
1286 EIGEN_UNUSED_VARIABLE(blB);
1287 EIGEN_UNUSED_VARIABLE(depth);
1288 EIGEN_UNUSED_VARIABLE(endk);
1289 EIGEN_UNUSED_VARIABLE(i);
1290 EIGEN_UNUSED_VARIABLE(j2);
1291 EIGEN_UNUSED_VARIABLE(alpha);
1292 EIGEN_UNUSED_VARIABLE(C0);
1293 }
1294};
1295
1296template <typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr,
1297 bool ConjugateLhs, bool ConjugateRhs>
1298struct last_row_process_16_packets<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs, 16> {
1299 using Traits = gebp_traits<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs, Architecture::Target>;
1300 using SwappedTraits = gebp_traits<RhsScalar, LhsScalar, ConjugateRhs, ConjugateLhs, Architecture::Target>;
1301
1302 using ResScalar = typename Traits::ResScalar;
1303 using SLhsPacket = typename SwappedTraits::LhsPacket;
1304 using SRhsPacket = typename SwappedTraits::RhsPacket;
1305 using SResPacket = typename SwappedTraits::ResPacket;
1306 using SAccPacket = typename SwappedTraits::AccPacket;
1307
1308 EIGEN_STRONG_INLINE void operator()(const DataMapper& res, SwappedTraits& straits, const LhsScalar* blA,
1309 const RhsScalar* blB, Index depth, const Index endk, Index i, Index j2,
1310 ResScalar alpha, SAccPacket& C0, bool overwrite = false) const {
1311 using SResPacketQuarter = typename unpacket_traits<typename unpacket_traits<SResPacket>::half>::half;
1312 using SLhsPacketQuarter = typename unpacket_traits<typename unpacket_traits<SLhsPacket>::half>::half;
1313 using SRhsPacketQuarter = typename unpacket_traits<typename unpacket_traits<SRhsPacket>::half>::half;
1314 using SAccPacketQuarter = typename unpacket_traits<typename unpacket_traits<SAccPacket>::half>::half;
1315
1316 SResPacketQuarter R =
1317 overwrite ? pset1<SResPacketQuarter>(ResScalar(0)) : res.template gatherPacket<SResPacketQuarter>(i, j2);
1318 SResPacketQuarter alphav = pset1<SResPacketQuarter>(alpha);
1319
1320 if (depth - endk > 0) {
1321 // We have to handle the last row(s) of the rhs, which
1322 // correspond to a half-packet
1323 SAccPacketQuarter c0 = predux_half(predux_half(C0));
1324
1325 for (Index kk = endk; kk < depth; kk++) {
1326 SLhsPacketQuarter a0;
1327 SRhsPacketQuarter b0;
1328 straits.loadLhsUnaligned(blB, a0);
1329 straits.loadRhs(blA, b0);
1330 straits.madd(a0, b0, c0, b0, fix<0>);
1331 blB += SwappedTraits::LhsProgress / 4;
1332 blA += 1;
1333 }
1334 straits.acc(c0, alphav, R);
1335 } else {
1336 straits.acc(predux_half(predux_half(C0)), alphav, R);
1337 }
1338 res.scatterPacket(i, j2, R);
1339 }
1340};
1341
1342// Compile-time recursive helper: processes RHS columns J..NrCols-1 for gebp_micro_onestep.
1343// For each column, loads/updates the RHS panel and does madd for all MrPackets LHS packets.
1344// The bool partial specialization terminates the recursion without requiring if constexpr.
1345template <int J, int MrPackets, int NrCols, bool Continue = (J < NrCols)>
1346struct gebp_rhs_cols;
1347
1348// Base case: J >= NrCols, do nothing.
1349template <int J, int MrPackets, int NrCols>
1350struct gebp_rhs_cols<J, MrPackets, NrCols, false> {
1351 template <typename GEBPTraits, typename LhsArray, typename RhsPanelType, typename RhsPacketType, typename AccArray,
1352 typename RhsScalar>
1353 static EIGEN_ALWAYS_INLINE void run(GEBPTraits&, const RhsScalar*, Index, LhsArray&, RhsPanelType&, RhsPacketType&,
1354 AccArray&) {}
1355};
1356
1357// Active case: J < NrCols.
1358template <int J, int MrPackets, int NrCols>
1359struct gebp_rhs_cols<J, MrPackets, NrCols, true> {
1360 template <typename GEBPTraits, typename LhsArray, typename RhsPanelType, typename RhsPacketType, typename AccArray,
1361 typename RhsScalar>
1362 static EIGEN_ALWAYS_INLINE void run(GEBPTraits& traits, const RhsScalar* blB, Index rhs_offset, LhsArray& A,
1363 RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
1364 constexpr int lane = J % 4;
1365 EIGEN_IF_CONSTEXPR (lane == 0)
1366 traits.loadRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
1367 else
1368 traits.updateRhs(blB + (J + rhs_offset) * GEBPTraits::RhsProgress, rhs_panel);
1369
1370 EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.madd(A[0], rhs_panel, C[J + 0 * NrCols], T0, fix<lane>);
1371 EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.madd(A[1], rhs_panel, C[J + 1 * NrCols], T0, fix<lane>);
1372 EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.madd(A[2], rhs_panel, C[J + 2 * NrCols], T0, fix<lane>);
1373
1374 gebp_rhs_cols<J + 1, MrPackets, NrCols>::run(traits, blB, rhs_offset, A, rhs_panel, T0, C);
1375 }
1376};
1377
1378// EIGEN_GEBP_NEON_GCC_WORKAROUNDS: AArch64 NEON with GCC >= 9, where the micro-kernel needs help keeping its
1379// accumulators in registers.
1380// EIGEN_GEBP_UNROLL_ACCUMULATORS: fully unrolls a loop over the accumulators (at most 3 x 8 = 24), exposing constant
1381// indices before GCC's scalar replacement pass at -O2.
1382#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_AT_LEAST(9, 0, 0)
1383#define EIGEN_GEBP_NEON_GCC_WORKAROUNDS
1384#define EIGEN_GEBP_UNROLL_ACCUMULATORS _Pragma("GCC unroll 24")
1385#else
1386#define EIGEN_GEBP_UNROLL_ACCUMULATORS
1387#endif
1388
1389// One step of the micro-kernel: loads MrPackets LHS packets at step K,
1390// then processes NrCols RHS columns via gebp_rhs_cols.
1391template <int K, int MrPackets, int NrCols>
1392struct gebp_micro_step {
1393 template <typename GEBPTraits, typename LhsScalar_, typename RhsScalar_, typename LhsArray, typename RhsPanelType,
1394 typename RhsPacketType, typename AccArray>
1395 static EIGEN_ALWAYS_INLINE void run(GEBPTraits& traits, const LhsScalar_* blA, const RhsScalar_* blB, LhsArray& A,
1396 RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C) {
1397 constexpr int LhsProg = GEBPTraits::LhsProgress;
1398
1399 EIGEN_IF_CONSTEXPR (MrPackets >= 1) traits.loadLhs(&blA[(0 + MrPackets * K) * LhsProg], A[0]);
1400 EIGEN_IF_CONSTEXPR (MrPackets >= 2) traits.loadLhs(&blA[(1 + MrPackets * K) * LhsProg], A[1]);
1401 EIGEN_IF_CONSTEXPR (MrPackets >= 3) traits.loadLhs(&blA[(2 + MrPackets * K) * LhsProg], A[2]);
1402
1403 gebp_rhs_cols<0, MrPackets, NrCols>::run(traits, blB, Index(NrCols * K), A, rhs_panel, T0, C);
1404#if defined(EIGEN_GEBP_NEON_GCC_WORKAROUNDS) && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING)
1405 using AccPacket = typename GEBPTraits::AccPacket;
1406 constexpr bool kRealPacket = std::is_same<AccPacket, Packet4f>::value || std::is_same<AccPacket, Packet2d>::value;
1407 EIGEN_IF_CONSTEXPR (MrPackets == 3 && NrCols == 8 && kRealPacket) {
1408 // Keep the next depth step's loads from extending live ranges beyond the 24 accumulators.
1409 __asm__ volatile("" ::: "memory");
1410#if EIGEN_GNUC_STRICT_AT_LEAST(16, 0, 0)
1411 // GCC 16 also needs the double accumulator updates to stay on their own depth step.
1412 EIGEN_IF_CONSTEXPR (std::is_same<AccPacket, Packet2d>::value) {
1413 EIGEN_GEBP_UNROLL_ACCUMULATORS
1414 for (int n = 0; n < MrPackets * NrCols; ++n) {
1415 EIGEN_OPTIMIZATION_BARRIER(C[n]);
1416 }
1417 }
1418#endif
1419 }
1420#endif
1421 }
1422};
1423// Compiler register allocation workarounds for the GEBP micro-kernel.
1424// GCC and clang can fail to keep array-based SIMD values in vector registers, causing
1425// excessive spilling. These helpers use inline asm constraints to pin values.
1426// Only applied when the scalar type is actually vectorizable (not custom types).
1427// See Eigen bugs 935, 1637, and 3059.
1428
1429// ARM64 NEON: pin 3 LHS packets in vector registers.
1430// Old GCC (< 9) misallocates registers for 3-packet paths without this hint.
1431template <int MrPackets, typename GEBPTraits_, typename FullLhsPacket_, typename LhsArray_>
1432EIGEN_ALWAYS_INLINE void gebp_neon_3p_workaround(LhsArray_& A) {
1433#if EIGEN_ARCH_ARM64 && defined(EIGEN_VECTORIZE_NEON) && EIGEN_GNUC_STRICT_LESS_THAN(9, 0, 0)
1434 using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
1435 constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets == 3 && std::is_same<LhsElement, FullLhsPacket_>::value;
1436 EIGEN_IF_CONSTEXPR (apply) {
1437 __asm__("" : "+w,m"(A[0]), "+w,m"(A[1]), "+w,m"(A[2]));
1438 }
1439#else
1440 EIGEN_UNUSED_VARIABLE(A);
1441#endif
1442}
1443
1444// GCC SSE: prevent register spilling for LHS packets and accumulators.
1445// C++17: pin accumulators with strict "+x" (if constexpr discards dead branches).
1446// C++14: pin LHS packets with relaxed "+x,m" (memory fallback for non-SSE types).
1447template <int MrPackets, int NrCols, typename GEBPTraits_, typename FullLhsPacket_, typename LhsArray_,
1448 typename AccArray_>
1449EIGEN_ALWAYS_INLINE void gebp_sse_spilling_workaround(LhsArray_& A, AccArray_& ACC) {
1450 EIGEN_UNUSED_VARIABLE(A);
1451 EIGEN_UNUSED_VARIABLE(ACC);
1452#if EIGEN_GNUC_STRICT_AT_LEAST(6, 0, 0) && defined(EIGEN_VECTORIZE_SSE)
1453 using LhsElement = std::remove_all_extents_t<std::remove_reference_t<LhsArray_>>;
1454 constexpr bool apply =
1455 GEBPTraits_::Vectorizable && MrPackets <= 2 && NrCols >= 4 && std::is_same<LhsElement, FullLhsPacket_>::value;
1456 EIGEN_IF_CONSTEXPR (apply) {
1457#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1458 using AccElement = std::decay_t<decltype(ACC[0])>;
1459 constexpr bool pin_acc = std::is_same<AccElement, FullLhsPacket_>::value && MrPackets == 2 && NrCols == 4;
1460 if constexpr (pin_acc) {
1461 __asm__(""
1462 : "+x"(ACC[0]), "+x"(ACC[1]), "+x"(ACC[2]), "+x"(ACC[3]), "+x"(ACC[4]), "+x"(ACC[5]), "+x"(ACC[6]),
1463 "+x"(ACC[7]));
1464 }
1465#else
1466 EIGEN_IF_CONSTEXPR (MrPackets == 2) {
1467 __asm__("" : "+x,m"(A[0]), "+x,m"(A[1]));
1468 }
1469#endif
1470 }
1471#endif
1472}
1473
1474// Clang AVX/AVX2 on x86-64: the 3pX4 tile needs all 16 ymm registers (12 accumulators, 3 LHS packets, 1 RHS
1475// broadcast), and clang keeps accumulators on the stack, reloading and storing them on every depth step: a dependency
1476// chain through store forwarding that roughly halves the kernel's throughput. Pinned accumulators make clang spill LHS
1477// packets instead, which are off those chains. 32-bit x86 has 8 ymm registers, so its accumulators spill anyway and
1478// pinning only adds stack traffic. A barrier sends a class through memory, so complex packets are pinned through their
1479// vector member. Not Packet8i: its madd needs a 17th register, and pinning made rare stack layouts up to 2.8x slower.
1480#if EIGEN_COMP_CLANG && EIGEN_ARCH_x86_64 && defined(EIGEN_VECTORIZE_AVX) && !defined(EIGEN_VECTORIZE_AVX512)
1481#define EIGEN_GEBP_AVX_CLANG_PIN_ACCUMULATORS
1482template <typename Packet>
1483EIGEN_ALWAYS_INLINE void gebp_avx_clang_pin_accumulator(Packet& x) {
1484 pbarrier(x);
1485}
1486EIGEN_ALWAYS_INLINE void gebp_avx_clang_pin_accumulator(Packet4cf& x) { pbarrier(x.v); }
1487EIGEN_ALWAYS_INLINE void gebp_avx_clang_pin_accumulator(Packet2cd& x) { pbarrier(x.v); }
1488#endif
1489template <int MrPackets, int NrCols, typename GEBPTraits_, typename FullLhsPacket_, typename AccArray_>
1490EIGEN_ALWAYS_INLINE void gebp_avx_clang_spilling_workaround(AccArray_& ACC) {
1491 EIGEN_UNUSED_VARIABLE(ACC);
1492#ifdef EIGEN_GEBP_AVX_CLANG_PIN_ACCUMULATORS
1493 using AccElement = std::decay_t<decltype(ACC[0])>;
1494 constexpr bool kFloatingPoint =
1495 std::is_same<AccElement, Packet8f>::value || std::is_same<AccElement, Packet4d>::value ||
1496 std::is_same<AccElement, Packet4cf>::value || std::is_same<AccElement, Packet2cd>::value;
1497 constexpr bool apply = GEBPTraits_::Vectorizable && MrPackets == 3 && NrCols == 4 && kFloatingPoint &&
1498 std::is_same<AccElement, FullLhsPacket_>::value;
1499 EIGEN_IF_CONSTEXPR (apply) {
1500 for (int n = 0; n < MrPackets * NrCols; ++n) gebp_avx_clang_pin_accumulator(ACC[n]);
1501 }
1502#endif
1503}
1504
1505// Double-accumulation trick for 1pX4 path to break FMA dependency chains. A DoublePacket
1506// accumulator already holds two chains, and with 16 registers doubling it would spill.
1507template <typename AccPacket, int MrPackets, int NrCols>
1508struct gebp_use_double_accum {
1509 static constexpr bool value = MrPackets == 1 && NrCols == 4
1510#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
1511 && !(is_double_packet<AccPacket>::value && EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS < 32)
1512#endif
1513 ;
1514};
1515
1516// Unrolled peeled loop body: calls gebp_micro_step for K=0..7, handling
1517// double-accumulation for 1pX4, prefetches, and compiler workarounds.
1518template <int MrPackets, int NrCols>
1519struct gebp_peeled_loop {
1520 template <typename GEBPTraits, typename LhsScalar_, typename RhsScalar_, typename LhsArray, typename RhsPanelType,
1521 typename RhsPacketType, typename AccArray, typename AccArrayD, typename FullLhsPacket>
1522 static EIGEN_ALWAYS_INLINE void run(GEBPTraits& traits, const LhsScalar_* blA, const RhsScalar_* blB, LhsArray& A,
1523 RhsPanelType& rhs_panel, RhsPacketType& T0, AccArray& C, AccArrayD& D) {
1524 constexpr bool use_double_accum = gebp_use_double_accum<typename GEBPTraits::AccPacket, MrPackets, NrCols>::value;
1525
1526 // Prefetch for 4-col paths
1527 EIGEN_IF_CONSTEXPR (NrCols == 4) {
1528 internal::prefetch(blB + (48 + 0));
1529 }
1530
1531 // Helper to do one step with workarounds
1532#define EIGEN_GEBP_DO_STEP(KVAL, ACC) \
1533 do { \
1534 gebp_micro_step<KVAL, MrPackets, NrCols>::run(traits, blA, blB, A, rhs_panel, T0, ACC); \
1535 gebp_neon_3p_workaround<MrPackets, GEBPTraits, FullLhsPacket>(A); \
1536 gebp_sse_spilling_workaround<MrPackets, NrCols, GEBPTraits, FullLhsPacket>(A, ACC); \
1537 gebp_avx_clang_spilling_workaround<MrPackets, NrCols, GEBPTraits, FullLhsPacket>(ACC); \
1538 /* LHS prefetch for 2pX4 and 3pX4 */ \
1539 EIGEN_IF_CONSTEXPR ((MrPackets == 2 || MrPackets == 3) && NrCols == 4) { \
1540 internal::prefetch(blA + (MrPackets * KVAL + 16) * GEBPTraits::LhsProgress); \
1541 if (EIGEN_ARCH_ARM || EIGEN_ARCH_MIPS) { \
1542 internal::prefetch(blB + (NrCols * KVAL + 16) * GEBPTraits::RhsProgress); \
1543 } \
1544 } \
1545 } while (false)
1546
1547 EIGEN_IF_CONSTEXPR (use_double_accum) {
1548 EIGEN_GEBP_DO_STEP(0, C);
1549 EIGEN_GEBP_DO_STEP(1, D);
1550 EIGEN_GEBP_DO_STEP(2, C);
1551 EIGEN_GEBP_DO_STEP(3, D);
1552 EIGEN_IF_CONSTEXPR (NrCols == 4) {
1553 internal::prefetch(blB + (48 + 16));
1554 }
1555 EIGEN_GEBP_DO_STEP(4, C);
1556 EIGEN_GEBP_DO_STEP(5, D);
1557 EIGEN_GEBP_DO_STEP(6, C);
1558 EIGEN_GEBP_DO_STEP(7, D);
1559 } else {
1560 EIGEN_GEBP_DO_STEP(0, C);
1561 EIGEN_GEBP_DO_STEP(1, C);
1562 EIGEN_GEBP_DO_STEP(2, C);
1563 EIGEN_GEBP_DO_STEP(3, C);
1564 EIGEN_IF_CONSTEXPR (NrCols == 4 && MrPackets == 2) {
1565 internal::prefetch(blB + (48 + 16));
1566 }
1567 EIGEN_GEBP_DO_STEP(4, C);
1568 EIGEN_GEBP_DO_STEP(5, C);
1569 EIGEN_GEBP_DO_STEP(6, C);
1570 EIGEN_GEBP_DO_STEP(7, C);
1571 }
1572
1573#undef EIGEN_GEBP_DO_STEP
1574 }
1575};
1576
1577// Unified micro-panel function: handles a MrPackets x NrCols register block.
1578// GEBPTraits determines the packet types (supports full/half/quarter sizes).
1579// Accumulator layout: C[j + p * NrCols] for column j, LHS packet p.
1580template <int MrPackets, int NrCols, typename GEBPTraits, typename LhsScalar_, typename RhsScalar_, typename ResScalar_,
1581 typename Index_, typename DataMapper_, typename LinearMapper_, typename FullLhsPacket>
1582EIGEN_ALWAYS_INLINE void gebp_micro_panel_impl(GEBPTraits& traits, const DataMapper_& res, const LhsScalar_* blockA,
1583 const RhsScalar_* blockB, ResScalar_ alpha, Index_ i, Index_ j2,
1584 Index_ depth, Index_ strideA, Index_ strideB, Index_ offsetA,
1585 Index_ offsetB, int prefetch_res_offset, Index_ peeled_kc, int pk,
1586 bool overwrite) {
1587 using LhsPacketLocal = typename GEBPTraits::LhsPacket;
1588 using RhsPacketLocal = typename GEBPTraits::RhsPacket;
1589 using ResPacketLocal = typename GEBPTraits::ResPacket;
1590 using AccPacketLocal = typename GEBPTraits::AccPacket;
1591 using RhsPacketx4Local = typename GEBPTraits::RhsPacketx4;
1592 constexpr int LhsProg = GEBPTraits::LhsProgress;
1593 constexpr int RhsProg = GEBPTraits::RhsProgress;
1594 constexpr int ResPacketSz = GEBPTraits::ResPacketSize;
1595
1596 // Determine RhsPanel type based on register pressure. A DoublePacket takes two registers.
1597 constexpr bool use_double_accum = gebp_use_double_accum<AccPacketLocal, MrPackets, NrCols>::value;
1598 constexpr int AccRegisters = is_double_packet<AccPacketLocal>::value ? 2 : 1;
1599 constexpr int RhsRegisters = is_double_packet<RhsPacketLocal>::value ? 2 : 1;
1600 constexpr int RegistersTaken = AccRegisters * (MrPackets + use_double_accum) * NrCols + MrPackets;
1601 using RhsPanelType =
1602 std::conditional_t<NrCols == 1, RhsPacketLocal,
1603 typename RhsPanelHelper<RhsPacketLocal, RhsPacketx4Local, RegistersTaken, RhsRegisters>::type>;
1604
1605 const LhsScalar_* blA = &blockA[i * strideA + offsetA * (MrPackets * LhsProg)];
1606 prefetch(&blA[0]);
1607
1608 // Accumulators: C[j + p * NrCols] for column j, LHS packet p.
1609 // With if constexpr (C++17) we use exact sizes; with plain if (C++14) we pad
1610 // to 3*NrCols so dead-branch array accesses in gebp_rhs_cols remain valid.
1611#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1612 constexpr int CSize = MrPackets * NrCols;
1613#else
1614 constexpr int CSize = 3 * NrCols > MrPackets * NrCols ? 3 * NrCols : MrPackets * NrCols;
1615#endif
1616 alignas(AccPacketLocal) AccPacketLocal C[CSize];
1617 EIGEN_GEBP_UNROLL_ACCUMULATORS
1618 for (int n = 0; n < MrPackets * NrCols; ++n) traits.initAcc(C[n]);
1619
1620#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1621 alignas(AccPacketLocal) AccPacketLocal D[use_double_accum ? NrCols : 1];
1622#else
1623 // Without if constexpr, we must allocate a larger array to satisfy the
1624 // compiler that D[n] is always in bounds for the use_double_accum path.
1625 alignas(AccPacketLocal) AccPacketLocal D[CSize];
1626#endif
1627 EIGEN_IF_CONSTEXPR (use_double_accum) {
1628 for (int n = 0; n < NrCols; ++n) traits.initAcc(D[n]);
1629 }
1630
1631 // Prefetch result memory
1632 for (int j = 0; j < NrCols; ++j) res.getLinearMapper(i, j2 + j).prefetch(NrCols > 1 ? prefetch_res_offset : 0);
1633
1634 // RHS pointer
1635 const RhsScalar_* blB = &blockB[j2 * strideB + offsetB * NrCols];
1636 prefetch(&blB[0]);
1637
1638 // LHS packet staging area. With if constexpr (C++17) we use exact sizes.
1639#ifdef EIGEN_HAS_CXX17_IFCONSTEXPR
1640 alignas(LhsPacketLocal) LhsPacketLocal A[MrPackets];
1641#else
1642 alignas(LhsPacketLocal) LhsPacketLocal A[3];
1643#endif
1644
1645 // ---- Peeled k-loop (pk=8 unrolled) ----
1646#if defined(EIGEN_VECTORIZE_RVV10) && EIGEN_GNUC_STRICT_AT_LEAST(15, 0, 0) && EIGEN_GNUC_STRICT_LESS_THAN(17, 0, 0)
1647 // GCC 15 and 16 miscompile scalar packet instantiations when this manually unrolled loop is vectorized.
1648#pragma GCC novector
1649#endif
1650 for (Index_ k = 0; k < peeled_kc; k += pk) {
1651 alignas(RhsPanelType) RhsPanelType rhs_panel;
1652 alignas(RhsPacketLocal) RhsPacketLocal T0;
1653
1654 gebp_peeled_loop<MrPackets, NrCols>::template run<GEBPTraits, LhsScalar_, RhsScalar_, decltype(A), RhsPanelType,
1655 RhsPacketLocal, decltype(C), decltype(D), FullLhsPacket>(
1656 traits, blA, blB, A, rhs_panel, T0, C, D);
1657
1658 blB += pk * NrCols * RhsProg;
1659 blA += pk * MrPackets * LhsProg;
1660 }
1661
1662 // Merge double accumulators
1663 EIGEN_IF_CONSTEXPR (use_double_accum) {
1664 EIGEN_GEBP_UNROLL_ACCUMULATORS
1665 for (int n = 0; n < NrCols; ++n) C[n] = padd(C[n], D[n]);
1666 }
1667
1668 // ---- Remainder k-loop ----
1669 for (Index_ k = peeled_kc; k < depth; k++) {
1670 alignas(RhsPanelType) RhsPanelType rhs_panel;
1671 alignas(RhsPacketLocal) RhsPacketLocal T0;
1672
1673 gebp_micro_step<0, MrPackets, NrCols>::run(traits, blA, blB, A, rhs_panel, T0, C);
1674
1675 blB += NrCols * RhsProg;
1676 blA += MrPackets * LhsProg;
1677 }
1678
1679 // ---- Store results: C[j + p * NrCols] -> res(i + p*ResPacketSz, j2 + j) ----
1680 alignas(ResPacketLocal) ResPacketLocal alphav = pset1<ResPacketLocal>(alpha);
1681 EIGEN_GEBP_UNROLL_ACCUMULATORS
1682 for (int j = 0; j < NrCols; ++j) {
1683 LinearMapper_ r = res.getLinearMapper(i, j2 + j);
1684 EIGEN_GEBP_UNROLL_ACCUMULATORS
1685 for (int p = 0; p < MrPackets; ++p) {
1686 alignas(ResPacketLocal) ResPacketLocal R =
1687 overwrite ? pset1<ResPacketLocal>(ResScalar_(0)) : r.template loadPacket<ResPacketLocal>(p * ResPacketSz);
1688 traits.acc(C[j + p * NrCols], alphav, R);
1689 r.storePacket(p * ResPacketSz, R);
1690 }
1691 }
1692}
1693#undef EIGEN_GEBP_UNROLL_ACCUMULATORS
1694#undef EIGEN_GEBP_NEON_GCC_WORKAROUNDS
1695#undef EIGEN_GEBP_AVX_CLANG_PIN_ACCUMULATORS
1696
1697// Workaround a GCC/AArch64 register-allocation issue (through at least GCC 14.3): the float
1698// kernel's mr=12 x nr=8 tile holds 24 of the 32 NEON registers, and GCC's pre-RA scheduler
1699// (-fschedule-insns, on at -O2/-O3) hoists loads past that limit, spilling ~20 vector regs and
1700// ~halving GEMM throughput (~2.1x fp32 on Cortex-X925; spills 20 -> 2 with it off). Clang
1701// allocates the same tile spill-free, so we keep nr=8 and just disable that GCC pass here.
1702// This is a known bug in GCC: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100697
1703// In-order cores can opt out via EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING, which also drops the
1704// depth-step barriers in gebp_micro_step.
1705#if EIGEN_COMP_GNUC_STRICT && EIGEN_ARCH_ARM64 && !defined(EIGEN_DONT_DISABLE_GEBP_INSN_SCHEDULING)
1706#pragma GCC push_options
1707#pragma GCC optimize("no-schedule-insns")
1708#define EIGEN_GEBP_DISABLED_INSN_SCHEDULING
1709#endif
1710template <typename LhsScalar, typename RhsScalar, typename Index, typename DataMapper, int mr, int nr,
1711 bool ConjugateLhs, bool ConjugateRhs>
1712EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE void
1713gebp_kernel<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::run(
1714 bool overwrite_, const DataMapper& res, const LhsScalar* blockA, const RhsScalar* blockB, Index rows, Index depth,
1715 Index cols, ResScalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) const {
1716 // Constant false where the scalar cannot overwrite, so that the overwriting stores fold away.
1717 const bool overwrite = CanOverwrite && overwrite_;
1718 Traits traits;
1719 SwappedTraits straits;
1720
1721 if (strideA == -1) strideA = depth;
1722 if (strideB == -1) strideB = depth;
1723 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
1724 Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
1725 Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0;
1726 const Index peeled_mc3 = mr >= 3 * Traits::LhsProgress ? (rows / (3 * LhsProgress)) * (3 * LhsProgress) : 0;
1727 const Index peeled_mc2 =
1728 mr >= 2 * Traits::LhsProgress ? peeled_mc3 + ((rows - peeled_mc3) / (2 * LhsProgress)) * (2 * LhsProgress) : 0;
1729 const Index peeled_mc1 =
1730 mr >= 1 * Traits::LhsProgress ? peeled_mc2 + ((rows - peeled_mc2) / (1 * LhsProgress)) * (1 * LhsProgress) : 0;
1731 const Index peeled_mc_half =
1732 mr >= LhsProgressHalf ? peeled_mc1 + ((rows - peeled_mc1) / (LhsProgressHalf)) * (LhsProgressHalf) : 0;
1733 const Index peeled_mc_quarter =
1734 mr >= LhsProgressQuarter
1735 ? peeled_mc_half + ((rows - peeled_mc_half) / (LhsProgressQuarter)) * (LhsProgressQuarter)
1736 : 0;
1737 enum { pk = 8 }; // NOTE Such a large peeling factor is important for large matrices (~ +5% when >1000 on Haswell)
1738 const Index peeled_kc = depth & ~(pk - 1);
1739 const int prefetch_res_offset = 32 / sizeof(ResScalar);
1740
1741 // Helper to invoke gebp_micro_panel_impl with the right types.
1742 // The always_inline attribute is critical: without it GCC outlines each
1743 // template instantiation of this generic lambda as a separate function,
1744 // adding call overhead that causes 10-17 % regressions in LLT/TRSM
1745 // for small-to-medium matrix sizes.
1746 auto micro_panel = [&](auto mrp_tag, auto nrc_tag, auto& local_traits, Index i, Index j2) EIGEN_LAMBDA_ALWAYS_INLINE {
1747 constexpr int MrP = decltype(mrp_tag)::value;
1748 constexpr int NrC = decltype(nrc_tag)::value;
1749 using LTraits = std::remove_reference_t<decltype(local_traits)>;
1750 gebp_micro_panel_impl<MrP, NrC, LTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1751 LhsPacket>(local_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB, offsetA,
1752 offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1753 };
1754
1755 // Budget (in bytes) for co-residency of the RHS block and a strip of the
1756 // LHS panel. On most architectures this is L1: the LHS streams sequentially
1757 // through L1 and we only need room for one micro-panel strip at a time.
1758 // Sub-blocking trades cache misses for extra passes over the RHS columns,
1759 // which hurts IPC and loop overhead. On modern x86, L1→L2 traffic is cheap
1760 // (~5 cycles) and hardware prefetchers absorb the LHS misses, so we use a
1761 // fraction of L2 instead — effectively disabling sub-blocking when the LHS
1762 // panel already fits in L2.
1763 Index lhs_budget;
1764 {
1765 std::ptrdiff_t l1, l2, l3;
1766 manage_caching_sizes(GetAction, &l1, &l2, &l3);
1767#if EIGEN_ARCH_i386_OR_x86_64
1768 lhs_budget = static_cast<Index>(l2 / 2);
1769#else
1770 lhs_budget = static_cast<Index>(l1);
1771#endif
1772 }
1773
1774 //---------- Process 3 * LhsProgress rows at once ----------
1775 EIGEN_IF_CONSTEXPR (mr >= 3 * Traits::LhsProgress) {
1776 const Index rhs_block = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
1777 const Index lhs_strip = depth * sizeof(LhsScalar) * 3 * LhsProgress;
1778 const Index lhs_avail = (lhs_budget > rhs_block) ? (lhs_budget - rhs_block) : 0;
1779 const Index actual_panel_rows = (lhs_avail >= peeled_mc3 * depth * static_cast<Index>(sizeof(LhsScalar)))
1780 ? peeled_mc3
1781 : (3 * LhsProgress) * std::max<Index>(1, lhs_avail / lhs_strip);
1782 for (Index i1 = 0; i1 < peeled_mc3; i1 += actual_panel_rows) {
1783 const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc3);
1784 EIGEN_IF_CONSTEXPR (nr >= 8) {
1785 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1786 for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
1787 micro_panel(fix<3>, fix<8>, traits, i, j2);
1788 }
1789 }
1790 }
1791 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1792 for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
1793 micro_panel(fix<3>, fix<4>, traits, i, j2);
1794 }
1795 }
1796 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1797 for (Index i = i1; i < actual_panel_end; i += 3 * LhsProgress) {
1798 micro_panel(fix<3>, fix<1>, traits, i, j2);
1799 }
1800 }
1801 }
1802 }
1803
1804 //---------- Process 2 * LhsProgress rows at once ----------
1805 EIGEN_IF_CONSTEXPR (mr >= 2 * Traits::LhsProgress) {
1806 const Index rhs_block2 = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
1807 const Index lhs_strip2 = depth * sizeof(LhsScalar) * 2 * LhsProgress;
1808 const Index lhs_avail2 = (lhs_budget > rhs_block2) ? (lhs_budget - rhs_block2) : 0;
1809 const Index mc2_range = peeled_mc2 - peeled_mc3;
1810 Index actual_panel_rows = (lhs_avail2 >= mc2_range * depth * static_cast<Index>(sizeof(LhsScalar)))
1811 ? mc2_range
1812 : (2 * LhsProgress) * std::max<Index>(1, lhs_avail2 / lhs_strip2);
1813 for (Index i1 = peeled_mc3; i1 < peeled_mc2; i1 += actual_panel_rows) {
1814 Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc2);
1815 EIGEN_IF_CONSTEXPR (nr >= 8) {
1816 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1817 for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
1818 micro_panel(fix<2>, fix<8>, traits, i, j2);
1819 }
1820 }
1821 }
1822 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1823 for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
1824 micro_panel(fix<2>, fix<4>, traits, i, j2);
1825 }
1826 }
1827 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1828 for (Index i = i1; i < actual_panel_end; i += 2 * LhsProgress) {
1829 micro_panel(fix<2>, fix<1>, traits, i, j2);
1830 }
1831 }
1832 }
1833 }
1834
1835 //---------- Process 1 * LhsProgress rows at once ----------
1836 // With a wider mr, at most one row panel is left here.
1837 EIGEN_IF_CONSTEXPR (mr > 1 * Traits::LhsProgress) {
1838 for (Index i = peeled_mc2; i < peeled_mc1; i += LhsProgress) {
1839 EIGEN_IF_CONSTEXPR (nr >= 8) {
1840 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1841 micro_panel(fix<1>, fix<8>, traits, i, j2);
1842 }
1843 }
1844 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1845 micro_panel(fix<1>, fix<4>, traits, i, j2);
1846 }
1847 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1848 micro_panel(fix<1>, fix<1>, traits, i, j2);
1849 }
1850 }
1851 } else EIGEN_IF_CONSTEXPR (mr == 1 * Traits::LhsProgress) {
1852 // Every row comes here, blocked like the wider panels above: rows inside columns keep the rhs panel in cache
1853 // and walk C down its columns.
1854 const Index rhs_block1 = sizeof(ResScalar) * mr * nr + depth * nr * sizeof(RhsScalar);
1855 const Index lhs_strip1 = depth * sizeof(LhsScalar) * LhsProgress;
1856 const Index lhs_avail1 = (lhs_budget > rhs_block1) ? (lhs_budget - rhs_block1) : 0;
1857 const Index mc1_range = peeled_mc1 - peeled_mc2;
1858 const Index actual_panel_rows = (lhs_avail1 >= mc1_range * depth * static_cast<Index>(sizeof(LhsScalar)))
1859 ? mc1_range
1860 : LhsProgress * std::max<Index>(1, lhs_avail1 / lhs_strip1);
1861 for (Index i1 = peeled_mc2; i1 < peeled_mc1; i1 += actual_panel_rows) {
1862 const Index actual_panel_end = (std::min)(i1 + actual_panel_rows, peeled_mc1);
1863 EIGEN_IF_CONSTEXPR (nr >= 8) {
1864 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1865 for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
1866 micro_panel(fix<1>, fix<8>, traits, i, j2);
1867 }
1868 }
1869 }
1870 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1871 for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
1872 micro_panel(fix<1>, fix<4>, traits, i, j2);
1873 }
1874 }
1875 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1876 for (Index i = i1; i < actual_panel_end; i += LhsProgress) {
1877 micro_panel(fix<1>, fix<1>, traits, i, j2);
1878 }
1879 }
1880 }
1881 }
1882
1883 //---------- Process LhsProgressHalf rows at once ----------
1884 EIGEN_IF_CONSTEXPR ((LhsProgressHalf < LhsProgress) && mr >= LhsProgressHalf) {
1885 HalfTraits half_traits;
1886 for (Index i = peeled_mc1; i < peeled_mc_half; i += LhsProgressHalf) {
1887 EIGEN_IF_CONSTEXPR (nr >= 8) {
1888 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1889 gebp_micro_panel_impl<1, 8, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1890 LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1891 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1892 }
1893 }
1894 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1895 gebp_micro_panel_impl<1, 4, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1896 LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1897 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1898 }
1899 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1900 gebp_micro_panel_impl<1, 1, HalfTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1901 LhsPacket>(half_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1902 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1903 }
1904 }
1905 }
1906
1907 //---------- Process LhsProgressQuarter rows at once ----------
1908 EIGEN_IF_CONSTEXPR ((LhsProgressQuarter < LhsProgressHalf) && mr >= LhsProgressQuarter) {
1909 QuarterTraits quarter_traits;
1910 for (Index i = peeled_mc_half; i < peeled_mc_quarter; i += LhsProgressQuarter) {
1911 EIGEN_IF_CONSTEXPR (nr >= 8) {
1912 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1913 gebp_micro_panel_impl<1, 8, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1914 LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1915 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1916 }
1917 }
1918 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1919 gebp_micro_panel_impl<1, 4, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1920 LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1921 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1922 }
1923 for (Index j2 = packet_cols4; j2 < cols; j2++) {
1924 gebp_micro_panel_impl<1, 1, QuarterTraits, LhsScalar, RhsScalar, ResScalar, Index, DataMapper, LinearMapper,
1925 LhsPacket>(quarter_traits, res, blockA, blockB, alpha, i, j2, depth, strideA, strideB,
1926 offsetA, offsetB, prefetch_res_offset, peeled_kc, pk, overwrite);
1927 }
1928 }
1929 }
1930
1931 //---------- Process remaining rows, 1 at once ----------
1932 if (peeled_mc_quarter < rows) {
1933 EIGEN_IF_CONSTEXPR (nr >= 8) {
1934 // loop on each panel of the rhs
1935 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
1936 // loop on each row of the lhs (1*LhsProgress x depth)
1937 for (Index i = peeled_mc_quarter; i < rows; i += 1) {
1938 const LhsScalar* blA = &blockA[i * strideA + offsetA];
1939 prefetch(&blA[0]);
1940 // gets a 1 x 1 res block as registers
1941 ResScalar C0(0), C1(0), C2(0), C3(0), C4(0), C5(0), C6(0), C7(0);
1942 const RhsScalar* blB = &blockB[j2 * strideB + offsetB * 8];
1943 for (Index k = 0; k < depth; k++) {
1944 LhsScalar A0 = blA[k];
1945 RhsScalar B_0;
1946
1947 B_0 = blB[0];
1948 C0 = cj.pmadd(A0, B_0, C0);
1949
1950 B_0 = blB[1];
1951 C1 = cj.pmadd(A0, B_0, C1);
1952
1953 B_0 = blB[2];
1954 C2 = cj.pmadd(A0, B_0, C2);
1955
1956 B_0 = blB[3];
1957 C3 = cj.pmadd(A0, B_0, C3);
1958
1959 B_0 = blB[4];
1960 C4 = cj.pmadd(A0, B_0, C4);
1961
1962 B_0 = blB[5];
1963 C5 = cj.pmadd(A0, B_0, C5);
1964
1965 B_0 = blB[6];
1966 C6 = cj.pmadd(A0, B_0, C6);
1967
1968 B_0 = blB[7];
1969 C7 = cj.pmadd(A0, B_0, C7);
1970
1971 blB += 8;
1972 }
1973 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 0), alpha, C0);
1974 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 1), alpha, C1);
1975 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 2), alpha, C2);
1976 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 3), alpha, C3);
1977 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 4), alpha, C4);
1978 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 5), alpha, C5);
1979 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 6), alpha, C6);
1980 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 7), alpha, C7);
1981 }
1982 }
1983 }
1984
1985 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
1986 // loop on each row of the lhs (1*LhsProgress x depth)
1987 for (Index i = peeled_mc_quarter; i < rows; i += 1) {
1988 const LhsScalar* blA = &blockA[i * strideA + offsetA];
1989 prefetch(&blA[0]);
1990 const RhsScalar* blB = &blockB[j2 * strideB + offsetB * 4];
1991
1992 // This loop packs groups of 4 columns, so the sub-packet holding them is 4
1993 // lanes wide regardless of nr, which is tunable (EIGEN_SVE_GEBP_NR and
1994 // friends) and 8 on several backends.
1995 constexpr int SResPacketHalfSize = unpacket_traits<typename unpacket_traits<SResPacket>::half>::size;
1996 constexpr int SResPacketQuarterSize =
1997 unpacket_traits<typename unpacket_traits<typename unpacket_traits<SResPacket>::half>::half>::size;
1998 // The following code assumes we can load SRhsPacket in such a way that
1999 // it multiplies blocks of 4 elements in SLhsPacket. This is not the
2000 // case for some customized kernels (i.e. NEON fp16). If the assumption
2001 // fails, drop down to the scalar path.
2002 constexpr bool kCanLoadSRhsQuad =
2003 (unpacket_traits<SLhsPacket>::size < 4) ||
2004 (unpacket_traits<SRhsPacket>::size % ((std::max<int>)(unpacket_traits<SLhsPacket>::size, 4) / 4)) == 0;
2005 EIGEN_IF_CONSTEXPR (kCanLoadSRhsQuad && (SwappedTraits::LhsProgress % 4) == 0 &&
2006 (SwappedTraits::LhsProgress <= 16) &&
2007 (SwappedTraits::LhsProgress != 8 || SResPacketHalfSize == 4) &&
2008 (SwappedTraits::LhsProgress != 16 || SResPacketQuarterSize == 4)) {
2009 SAccPacket C0, C1, C2, C3;
2010 straits.initAcc(C0);
2011 straits.initAcc(C1);
2012 straits.initAcc(C2);
2013 straits.initAcc(C3);
2014
2015 const Index spk = (std::max)(1, SwappedTraits::LhsProgress / 4);
2016 const Index endk = (depth / spk) * spk;
2017 const Index endk4 = (depth / (spk * 4)) * (spk * 4);
2018
2019 Index k = 0;
2020 for (; k < endk4; k += 4 * spk) {
2021 SLhsPacket A0, A1;
2022 SRhsPacket B_0, B_1;
2023
2024 straits.loadLhsUnaligned(blB + 0 * SwappedTraits::LhsProgress, A0);
2025 straits.loadLhsUnaligned(blB + 1 * SwappedTraits::LhsProgress, A1);
2026
2027 straits.loadRhsQuad(blA + 0 * spk, B_0);
2028 straits.loadRhsQuad(blA + 1 * spk, B_1);
2029 straits.madd(A0, B_0, C0, B_0, fix<0>);
2030 straits.madd(A1, B_1, C1, B_1, fix<0>);
2031
2032 straits.loadLhsUnaligned(blB + 2 * SwappedTraits::LhsProgress, A0);
2033 straits.loadLhsUnaligned(blB + 3 * SwappedTraits::LhsProgress, A1);
2034 straits.loadRhsQuad(blA + 2 * spk, B_0);
2035 straits.loadRhsQuad(blA + 3 * spk, B_1);
2036 straits.madd(A0, B_0, C2, B_0, fix<0>);
2037 straits.madd(A1, B_1, C3, B_1, fix<0>);
2038
2039 blB += 4 * SwappedTraits::LhsProgress;
2040 blA += 4 * spk;
2041 }
2042 C0 = padd(padd(C0, C1), padd(C2, C3));
2043 for (; k < endk; k += spk) {
2044 SLhsPacket A0;
2045 SRhsPacket B_0;
2046
2047 straits.loadLhsUnaligned(blB, A0);
2048 straits.loadRhsQuad(blA, B_0);
2049 straits.madd(A0, B_0, C0, B_0, fix<0>);
2050
2051 blB += SwappedTraits::LhsProgress;
2052 blA += spk;
2053 }
2054 if (SwappedTraits::LhsProgress == 8) {
2055 // Special case where we have to first reduce the accumulation register C0
2056 using SResPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2057 typename unpacket_traits<SResPacket>::half, SResPacket>;
2058 using SLhsPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2059 typename unpacket_traits<SLhsPacket>::half, SLhsPacket>;
2060 using SRhsPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2061 typename unpacket_traits<SRhsPacket>::half, SRhsPacket>;
2062 using SAccPacketHalf = std::conditional_t<SwappedTraits::LhsProgress >= 8,
2063 typename unpacket_traits<SAccPacket>::half, SAccPacket>;
2064
2065 SResPacketHalf R =
2066 overwrite ? pset1<SResPacketHalf>(ResScalar(0)) : res.template gatherPacket<SResPacketHalf>(i, j2);
2067 SResPacketHalf alphav = pset1<SResPacketHalf>(alpha);
2068
2069 if (depth - endk > 0) {
2070 // We have to handle the last row of the rhs which corresponds to a half-packet
2071 SLhsPacketHalf a0;
2072 SRhsPacketHalf b0;
2073 straits.loadLhsUnaligned(blB, a0);
2074 straits.loadRhs(blA, b0);
2075 SAccPacketHalf c0 = predux_half(C0);
2076 straits.madd(a0, b0, c0, b0, fix<0>);
2077 straits.acc(c0, alphav, R);
2078 } else {
2079 straits.acc(predux_half(C0), alphav, R);
2080 }
2081 res.scatterPacket(i, j2, R);
2082 } else if (SwappedTraits::LhsProgress == 16) {
2083 // Special case where we have to first reduce the
2084 // accumulation register C0. We specialize the block in
2085 // template form, so that LhsProgress < 16 paths don't
2086 // fail to compile
2087 last_row_process_16_packets<LhsScalar, RhsScalar, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> p;
2088 p(res, straits, blA, blB, depth, endk, i, j2, alpha, C0, overwrite);
2089 } else {
2090 SResPacket R = overwrite ? pset1<SResPacket>(ResScalar(0)) : res.template gatherPacket<SResPacket>(i, j2);
2091 SResPacket alphav = pset1<SResPacket>(alpha);
2092 straits.acc(C0, alphav, R);
2093 res.scatterPacket(i, j2, R);
2094 }
2095 } else // scalar path
2096 {
2097 // get a 1 x 4 res block as registers
2098 ResScalar C0(0), C1(0), C2(0), C3(0);
2099
2100 for (Index k = 0; k < depth; k++) {
2101 LhsScalar A0;
2102 RhsScalar B_0, B_1;
2103
2104 A0 = blA[k];
2105
2106 B_0 = blB[0];
2107 B_1 = blB[1];
2108 C0 = cj.pmadd(A0, B_0, C0);
2109 C1 = cj.pmadd(A0, B_1, C1);
2110
2111 B_0 = blB[2];
2112 B_1 = blB[3];
2113 C2 = cj.pmadd(A0, B_0, C2);
2114 C3 = cj.pmadd(A0, B_1, C3);
2115
2116 blB += 4;
2117 }
2118 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 0), alpha, C0);
2119 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 1), alpha, C1);
2120 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 2), alpha, C2);
2121 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2 + 3), alpha, C3);
2122 }
2123 }
2124 }
2125 // remaining columns
2126 for (Index j2 = packet_cols4; j2 < cols; j2++) {
2127 // loop on each row of the lhs (1*LhsProgress x depth)
2128 for (Index i = peeled_mc_quarter; i < rows; i += 1) {
2129 const LhsScalar* blA = &blockA[i * strideA + offsetA];
2130 prefetch(&blA[0]);
2131 // gets a 1 x 1 res block as registers
2132 ResScalar C0(0);
2133 const RhsScalar* blB = &blockB[j2 * strideB + offsetB];
2134 for (Index k = 0; k < depth; k++) {
2135 LhsScalar A0 = blA[k];
2136 RhsScalar B_0 = blB[k];
2137 C0 = cj.pmadd(A0, B_0, C0);
2138 }
2139 gebp_store_scalar<CanOverwrite>(overwrite, res(i, j2), alpha, C0);
2140 }
2141 }
2142 }
2143}
2144#ifdef EIGEN_GEBP_DISABLED_INSN_SCHEDULING
2145#pragma GCC pop_options
2146#undef EIGEN_GEBP_DISABLED_INSN_SCHEDULING
2147#endif
2148
2149// pack a block of the lhs
2150// The traversal is as follow (mr==4):
2151// 0 4 8 12 ...
2152// 1 5 9 13 ...
2153// 2 6 10 14 ...
2154// 3 7 11 15 ...
2155//
2156// 16 20 24 28 ...
2157// 17 21 25 29 ...
2158// 18 22 26 30 ...
2159// 19 23 27 31 ...
2160//
2161// 32 33 34 35 ...
2162// 36 37 38 39 ...
2163template <typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate,
2164 bool PanelMode>
2165struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> {
2166 using LinearMapper = typename DataMapper::LinearMapper;
2167 EIGEN_DONT_INLINE void operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
2168 Index offset = 0) const;
2169};
2170
2171template <typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate,
2172 bool PanelMode>
2173EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate,
2174 PanelMode>::operator()(Scalar* blockA, const DataMapper& lhs, Index depth,
2175 Index rows, Index stride, Index offset) const {
2176 using HalfPacket = typename unpacket_traits<Packet>::half;
2177 using QuarterPacket = typename unpacket_traits<typename unpacket_traits<Packet>::half>::half;
2178 enum {
2179 PacketSize = unpacket_traits<Packet>::size,
2180 HalfPacketSize = unpacket_traits<HalfPacket>::size,
2181 QuarterPacketSize = unpacket_traits<QuarterPacket>::size,
2182 HasHalf = (int)HalfPacketSize < (int)PacketSize,
2183 HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize
2184 };
2185
2186 EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK LHS");
2187 EIGEN_UNUSED_VARIABLE(stride);
2188 EIGEN_UNUSED_VARIABLE(offset);
2189 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2190 eigen_assert(((Pack1 % PacketSize) == 0 && Pack1 <= 4 * PacketSize) || (Pack1 <= 4) || (Pack1 < PacketSize));
2191 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2192 Index count = 0;
2193
2194 const Index peeled_mc3 = Pack1 >= 3 * PacketSize ? (rows / (3 * PacketSize)) * (3 * PacketSize) : 0;
2195 const Index peeled_mc2 =
2196 Pack1 >= 2 * PacketSize ? peeled_mc3 + ((rows - peeled_mc3) / (2 * PacketSize)) * (2 * PacketSize) : 0;
2197 const Index peeled_mc1 =
2198 Pack1 >= 1 * PacketSize ? peeled_mc2 + ((rows - peeled_mc2) / (1 * PacketSize)) * (1 * PacketSize) : 0;
2199 const Index peeled_mc_half =
2200 Pack1 >= HalfPacketSize ? peeled_mc1 + ((rows - peeled_mc1) / (HalfPacketSize)) * (HalfPacketSize) : 0;
2201 const Index peeled_mc_quarter = Pack1 >= QuarterPacketSize ? (rows / (QuarterPacketSize)) * (QuarterPacketSize) : 0;
2202 const Index last_lhs_progress = rows > peeled_mc_quarter ? (rows - peeled_mc_quarter) & ~1 : 0;
2203 const Index peeled_mc0 = Pack2 >= PacketSize ? peeled_mc_quarter
2204 : Pack2 > 1 && last_lhs_progress ? (rows / last_lhs_progress) * last_lhs_progress
2205 : 0;
2206
2207 Index i = 0;
2208
2209 // Pack 3 packets
2210 EIGEN_IF_CONSTEXPR (Pack1 >= 3 * PacketSize) {
2211 for (; i < peeled_mc3; i += 3 * PacketSize) {
2212 EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * offset;
2213
2214 for (Index k = 0; k < depth; k++) {
2215 Packet A, B, C;
2216 A = lhs.template loadPacket<Packet>(i + 0 * PacketSize, k);
2217 B = lhs.template loadPacket<Packet>(i + 1 * PacketSize, k);
2218 C = lhs.template loadPacket<Packet>(i + 2 * PacketSize, k);
2219 pstore(blockA + count, cj.pconj(A));
2220 count += PacketSize;
2221 pstore(blockA + count, cj.pconj(B));
2222 count += PacketSize;
2223 pstore(blockA + count, cj.pconj(C));
2224 count += PacketSize;
2225 }
2226 EIGEN_IF_CONSTEXPR (PanelMode) count += (3 * PacketSize) * (stride - offset - depth);
2227 }
2228 }
2229 // Pack 2 packets
2230 EIGEN_IF_CONSTEXPR (Pack1 >= 2 * PacketSize) {
2231 for (; i < peeled_mc2; i += 2 * PacketSize) {
2232 EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * offset;
2233
2234 for (Index k = 0; k < depth; k++) {
2235 Packet A, B;
2236 A = lhs.template loadPacket<Packet>(i + 0 * PacketSize, k);
2237 B = lhs.template loadPacket<Packet>(i + 1 * PacketSize, k);
2238 pstore(blockA + count, cj.pconj(A));
2239 count += PacketSize;
2240 pstore(blockA + count, cj.pconj(B));
2241 count += PacketSize;
2242 }
2243 EIGEN_IF_CONSTEXPR (PanelMode) count += (2 * PacketSize) * (stride - offset - depth);
2244 }
2245 }
2246 // Pack 1 packets
2247 EIGEN_IF_CONSTEXPR (Pack1 >= 1 * PacketSize) {
2248 for (; i < peeled_mc1; i += 1 * PacketSize) {
2249 EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * offset;
2250
2251 for (Index k = 0; k < depth; k++) {
2252 Packet A;
2253 A = lhs.template loadPacket<Packet>(i + 0 * PacketSize, k);
2254 pstore(blockA + count, cj.pconj(A));
2255 count += PacketSize;
2256 }
2257 EIGEN_IF_CONSTEXPR (PanelMode) count += (1 * PacketSize) * (stride - offset - depth);
2258 }
2259 }
2260 // Pack half packets
2261 EIGEN_IF_CONSTEXPR (HasHalf && Pack1 >= HalfPacketSize) {
2262 for (; i < peeled_mc_half; i += HalfPacketSize) {
2263 EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize)*offset;
2264
2265 for (Index k = 0; k < depth; k++) {
2266 HalfPacket A;
2267 A = lhs.template loadPacket<HalfPacket>(i + 0 * (HalfPacketSize), k);
2268 pstoreu(blockA + count, cj.pconj(A));
2269 count += HalfPacketSize;
2270 }
2271 EIGEN_IF_CONSTEXPR (PanelMode) count += (HalfPacketSize) * (stride - offset - depth);
2272 }
2273 }
2274 // Pack quarter packets
2275 EIGEN_IF_CONSTEXPR (HasQuarter && Pack1 >= QuarterPacketSize) {
2276 for (; i < peeled_mc_quarter; i += QuarterPacketSize) {
2277 EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize)*offset;
2278
2279 for (Index k = 0; k < depth; k++) {
2280 QuarterPacket A;
2281 A = lhs.template loadPacket<QuarterPacket>(i + 0 * (QuarterPacketSize), k);
2282 pstoreu(blockA + count, cj.pconj(A));
2283 count += QuarterPacketSize;
2284 }
2285 EIGEN_IF_CONSTEXPR (PanelMode) count += (QuarterPacketSize) * (stride - offset - depth);
2286 }
2287 }
2288 // Pack2 may be *smaller* than PacketSize—that happens for
2289 // products like real * complex, where we have to go half the
2290 // progress on the lhs in order to duplicate those operands to
2291 // address both real & imaginary parts on the rhs. This portion will
2292 // pack those half ones until they match the number expected on the
2293 // last peeling loop at this point (for the rhs).
2294 //
2295 // When there are no half/quarter packet types (HasHalf and HasQuarter
2296 // are both false), last_lhs_progress can exceed Pack2, producing
2297 // interleaved groups that the GEBP micro-kernel cannot consume. In
2298 // that case we use exactly Pack2 rows per group so the kernel's main
2299 // loop (which reads Pack2 = LhsProgress values via ploaddup) can
2300 // handle them; remaining rows fall through to the scalar loop below.
2301 EIGEN_IF_CONSTEXPR (Pack2 < PacketSize && Pack2 > 1) {
2302 const Index pack2_progress = (HasHalf || HasQuarter) ? last_lhs_progress : Pack2;
2303 const Index peeled = (HasHalf || HasQuarter) ? peeled_mc0 : (rows / Pack2) * Pack2;
2304 for (; i < peeled; i += pack2_progress) {
2305 EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * offset;
2306
2307 for (Index k = 0; k < depth; k++)
2308 for (Index w = 0; w < pack2_progress; w++) blockA[count++] = cj(lhs(i + w, k));
2309
2310 EIGEN_IF_CONSTEXPR (PanelMode) count += pack2_progress * (stride - offset - depth);
2311 }
2312 }
2313 // Pack scalars
2314 for (; i < rows; i++) {
2315 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2316 for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
2317 EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
2318 }
2319}
2320
2321template <typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate,
2322 bool PanelMode>
2323struct gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> {
2324 using LinearMapper = typename DataMapper::LinearMapper;
2325 EIGEN_DONT_INLINE void operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
2326 Index offset = 0) const;
2327};
2328
2329template <typename Scalar, typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate,
2330 bool PanelMode>
2331EIGEN_DONT_INLINE void gemm_pack_lhs<Scalar, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate,
2332 PanelMode>::operator()(Scalar* blockA, const DataMapper& lhs, Index depth,
2333 Index rows, Index stride, Index offset) const {
2334 using HalfPacket = typename unpacket_traits<Packet>::half;
2335 using QuarterPacket = typename unpacket_traits<typename unpacket_traits<Packet>::half>::half;
2336 enum {
2337 PacketSize = unpacket_traits<Packet>::size,
2338 HalfPacketSize = unpacket_traits<HalfPacket>::size,
2339 QuarterPacketSize = unpacket_traits<QuarterPacket>::size,
2340 HasHalf = (int)HalfPacketSize < (int)PacketSize,
2341 HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize
2342 };
2343
2344 EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK LHS");
2345 EIGEN_UNUSED_VARIABLE(stride);
2346 EIGEN_UNUSED_VARIABLE(offset);
2347 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2348 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2349 Index count = 0;
2350 bool gone_half = false, gone_quarter = false, gone_last = false;
2351
2352 Index i = 0;
2353 Index pack = Pack1;
2354 Index psize = PacketSize;
2355 while (pack > 0) {
2356 Index remaining_rows = rows - i;
2357 Index peeled_mc = gone_last ? Pack2 > 1 ? (rows / pack) * pack : 0 : i + (remaining_rows / pack) * pack;
2358 Index starting_pos = i;
2359 for (; i < peeled_mc; i += pack) {
2360 EIGEN_IF_CONSTEXPR (PanelMode) count += pack * offset;
2361
2362 Index k = 0;
2363 if (pack >= psize && psize >= QuarterPacketSize) {
2364 const Index peeled_k = (depth / psize) * psize;
2365 for (; k < peeled_k; k += psize) {
2366 for (Index m = 0; m < pack; m += psize) {
2367 if (psize == PacketSize) {
2368 PacketBlock<Packet> kernel;
2369 for (Index p = 0; p < psize; ++p) kernel.packet[p] = lhs.template loadPacket<Packet>(i + p + m, k);
2370 ptranspose(kernel);
2371 for (Index p = 0; p < psize; ++p) pstore(blockA + count + m + (pack)*p, cj.pconj(kernel.packet[p]));
2372 } else if (HasHalf && psize == HalfPacketSize) {
2373 gone_half = true;
2374 PacketBlock<HalfPacket> kernel_half;
2375 for (Index p = 0; p < psize; ++p)
2376 kernel_half.packet[p] = lhs.template loadPacket<HalfPacket>(i + p + m, k);
2377 ptranspose(kernel_half);
2378 for (Index p = 0; p < psize; ++p) pstore(blockA + count + m + (pack)*p, cj.pconj(kernel_half.packet[p]));
2379 } else if (HasQuarter && psize == QuarterPacketSize) {
2380 gone_quarter = true;
2381 PacketBlock<QuarterPacket> kernel_quarter;
2382 for (Index p = 0; p < psize; ++p)
2383 kernel_quarter.packet[p] = lhs.template loadPacket<QuarterPacket>(i + p + m, k);
2384 ptranspose(kernel_quarter);
2385 for (Index p = 0; p < psize; ++p)
2386 pstore(blockA + count + m + (pack)*p, cj.pconj(kernel_quarter.packet[p]));
2387 }
2388 }
2389 count += psize * pack;
2390 }
2391 }
2392
2393 for (; k < depth; k++) {
2394 Index w = 0;
2395 for (; w < pack - 3; w += 4) {
2396 Scalar a(cj(lhs(i + w + 0, k))), b(cj(lhs(i + w + 1, k))), c(cj(lhs(i + w + 2, k))), d(cj(lhs(i + w + 3, k)));
2397 blockA[count++] = a;
2398 blockA[count++] = b;
2399 blockA[count++] = c;
2400 blockA[count++] = d;
2401 }
2402 if (pack % 4)
2403 for (; w < pack; ++w) blockA[count++] = cj(lhs(i + w, k));
2404 }
2405
2406 EIGEN_IF_CONSTEXPR (PanelMode) count += pack * (stride - offset - depth);
2407 }
2408
2409 pack -= psize;
2410 Index left = rows - i;
2411 if (pack <= 0) {
2412 if (!gone_last && (starting_pos == i || left >= psize / 2 || left >= psize / 4) &&
2413 ((psize / 2 == HalfPacketSize && HasHalf && !gone_half) ||
2414 (psize / 2 == QuarterPacketSize && HasQuarter && !gone_quarter))) {
2415 psize /= 2;
2416 pack = psize;
2417 continue;
2418 }
2419 // Pack2 may be *smaller* than PacketSize—that happens for
2420 // products like real * complex, where we have to go half the
2421 // progress on the lhs in order to duplicate those operands to
2422 // address both real & imaginary parts on the rhs. This portion will
2423 // pack those half ones until they match the number expected on the
2424 // last peeling loop at this point (for the rhs).
2425 //
2426 // When there are no half/quarter packet types (HasHalf and HasQuarter
2427 // are both false), last_lhs_progress can exceed Pack2, producing
2428 // interleaved groups that the GEBP micro-kernel cannot consume. In
2429 // that case we use exactly Pack2 rows per group so the kernel's main
2430 // loop (which reads Pack2 = LhsProgress values via ploaddup) can
2431 // handle them; remaining rows fall through to the scalar loop below.
2432 EIGEN_IF_CONSTEXPR (Pack2 < PacketSize) {
2433 if (!gone_last) {
2434 gone_last = true;
2435 psize = pack = (HasHalf || HasQuarter) ? (left & ~1) : Pack2;
2436 }
2437 }
2438 }
2439 }
2440
2441 for (; i < rows; i++) {
2442 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2443 for (Index k = 0; k < depth; k++) blockA[count++] = cj(lhs(i, k));
2444 EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
2445 }
2446}
2447
2448// copy a complete panel of the rhs
2449// this version is optimized for column major matrices
2450// The traversal order is as follow: (nr==4):
2451// 0 1 2 3 12 13 14 15 24 27
2452// 4 5 6 7 16 17 18 19 25 28
2453// 8 9 10 11 20 21 22 23 26 29
2454// . . . . . . . . . .
2455template <typename Scalar, typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
2456struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> {
2457 using Packet = typename packet_traits<Scalar>::type;
2458 using LinearMapper = typename DataMapper::LinearMapper;
2459 enum { PacketSize = packet_traits<Scalar>::size };
2460 EIGEN_DONT_INLINE void operator()(Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
2461 Index offset = 0) const;
2462};
2463
2464template <typename Scalar, typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
2465EIGEN_DONT_INLINE void gemm_pack_rhs<Scalar, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
2466 Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) const {
2467 EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS COLMAJOR");
2468 EIGEN_UNUSED_VARIABLE(stride);
2469 EIGEN_UNUSED_VARIABLE(offset);
2470 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2471 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2472 Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0;
2473 Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
2474 Index count = 0;
2475 const Index peeled_k = (depth / PacketSize) * PacketSize;
2476
2477 EIGEN_IF_CONSTEXPR (nr >= 8) {
2478 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
2479 // skip what we have before
2480 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
2481 const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
2482 const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
2483 const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
2484 const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
2485 const LinearMapper dm4 = rhs.getLinearMapper(0, j2 + 4);
2486 const LinearMapper dm5 = rhs.getLinearMapper(0, j2 + 5);
2487 const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6);
2488 const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7);
2489 Index k = 0;
2490 EIGEN_IF_CONSTEXPR (PacketSize % 2 == 0 && PacketSize <= 8) // 2 4 8
2491 {
2492 for (; k < peeled_k; k += PacketSize) {
2493 EIGEN_IF_CONSTEXPR (PacketSize == 2) {
2494 PacketBlock<Packet, PacketSize == 2 ? 2 : PacketSize> kernel0, kernel1, kernel2, kernel3;
2495 kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
2496 kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
2497 kernel1.packet[0 % PacketSize] = dm2.template loadPacket<Packet>(k);
2498 kernel1.packet[1 % PacketSize] = dm3.template loadPacket<Packet>(k);
2499 kernel2.packet[0 % PacketSize] = dm4.template loadPacket<Packet>(k);
2500 kernel2.packet[1 % PacketSize] = dm5.template loadPacket<Packet>(k);
2501 kernel3.packet[0 % PacketSize] = dm6.template loadPacket<Packet>(k);
2502 kernel3.packet[1 % PacketSize] = dm7.template loadPacket<Packet>(k);
2503 ptranspose(kernel0);
2504 ptranspose(kernel1);
2505 ptranspose(kernel2);
2506 ptranspose(kernel3);
2507
2508 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel0.packet[0 % PacketSize]));
2509 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel1.packet[0 % PacketSize]));
2510 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel2.packet[0 % PacketSize]));
2511 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel3.packet[0 % PacketSize]));
2512
2513 pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel0.packet[1 % PacketSize]));
2514 pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel1.packet[1 % PacketSize]));
2515 pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel2.packet[1 % PacketSize]));
2516 pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel3.packet[1 % PacketSize]));
2517 count += 8 * PacketSize;
2518 } else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
2519 PacketBlock<Packet, PacketSize == 4 ? 4 : PacketSize> kernel0, kernel1;
2520
2521 kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
2522 kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
2523 kernel0.packet[2 % PacketSize] = dm2.template loadPacket<Packet>(k);
2524 kernel0.packet[3 % PacketSize] = dm3.template loadPacket<Packet>(k);
2525 kernel1.packet[0 % PacketSize] = dm4.template loadPacket<Packet>(k);
2526 kernel1.packet[1 % PacketSize] = dm5.template loadPacket<Packet>(k);
2527 kernel1.packet[2 % PacketSize] = dm6.template loadPacket<Packet>(k);
2528 kernel1.packet[3 % PacketSize] = dm7.template loadPacket<Packet>(k);
2529 ptranspose(kernel0);
2530 ptranspose(kernel1);
2531
2532 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel0.packet[0 % PacketSize]));
2533 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel1.packet[0 % PacketSize]));
2534 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel0.packet[1 % PacketSize]));
2535 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel1.packet[1 % PacketSize]));
2536 pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel0.packet[2 % PacketSize]));
2537 pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel1.packet[2 % PacketSize]));
2538 pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel0.packet[3 % PacketSize]));
2539 pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel1.packet[3 % PacketSize]));
2540 count += 8 * PacketSize;
2541 } else EIGEN_IF_CONSTEXPR (PacketSize == 8) {
2542 PacketBlock<Packet, PacketSize == 8 ? 8 : PacketSize> kernel0;
2543
2544 kernel0.packet[0 % PacketSize] = dm0.template loadPacket<Packet>(k);
2545 kernel0.packet[1 % PacketSize] = dm1.template loadPacket<Packet>(k);
2546 kernel0.packet[2 % PacketSize] = dm2.template loadPacket<Packet>(k);
2547 kernel0.packet[3 % PacketSize] = dm3.template loadPacket<Packet>(k);
2548 kernel0.packet[4 % PacketSize] = dm4.template loadPacket<Packet>(k);
2549 kernel0.packet[5 % PacketSize] = dm5.template loadPacket<Packet>(k);
2550 kernel0.packet[6 % PacketSize] = dm6.template loadPacket<Packet>(k);
2551 kernel0.packet[7 % PacketSize] = dm7.template loadPacket<Packet>(k);
2552 ptranspose(kernel0);
2553
2554 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel0.packet[0 % PacketSize]));
2555 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel0.packet[1 % PacketSize]));
2556 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel0.packet[2 % PacketSize]));
2557 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel0.packet[3 % PacketSize]));
2558 pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel0.packet[4 % PacketSize]));
2559 pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel0.packet[5 % PacketSize]));
2560 pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel0.packet[6 % PacketSize]));
2561 pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel0.packet[7 % PacketSize]));
2562 count += 8 * PacketSize;
2563 }
2564 }
2565 }
2566
2567 for (; k < depth; k++) {
2568 blockB[count + 0] = cj(dm0(k));
2569 blockB[count + 1] = cj(dm1(k));
2570 blockB[count + 2] = cj(dm2(k));
2571 blockB[count + 3] = cj(dm3(k));
2572 blockB[count + 4] = cj(dm4(k));
2573 blockB[count + 5] = cj(dm5(k));
2574 blockB[count + 6] = cj(dm6(k));
2575 blockB[count + 7] = cj(dm7(k));
2576 count += 8;
2577 }
2578 // skip what we have after
2579 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
2580 }
2581 }
2582
2583 EIGEN_IF_CONSTEXPR (nr >= 4) {
2584 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
2585 // skip what we have before
2586 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
2587 const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
2588 const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
2589 const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
2590 const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
2591
2592 Index k = 0;
2593 EIGEN_IF_CONSTEXPR ((PacketSize % 4) == 0 || PacketSize == 2) {
2594 for (; k < peeled_k; k += PacketSize) {
2595 PacketBlock<Packet, 4> kernel;
2596 kernel.packet[0] = dm0.template loadPacket<Packet>(k);
2597 kernel.packet[1] = dm1.template loadPacket<Packet>(k);
2598 kernel.packet[2] = dm2.template loadPacket<Packet>(k);
2599 kernel.packet[3] = dm3.template loadPacket<Packet>(k);
2600 EIGEN_IF_CONSTEXPR (PacketSize == 2) {
2601 // For PacketSize==2 we cannot ptranspose 4 packets directly; compose two
2602 // 2-packet transposes and re-interleave so the 4 stores produce the
2603 // packed-rhs layout (each store writing one half-row of the panel).
2604 PacketBlock<Packet, 2> tmp01;
2605 tmp01.packet[0] = kernel.packet[0];
2606 tmp01.packet[1] = kernel.packet[1];
2607 ptranspose(tmp01);
2608 PacketBlock<Packet, 2> tmp23;
2609 tmp23.packet[0] = kernel.packet[2];
2610 tmp23.packet[1] = kernel.packet[3];
2611 ptranspose(tmp23);
2612 kernel.packet[0] = tmp01.packet[0];
2613 kernel.packet[1] = tmp23.packet[0];
2614 kernel.packet[2] = tmp01.packet[1];
2615 kernel.packet[3] = tmp23.packet[1];
2616 } else {
2617 ptranspose(kernel);
2618 }
2619 pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0]));
2620 pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel.packet[1]));
2621 pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel.packet[2]));
2622 pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel.packet[3]));
2623 count += 4 * PacketSize;
2624 }
2625 }
2626 for (; k < depth; k++) {
2627 blockB[count + 0] = cj(dm0(k));
2628 blockB[count + 1] = cj(dm1(k));
2629 blockB[count + 2] = cj(dm2(k));
2630 blockB[count + 3] = cj(dm3(k));
2631 count += 4;
2632 }
2633 // skip what we have after
2634 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
2635 }
2636 }
2637
2638 // copy the remaining columns one at a time (nr==1)
2639 for (Index j2 = packet_cols4; j2 < cols; ++j2) {
2640 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2641 const LinearMapper dm0 = rhs.getLinearMapper(0, j2);
2642 for (Index k = 0; k < depth; k++) {
2643 blockB[count] = cj(dm0(k));
2644 count += 1;
2645 }
2646 EIGEN_IF_CONSTEXPR (PanelMode) count += (stride - offset - depth);
2647 }
2648}
2649
2650// this version is optimized for row major matrices
2651template <typename Scalar, typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
2652struct gemm_pack_rhs<Scalar, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> {
2653 using Packet = typename packet_traits<Scalar>::type;
2654 using HalfPacket = typename unpacket_traits<Packet>::half;
2655 using QuarterPacket = typename unpacket_traits<typename unpacket_traits<Packet>::half>::half;
2656 using LinearMapper = typename DataMapper::LinearMapper;
2657 enum {
2658 PacketSize = packet_traits<Scalar>::size,
2659 HalfPacketSize = unpacket_traits<HalfPacket>::size,
2660 QuarterPacketSize = unpacket_traits<QuarterPacket>::size
2661 };
2662 EIGEN_DONT_INLINE void operator()(Scalar* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
2663 Index offset = 0) const {
2664 EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS ROWMAJOR");
2665 EIGEN_UNUSED_VARIABLE(stride);
2666 EIGEN_UNUSED_VARIABLE(offset);
2667 eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride));
2668 constexpr bool HasHalf = (int)HalfPacketSize < (int)PacketSize;
2669 constexpr bool HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize;
2670 conj_if<NumTraits<Scalar>::IsComplex && Conjugate> cj;
2671 Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0;
2672 Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0;
2673 Index count = 0;
2674
2675 EIGEN_IF_CONSTEXPR (nr >= 8) {
2676 for (Index j2 = 0; j2 < packet_cols8; j2 += 8) {
2677 // skip what we have before
2678 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * offset;
2679 for (Index k = 0; k < depth; k++) {
2680 EIGEN_IF_CONSTEXPR (PacketSize == 8) {
2681 Packet A = rhs.template loadPacket<Packet>(k, j2);
2682 pstoreu(blockB + count, cj.pconj(A));
2683 count += PacketSize;
2684 } else EIGEN_IF_CONSTEXPR (PacketSize == 4) {
2685 Packet A = rhs.template loadPacket<Packet>(k, j2);
2686 Packet B = rhs.template loadPacket<Packet>(k, j2 + 4);
2687 pstoreu(blockB + count, cj.pconj(A));
2688 pstoreu(blockB + count + PacketSize, cj.pconj(B));
2689 count += 2 * PacketSize;
2690 } else {
2691 const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
2692 blockB[count + 0] = cj(dm0(0));
2693 blockB[count + 1] = cj(dm0(1));
2694 blockB[count + 2] = cj(dm0(2));
2695 blockB[count + 3] = cj(dm0(3));
2696 blockB[count + 4] = cj(dm0(4));
2697 blockB[count + 5] = cj(dm0(5));
2698 blockB[count + 6] = cj(dm0(6));
2699 blockB[count + 7] = cj(dm0(7));
2700 count += 8;
2701 }
2702 }
2703 // skip what we have after
2704 EIGEN_IF_CONSTEXPR (PanelMode) count += 8 * (stride - offset - depth);
2705 }
2706 }
2707
2708 EIGEN_IF_CONSTEXPR (nr >= 4) {
2709 for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) {
2710 // skip what we have before
2711 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * offset;
2712 for (Index k = 0; k < depth; k++) {
2713 EIGEN_IF_CONSTEXPR (PacketSize == 4) {
2714 Packet A = rhs.template loadPacket<Packet>(k, j2);
2715 pstoreu(blockB + count, cj.pconj(A));
2716 count += PacketSize;
2717 } else EIGEN_IF_CONSTEXPR (HasHalf && HalfPacketSize == 4) {
2718 HalfPacket A = rhs.template loadPacket<HalfPacket>(k, j2);
2719 pstoreu(blockB + count, cj.pconj(A));
2720 count += HalfPacketSize;
2721 } else EIGEN_IF_CONSTEXPR (HasQuarter && QuarterPacketSize == 4) {
2722 QuarterPacket A = rhs.template loadPacket<QuarterPacket>(k, j2);
2723 pstoreu(blockB + count, cj.pconj(A));
2724 count += QuarterPacketSize;
2725 } else {
2726 const LinearMapper dm0 = rhs.getLinearMapper(k, j2);
2727 blockB[count + 0] = cj(dm0(0));
2728 blockB[count + 1] = cj(dm0(1));
2729 blockB[count + 2] = cj(dm0(2));
2730 blockB[count + 3] = cj(dm0(3));
2731 count += 4;
2732 }
2733 }
2734 // skip what we have after
2735 EIGEN_IF_CONSTEXPR (PanelMode) count += 4 * (stride - offset - depth);
2736 }
2737 }
2738 // copy the remaining columns one at a time (nr==1)
2739 for (Index j2 = packet_cols4; j2 < cols; ++j2) {
2740 EIGEN_IF_CONSTEXPR (PanelMode) count += offset;
2741 for (Index k = 0; k < depth; k++) {
2742 blockB[count] = cj(rhs(k, j2));
2743 count += 1;
2744 }
2745 EIGEN_IF_CONSTEXPR (PanelMode) count += stride - offset - depth;
2746 }
2747 }
2748};
2749
2750} // end namespace internal
2751
2754inline std::ptrdiff_t l1CacheSize() {
2755 std::ptrdiff_t l1, l2, l3;
2756 internal::manage_caching_sizes(GetAction, &l1, &l2, &l3);
2757 return l1;
2758}
2759
2762inline std::ptrdiff_t l2CacheSize() {
2763 std::ptrdiff_t l1, l2, l3;
2764 internal::manage_caching_sizes(GetAction, &l1, &l2, &l3);
2765 return l2;
2766}
2767
2770inline std::ptrdiff_t l3CacheSize() {
2771 std::ptrdiff_t l1, l2, l3;
2772 internal::manage_caching_sizes(GetAction, &l1, &l2, &l3);
2773 return l3;
2774}
2775
2782inline void setCpuCacheSizes(std::ptrdiff_t l1, std::ptrdiff_t l2, std::ptrdiff_t l3) {
2783 internal::manage_caching_sizes(SetAction, &l1, &l2, &l3);
2784}
2785
2786} // end namespace Eigen
2787
2788#if EIGEN_COMP_MSVC
2789#pragma warning(pop)
2790#endif
2791
2792#endif // EIGEN_GENERAL_BLOCK_PANEL_H
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321