Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GeneralMatrixMatrix.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_GENERAL_MATRIX_MATRIX_H
12#define EIGEN_GENERAL_MATRIX_MATRIX_H
13
14// IWYU pragma: private
15#include "../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21template <typename LhsScalar_, typename RhsScalar_>
22class level3_blocking;
23
24// Coalesce two heap packing buffers so glibc's dynamic thresholds learn the whole workspace size (#3201).
25// Leave stack allocations and caller-provided buffers on their existing paths.
26template <typename LhsScalar, typename RhsScalar>
27class gemm_packing_space {
28 LhsScalar* m_blockA;
29 RhsScalar* m_blockB;
30 aligned_stack_memory_handler<unsigned char> m_storage;
31 aligned_stack_memory_handler<LhsScalar> m_lhs_lifetime;
32 aligned_stack_memory_handler<RhsScalar> m_rhs_lifetime;
33
34 static EIGEN_DONT_INLINE unsigned char* allocateCombined(std::size_t sizeA, std::size_t sizeB, LhsScalar*& blockA,
35 RhsScalar*& blockB) {
36 check_size_for_overflow<LhsScalar>(sizeA);
37 check_size_for_overflow<RhsScalar>(sizeB);
38 // Not numext::maxi: under SYCL_DEVICE_ONLY its cl_ulong (std::size_t) specialization is not constexpr.
39 constexpr std::size_t alignment = (std::max)(std::size_t{EIGEN_STACK_ALIGN_BYTES}, alignof(RhsScalar));
40 const std::size_t max_bytes = (std::numeric_limits<std::ptrdiff_t>::max)();
41 const std::size_t bytesA = sizeA * sizeof(LhsScalar);
42 const std::size_t bytesB = sizeB * sizeof(RhsScalar);
43 if (bytesA > max_bytes - (alignment - 1)) throw_std_bad_alloc();
44 const std::size_t offsetB = first_multiple(bytesA, alignment);
45 if (bytesB > max_bytes - offsetB) throw_std_bad_alloc();
46 void* storage = scratch_malloc(offsetB + bytesB);
47 void* storageB = static_cast<unsigned char*>(storage) + offsetB;
48 blockA = static_cast<LhsScalar*>(storage);
49 blockB = static_cast<RhsScalar*>(storageB);
50 return static_cast<unsigned char*>(storage);
51 }
52
53 static unsigned char* allocate(std::size_t sizeA, std::size_t sizeB, LhsScalar*& blockA, RhsScalar*& blockB) {
54 if (blockA != nullptr || blockB != nullptr || sizeA == 0 || sizeB == 0) return nullptr;
55#if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA)
56 if (sizeA <= EIGEN_STACK_ALLOCATION_LIMIT / sizeof(LhsScalar) ||
57 sizeB <= EIGEN_STACK_ALLOCATION_LIMIT / sizeof(RhsScalar))
58 return nullptr;
59#endif
60 // scratch_malloc guarantees only EIGEN_STACK_ALIGN_BYTES, or malloc's alignment when that is 0.
61 constexpr std::size_t base_alignment =
62 EIGEN_STACK_ALIGN_BYTES > 0 ? EIGEN_STACK_ALIGN_BYTES : alignof(std::max_align_t);
63 if ((std::max)(alignof(LhsScalar), alignof(RhsScalar)) > base_alignment) return nullptr;
64 return allocateCombined(sizeA, sizeB, blockA, blockB);
65 }
66
67 public:
68 gemm_packing_space(std::size_t sizeA, std::size_t sizeB, LhsScalar* blockA, RhsScalar* blockB)
69 : m_blockA(blockA),
70 m_blockB(blockB),
71 m_storage(allocate(sizeA, sizeB, m_blockA, m_blockB), 0, true),
72 m_lhs_lifetime(blockA == nullptr ? m_blockA : nullptr, sizeA, false),
73 m_rhs_lifetime(blockB == nullptr ? m_blockB : nullptr, sizeB, false) {}
74
75 LhsScalar* blockA() const { return m_blockA; }
76 RhsScalar* blockB() const { return m_blockB; }
77};
78
79// Runs one gebp call, as C = alpha * A' * B' when `overwrite` holds and the kernel can (gebp_kernel_overwrites), and
80// as C += alpha * A' * B' otherwise. It and the loop policies are forced inline into the driver: GCC 13 otherwise
81// clones a policy (IPA-SRA) with the kernel call still in it, and emits kernels that only that call's dead branches
82// reach (the AVX-512 kernel's non-unit-increment ones).
83template <typename Gebp, typename... Args>
84EIGEN_ALWAYS_INLINE void gebp_run(std::true_type, Gebp& gebp, bool overwrite, const Args&... args) {
85 if (overwrite)
86 gebp(gebp_overwrite_t(), args...);
87 else
88 gebp(args...);
89}
90template <typename Gebp, typename... Args>
91EIGEN_ALWAYS_INLINE void gebp_run(std::false_type, Gebp& gebp, bool, const Args&... args) {
92 gebp(args...);
93}
94
95// Zeroes the rows x cols column-major result block at `res`.
96template <typename Scalar, typename Index>
97void gemm_zero_result(Scalar* res, Index rows, Index cols, Index resIncr, Index resStride) {
98 using Block = Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
99 if (resIncr == 1 && resStride == rows)
100 Map<Matrix<Scalar, Dynamic, 1>>(res, rows * cols).setZero();
101 else if (resIncr == 1)
102 Map<Block, 0, OuterStride<>>(res, rows, cols, OuterStride<>(resStride)).setZero();
103 else
104 Map<Block, 0, Stride<Dynamic, Dynamic>>(res, rows, cols, Stride<Dynamic, Dynamic>(resStride, resIncr)).setZero();
105}
106
107// LHS-first loop order: mc -> kc -> nc. This is Eigen's usual sequential
108// schedule, with a fast path that can pack RHS once for tall skinny blocking.
109struct gemm_pack_lhs_first_loop_policy {
110 template <typename Index, typename LhsScalar, typename RhsScalar, typename ResScalar, typename LhsMapper,
111 typename RhsMapper, typename ResMapper, typename PackLhs, typename PackRhs, typename Gebp>
112 static EIGEN_ALWAYS_INLINE void run(Index rows, Index cols, Index depth, Index kc, Index mc, Index nc,
113 const LhsMapper& lhs, const RhsMapper& rhs, ResMapper& res, PackLhs& pack_lhs,
114 PackRhs& pack_rhs, Gebp& gebp, LhsScalar* blockA, RhsScalar* blockB,
115 ResScalar alpha, bool overwrite) {
116 const bool pack_rhs_once = mc != rows && kc == depth && nc == cols;
117
118 // For each horizontal panel of the rhs, and corresponding panel of the lhs...
119 for (Index i2 = 0; i2 < rows; i2 += mc) {
120 const Index actual_mc = (std::min)(i2 + mc, rows) - i2;
121
122 for (Index k2 = 0; k2 < depth; k2 += kc) {
123 const Index actual_kc = (std::min)(k2 + kc, depth) - k2;
124
125 // OK, here we have selected one horizontal panel of rhs and one vertical panel of lhs.
126 // => Pack lhs's panel into a sequential chunk of memory (L2/L3 caching)
127 // Note that this panel will be read as many times as the number of blocks in the rhs's
128 // horizontal panel which is, in practice, a very low number.
129 pack_lhs(blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc);
130
131 // For each kc x nc block of the rhs's horizontal panel...
132 for (Index j2 = 0; j2 < cols; j2 += nc) {
133 const Index actual_nc = (std::min)(j2 + nc, cols) - j2;
134
135 // We pack the rhs's block into a sequential chunk of memory (L2 caching)
136 // Note that this block will be read a very high number of times, which is equal to the number of
137 // micro horizontal panel of the large rhs's panel (e.g., rows/12 times).
138 if ((!pack_rhs_once) || i2 == 0) pack_rhs(blockB, rhs.getSubMapper(k2, j2), actual_kc, actual_nc);
139
140 // Everything is packed, we can now call the panel * block kernel:
141 gebp_run(bool_constant<gebp_kernel_overwrites<Gebp>::value>(), gebp, overwrite && k2 == 0,
142 res.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, alpha);
143 }
144 }
145 }
146 }
147};
148
149#ifdef EIGEN_VECTORIZE_SME
150// Defined in arch/SME/GeneralBlockPanelKernel.h, included after this header:
151// whether the SME kernel can read this ColMajor LHS block straight from its
152// source instead of a packed panel.
153template <typename Scalar, typename Index>
154bool sme_direct_lhs_ok(Index lhsStride, Index rows, Index depth, Index cols);
155// True for the unit-stride ColMajor mapper the GEMM driver hands the packers.
156template <typename Mapper>
157struct sme_direct_lhs_mapper : std::false_type {};
158template <typename Scalar, typename Index>
159struct sme_direct_lhs_mapper<const_blas_data_mapper<Scalar, Index, ColMajor>>
160 : bool_constant<!NumTraits<Scalar>::IsComplex> {};
161// Runs the block on the in-place LHS when the kernel can take it; the false
162// overload keeps the call out of every other instantiation.
163// The in-place call, tagged to overwrite only for a kernel that can (gebp_run).
164template <typename Gebp, typename... Args>
165EIGEN_ALWAYS_INLINE void sme_direct_lhs_call(std::true_type, Gebp& gebp, bool overwrite, const Args&... args) {
166 if (overwrite)
167 gebp.run_direct_lhs(gebp_overwrite_t(), args...);
168 else
169 gebp.run_direct_lhs(args...);
170}
171template <typename Gebp, typename... Args>
172EIGEN_ALWAYS_INLINE void sme_direct_lhs_call(std::false_type, Gebp& gebp, bool, const Args&... args) {
173 gebp.run_direct_lhs(args...);
174}
175template <typename Gebp, typename ResMapper, typename LhsMapper, typename Scalar, typename ResScalar, typename Index>
176EIGEN_ALWAYS_INLINE bool sme_run_direct_lhs(std::true_type, Gebp& gebp, const ResMapper& res, const LhsMapper& lhs,
177 Index i2, Index k2, const Scalar* blockB, Index mc, Index kc, Index nc,
178 ResScalar alpha, bool overwrite) {
179 if (!sme_direct_lhs_ok<Scalar>(lhs.stride(), mc, kc, nc)) return false;
180 sme_direct_lhs_call(bool_constant<gebp_kernel_overwrites<Gebp>::value>(), gebp, overwrite, res, &lhs(i2, k2),
181 lhs.stride(), blockB, mc, kc, nc, alpha);
182 return true;
183}
184template <typename Scalar, int LhsOrder, int RhsOrder, typename Index>
185bool sme_tiny_gemm(Index rows, Index cols, Index depth, const Scalar* lhs, Index lhsStride, const Scalar* rhs,
186 Index rhsStride, Scalar* res, Index resIncr, Index resStride, Scalar alpha);
187// Where the tiny-result kernel beats both the coeff-based product and the packed paths, as tuned on Apple M4: short
188// depths keep the coeff-based product, and a full 2 * ps x 8 result over a long depth the SME kernel. The NEON
189// small-block switches pin the paths they select, so they turn it off.
190template <typename Scalar, typename Index>
191EIGEN_ALWAYS_INLINE bool sme_tiny_gemm_wins(Index rows, Index cols, Index depth) {
192#if defined(EIGEN_SME_NO_NEON_SMALL_BLOCKS) || defined(EIGEN_SME_FORCE_NEON_SMALL_BLOCKS) || defined(EIGEN_USE_BLAS)
193 EIGEN_UNUSED_VARIABLE(rows);
194 EIGEN_UNUSED_VARIABLE(cols);
195 EIGEN_UNUSED_VARIABLE(depth);
196 return false;
197#else
198 const Index ps = Index(16 / sizeof(Scalar)); // scalars per NEON vector
199 if (rows < 2 || cols < 2 || rows > 2 * ps || cols > 8) return false;
200 if (depth < 16 && (depth < 6 || rows * cols < 24)) return false;
201 return !(cols == 8 && rows > ps && depth >= 2048);
202#endif
203}
204// Real float and double on the SME kernel take the tiny-result kernel; the false overload keeps it out of every
205// other pair, including double without FEAT_SME_F64F64.
206template <typename LhsScalar, typename RhsScalar>
207struct sme_tiny_gemm_pair
208 : bool_constant<std::is_same<LhsScalar, RhsScalar>::value &&
209 (std::is_same<LhsScalar, float>::value || std::is_same<LhsScalar, double>::value) &&
210 sme_has_gebp_kernel<LhsScalar, RhsScalar>::value> {};
211template <int LhsOrder, int RhsOrder, typename Scalar, typename Index>
212EIGEN_ALWAYS_INLINE bool sme_run_tiny_gemm(std::true_type, Index rows, Index cols, Index depth, const Scalar* lhs,
213 Index lhsStride, const Scalar* rhs, Index rhsStride, Scalar* res,
214 Index resIncr, Index resStride, Scalar alpha) {
215 return sme_tiny_gemm<Scalar, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr,
216 resStride, alpha);
217}
218template <int LhsOrder, int RhsOrder, typename LhsScalar, typename RhsScalar, typename ResScalar, typename Index>
219EIGEN_ALWAYS_INLINE bool sme_run_tiny_gemm(std::false_type, Index, Index, Index, const LhsScalar*, Index,
220 const RhsScalar*, Index, ResScalar*, Index, Index, ResScalar) {
221 return false;
222}
223template <typename Gebp, typename ResMapper, typename LhsMapper, typename Scalar, typename ResScalar, typename Index>
224EIGEN_ALWAYS_INLINE bool sme_run_direct_lhs(std::false_type, Gebp&, const ResMapper&, const LhsMapper&, Index, Index,
225 const Scalar*, Index, Index, Index, ResScalar, bool) {
226 return false;
227}
228template <typename Scalar, typename Index>
229EIGEN_ALWAYS_INLINE bool sme_small_gemm_wins(Index rows, Index cols, Index depth, bool packed_rhs);
230template <typename Scalar, int LhsOrder, int RhsOrder, bool ConjLhs, bool ConjRhs, int ResInnerStride, typename Index>
231EIGEN_DONT_INLINE void sme_small_gemm(Index rows, Index cols, Index depth, const Scalar* lhs, Index lhsStride,
232 const Scalar* rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
233 Scalar alpha, Scalar* blockA, Scalar* blockB);
234// The test stays inline: a call into the NEON path's frame dirties stack lines the ZA path then packs into.
235template <int LhsOrder, int RhsOrder, bool ConjLhs, bool ConjRhs, int ResInnerStride, typename Scalar, typename Index,
236 typename Blocking>
237EIGEN_ALWAYS_INLINE bool sme_run_small_gemm(std::true_type, bool overwrite, Index rows, Index cols, Index depth,
238 const Scalar* lhs, Index lhsStride, const Scalar* rhs, Index rhsStride,
239 Scalar* res, Index resIncr, Index resStride, Scalar alpha,
240 Blocking& blocking) {
241 if (!sme_small_gemm_wins<Scalar>(rows, cols, depth, RhsOrder == ColMajor)) return false;
242 if (overwrite) gemm_zero_result(res, rows, cols, resIncr, resStride);
243 const bool fits = depth <= blocking.kc() && rows <= blocking.mc() && cols <= blocking.nc();
244 sme_small_gemm<Scalar, LhsOrder, RhsOrder, ConjLhs, ConjRhs, ResInnerStride>(
245 rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr, resStride, alpha,
246 fits ? blocking.blockA() : nullptr, fits ? blocking.blockB() : nullptr);
247 return true;
248}
249template <int LhsOrder, int RhsOrder, bool ConjLhs, bool ConjRhs, int ResInnerStride, typename LhsScalar,
250 typename RhsScalar, typename ResScalar, typename Index, typename Blocking>
251EIGEN_ALWAYS_INLINE bool sme_run_small_gemm(std::false_type, bool, Index, Index, Index, const LhsScalar*, Index,
252 const RhsScalar*, Index, ResScalar*, Index, Index, ResScalar, Blocking&) {
253 return false;
254}
255#endif
256
257// RHS-first loop order: nc -> kc -> mc. Used by SME to stream ColMajor result
258// stores through adjacent row panels; a ColMajor LHS whose MR-row slices are
259// already contiguous per depth step is read from its source (no LHS packing).
260struct gemm_pack_rhs_first_loop_policy {
261 template <typename Index, typename LhsScalar, typename RhsScalar, typename ResScalar, typename LhsMapper,
262 typename RhsMapper, typename ResMapper, typename PackLhs, typename PackRhs, typename Gebp>
263 static EIGEN_ALWAYS_INLINE void run(Index rows, Index cols, Index depth, Index kc, Index mc, Index nc,
264 const LhsMapper& lhs, const RhsMapper& rhs, ResMapper& res, PackLhs& pack_lhs,
265 PackRhs& pack_rhs, Gebp& gebp, LhsScalar* blockA, RhsScalar* blockB,
266 ResScalar alpha, bool overwrite) {
267 // Mirror of pack_rhs_once: reuse one full LHS panel across column blocks. The in-place read is decided per column
268 // block, so a block that falls back to packing packs unless an earlier one already did.
269 const bool pack_lhs_once = nc != cols && kc == depth && mc == rows;
270 bool lhs_packed = false;
271
272 for (Index j2 = 0; j2 < cols; j2 += nc) {
273 const Index actual_nc = (std::min)(j2 + nc, cols) - j2;
274
275 for (Index k2 = 0; k2 < depth; k2 += kc) {
276 const Index actual_kc = (std::min)(k2 + kc, depth) - k2;
277
278 // Pack one nc-strip of RHS and reuse it across all row panels.
279 pack_rhs(blockB, rhs.getSubMapper(k2, j2), actual_kc, actual_nc);
280
281 for (Index i2 = 0; i2 < rows; i2 += mc) {
282 const Index actual_mc = (std::min)(i2 + mc, rows) - i2;
283#ifdef EIGEN_VECTORIZE_SME
284 if (sme_run_direct_lhs(bool_constant<sme_direct_lhs_mapper<LhsMapper>::value>(), gebp,
285 res.getSubMapper(i2, j2), lhs, i2, k2, blockB, actual_mc, actual_kc, actual_nc, alpha,
286 overwrite && k2 == 0))
287 continue;
288#endif
289 if (!pack_lhs_once || !lhs_packed) {
290 pack_lhs(blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc);
291 lhs_packed = true;
292 }
293 gebp_run(bool_constant<gebp_kernel_overwrites<Gebp>::value>(), gebp, overwrite && k2 == 0,
294 res.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, alpha);
295 }
296 }
297 }
298 }
299};
300
301// Whether Gemm::run_overwrite writes the result without zeroing it first (Gemm::CanOverwrite), so that assignment
302// should take it instead of zeroing the destination and accumulating.
303template <typename Gemm, typename = void>
304struct gemm_can_overwrite : std::false_type {};
305template <typename Gemm>
306struct gemm_can_overwrite<Gemm, std::enable_if_t<Gemm::CanOverwrite>> : std::true_type {};
307
308// Overwrites the result through Gemm::run_overwrite, or, for a general_matrix_matrix_product specialization without
309// one, by zeroing it and accumulating with Gemm::run.
310template <typename Gemm, typename = void>
311struct gemm_has_run_overwrite : std::false_type {};
312template <typename Gemm>
313struct gemm_has_run_overwrite<Gemm, void_t<decltype(&Gemm::run_overwrite)>> : std::true_type {};
314template <typename Gemm, typename Index, typename LhsScalar, typename RhsScalar, typename ResScalar, typename Blocking>
315EIGEN_STRONG_INLINE void gemm_run_overwrite_impl(std::true_type, Index rows, Index cols, Index depth,
316 const LhsScalar* lhs, Index lhsStride, const RhsScalar* rhs,
317 Index rhsStride, ResScalar* res, Index resIncr, Index resStride,
318 ResScalar alpha, Blocking& blocking, GemmParallelInfo<Index>* info) {
319 Gemm::run_overwrite(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr, resStride, alpha, blocking,
320 info);
321}
322template <typename Gemm, typename Index, typename LhsScalar, typename RhsScalar, typename ResScalar, typename Blocking>
323EIGEN_STRONG_INLINE void gemm_run_overwrite_impl(std::false_type, Index rows, Index cols, Index depth,
324 const LhsScalar* lhs, Index lhsStride, const RhsScalar* rhs,
325 Index rhsStride, ResScalar* res, Index resIncr, Index resStride,
326 ResScalar alpha, Blocking& blocking, GemmParallelInfo<Index>* info) {
327 // A cooperative session needs every thread in Gemm::run, so only a sequential call returns on an empty result.
328 if (info == nullptr && (rows <= 0 || cols <= 0)) return;
329 gemm_zero_result(res, rows, cols, resIncr, resStride);
330 if (numext::is_exactly_zero(alpha) || depth <= 0) return;
331 Gemm::run(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr, resStride, alpha, blocking, info);
332}
333template <typename Gemm, typename Index, typename LhsScalar, typename RhsScalar, typename ResScalar, typename Blocking>
334EIGEN_STRONG_INLINE void gemm_run_overwrite(Index rows, Index cols, Index depth, const LhsScalar* lhs, Index lhsStride,
335 const RhsScalar* rhs, Index rhsStride, ResScalar* res, Index resIncr,
336 Index resStride, ResScalar alpha, Blocking& blocking,
337 GemmParallelInfo<Index>* info) {
338 gemm_run_overwrite_impl<Gemm>(gemm_has_run_overwrite<Gemm>(), rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res,
339 resIncr, resStride, alpha, blocking, info);
340}
341
342/* Specialization for a row-major destination matrix => simple transposition of the product */
343template <typename Index, typename LhsScalar, int LhsStorageOrder, bool ConjugateLhs, typename RhsScalar,
344 int RhsStorageOrder, bool ConjugateRhs, int ResInnerStride, typename LhsStorage, typename RhsStorage>
345struct general_matrix_matrix_product<Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder,
346 ConjugateRhs, RowMajor, ResInnerStride, LhsStorage, RhsStorage> {
347 using Traits = gebp_traits<RhsScalar, LhsScalar>;
348 using Transposed =
349 general_matrix_matrix_product<Index, RhsScalar, RhsStorageOrder == RowMajor ? ColMajor : RowMajor, ConjugateRhs,
350 LhsScalar, LhsStorageOrder == RowMajor ? ColMajor : RowMajor, ConjugateLhs,
351 ColMajor, ResInnerStride, RhsStorage, LhsStorage>;
352
353 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
354 static constexpr bool CanOverwrite = gemm_can_overwrite<Transposed>::value;
355 static EIGEN_STRONG_INLINE void run(Index rows, Index cols, Index depth, const LhsStorage* lhs, Index lhsStride,
356 const RhsStorage* rhs, Index rhsStride, ResScalar* res, Index resIncr,
357 Index resStride, ResScalar alpha, level3_blocking<RhsScalar, LhsScalar>& blocking,
358 GemmParallelInfo<Index>* info = 0) {
359 // transpose the product such that the result is column major
360 Transposed::run(cols, rows, depth, rhs, rhsStride, lhs, lhsStride, res, resIncr, resStride, alpha, blocking, info);
361 }
362 // As run, but overwrites the result instead of accumulating into it.
363 static EIGEN_STRONG_INLINE void run_overwrite(Index rows, Index cols, Index depth, const LhsStorage* lhs,
364 Index lhsStride, const RhsStorage* rhs, Index rhsStride, ResScalar* res,
365 Index resIncr, Index resStride, ResScalar alpha,
366 level3_blocking<RhsScalar, LhsScalar>& blocking,
367 GemmParallelInfo<Index>* info = 0) {
368 gemm_run_overwrite<Transposed>(cols, rows, depth, rhs, rhsStride, lhs, lhsStride, res, resIncr, resStride, alpha,
369 blocking, info);
370 }
371};
372
373/* Specialization for a col-major destination matrix
374 * => Blocking algorithm following Goto's paper */
375template <typename Index, typename LhsScalar, int LhsStorageOrder, bool ConjugateLhs, typename RhsScalar,
376 int RhsStorageOrder, bool ConjugateRhs, int ResInnerStride, typename LhsStorage, typename RhsStorage>
377struct general_matrix_matrix_product<Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder,
378 ConjugateRhs, ColMajor, ResInnerStride, LhsStorage, RhsStorage> {
379 using Traits = gebp_traits<LhsScalar, RhsScalar>;
380
381 private:
382 using Gebp = gebp_kernel<LhsScalar, RhsScalar, Index,
383 blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>,
384 Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs>;
385
386 public:
387 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
388 static constexpr bool CanOverwrite = gebp_kernel_overwrites<Gebp>::value;
389 static void run(Index rows, Index cols, Index depth, const LhsStorage* lhs_, Index lhsStride, const RhsStorage* rhs_,
390 Index rhsStride, ResScalar* res_, Index resIncr, Index resStride, ResScalar alpha,
391 level3_blocking<LhsScalar, RhsScalar>& blocking, GemmParallelInfo<Index>* info = 0) {
392 run_impl(false, rows, cols, depth, lhs_, lhsStride, rhs_, rhsStride, res_, resIncr, resStride, alpha, blocking,
393 info);
394 }
395 // As run, but assigns the result instead of accumulating into it, independently of what res held before.
396 static void run_overwrite(Index rows, Index cols, Index depth, const LhsStorage* lhs_, Index lhsStride,
397 const RhsStorage* rhs_, Index rhsStride, ResScalar* res_, Index resIncr, Index resStride,
398 ResScalar alpha, level3_blocking<LhsScalar, RhsScalar>& blocking,
399 GemmParallelInfo<Index>* info = 0) {
400 run_impl(true, rows, cols, depth, lhs_, lhsStride, rhs_, rhsStride, res_, resIncr, resStride, alpha, blocking,
401 info);
402 }
403
404 private:
405 // One body for both: `overwrite` is a runtime flag, so that assignment and accumulation share one
406 // instantiation.
407 static EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE void run_impl(bool overwrite, Index rows, Index cols, Index depth,
408 const LhsStorage* lhs_, Index lhsStride,
409 const RhsStorage* rhs_, Index rhsStride, ResScalar* res_,
410 Index resIncr, Index resStride, ResScalar alpha,
411 level3_blocking<LhsScalar, RhsScalar>& blocking,
412 GemmParallelInfo<Index>* info) {
413 // A cooperative session needs every thread to pack and publish its part of A', so only a sequential call returns.
414 if (info == nullptr && (rows <= 0 || cols <= 0)) return;
415 // BLAS contract: if alpha == 0, the result is unchanged (and lhs/rhs need not be read); overwriting, it is zero.
416 // A kernel that only accumulates or finds it does not pay, and an empty depth, overwrite by zeroing first.
417 if (overwrite && (numext::is_exactly_zero(alpha) || depth == 0 || !gebp_kernel_overwrites<Gebp>::value ||
418 !gebp_overwrite_pays<Gebp>(rows, cols, depth))) {
419 gemm_zero_result(res_, rows, cols, resIncr, resStride);
420 overwrite = false;
421 }
422 if (numext::is_exactly_zero(alpha) || depth == 0) return;
423#ifdef EIGEN_VECTORIZE_SME
424 // A parallel session needs every thread to take the packed path; scaleAndAddTo runs tiny results on one thread.
425 // The tiny-result and NEON kernels accumulate, so overwriting zeroes their result first; the tiny one runs exactly
426 // when sme_tiny_gemm_wins holds.
427 if (info == nullptr) {
428 constexpr bool tiny_pair = sme_tiny_gemm_pair<LhsScalar, RhsScalar>::value;
429 if (overwrite && tiny_pair && sme_tiny_gemm_wins<ResScalar>(rows, cols, depth))
430 gemm_zero_result(res_, rows, cols, resIncr, resStride);
431 if (sme_run_tiny_gemm<LhsStorageOrder, RhsStorageOrder>(bool_constant<tiny_pair>(), rows, cols, depth, lhs_,
432 lhsStride, rhs_, rhsStride, res_, resIncr, resStride,
433 alpha) ||
434 sme_run_small_gemm<LhsStorageOrder, RhsStorageOrder, ConjugateLhs, ConjugateRhs, ResInnerStride>(
435 bool_constant<sme_has_gebp_kernel<LhsScalar, RhsScalar>::value>(), overwrite, rows, cols, depth, lhs_,
436 lhsStride, rhs_, rhsStride, res_, resIncr, resStride, alpha, blocking))
437 return;
438 }
439#endif
440
441 using LhsMapper = gemm_operand_mapper<LhsScalar, LhsStorage, Index, LhsStorageOrder>;
442 using RhsMapper = gemm_operand_mapper<RhsScalar, RhsStorage, Index, RhsStorageOrder>;
443 using ResMapper = blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>;
444 LhsMapper lhs(lhs_, lhsStride);
445 RhsMapper rhs(rhs_, rhsStride);
446 ResMapper res(res_, resStride, resIncr);
447
448 Index kc = blocking.kc(); // cache block size along the K direction
449 Index mc = (std::min)(rows, blocking.mc()); // cache block size along the M direction
450 Index nc = (std::min)(cols, blocking.nc()); // cache block size along the N direction
451
452 gemm_pack_lhs<LhsScalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
453 LhsStorageOrder>
454 pack_lhs;
455 gemm_pack_rhs<RhsScalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
456 Gebp gebp;
457
458#if !defined(EIGEN_USE_BLAS) && (defined(EIGEN_HAS_OPENMP) || defined(EIGEN_GEMM_THREADPOOL))
459 constexpr bool kOverwrites = gebp_kernel_overwrites<Gebp>::value;
460 if (info) {
461 // this is the parallel version!
462 int tid = info->logical_thread_id;
463 int threads = info->num_threads;
464
465 LhsScalar* blockA = blocking.blockA();
466 eigen_internal_assert(blockA != 0);
467
468 std::size_t sizeB = kc * nc;
469 ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, 0);
470
471 // For each horizontal panel of the rhs, and corresponding vertical panel of the lhs...
472 for (Index k = 0; k < depth; k += kc) {
473 const Index actual_kc = (std::min)(k + kc, depth) - k; // => rows of B', and cols of the A'
474
475 // In order to reduce the chance that a thread has to wait for the other,
476 // let's start by packing B'.
477 pack_rhs(blockB, rhs.getSubMapper(k, 0), actual_kc, nc);
478
479 // Pack A_k to A' in a parallel fashion:
480 // each thread packs the sub block A_k,i to A'_i where i is the thread id.
481
482 // However, before copying to A'_i, we have to make sure that no other thread is still using it,
483 // i.e., we test that info->task_info[tid].users equals 0.
484 // Then, we set info->task_info[tid].users to the number of threads to mark that all other threads are going to
485 // use it.
486 while (info->task_info[tid].users != 0) {
487 std::this_thread::yield();
488 }
489 info->task_info[tid].users = threads;
490
491 pack_lhs(blockA + info->task_info[tid].lhs_start * actual_kc,
492 lhs.getSubMapper(info->task_info[tid].lhs_start, k), actual_kc, info->task_info[tid].lhs_length);
493
494 // Notify the other threads that the part A'_i is ready to go.
495 info->task_info[tid].sync = k;
496
497 // Computes C_i += A' * B' per A'_i
498 for (int shift = 0; shift < threads; ++shift) {
499 int i = (tid + shift) % threads;
500
501 // At this point we have to make sure that A'_i has been updated by the thread i,
502 // we use testAndSetOrdered to mimic a volatile access.
503 // However, no need to wait for the B' part which has been updated by the current thread!
504 if (shift > 0) {
505 while (info->task_info[i].sync != k) {
506 std::this_thread::yield();
507 }
508 }
509
510 gebp_run(bool_constant<kOverwrites>(), gebp, overwrite && k == 0,
511 res.getSubMapper(info->task_info[i].lhs_start, 0), blockA + info->task_info[i].lhs_start * actual_kc,
512 blockB, info->task_info[i].lhs_length, actual_kc, nc, alpha);
513 }
514
515 // Then keep going as usual with the remaining B'
516 for (Index j = nc; j < cols; j += nc) {
517 const Index actual_nc = (std::min)(j + nc, cols) - j;
518
519 // pack B_k,j to B'
520 pack_rhs(blockB, rhs.getSubMapper(k, j), actual_kc, actual_nc);
521
522 // C_j += A' * B'
523 gebp_run(bool_constant<kOverwrites>(), gebp, overwrite && k == 0, res.getSubMapper(0, j), blockA, blockB,
524 rows, actual_kc, actual_nc, alpha);
525 }
526
527 // Release all the sub blocks A'_i of A' for the current thread,
528 // i.e., we simply decrement the number of users by 1
529 for (Index i = 0; i < threads; ++i) info->task_info[i].users -= 1;
530 }
531 } else
532#endif // !defined(EIGEN_USE_BLAS) && (defined(EIGEN_HAS_OPENMP) || defined(EIGEN_GEMM_THREADPOOL))
533 {
534 EIGEN_UNUSED_VARIABLE(info);
535
536 // this is the sequential version!
537 std::size_t sizeA = kc * mc;
538 std::size_t sizeB = kc * nc;
539
540 gemm_packing_space<LhsScalar, RhsScalar> packing(sizeA, sizeB, blocking.blockA(), blocking.blockB());
541 ei_declare_aligned_stack_constructed_variable(LhsScalar, blockA, sizeA, packing.blockA());
542 ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, packing.blockB());
543
544 // The SME kernel uses RHS-first order so consecutive gebp calls stream
545 // through adjacent row panels of a ColMajor result. Other kernels --
546 // including the scalar pairs SME does not specialize -- keep Eigen's
547 // default LHS-first order.
548#ifdef EIGEN_VECTORIZE_SME
549 using SequentialGemmLoop = std::conditional_t<sme_has_gebp_kernel<LhsScalar, RhsScalar>::value,
550 gemm_pack_rhs_first_loop_policy, gemm_pack_lhs_first_loop_policy>;
551#else
552 using SequentialGemmLoop = gemm_pack_lhs_first_loop_policy;
553#endif
554
555 SequentialGemmLoop::run(rows, cols, depth, kc, mc, nc, lhs, rhs, res, pack_lhs, pack_rhs, gebp, blockA, blockB,
556 alpha, overwrite);
557 }
558 }
559};
560
561/*********************************************************************************
562 * Specialization of generic_product_impl for "large" GEMM, i.e.,
563 * implementation of the high level wrapper to general_matrix_matrix_product
564 **********************************************************************************/
565
566// overwrite: the product is assigned to dest, independently of its prior contents, instead of added to it.
567template <typename Scalar, typename Index, typename Gemm, typename Lhs, typename Rhs, typename Dest,
568 typename BlockingType>
569struct gemm_functor {
570 gemm_functor(const Lhs& lhs, const Rhs& rhs, Dest& dest, const Scalar& actualAlpha, BlockingType& blocking,
571 bool overwrite = false)
572 : m_lhs(lhs),
573 m_rhs(rhs),
574 m_dest(dest),
575 m_actualAlpha(actualAlpha),
576 m_blocking(blocking),
577 m_overwrite(overwrite) {}
578
579 void initParallelSession(Index num_threads) const {
580 m_blocking.initParallel(m_lhs.rows(), m_rhs.cols(), m_lhs.cols(), num_threads);
581 m_blocking.allocateA();
582 }
583
584 // Whether the blocking holds no preallocated buffers, so that disjoint parts of the product can run concurrently.
585 bool ownsNoBuffers() const { return m_blocking.blockA() == nullptr && m_blocking.blockB() == nullptr; }
586
587 void operator()(Index row, Index rows, Index col = 0, Index cols = -1, GemmParallelInfo<Index>* info = 0) const {
588 if (cols == -1) cols = m_rhs.cols();
589
590 if (m_overwrite) {
591 gemm_run_overwrite<Gemm>(rows, cols, m_lhs.cols(), &m_lhs.coeffRef(row, 0), m_lhs.outerStride(),
592 &m_rhs.coeffRef(0, col), m_rhs.outerStride(), (Scalar*)&(m_dest.coeffRef(row, col)),
593 m_dest.innerStride(), m_dest.outerStride(), m_actualAlpha, m_blocking, info);
594 } else {
595 Gemm::run(rows, cols, m_lhs.cols(), &m_lhs.coeffRef(row, 0), m_lhs.outerStride(), &m_rhs.coeffRef(0, col),
596 m_rhs.outerStride(), (Scalar*)&(m_dest.coeffRef(row, col)), m_dest.innerStride(), m_dest.outerStride(),
597 m_actualAlpha, m_blocking, info);
598 }
599 }
600
601 using Traits = typename Gemm::Traits;
602
603 protected:
604 const Lhs& m_lhs;
605 const Rhs& m_rhs;
606 Dest& m_dest;
607 Scalar m_actualAlpha;
608 BlockingType& m_blocking;
609 bool m_overwrite;
610};
611
612template <int StorageOrder, typename LhsScalar, typename RhsScalar, int MaxRows, int MaxCols, int MaxDepth,
613 int KcFactor = 1, bool FiniteAtCompileTime = MaxRows != Dynamic && MaxCols != Dynamic && MaxDepth != Dynamic>
614class gemm_blocking_space;
615
616template <typename LhsScalar_, typename RhsScalar_>
617class level3_blocking {
618 using LhsScalar = LhsScalar_;
619 using RhsScalar = RhsScalar_;
620
621 protected:
622 LhsScalar* m_blockA = nullptr;
623 RhsScalar* m_blockB = nullptr;
624
625 Index m_mc = 0;
626 Index m_nc = 0;
627 Index m_kc = 0;
628
629 public:
630 level3_blocking() = default;
631
632 inline Index mc() const { return m_mc; }
633 inline Index nc() const { return m_nc; }
634 inline Index kc() const { return m_kc; }
635
636 inline LhsScalar* blockA() { return m_blockA; }
637 inline RhsScalar* blockB() { return m_blockB; }
638};
639
640template <int StorageOrder, typename LhsScalar_, typename RhsScalar_, int MaxRows, int MaxCols, int MaxDepth,
641 int KcFactor>
642class gemm_blocking_space<StorageOrder, LhsScalar_, RhsScalar_, MaxRows, MaxCols, MaxDepth, KcFactor,
643 true /* == FiniteAtCompileTime */>
644 : public level3_blocking<std::conditional_t<StorageOrder == RowMajor, RhsScalar_, LhsScalar_>,
645 std::conditional_t<StorageOrder == RowMajor, LhsScalar_, RhsScalar_>> {
646 enum {
647 Transpose = StorageOrder == RowMajor,
648 ActualRows = Transpose ? MaxCols : MaxRows,
649 ActualCols = Transpose ? MaxRows : MaxCols
650 };
651 using LhsScalar = std::conditional_t<Transpose, RhsScalar_, LhsScalar_>;
652 using RhsScalar = std::conditional_t<Transpose, LhsScalar_, RhsScalar_>;
653 // A zero-sized fixed dimension must not declare a zero-length array, which MSVC rejects (C2229); every caller
654 // returns before it touches the blocks of an empty product.
655 static constexpr int SizeA = plain_enum_max(int(ActualRows) * MaxDepth, 1);
656 // One packet of slack: product_triangular_matrix_matrix with a triangular rhs realigns its packed general panel
657 // inside blockB to a packet boundary.
658 static constexpr int SizeB = plain_enum_max(int(ActualCols) * MaxDepth, 1) + int(packet_traits<RhsScalar>::size);
659
660#if EIGEN_MAX_STATIC_ALIGN_BYTES >= EIGEN_DEFAULT_ALIGN_BYTES
661 EIGEN_ALIGN_MAX LhsScalar m_staticA[SizeA > 0 ? SizeA : 1];
662 EIGEN_ALIGN_MAX RhsScalar m_staticB[SizeB > 0 ? SizeB : 1];
663#else
664 EIGEN_ALIGN_MAX char m_staticA[(SizeA > 0 ? SizeA : 1) * sizeof(LhsScalar) + EIGEN_DEFAULT_ALIGN_BYTES - 1];
665 EIGEN_ALIGN_MAX char m_staticB[(SizeB > 0 ? SizeB : 1) * sizeof(RhsScalar) + EIGEN_DEFAULT_ALIGN_BYTES - 1];
666#endif
667
668 public:
669 gemm_blocking_space(Index /*rows*/, Index /*cols*/, Index /*depth*/, Index /*num_threads*/,
670 bool /*full_rows = false*/) {
671 this->m_mc = ActualRows;
672 this->m_nc = ActualCols;
673 this->m_kc = MaxDepth;
674#if EIGEN_MAX_STATIC_ALIGN_BYTES >= EIGEN_DEFAULT_ALIGN_BYTES
675 this->m_blockA = m_staticA;
676 this->m_blockB = m_staticB;
677#else
678 this->m_blockA = reinterpret_cast<LhsScalar*>((std::uintptr_t(m_staticA) + (EIGEN_DEFAULT_ALIGN_BYTES - 1)) &
679 ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES - 1));
680 this->m_blockB = reinterpret_cast<RhsScalar*>((std::uintptr_t(m_staticB) + (EIGEN_DEFAULT_ALIGN_BYTES - 1)) &
681 ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES - 1));
682#endif
683 }
684
685 void initParallel(Index, Index, Index, Index) {}
686
687 inline void allocateA() {}
688 inline void allocateB() {}
689 inline void allocateAll() {}
690};
691
692template <int StorageOrder, typename LhsScalar_, typename RhsScalar_, int MaxRows, int MaxCols, int MaxDepth,
693 int KcFactor>
694class gemm_blocking_space<StorageOrder, LhsScalar_, RhsScalar_, MaxRows, MaxCols, MaxDepth, KcFactor, false>
695 : public level3_blocking<std::conditional_t<StorageOrder == RowMajor, RhsScalar_, LhsScalar_>,
696 std::conditional_t<StorageOrder == RowMajor, LhsScalar_, RhsScalar_>> {
697 enum { Transpose = StorageOrder == RowMajor };
698 using LhsScalar = std::conditional_t<Transpose, RhsScalar_, LhsScalar_>;
699 using RhsScalar = std::conditional_t<Transpose, LhsScalar_, RhsScalar_>;
700
701 Index m_sizeA;
702 Index m_sizeB;
703
704 public:
705 gemm_blocking_space(Index rows, Index cols, Index depth, Index num_threads, bool l3_blocking) {
706 this->m_mc = Transpose ? cols : rows;
707 this->m_nc = Transpose ? rows : cols;
708 this->m_kc = depth;
709
710 if (l3_blocking) {
711 computeProductBlockingSizes<LhsScalar, RhsScalar, KcFactor>(this->m_kc, this->m_mc, this->m_nc, num_threads);
712 } else // no l3 blocking
713 {
714 Index n = this->m_nc;
715 computeProductBlockingSizes<LhsScalar, RhsScalar, KcFactor>(this->m_kc, this->m_mc, n, num_threads);
716 }
717
718 m_sizeA = this->m_mc * this->m_kc;
719 m_sizeB = this->m_kc * this->m_nc;
720 }
721
722 void initParallel(Index rows, Index cols, Index depth, Index num_threads) {
723 this->m_mc = Transpose ? cols : rows;
724 this->m_nc = Transpose ? rows : cols;
725 this->m_kc = depth;
726
727 eigen_internal_assert(this->m_blockA == 0 && this->m_blockB == 0);
728 Index m = this->m_mc;
729 computeProductBlockingSizes<LhsScalar, RhsScalar, KcFactor>(this->m_kc, m, this->m_nc, num_threads);
730 m_sizeA = this->m_mc * this->m_kc;
731 m_sizeB = this->m_kc * this->m_nc;
732 }
733
734 // The blocking buffers get the temporaries' alignment (64 bytes in SME builds, EIGEN_STACK_ALIGN_BYTES).
735 void allocateA() {
736 if (this->m_blockA == 0) this->m_blockA = scratch_new<LhsScalar>(m_sizeA);
737 }
738
739 void allocateB() {
740 if (this->m_blockB == 0) this->m_blockB = scratch_new<RhsScalar>(m_sizeB);
741 }
742
743 void allocateAll() {
744 allocateA();
745 allocateB();
746 }
747
748 ~gemm_blocking_space() {
749 scratch_delete(this->m_blockA, m_sizeA);
750 scratch_delete(this->m_blockB, m_sizeB);
751 }
752};
753
754} // end namespace internal
755
756namespace internal {
757
758// Whether a GEMM of LhsScalar by RhsScalar runs in float on the converted operands and rounds each result coefficient
759// once. Summed in the operand type, half by half and bfloat16 by bfloat16 round every multiply-add, or every kc block,
760// to 11 or 8 significand bits (#3158).
761template <typename LhsScalar, typename RhsScalar>
762struct gemm_accumulates_in_float : std::false_type {};
763template <>
764struct gemm_accumulates_in_float<half, half> : std::true_type {};
765// OpenBLAS's sbgemm already accumulates bfloat16 in float; Eigen/Core reaches it only under EIGEN_USE_BLAS.
766#if !(defined(EIGEN_USE_BLAS) && defined(EIGEN_USE_OPENBLAS_BFLOAT16) && EIGEN_USE_OPENBLAS_BFLOAT16)
767template <>
768struct gemm_accumulates_in_float<bfloat16, bfloat16> : std::true_type {};
769#endif
770
771// blas_traits of a GEMM operand, which also sees through a cast of a real operand with direct access: the packers then
772// read the operand's StorageScalar coefficients and convert them (gemm_operand_mapper), instead of the product
773// materializing the cast. A scalar factor below the cast stays below it, so such an operand is materialized as before.
774// So is a cast to a type whose GEMM sums in float, which would otherwise skip the rounding to that type.
775// With EIGEN_USE_BLAS the cast is materialized, so that the product still reaches the external ?gemm, and so it is on
776// SME, whose tiny, small and direct-packing paths read the operand in the kernel's scalar type.
777template <typename Xpr, typename = void>
778struct gemm_operand_traits : blas_traits<Xpr> {
779 using StorageScalar = typename traits<Xpr>::Scalar;
780};
781#if !defined(EIGEN_USE_BLAS) && !defined(EIGEN_VECTORIZE_SME)
782template <typename SrcScalar, typename DstScalar, typename NestedXpr>
783struct gemm_operand_traits<
784 CwiseUnaryOp<core_cast_op<SrcScalar, DstScalar>, NestedXpr>,
785 std::enable_if_t<bool(blas_traits<remove_all_t<NestedXpr>>::HasUsableDirectAccess) &&
786 !bool(blas_traits<remove_all_t<NestedXpr>>::HasScalarFactor) && !NumTraits<SrcScalar>::IsComplex &&
787 !NumTraits<DstScalar>::IsComplex && !gemm_accumulates_in_float<DstScalar, DstScalar>::value>>
788 : blas_traits<remove_all_t<NestedXpr>> {
789 using Base = blas_traits<remove_all_t<NestedXpr>>;
790 using XprType = CwiseUnaryOp<core_cast_op<SrcScalar, DstScalar>, NestedXpr>;
791 using StorageScalar = SrcScalar;
792 using Scalar = DstScalar;
793 static constexpr bool IsComplex = false;
794 static constexpr bool NeedToConjugate = false;
795 static EIGEN_DEVICE_FUNC inline typename Base::ExtractType extract(const XprType& x) {
796 return Base::extract(x.nestedExpression());
797 }
798 static EIGEN_DEVICE_FUNC inline Scalar extractScalarFactor(const XprType&) { return Scalar(1); }
799};
800#endif
801template <typename Xpr>
802struct gemm_operand_traits<const Xpr> : gemm_operand_traits<Xpr> {};
803
804template <typename Xpr, int Order = Xpr::Flags & RowMajorBit>
805struct gemm_product_workspace {
806 static constexpr bool Fits =
807 Xpr::MaxSizeAtCompileTime != Dynamic &&
808 (EIGEN_STACK_ALLOCATION_LIMIT == 0 ||
809 std::size_t(Xpr::MaxSizeAtCompileTime) <= EIGEN_STACK_ALLOCATION_LIMIT / sizeof(typename Xpr::Scalar));
810 using type =
811 Matrix<typename Xpr::Scalar, Fits ? Xpr::RowsAtCompileTime : Dynamic, Fits ? Xpr::ColsAtCompileTime : Dynamic,
812 Fits && Xpr::RowsAtCompileTime == 1 && Xpr::ColsAtCompileTime != 1 ? RowMajor : Order,
813 Fits ? Xpr::MaxRowsAtCompileTime : Dynamic, Fits ? Xpr::MaxColsAtCompileTime : Dynamic>;
814};
815
816template <typename Lhs, typename Rhs,
817 bool Commutative = blas_product_is_commutative<typename Lhs::Scalar, typename Rhs::Scalar>::value>
818struct gemm_product_impl;
819
820// The column-major kernel preserves coefficient order when factors stay in the operands and alpha is one.
821template <typename Lhs, typename Rhs>
822struct gemm_product_impl<Lhs, Rhs, false>
823 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemmProduct>> {
824 using Scalar = typename Product<Lhs, Rhs>::Scalar;
825
826 template <typename Dest>
827 static void evalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
828 run<true>(dst, lhs, rhs, Scalar(1));
829 }
830
831 template <typename Dest>
832 static void scaleAndAddTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
833 run<false>(dst, lhs, rhs, alpha);
834 }
835
836 template <typename Dest>
837 static void scaleAndEvalTo(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
838 run<true>(dst, lhs, rhs, alpha);
839 }
840
841 template <typename Dest>
842 static void subTo(Dest& dst, const Lhs& lhs, const Rhs& rhs) {
843 run<false, true>(dst, lhs, rhs, Scalar(1));
844 }
845
846 private:
847 template <bool Assign, bool Subtract = false, typename Dest>
848 static void run(Dest& dst, const Lhs& lhs, const Rhs& rhs, const Scalar& alpha) {
849 // Test factors before materializing operands: zero times a nonfinite coefficient need not be evaluated.
850 if (lhs.cols() == 0 || numext::is_exactly_zero(alpha) || blas_traits<Lhs>::hasZeroScalarFactor(lhs) ||
851 blas_traits<Rhs>::hasZeroScalarFactor(rhs)) {
852 EIGEN_IF_CONSTEXPR (Assign) dst.setZero();
853 return;
854 }
855 // Preserve bounded storage when it fits; oversized fixed maps still need dynamic workspaces.
856 using LhsPlain = typename gemm_product_workspace<Lhs>::type;
857 using RhsPlain = typename gemm_product_workspace<Rhs>::type;
858 using ActualLhs = std::conditional_t<blas_traits<Lhs>::HasScalarFactor, LhsPlain, Lhs>;
859 using ActualRhs = std::conditional_t<blas_traits<Rhs>::HasScalarFactor, RhsPlain, Rhs>;
860 const ActualLhs& actualLhs = lhs;
861 const ActualRhs& actualRhs = rhs;
862 using Packed = gemm_product_impl<ActualLhs, ActualRhs, true>;
863 EIGEN_IF_CONSTEXPR ((Dest::Flags & RowMajorBit) == 0) {
864 EIGEN_IF_CONSTEXPR (Assign) {
865 Packed::evalTo(dst, actualLhs, actualRhs);
866 if (!numext::is_exactly_one(alpha)) dst = alpha * dst;
867 return;
868 } else if (numext::is_exactly_one(alpha)) {
869 EIGEN_IF_CONSTEXPR (Subtract) {
870 Packed::subTo(dst, actualLhs, actualRhs);
871 } else {
872 Packed::addTo(dst, actualLhs, actualRhs);
873 }
874 return;
875 }
876 }
877 typename gemm_product_workspace<Product<Lhs, Rhs>, ColMajor>::type product(dst.rows(), dst.cols());
878 Packed::evalTo(product, actualLhs, actualRhs);
879 using AssignOp = std::conditional_t<Assign, assign_op<typename Dest::Scalar, Scalar>,
880 std::conditional_t<Subtract, sub_assign_op<typename Dest::Scalar, Scalar>,
881 add_assign_op<typename Dest::Scalar, Scalar>>>;
882 if (numext::is_exactly_one(alpha)) {
883 call_assignment_no_alias(dst, product, AssignOp());
884 } else {
885 call_assignment_no_alias(dst, alpha * product, AssignOp());
886 }
887 }
888};
889
890template <typename Lhs, typename Rhs>
891struct gemm_product_impl<Lhs, Rhs, true>
892 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemmProduct>> {
893 using Scalar = typename Product<Lhs, Rhs>::Scalar;
894 using LhsScalar = typename Lhs::Scalar;
895 using RhsScalar = typename Rhs::Scalar;
896
897 using LhsBlasTraits = internal::gemm_operand_traits<Lhs>;
898 using ActualLhsType = typename LhsBlasTraits::DirectLinearAccessType;
899 using ActualLhsTypeCleaned = internal::remove_all_t<ActualLhsType>;
900
901 using RhsBlasTraits = internal::gemm_operand_traits<Rhs>;
902 using ActualRhsType = typename RhsBlasTraits::DirectLinearAccessType;
903 using ActualRhsTypeCleaned = internal::remove_all_t<ActualRhsType>;
904
905 enum { MaxDepthAtCompileTime = min_size_prefer_fixed(Lhs::MaxColsAtCompileTime, Rhs::MaxRowsAtCompileTime) };
906
907 using lazyproduct = generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode>;
908
909 // The runtime-size heuristic comes from bug 404 and was tuned with a
910 // helper program on Haswell. The threshold belongs to the kernel the GEMM
911 // path would select, not to the path itself, and for the SME kernel to the
912 // scalar type as well: it needs a larger product before it beats the
913 // coeff-based one, by an amount that falls as the scalar widens (see
914 // GeneralProduct.h). The rhs.rows() > 0 guard preserves the historical
915 // empty-product path through scaleAndAddTo().
916 static constexpr int kCoeffBasedThreshold =
917#ifdef EIGEN_VECTORIZE_SME
918 sme_has_gebp_kernel<LhsScalar, RhsScalar>::value ? sme_gemm_to_coeffbased_threshold<Scalar>::value :
919#endif
920 EIGEN_GEMM_TO_COEFFBASED_THRESHOLD;
921
922#ifdef EIGEN_VECTORIZE_SME
923 // Second bound for the SME kernel only: a small output over a long depth.
924 // The sum above grows with the depth and so never catches it, while the ZA
925 // grid this kernel fills is sized by the output (see GeneralProduct.h).
926 // Vector shapes are excluded -- scaleAndAddTo() routes those to GEMV, which
927 // is not the path being compared here.
928 static constexpr Index kCoeffBasedOutputArea = sme_has_gebp_kernel<LhsScalar, RhsScalar>::value
929 ? Index(EIGEN_SME_GEMM_TO_COEFFBASED_OUTPUT_AREA_THRESHOLD(Scalar))
930 : Index(0);
931
932 template <typename Dst>
933 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool outputAreaBelowThreshold(const Dst& dst) {
934 // rows * cols is dst.size(), which fits Index by construction.
935 return dst.rows() > 1 && dst.cols() > 1 && dst.size() <= kCoeffBasedOutputArea;
936 }
937#endif
938
939#ifdef EIGEN_VECTORIZE_SME
940 // Whether the tiny-result kernel takes a product the coeff-based one would; the GEMM driver sees a RowMajor
941 // result transposed.
942 template <typename Dst>
943 static EIGEN_STRONG_INLINE bool tinyKernelWins(const Dst& dst, const Rhs& rhs) {
944 constexpr bool row_major = (Dst::Flags & RowMajorBit) != 0;
945 return sme_tiny_gemm_pair<LhsScalar, RhsScalar>::value &&
946 sme_tiny_gemm_wins<Scalar>(row_major ? dst.cols() : dst.rows(), row_major ? dst.rows() : dst.cols(),
947 rhs.rows());
948 }
949#endif
950
951 template <typename Dst>
952 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool useRuntimeCoeffBasedProduct(const Dst& dst, const Rhs& rhs) {
953 if (rhs.rows() <= 0) return false;
954#ifdef EIGEN_VECTORIZE_SME
955 if ((rhs.rows() + dst.rows() + dst.cols()) < kCoeffBasedThreshold || outputAreaBelowThreshold(dst))
956 return !tinyKernelWins(dst, rhs);
957#else
958 if ((rhs.rows() + dst.rows() + dst.cols()) < kCoeffBasedThreshold) return true;
959#endif
960 return false;
961 }
962
963 // BLAS contract: a zero scalar factor leaves the destination unchanged and
964 // neither operand need be read, so that a non-finite coefficient cannot taint
965 // the result through 0 * Inf. general_matrix_matrix_product::run enforces it
966 // for the GEMM path, but the coeff-based path below has no such exit and
967 // would evaluate the product, so the factor is tested before the dispatch
968 // rather than inside either kernel.
969 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool scalarFactorIsZero(const Lhs& lhs, const Rhs& rhs) {
970 return scalarFactorIsZero(gemm_accumulates_in_float<LhsScalar, RhsScalar>(), lhs, rhs);
971 }
972 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool scalarFactorIsZero(std::false_type, const Lhs& lhs,
973 const Rhs& rhs) {
974 return numext::is_exactly_zero(combine_scalar_factors<Scalar>(lhs, rhs));
975 }
976 // The float GEMM combines the factors in float, where their product may not underflow. The coefficient-based path
977 // recombines them in Scalar and would then evaluate 0 * lazyProduct, so such a product takes the GEMM.
978 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool scalarFactorIsZero(std::true_type, const Lhs& lhs, const Rhs& rhs) {
979 return numext::is_exactly_zero(float(LhsBlasTraits::extractScalarFactor(lhs)) *
980 float(RhsBlasTraits::extractScalarFactor(rhs)));
981 }
982
983 // A factor that underflows in Scalar but not in float takes the float GEMM (see scalarFactorIsZero).
984 template <typename Dst>
985 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool useCoeffBasedPath(const Dst& dst, const Lhs& lhs, const Rhs& rhs) {
986 return useRuntimeCoeffBasedProduct(dst, rhs) && !numext::is_exactly_zero(combine_scalar_factors<Scalar>(lhs, rhs));
987 }
988
989 template <typename Dst>
990 static void evalTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
991 if (scalarFactorIsZero(lhs, rhs)) {
992 dst.setZero();
993 return;
994 }
995 if (useCoeffBasedPath(dst, lhs, rhs))
996 lazyproduct::eval_dynamic(dst, lhs, rhs, internal::assign_op<typename Dst::Scalar, Scalar>());
997 else
998 scaleAndEvalTo(dst, lhs, rhs, Scalar(1));
999 }
1000
1001 template <typename Dst>
1002 static void addTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
1003 if (scalarFactorIsZero(lhs, rhs)) return;
1004 if (useCoeffBasedPath(dst, lhs, rhs))
1005 lazyproduct::eval_dynamic(dst, lhs, rhs, internal::add_assign_op<typename Dst::Scalar, Scalar>());
1006 else
1007 scaleAndAddTo(dst, lhs, rhs, Scalar(1));
1008 }
1009
1010 template <typename Dst>
1011 static void subTo(Dst& dst, const Lhs& lhs, const Rhs& rhs) {
1012 if (scalarFactorIsZero(lhs, rhs)) return;
1013 if (useCoeffBasedPath(dst, lhs, rhs))
1014 lazyproduct::eval_dynamic(dst, lhs, rhs, internal::sub_assign_op<typename Dst::Scalar, Scalar>());
1015 else
1016 scaleAndAddTo(dst, lhs, rhs, Scalar(-1));
1017 }
1018
1019 template <typename Dest>
1020 static void scaleAndAddTo(Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha) {
1021 scaleAndAddOrEvalTo(dst, a_lhs, a_rhs, alpha, false);
1022 }
1023
1024 // dst = alpha * a_lhs * a_rhs, independently of dst's prior contents and, where the GEMM kernel can overwrite,
1025 // without zeroing it first.
1026 template <typename Dest>
1027 static void scaleAndEvalTo(Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha) {
1028 scaleAndAddOrEvalTo(dst, a_lhs, a_rhs, alpha, true);
1029 }
1030
1031 private:
1032 // One body for both: `overwrite` is a runtime flag, so that assignment and accumulation share one
1033 // instantiation.
1034 template <typename Dest>
1035 static void scaleAndAddOrEvalTo(Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha, bool overwrite) {
1036 scaleAndAddOrEvalTo(gemm_accumulates_in_float<LhsScalar, RhsScalar>(), dst, a_lhs, a_rhs, alpha, overwrite);
1037 }
1038
1039 // The products the GEMM kernel does not take: empty operands, zero depth and a runtime vector result, which goes to
1040 // GEMV. They run on the operand type also where the GEMM sums in float, so that they leave an empty product's dst
1041 // untouched and do not allocate. Returns whether dst is done.
1042 template <typename Dest>
1043 static bool evalWithoutGemm(Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha, bool overwrite) {
1044 if (a_lhs.rows() == 0 || a_rhs.cols() == 0) return true;
1045 if (a_lhs.cols() != 0 && dst.cols() != 1 && dst.rows() != 1) return false;
1046 // Zero depth and GEMV accumulate, so overwriting zeroes first.
1047 if (overwrite) dst.setZero();
1048 if (a_lhs.cols() == 0) return true;
1049 if (dst.cols() == 1) {
1050 typename Dest::ColXpr dst_vec(dst.col(0));
1051 internal::generic_product_impl<Lhs, typename Rhs::ConstColXpr, DenseShape, DenseShape,
1052 GemvProduct>::scaleAndAddTo(dst_vec, a_lhs, a_rhs.col(0), alpha);
1053 } else {
1054 typename Dest::RowXpr dst_vec(dst.row(0));
1055 internal::generic_product_impl<typename Lhs::ConstRowXpr, Rhs, DenseShape, DenseShape,
1056 GemvProduct>::scaleAndAddTo(dst_vec, a_lhs.row(0), a_rhs, alpha);
1057 }
1058 return true;
1059 }
1060
1061 // dst (+)= alpha * a_lhs * a_rhs as the float product of the converted operands, rounded once into dst. The packers
1062 // convert the operands; only the result needs a float workspace, which goes where the packed blocks do: on the stack
1063 // up to EIGEN_STACK_ALLOCATION_LIMIT, and on the heap above it, also for a fixed-size result.
1064 template <typename Dest>
1065 static void scaleAndAddOrEvalTo(std::true_type, Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha,
1066 bool overwrite) {
1067 EIGEN_INTERNAL_ASSERT_SAME_ROWS(dst, a_lhs);
1068 EIGEN_INTERNAL_ASSERT_SAME_COLS(dst, a_rhs);
1069 if (evalWithoutGemm(dst, a_lhs, a_rhs, alpha, overwrite)) return;
1070 using Real = float;
1071 add_const_on_value_type_t<ActualLhsType> lhs = LhsBlasTraits::extract(a_lhs);
1072 add_const_on_value_type_t<ActualRhsType> rhs = RhsBlasTraits::extract(a_rhs);
1073 // A cast the float GEMM does not pack in place (under EIGEN_USE_BLAS or on SME) is materialized at run-time size,
1074 // so that a fixed-size operand does not land on the stack.
1075 using CastLhsXpr = std::remove_const_t<decltype(lhs.template cast<Real>())>;
1076 using CastRhsXpr = std::remove_const_t<decltype(rhs.template cast<Real>())>;
1077 using CastLhs = std::conditional_t<
1078 std::is_same<typename gemm_operand_traits<CastLhsXpr>::StorageScalar, Real>::value,
1079 Matrix<Real, Dynamic, Dynamic, (ActualLhsTypeCleaned::Flags & RowMajorBit) ? RowMajor : ColMajor>, CastLhsXpr>;
1080 using CastRhs = std::conditional_t<
1081 std::is_same<typename gemm_operand_traits<CastRhsXpr>::StorageScalar, Real>::value,
1082 Matrix<Real, Dynamic, Dynamic, (ActualRhsTypeCleaned::Flags & RowMajorBit) ? RowMajor : ColMajor>, CastRhsXpr>;
1083 CastLhs cast_lhs = lhs.template cast<Real>();
1084 CastRhs cast_rhs = rhs.template cast<Real>();
1085 // The factors combine in float, so that (s * A) * (t * B) keeps an s * t below half's range.
1086 Real actualAlpha =
1087 Real(alpha) * Real(LhsBlasTraits::extractScalarFactor(a_lhs)) * Real(RhsBlasTraits::extractScalarFactor(a_rhs));
1088
1089 using FloatGemm = generic_product_impl<CastLhs, CastRhs, DenseShape, DenseShape, GemmProduct>;
1090 ei_declare_aligned_stack_constructed_variable(Real, acc_data, dst.size(), nullptr);
1091 Map<Matrix<Real, Dynamic, Dynamic, (Dest::Flags & RowMajorBit) ? RowMajor : ColMajor>> acc(acc_data, dst.rows(),
1092 dst.cols());
1093 if (overwrite) {
1094 FloatGemm::scaleAndEvalTo(acc, cast_lhs, cast_rhs, actualAlpha);
1095 } else {
1096 acc = dst.template cast<Real>();
1097 FloatGemm::scaleAndAddTo(acc, cast_lhs, cast_rhs, actualAlpha);
1098 }
1099 dst = acc.template cast<typename Dest::Scalar>();
1100 }
1101
1102 template <typename Dest>
1103 static void scaleAndAddOrEvalTo(std::false_type, Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const Scalar& alpha,
1104 bool overwrite) {
1105 EIGEN_INTERNAL_ASSERT_SAME_ROWS(dst, a_lhs);
1106 EIGEN_INTERNAL_ASSERT_SAME_COLS(dst, a_rhs);
1107 using Gemm = internal::general_matrix_matrix_product<
1108 Index, LhsScalar, (ActualLhsTypeCleaned::Flags & RowMajorBit) ? RowMajor : ColMajor,
1109 bool(LhsBlasTraits::NeedToConjugate), RhsScalar,
1110 (ActualRhsTypeCleaned::Flags & RowMajorBit) ? RowMajor : ColMajor, bool(RhsBlasTraits::NeedToConjugate),
1111 (Dest::Flags & RowMajorBit) ? RowMajor : ColMajor, Dest::InnerStrideAtCompileTime,
1112 typename LhsBlasTraits::StorageScalar, typename RhsBlasTraits::StorageScalar>;
1113 if (evalWithoutGemm(dst, a_lhs, a_rhs, alpha, overwrite)) return;
1114 // A GEMM that cannot overwrite accumulates, so overwriting zeroes first.
1115 if (overwrite && !internal::gemm_can_overwrite<Gemm>::value) {
1116 dst.setZero();
1117 overwrite = false;
1118 }
1119
1120 add_const_on_value_type_t<ActualLhsType> lhs = LhsBlasTraits::extract(a_lhs);
1121 add_const_on_value_type_t<ActualRhsType> rhs = RhsBlasTraits::extract(a_rhs);
1122
1123 Scalar actualAlpha = combine_scalar_factors(alpha, a_lhs, a_rhs);
1124
1125 using BlockingType =
1126 internal::gemm_blocking_space<(Dest::Flags & RowMajorBit) ? RowMajor : ColMajor, LhsScalar, RhsScalar,
1127 Dest::MaxRowsAtCompileTime, Dest::MaxColsAtCompileTime, MaxDepthAtCompileTime>;
1128
1129 using GemmFunctor =
1130 internal::gemm_functor<Scalar, Index, Gemm, ActualLhsTypeCleaned, ActualRhsTypeCleaned, Dest, BlockingType>;
1131
1132 BlockingType blocking(dst.rows(), dst.cols(), lhs.cols(), 1, true);
1133#ifdef EIGEN_VECTORIZE_SME
1134 // A result the tiny-result kernel takes is never worth a parallel session.
1135 if (tinyKernelWins(dst, a_rhs))
1136 return internal::parallelize_gemm<false>(GemmFunctor(lhs, rhs, dst, actualAlpha, blocking, overwrite),
1137 a_lhs.rows(), a_rhs.cols(), a_lhs.cols(), Dest::Flags & RowMajorBit);
1138#endif
1139 internal::parallelize_gemm<(Dest::MaxRowsAtCompileTime > 32 || Dest::MaxRowsAtCompileTime == Dynamic)>(
1140 GemmFunctor(lhs, rhs, dst, actualAlpha, blocking, overwrite), a_lhs.rows(), a_rhs.cols(), a_lhs.cols(),
1141 Dest::Flags & RowMajorBit);
1142 }
1143};
1144
1145template <typename Lhs, typename Rhs>
1146struct generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemmProduct> : gemm_product_impl<Lhs, Rhs> {};
1147
1148} // end namespace internal
1149
1150} // end namespace Eigen
1151
1152#endif // EIGEN_GENERAL_MATRIX_MATRIX_H
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
constexpr unsigned int RowMajorBit
Definition Constants.h:71