11#ifndef EIGEN_GENERAL_MATRIX_MATRIX_H
12#define EIGEN_GENERAL_MATRIX_MATRIX_H
15#include "../InternalHeaderCheck.h"
21template <
typename LhsScalar_,
typename RhsScalar_>
26template <
typename LhsScalar,
typename RhsScalar>
27class gemm_packing_space {
30 aligned_stack_memory_handler<unsigned char> m_storage;
31 aligned_stack_memory_handler<LhsScalar> m_lhs_lifetime;
32 aligned_stack_memory_handler<RhsScalar> m_rhs_lifetime;
34 static EIGEN_DONT_INLINE
unsigned char* allocateCombined(std::size_t sizeA, std::size_t sizeB, LhsScalar*& blockA,
36 check_size_for_overflow<LhsScalar>(sizeA);
37 check_size_for_overflow<RhsScalar>(sizeB);
39 constexpr std::size_t alignment = (std::max)(std::size_t{EIGEN_STACK_ALIGN_BYTES},
alignof(RhsScalar));
40 const std::size_t max_bytes = (std::numeric_limits<std::ptrdiff_t>::max)();
41 const std::size_t bytesA = sizeA *
sizeof(LhsScalar);
42 const std::size_t bytesB = sizeB *
sizeof(RhsScalar);
43 if (bytesA > max_bytes - (alignment - 1)) throw_std_bad_alloc();
44 const std::size_t offsetB = first_multiple(bytesA, alignment);
45 if (bytesB > max_bytes - offsetB) throw_std_bad_alloc();
46 void* storage = scratch_malloc(offsetB + bytesB);
47 void* storageB =
static_cast<unsigned char*
>(storage) + offsetB;
48 blockA =
static_cast<LhsScalar*
>(storage);
49 blockB =
static_cast<RhsScalar*
>(storageB);
50 return static_cast<unsigned char*
>(storage);
53 static unsigned char* allocate(std::size_t sizeA, std::size_t sizeB, LhsScalar*& blockA, RhsScalar*& blockB) {
54 if (blockA !=
nullptr || blockB !=
nullptr || sizeA == 0 || sizeB == 0)
return nullptr;
55#if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA)
56 if (sizeA <= EIGEN_STACK_ALLOCATION_LIMIT /
sizeof(LhsScalar) ||
57 sizeB <= EIGEN_STACK_ALLOCATION_LIMIT /
sizeof(RhsScalar))
61 constexpr std::size_t base_alignment =
62 EIGEN_STACK_ALIGN_BYTES > 0 ? EIGEN_STACK_ALIGN_BYTES :
alignof(std::max_align_t);
63 if ((std::max)(
alignof(LhsScalar),
alignof(RhsScalar)) > base_alignment)
return nullptr;
64 return allocateCombined(sizeA, sizeB, blockA, blockB);
68 gemm_packing_space(std::size_t sizeA, std::size_t sizeB, LhsScalar* blockA, RhsScalar* blockB)
71 m_storage(allocate(sizeA, sizeB, m_blockA, m_blockB), 0, true),
72 m_lhs_lifetime(blockA == nullptr ? m_blockA : nullptr, sizeA, false),
73 m_rhs_lifetime(blockB == nullptr ? m_blockB : nullptr, sizeB, false) {}
75 LhsScalar* blockA()
const {
return m_blockA; }
76 RhsScalar* blockB()
const {
return m_blockB; }
83template <
typename Gebp,
typename... Args>
84EIGEN_ALWAYS_INLINE
void gebp_run(std::true_type, Gebp& gebp,
bool overwrite,
const Args&... args) {
86 gebp(gebp_overwrite_t(), args...);
90template <
typename Gebp,
typename... Args>
91EIGEN_ALWAYS_INLINE
void gebp_run(std::false_type, Gebp& gebp,
bool,
const Args&... args) {
96template <
typename Scalar,
typename Index>
97void gemm_zero_result(Scalar* res, Index rows, Index cols, Index resIncr, Index resStride) {
98 using Block = Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
99 if (resIncr == 1 && resStride == rows)
100 Map<Matrix<Scalar, Dynamic, 1>>(res, rows * cols).setZero();
101 else if (resIncr == 1)
102 Map<Block, 0, OuterStride<>>(res, rows, cols, OuterStride<>(resStride)).setZero();
104 Map<Block, 0, Stride<Dynamic, Dynamic>>(res, rows, cols, Stride<Dynamic, Dynamic>(resStride, resIncr)).setZero();
109struct gemm_pack_lhs_first_loop_policy {
110 template <
typename Index,
typename LhsScalar,
typename RhsScalar,
typename ResScalar,
typename LhsMapper,
111 typename RhsMapper,
typename ResMapper,
typename PackLhs,
typename PackRhs,
typename Gebp>
112 static EIGEN_ALWAYS_INLINE
void run(Index rows, Index cols, Index depth, Index kc, Index mc, Index nc,
113 const LhsMapper& lhs,
const RhsMapper& rhs, ResMapper& res, PackLhs& pack_lhs,
114 PackRhs& pack_rhs, Gebp& gebp, LhsScalar* blockA, RhsScalar* blockB,
115 ResScalar alpha,
bool overwrite) {
116 const bool pack_rhs_once = mc != rows && kc == depth && nc == cols;
119 for (Index i2 = 0; i2 < rows; i2 += mc) {
120 const Index actual_mc = (std::min)(i2 + mc, rows) - i2;
122 for (Index k2 = 0; k2 < depth; k2 += kc) {
123 const Index actual_kc = (std::min)(k2 + kc, depth) - k2;
129 pack_lhs(blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc);
132 for (Index j2 = 0; j2 < cols; j2 += nc) {
133 const Index actual_nc = (std::min)(j2 + nc, cols) - j2;
138 if ((!pack_rhs_once) || i2 == 0) pack_rhs(blockB, rhs.getSubMapper(k2, j2), actual_kc, actual_nc);
141 gebp_run(bool_constant<gebp_kernel_overwrites<Gebp>::value>(), gebp, overwrite && k2 == 0,
142 res.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, alpha);
149#ifdef EIGEN_VECTORIZE_SME
153template <
typename Scalar,
typename Index>
154bool sme_direct_lhs_ok(Index lhsStride, Index rows, Index depth, Index cols);
156template <
typename Mapper>
157struct sme_direct_lhs_mapper : std::false_type {};
158template <
typename Scalar,
typename Index>
159struct sme_direct_lhs_mapper<const_blas_data_mapper<Scalar, Index,
ColMajor>>
160 : bool_constant<!NumTraits<Scalar>::IsComplex> {};
164template <
typename Gebp,
typename... Args>
165EIGEN_ALWAYS_INLINE
void sme_direct_lhs_call(std::true_type, Gebp& gebp,
bool overwrite,
const Args&... args) {
167 gebp.run_direct_lhs(gebp_overwrite_t(), args...);
169 gebp.run_direct_lhs(args...);
171template <
typename Gebp,
typename... Args>
172EIGEN_ALWAYS_INLINE
void sme_direct_lhs_call(std::false_type, Gebp& gebp,
bool,
const Args&... args) {
173 gebp.run_direct_lhs(args...);
175template <
typename Gebp,
typename ResMapper,
typename LhsMapper,
typename Scalar,
typename ResScalar,
typename Index>
176EIGEN_ALWAYS_INLINE
bool sme_run_direct_lhs(std::true_type, Gebp& gebp,
const ResMapper& res,
const LhsMapper& lhs,
177 Index i2, Index k2,
const Scalar* blockB, Index mc, Index kc, Index nc,
178 ResScalar alpha,
bool overwrite) {
179 if (!sme_direct_lhs_ok<Scalar>(lhs.stride(), mc, kc, nc))
return false;
180 sme_direct_lhs_call(bool_constant<gebp_kernel_overwrites<Gebp>::value>(), gebp, overwrite, res, &lhs(i2, k2),
181 lhs.stride(), blockB, mc, kc, nc, alpha);
184template <
typename Scalar,
int LhsOrder,
int RhsOrder,
typename Index>
185bool sme_tiny_gemm(Index rows, Index cols, Index depth,
const Scalar* lhs, Index lhsStride,
const Scalar* rhs,
186 Index rhsStride, Scalar* res, Index resIncr, Index resStride, Scalar alpha);
190template <
typename Scalar,
typename Index>
191EIGEN_ALWAYS_INLINE
bool sme_tiny_gemm_wins(Index rows, Index cols, Index depth) {
192#if defined(EIGEN_SME_NO_NEON_SMALL_BLOCKS) || defined(EIGEN_SME_FORCE_NEON_SMALL_BLOCKS) || defined(EIGEN_USE_BLAS)
193 EIGEN_UNUSED_VARIABLE(rows);
194 EIGEN_UNUSED_VARIABLE(cols);
195 EIGEN_UNUSED_VARIABLE(depth);
198 const Index ps = Index(16 /
sizeof(Scalar));
199 if (rows < 2 || cols < 2 || rows > 2 * ps || cols > 8)
return false;
200 if (depth < 16 && (depth < 6 || rows * cols < 24))
return false;
201 return !(cols == 8 && rows > ps && depth >= 2048);
206template <
typename LhsScalar,
typename RhsScalar>
207struct sme_tiny_gemm_pair
208 : bool_constant<std::is_same<LhsScalar, RhsScalar>::value &&
209 (std::is_same<LhsScalar, float>::value || std::is_same<LhsScalar, double>::value) &&
210 sme_has_gebp_kernel<LhsScalar, RhsScalar>::value> {};
211template <
int LhsOrder,
int RhsOrder,
typename Scalar,
typename Index>
212EIGEN_ALWAYS_INLINE
bool sme_run_tiny_gemm(std::true_type, Index rows, Index cols, Index depth,
const Scalar* lhs,
213 Index lhsStride,
const Scalar* rhs, Index rhsStride, Scalar* res,
214 Index resIncr, Index resStride, Scalar alpha) {
215 return sme_tiny_gemm<Scalar, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr,
218template <
int LhsOrder,
int RhsOrder,
typename LhsScalar,
typename RhsScalar,
typename ResScalar,
typename Index>
219EIGEN_ALWAYS_INLINE
bool sme_run_tiny_gemm(std::false_type, Index, Index, Index,
const LhsScalar*, Index,
220 const RhsScalar*, Index, ResScalar*, Index, Index, ResScalar) {
223template <
typename Gebp,
typename ResMapper,
typename LhsMapper,
typename Scalar,
typename ResScalar,
typename Index>
224EIGEN_ALWAYS_INLINE
bool sme_run_direct_lhs(std::false_type, Gebp&,
const ResMapper&,
const LhsMapper&, Index, Index,
225 const Scalar*, Index, Index, Index, ResScalar,
bool) {
228template <
typename Scalar,
typename Index>
229EIGEN_ALWAYS_INLINE
bool sme_small_gemm_wins(Index rows, Index cols, Index depth,
bool packed_rhs);
230template <
typename Scalar,
int LhsOrder,
int RhsOrder,
bool ConjLhs,
bool ConjRhs,
int ResInnerStr
ide,
typename Index>
231EIGEN_DONT_INLINE
void sme_small_gemm(Index rows, Index cols, Index depth,
const Scalar* lhs, Index lhsStride,
232 const Scalar* rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
233 Scalar alpha, Scalar* blockA, Scalar* blockB);
235template <
int LhsOrder,
int RhsOrder,
bool ConjLhs,
bool ConjRhs,
int ResInnerStride,
typename Scalar,
typename Index,
237EIGEN_ALWAYS_INLINE
bool sme_run_small_gemm(std::true_type,
bool overwrite, Index rows, Index cols, Index depth,
238 const Scalar* lhs, Index lhsStride,
const Scalar* rhs, Index rhsStride,
239 Scalar* res, Index resIncr, Index resStride, Scalar alpha,
240 Blocking& blocking) {
241 if (!sme_small_gemm_wins<Scalar>(rows, cols, depth, RhsOrder ==
ColMajor))
return false;
242 if (overwrite) gemm_zero_result(res, rows, cols, resIncr, resStride);
243 const bool fits = depth <= blocking.kc() && rows <= blocking.mc() && cols <= blocking.nc();
244 sme_small_gemm<Scalar, LhsOrder, RhsOrder, ConjLhs, ConjRhs, ResInnerStride>(
245 rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr, resStride, alpha,
246 fits ? blocking.blockA() :
nullptr, fits ? blocking.blockB() :
nullptr);
249template <
int LhsOrder,
int RhsOrder,
bool ConjLhs,
bool ConjRhs,
int ResInnerStride,
typename LhsScalar,
250 typename RhsScalar,
typename ResScalar,
typename Index,
typename Blocking>
251EIGEN_ALWAYS_INLINE
bool sme_run_small_gemm(std::false_type,
bool, Index, Index, Index,
const LhsScalar*, Index,
252 const RhsScalar*, Index, ResScalar*, Index, Index, ResScalar, Blocking&) {
260struct gemm_pack_rhs_first_loop_policy {
261 template <
typename Index,
typename LhsScalar,
typename RhsScalar,
typename ResScalar,
typename LhsMapper,
262 typename RhsMapper,
typename ResMapper,
typename PackLhs,
typename PackRhs,
typename Gebp>
263 static EIGEN_ALWAYS_INLINE
void run(Index rows, Index cols, Index depth, Index kc, Index mc, Index nc,
264 const LhsMapper& lhs,
const RhsMapper& rhs, ResMapper& res, PackLhs& pack_lhs,
265 PackRhs& pack_rhs, Gebp& gebp, LhsScalar* blockA, RhsScalar* blockB,
266 ResScalar alpha,
bool overwrite) {
269 const bool pack_lhs_once = nc != cols && kc == depth && mc == rows;
270 bool lhs_packed =
false;
272 for (Index j2 = 0; j2 < cols; j2 += nc) {
273 const Index actual_nc = (std::min)(j2 + nc, cols) - j2;
275 for (Index k2 = 0; k2 < depth; k2 += kc) {
276 const Index actual_kc = (std::min)(k2 + kc, depth) - k2;
279 pack_rhs(blockB, rhs.getSubMapper(k2, j2), actual_kc, actual_nc);
281 for (Index i2 = 0; i2 < rows; i2 += mc) {
282 const Index actual_mc = (std::min)(i2 + mc, rows) - i2;
283#ifdef EIGEN_VECTORIZE_SME
284 if (sme_run_direct_lhs(bool_constant<sme_direct_lhs_mapper<LhsMapper>::value>(), gebp,
285 res.getSubMapper(i2, j2), lhs, i2, k2, blockB, actual_mc, actual_kc, actual_nc, alpha,
286 overwrite && k2 == 0))
289 if (!pack_lhs_once || !lhs_packed) {
290 pack_lhs(blockA, lhs.getSubMapper(i2, k2), actual_kc, actual_mc);
293 gebp_run(bool_constant<gebp_kernel_overwrites<Gebp>::value>(), gebp, overwrite && k2 == 0,
294 res.getSubMapper(i2, j2), blockA, blockB, actual_mc, actual_kc, actual_nc, alpha);
303template <
typename Gemm,
typename =
void>
304struct gemm_can_overwrite : std::false_type {};
305template <
typename Gemm>
306struct gemm_can_overwrite<Gemm, std::enable_if_t<Gemm::CanOverwrite>> : std::true_type {};
310template <
typename Gemm,
typename =
void>
311struct gemm_has_run_overwrite : std::false_type {};
312template <
typename Gemm>
313struct gemm_has_run_overwrite<Gemm, void_t<decltype(&Gemm::run_overwrite)>> : std::true_type {};
314template <
typename Gemm,
typename Index,
typename LhsScalar,
typename RhsScalar,
typename ResScalar,
typename Blocking>
315EIGEN_STRONG_INLINE
void gemm_run_overwrite_impl(std::true_type, Index rows, Index cols, Index depth,
316 const LhsScalar* lhs, Index lhsStride,
const RhsScalar* rhs,
317 Index rhsStride, ResScalar* res, Index resIncr, Index resStride,
318 ResScalar alpha, Blocking& blocking, GemmParallelInfo<Index>* info) {
319 Gemm::run_overwrite(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr, resStride, alpha, blocking,
322template <
typename Gemm,
typename Index,
typename LhsScalar,
typename RhsScalar,
typename ResScalar,
typename Blocking>
323EIGEN_STRONG_INLINE
void gemm_run_overwrite_impl(std::false_type, Index rows, Index cols, Index depth,
324 const LhsScalar* lhs, Index lhsStride,
const RhsScalar* rhs,
325 Index rhsStride, ResScalar* res, Index resIncr, Index resStride,
326 ResScalar alpha, Blocking& blocking, GemmParallelInfo<Index>* info) {
328 if (info ==
nullptr && (rows <= 0 || cols <= 0))
return;
329 gemm_zero_result(res, rows, cols, resIncr, resStride);
330 if (numext::is_exactly_zero(alpha) || depth <= 0)
return;
331 Gemm::run(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res, resIncr, resStride, alpha, blocking, info);
333template <
typename Gemm,
typename Index,
typename LhsScalar,
typename RhsScalar,
typename ResScalar,
typename Blocking>
334EIGEN_STRONG_INLINE
void gemm_run_overwrite(Index rows, Index cols, Index depth,
const LhsScalar* lhs, Index lhsStride,
335 const RhsScalar* rhs, Index rhsStride, ResScalar* res, Index resIncr,
336 Index resStride, ResScalar alpha, Blocking& blocking,
337 GemmParallelInfo<Index>* info) {
338 gemm_run_overwrite_impl<Gemm>(gemm_has_run_overwrite<Gemm>(), rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res,
339 resIncr, resStride, alpha, blocking, info);
343template <
typename Index,
typename LhsScalar,
int LhsStorageOrder,
bool ConjugateLhs,
typename RhsScalar,
344 int RhsStorageOrder,
bool ConjugateRhs,
int ResInnerStride,
typename LhsStorage,
typename RhsStorage>
345struct general_matrix_matrix_product<Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder,
346 ConjugateRhs,
RowMajor, ResInnerStride, LhsStorage, RhsStorage> {
347 using Traits = gebp_traits<RhsScalar, LhsScalar>;
351 ColMajor, ResInnerStride, RhsStorage, LhsStorage>;
353 using ResScalar =
typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
354 static constexpr bool CanOverwrite = gemm_can_overwrite<Transposed>::value;
355 static EIGEN_STRONG_INLINE
void run(Index rows, Index cols, Index depth,
const LhsStorage* lhs, Index lhsStride,
356 const RhsStorage* rhs, Index rhsStride, ResScalar* res, Index resIncr,
357 Index resStride, ResScalar alpha, level3_blocking<RhsScalar, LhsScalar>& blocking,
358 GemmParallelInfo<Index>* info = 0) {
360 Transposed::run(cols, rows, depth, rhs, rhsStride, lhs, lhsStride, res, resIncr, resStride, alpha, blocking, info);
363 static EIGEN_STRONG_INLINE
void run_overwrite(Index rows, Index cols, Index depth,
const LhsStorage* lhs,
364 Index lhsStride,
const RhsStorage* rhs, Index rhsStride, ResScalar* res,
365 Index resIncr, Index resStride, ResScalar alpha,
366 level3_blocking<RhsScalar, LhsScalar>& blocking,
367 GemmParallelInfo<Index>* info = 0) {
368 gemm_run_overwrite<Transposed>(cols, rows, depth, rhs, rhsStride, lhs, lhsStride, res, resIncr, resStride, alpha,
375template <
typename Index,
typename LhsScalar,
int LhsStorageOrder,
bool ConjugateLhs,
typename RhsScalar,
376 int RhsStorageOrder,
bool ConjugateRhs,
int ResInnerStride,
typename LhsStorage,
typename RhsStorage>
377struct general_matrix_matrix_product<Index, LhsScalar, LhsStorageOrder, ConjugateLhs, RhsScalar, RhsStorageOrder,
378 ConjugateRhs,
ColMajor, ResInnerStride, LhsStorage, RhsStorage> {
379 using Traits = gebp_traits<LhsScalar, RhsScalar>;
382 using Gebp = gebp_kernel<LhsScalar, RhsScalar, Index,
383 blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>,
384 Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs>;
387 using ResScalar =
typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
388 static constexpr bool CanOverwrite = gebp_kernel_overwrites<Gebp>::value;
389 static void run(Index rows, Index cols, Index depth,
const LhsStorage* lhs_, Index lhsStride,
const RhsStorage* rhs_,
390 Index rhsStride, ResScalar* res_, Index resIncr, Index resStride, ResScalar alpha,
391 level3_blocking<LhsScalar, RhsScalar>& blocking, GemmParallelInfo<Index>* info = 0) {
392 run_impl(
false, rows, cols, depth, lhs_, lhsStride, rhs_, rhsStride, res_, resIncr, resStride, alpha, blocking,
396 static void run_overwrite(Index rows, Index cols, Index depth,
const LhsStorage* lhs_, Index lhsStride,
397 const RhsStorage* rhs_, Index rhsStride, ResScalar* res_, Index resIncr, Index resStride,
398 ResScalar alpha, level3_blocking<LhsScalar, RhsScalar>& blocking,
399 GemmParallelInfo<Index>* info = 0) {
400 run_impl(
true, rows, cols, depth, lhs_, lhsStride, rhs_, rhsStride, res_, resIncr, resStride, alpha, blocking,
407 static EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE
void run_impl(
bool overwrite, Index rows, Index cols, Index depth,
408 const LhsStorage* lhs_, Index lhsStride,
409 const RhsStorage* rhs_, Index rhsStride, ResScalar* res_,
410 Index resIncr, Index resStride, ResScalar alpha,
411 level3_blocking<LhsScalar, RhsScalar>& blocking,
412 GemmParallelInfo<Index>* info) {
414 if (info ==
nullptr && (rows <= 0 || cols <= 0))
return;
417 if (overwrite && (numext::is_exactly_zero(alpha) || depth == 0 || !gebp_kernel_overwrites<Gebp>::value ||
418 !gebp_overwrite_pays<Gebp>(rows, cols, depth))) {
419 gemm_zero_result(res_, rows, cols, resIncr, resStride);
422 if (numext::is_exactly_zero(alpha) || depth == 0)
return;
423#ifdef EIGEN_VECTORIZE_SME
427 if (info ==
nullptr) {
428 constexpr bool tiny_pair = sme_tiny_gemm_pair<LhsScalar, RhsScalar>::value;
429 if (overwrite && tiny_pair && sme_tiny_gemm_wins<ResScalar>(rows, cols, depth))
430 gemm_zero_result(res_, rows, cols, resIncr, resStride);
431 if (sme_run_tiny_gemm<LhsStorageOrder, RhsStorageOrder>(bool_constant<tiny_pair>(), rows, cols, depth, lhs_,
432 lhsStride, rhs_, rhsStride, res_, resIncr, resStride,
434 sme_run_small_gemm<LhsStorageOrder, RhsStorageOrder, ConjugateLhs, ConjugateRhs, ResInnerStride>(
435 bool_constant<sme_has_gebp_kernel<LhsScalar, RhsScalar>::value>(), overwrite, rows, cols, depth, lhs_,
436 lhsStride, rhs_, rhsStride, res_, resIncr, resStride, alpha, blocking))
441 using LhsMapper = gemm_operand_mapper<LhsScalar, LhsStorage, Index, LhsStorageOrder>;
442 using RhsMapper = gemm_operand_mapper<RhsScalar, RhsStorage, Index, RhsStorageOrder>;
443 using ResMapper = blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>;
444 LhsMapper lhs(lhs_, lhsStride);
445 RhsMapper rhs(rhs_, rhsStride);
446 ResMapper res(res_, resStride, resIncr);
448 Index kc = blocking.kc();
449 Index mc = (std::min)(rows, blocking.mc());
450 Index nc = (std::min)(cols, blocking.nc());
452 gemm_pack_lhs<LhsScalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress,
typename Traits::LhsPacket4Packing,
455 gemm_pack_rhs<RhsScalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
458#if !defined(EIGEN_USE_BLAS) && (defined(EIGEN_HAS_OPENMP) || defined(EIGEN_GEMM_THREADPOOL))
459 constexpr bool kOverwrites = gebp_kernel_overwrites<Gebp>::value;
462 int tid = info->logical_thread_id;
463 int threads = info->num_threads;
465 LhsScalar* blockA = blocking.blockA();
466 eigen_internal_assert(blockA != 0);
468 std::size_t sizeB = kc * nc;
469 ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, 0);
472 for (Index k = 0; k < depth; k += kc) {
473 const Index actual_kc = (std::min)(k + kc, depth) - k;
477 pack_rhs(blockB, rhs.getSubMapper(k, 0), actual_kc, nc);
486 while (info->task_info[tid].users != 0) {
487 std::this_thread::yield();
489 info->task_info[tid].users = threads;
491 pack_lhs(blockA + info->task_info[tid].lhs_start * actual_kc,
492 lhs.getSubMapper(info->task_info[tid].lhs_start, k), actual_kc, info->task_info[tid].lhs_length);
495 info->task_info[tid].sync = k;
498 for (
int shift = 0; shift < threads; ++shift) {
499 int i = (tid + shift) % threads;
505 while (info->task_info[i].sync != k) {
506 std::this_thread::yield();
510 gebp_run(bool_constant<kOverwrites>(), gebp, overwrite && k == 0,
511 res.getSubMapper(info->task_info[i].lhs_start, 0), blockA + info->task_info[i].lhs_start * actual_kc,
512 blockB, info->task_info[i].lhs_length, actual_kc, nc, alpha);
516 for (Index j = nc; j < cols; j += nc) {
517 const Index actual_nc = (std::min)(j + nc, cols) - j;
520 pack_rhs(blockB, rhs.getSubMapper(k, j), actual_kc, actual_nc);
523 gebp_run(bool_constant<kOverwrites>(), gebp, overwrite && k == 0, res.getSubMapper(0, j), blockA, blockB,
524 rows, actual_kc, actual_nc, alpha);
529 for (Index i = 0; i < threads; ++i) info->task_info[i].users -= 1;
534 EIGEN_UNUSED_VARIABLE(info);
537 std::size_t sizeA = kc * mc;
538 std::size_t sizeB = kc * nc;
540 gemm_packing_space<LhsScalar, RhsScalar> packing(sizeA, sizeB, blocking.blockA(), blocking.blockB());
541 ei_declare_aligned_stack_constructed_variable(LhsScalar, blockA, sizeA, packing.blockA());
542 ei_declare_aligned_stack_constructed_variable(RhsScalar, blockB, sizeB, packing.blockB());
548#ifdef EIGEN_VECTORIZE_SME
549 using SequentialGemmLoop = std::conditional_t<sme_has_gebp_kernel<LhsScalar, RhsScalar>::value,
550 gemm_pack_rhs_first_loop_policy, gemm_pack_lhs_first_loop_policy>;
552 using SequentialGemmLoop = gemm_pack_lhs_first_loop_policy;
555 SequentialGemmLoop::run(rows, cols, depth, kc, mc, nc, lhs, rhs, res, pack_lhs, pack_rhs, gebp, blockA, blockB,
567template <
typename Scalar,
typename Index,
typename Gemm,
typename Lhs,
typename Rhs,
typename Dest,
568 typename BlockingType>
570 gemm_functor(
const Lhs& lhs,
const Rhs& rhs, Dest& dest,
const Scalar& actualAlpha, BlockingType& blocking,
571 bool overwrite =
false)
575 m_actualAlpha(actualAlpha),
576 m_blocking(blocking),
577 m_overwrite(overwrite) {}
579 void initParallelSession(Index num_threads)
const {
580 m_blocking.initParallel(m_lhs.rows(), m_rhs.cols(), m_lhs.cols(), num_threads);
581 m_blocking.allocateA();
585 bool ownsNoBuffers()
const {
return m_blocking.blockA() ==
nullptr && m_blocking.blockB() ==
nullptr; }
587 void operator()(Index row, Index rows, Index col = 0, Index cols = -1, GemmParallelInfo<Index>* info = 0)
const {
588 if (cols == -1) cols = m_rhs.cols();
591 gemm_run_overwrite<Gemm>(rows, cols, m_lhs.cols(), &m_lhs.coeffRef(row, 0), m_lhs.outerStride(),
592 &m_rhs.coeffRef(0, col), m_rhs.outerStride(), (Scalar*)&(m_dest.coeffRef(row, col)),
593 m_dest.innerStride(), m_dest.outerStride(), m_actualAlpha, m_blocking, info);
595 Gemm::run(rows, cols, m_lhs.cols(), &m_lhs.coeffRef(row, 0), m_lhs.outerStride(), &m_rhs.coeffRef(0, col),
596 m_rhs.outerStride(), (Scalar*)&(m_dest.coeffRef(row, col)), m_dest.innerStride(), m_dest.outerStride(),
597 m_actualAlpha, m_blocking, info);
601 using Traits =
typename Gemm::Traits;
607 Scalar m_actualAlpha;
608 BlockingType& m_blocking;
612template <
int StorageOrder,
typename LhsScalar,
typename RhsScalar,
int MaxRows,
int MaxCols,
int MaxDepth,
613 int KcFactor = 1,
bool FiniteAtCompileTime = MaxRows != Dynamic && MaxCols != Dynamic && MaxDepth != Dynamic>
614class gemm_blocking_space;
616template <
typename LhsScalar_,
typename RhsScalar_>
617class level3_blocking {
618 using LhsScalar = LhsScalar_;
619 using RhsScalar = RhsScalar_;
622 LhsScalar* m_blockA =
nullptr;
623 RhsScalar* m_blockB =
nullptr;
630 level3_blocking() =
default;
632 inline Index mc()
const {
return m_mc; }
633 inline Index nc()
const {
return m_nc; }
634 inline Index kc()
const {
return m_kc; }
636 inline LhsScalar* blockA() {
return m_blockA; }
637 inline RhsScalar* blockB() {
return m_blockB; }
640template <
int StorageOrder,
typename LhsScalar_,
typename RhsScalar_,
int MaxRows,
int MaxCols,
int MaxDepth,
642class gemm_blocking_space<StorageOrder, LhsScalar_, RhsScalar_, MaxRows, MaxCols, MaxDepth, KcFactor,
644 :
public level3_blocking<std::conditional_t<StorageOrder == RowMajor, RhsScalar_, LhsScalar_>,
645 std::conditional_t<StorageOrder == RowMajor, LhsScalar_, RhsScalar_>> {
647 Transpose = StorageOrder ==
RowMajor,
648 ActualRows = Transpose ? MaxCols : MaxRows,
649 ActualCols = Transpose ? MaxRows : MaxCols
651 using LhsScalar = std::conditional_t<Transpose, RhsScalar_, LhsScalar_>;
652 using RhsScalar = std::conditional_t<Transpose, LhsScalar_, RhsScalar_>;
655 static constexpr int SizeA = plain_enum_max(
int(ActualRows) * MaxDepth, 1);
658 static constexpr int SizeB = plain_enum_max(
int(ActualCols) * MaxDepth, 1) + int(packet_traits<RhsScalar>::size);
660#if EIGEN_MAX_STATIC_ALIGN_BYTES >= EIGEN_DEFAULT_ALIGN_BYTES
661 EIGEN_ALIGN_MAX LhsScalar m_staticA[SizeA > 0 ? SizeA : 1];
662 EIGEN_ALIGN_MAX RhsScalar m_staticB[SizeB > 0 ? SizeB : 1];
664 EIGEN_ALIGN_MAX
char m_staticA[(SizeA > 0 ? SizeA : 1) *
sizeof(LhsScalar) + EIGEN_DEFAULT_ALIGN_BYTES - 1];
665 EIGEN_ALIGN_MAX
char m_staticB[(SizeB > 0 ? SizeB : 1) *
sizeof(RhsScalar) + EIGEN_DEFAULT_ALIGN_BYTES - 1];
669 gemm_blocking_space(Index , Index , Index , Index ,
671 this->m_mc = ActualRows;
672 this->m_nc = ActualCols;
673 this->m_kc = MaxDepth;
674#if EIGEN_MAX_STATIC_ALIGN_BYTES >= EIGEN_DEFAULT_ALIGN_BYTES
675 this->m_blockA = m_staticA;
676 this->m_blockB = m_staticB;
678 this->m_blockA =
reinterpret_cast<LhsScalar*
>((std::uintptr_t(m_staticA) + (EIGEN_DEFAULT_ALIGN_BYTES - 1)) &
679 ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES - 1));
680 this->m_blockB =
reinterpret_cast<RhsScalar*
>((std::uintptr_t(m_staticB) + (EIGEN_DEFAULT_ALIGN_BYTES - 1)) &
681 ~std::size_t(EIGEN_DEFAULT_ALIGN_BYTES - 1));
685 void initParallel(Index, Index, Index, Index) {}
687 inline void allocateA() {}
688 inline void allocateB() {}
689 inline void allocateAll() {}
692template <
int StorageOrder,
typename LhsScalar_,
typename RhsScalar_,
int MaxRows,
int MaxCols,
int MaxDepth,
694class gemm_blocking_space<StorageOrder, LhsScalar_, RhsScalar_, MaxRows, MaxCols, MaxDepth, KcFactor, false>
695 :
public level3_blocking<std::conditional_t<StorageOrder == RowMajor, RhsScalar_, LhsScalar_>,
696 std::conditional_t<StorageOrder == RowMajor, LhsScalar_, RhsScalar_>> {
697 enum { Transpose = StorageOrder ==
RowMajor };
698 using LhsScalar = std::conditional_t<Transpose, RhsScalar_, LhsScalar_>;
699 using RhsScalar = std::conditional_t<Transpose, LhsScalar_, RhsScalar_>;
705 gemm_blocking_space(Index rows, Index cols, Index depth, Index num_threads,
bool l3_blocking) {
706 this->m_mc = Transpose ? cols : rows;
707 this->m_nc = Transpose ? rows : cols;
711 computeProductBlockingSizes<LhsScalar, RhsScalar, KcFactor>(this->m_kc, this->m_mc, this->m_nc, num_threads);
714 Index n = this->m_nc;
715 computeProductBlockingSizes<LhsScalar, RhsScalar, KcFactor>(this->m_kc, this->m_mc, n, num_threads);
718 m_sizeA = this->m_mc * this->m_kc;
719 m_sizeB = this->m_kc * this->m_nc;
722 void initParallel(Index rows, Index cols, Index depth, Index num_threads) {
723 this->m_mc = Transpose ? cols : rows;
724 this->m_nc = Transpose ? rows : cols;
727 eigen_internal_assert(this->m_blockA == 0 && this->m_blockB == 0);
728 Index m = this->m_mc;
729 computeProductBlockingSizes<LhsScalar, RhsScalar, KcFactor>(this->m_kc, m, this->m_nc, num_threads);
730 m_sizeA = this->m_mc * this->m_kc;
731 m_sizeB = this->m_kc * this->m_nc;
736 if (this->m_blockA == 0) this->m_blockA = scratch_new<LhsScalar>(m_sizeA);
740 if (this->m_blockB == 0) this->m_blockB = scratch_new<RhsScalar>(m_sizeB);
748 ~gemm_blocking_space() {
749 scratch_delete(this->m_blockA, m_sizeA);
750 scratch_delete(this->m_blockB, m_sizeB);
761template <
typename LhsScalar,
typename RhsScalar>
762struct gemm_accumulates_in_float : std::false_type {};
764struct gemm_accumulates_in_float<half, half> : std::true_type {};
766#if !(defined(EIGEN_USE_BLAS) && defined(EIGEN_USE_OPENBLAS_BFLOAT16) && EIGEN_USE_OPENBLAS_BFLOAT16)
768struct gemm_accumulates_in_float<bfloat16, bfloat16> : std::true_type {};
777template <
typename Xpr,
typename =
void>
778struct gemm_operand_traits : blas_traits<Xpr> {
779 using StorageScalar =
typename traits<Xpr>::Scalar;
781#if !defined(EIGEN_USE_BLAS) && !defined(EIGEN_VECTORIZE_SME)
782template <
typename SrcScalar,
typename DstScalar,
typename NestedXpr>
783struct gemm_operand_traits<
784 CwiseUnaryOp<core_cast_op<SrcScalar, DstScalar>, NestedXpr>,
785 std::enable_if_t<bool(blas_traits<remove_all_t<NestedXpr>>::HasUsableDirectAccess) &&
786 !bool(blas_traits<remove_all_t<NestedXpr>>::HasScalarFactor) && !NumTraits<SrcScalar>::IsComplex &&
787 !NumTraits<DstScalar>::IsComplex && !gemm_accumulates_in_float<DstScalar, DstScalar>::value>>
788 : blas_traits<remove_all_t<NestedXpr>> {
789 using Base = blas_traits<remove_all_t<NestedXpr>>;
790 using XprType = CwiseUnaryOp<core_cast_op<SrcScalar, DstScalar>, NestedXpr>;
791 using StorageScalar = SrcScalar;
792 using Scalar = DstScalar;
793 static constexpr bool IsComplex =
false;
794 static constexpr bool NeedToConjugate =
false;
795 static EIGEN_DEVICE_FUNC
inline typename Base::ExtractType extract(
const XprType& x) {
796 return Base::extract(x.nestedExpression());
798 static EIGEN_DEVICE_FUNC
inline Scalar extractScalarFactor(
const XprType&) {
return Scalar(1); }
801template <
typename Xpr>
802struct gemm_operand_traits<const Xpr> : gemm_operand_traits<Xpr> {};
804template <
typename Xpr,
int Order = Xpr::Flags & RowMajorBit>
805struct gemm_product_workspace {
806 static constexpr bool Fits =
807 Xpr::MaxSizeAtCompileTime != Dynamic &&
808 (EIGEN_STACK_ALLOCATION_LIMIT == 0 ||
809 std::size_t(Xpr::MaxSizeAtCompileTime) <= EIGEN_STACK_ALLOCATION_LIMIT /
sizeof(
typename Xpr::Scalar));
811 Matrix<
typename Xpr::Scalar, Fits ? Xpr::RowsAtCompileTime : Dynamic, Fits ? Xpr::ColsAtCompileTime : Dynamic,
812 Fits && Xpr::RowsAtCompileTime == 1 && Xpr::ColsAtCompileTime != 1 ?
RowMajor : Order,
813 Fits ? Xpr::MaxRowsAtCompileTime : Dynamic, Fits ? Xpr::MaxColsAtCompileTime : Dynamic>;
816template <
typename Lhs,
typename Rhs,
817 bool Commutative = blas_product_is_commutative<typename Lhs::Scalar, typename Rhs::Scalar>::value>
818struct gemm_product_impl;
821template <
typename Lhs,
typename Rhs>
822struct gemm_product_impl<Lhs, Rhs, false>
823 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemmProduct>> {
824 using Scalar =
typename Product<Lhs, Rhs>::Scalar;
826 template <
typename Dest>
827 static void evalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs) {
828 run<true>(dst, lhs, rhs, Scalar(1));
831 template <
typename Dest>
832 static void scaleAndAddTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha) {
833 run<false>(dst, lhs, rhs, alpha);
836 template <
typename Dest>
837 static void scaleAndEvalTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha) {
838 run<true>(dst, lhs, rhs, alpha);
841 template <
typename Dest>
842 static void subTo(Dest& dst,
const Lhs& lhs,
const Rhs& rhs) {
843 run<false, true>(dst, lhs, rhs, Scalar(1));
847 template <
bool Assign,
bool Subtract = false,
typename Dest>
848 static void run(Dest& dst,
const Lhs& lhs,
const Rhs& rhs,
const Scalar& alpha) {
850 if (lhs.cols() == 0 || numext::is_exactly_zero(alpha) || blas_traits<Lhs>::hasZeroScalarFactor(lhs) ||
851 blas_traits<Rhs>::hasZeroScalarFactor(rhs)) {
852 EIGEN_IF_CONSTEXPR (Assign) dst.setZero();
856 using LhsPlain =
typename gemm_product_workspace<Lhs>::type;
857 using RhsPlain =
typename gemm_product_workspace<Rhs>::type;
858 using ActualLhs = std::conditional_t<blas_traits<Lhs>::HasScalarFactor, LhsPlain, Lhs>;
859 using ActualRhs = std::conditional_t<blas_traits<Rhs>::HasScalarFactor, RhsPlain, Rhs>;
860 const ActualLhs& actualLhs = lhs;
861 const ActualRhs& actualRhs = rhs;
862 using Packed = gemm_product_impl<ActualLhs, ActualRhs, true>;
863 EIGEN_IF_CONSTEXPR ((Dest::Flags &
RowMajorBit) == 0) {
864 EIGEN_IF_CONSTEXPR (Assign) {
865 Packed::evalTo(dst, actualLhs, actualRhs);
866 if (!numext::is_exactly_one(alpha)) dst = alpha * dst;
868 }
else if (numext::is_exactly_one(alpha)) {
869 EIGEN_IF_CONSTEXPR (Subtract) {
870 Packed::subTo(dst, actualLhs, actualRhs);
872 Packed::addTo(dst, actualLhs, actualRhs);
877 typename gemm_product_workspace<Product<Lhs, Rhs>,
ColMajor>::type product(dst.rows(), dst.cols());
878 Packed::evalTo(product, actualLhs, actualRhs);
879 using AssignOp = std::conditional_t<Assign, assign_op<typename Dest::Scalar, Scalar>,
880 std::conditional_t<Subtract, sub_assign_op<typename Dest::Scalar, Scalar>,
881 add_assign_op<typename Dest::Scalar, Scalar>>>;
882 if (numext::is_exactly_one(alpha)) {
883 call_assignment_no_alias(dst, product, AssignOp());
885 call_assignment_no_alias(dst, alpha * product, AssignOp());
890template <
typename Lhs,
typename Rhs>
891struct gemm_product_impl<Lhs, Rhs, true>
892 : generic_product_impl_base<Lhs, Rhs, generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemmProduct>> {
893 using Scalar =
typename Product<Lhs, Rhs>::Scalar;
894 using LhsScalar =
typename Lhs::Scalar;
895 using RhsScalar =
typename Rhs::Scalar;
897 using LhsBlasTraits = internal::gemm_operand_traits<Lhs>;
898 using ActualLhsType =
typename LhsBlasTraits::DirectLinearAccessType;
899 using ActualLhsTypeCleaned = internal::remove_all_t<ActualLhsType>;
901 using RhsBlasTraits = internal::gemm_operand_traits<Rhs>;
902 using ActualRhsType =
typename RhsBlasTraits::DirectLinearAccessType;
903 using ActualRhsTypeCleaned = internal::remove_all_t<ActualRhsType>;
905 enum { MaxDepthAtCompileTime = min_size_prefer_fixed(Lhs::MaxColsAtCompileTime, Rhs::MaxRowsAtCompileTime) };
907 using lazyproduct = generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, CoeffBasedProductMode>;
916 static constexpr int kCoeffBasedThreshold =
917#ifdef EIGEN_VECTORIZE_SME
918 sme_has_gebp_kernel<LhsScalar, RhsScalar>::value ? sme_gemm_to_coeffbased_threshold<Scalar>::value :
920 EIGEN_GEMM_TO_COEFFBASED_THRESHOLD;
922#ifdef EIGEN_VECTORIZE_SME
928 static constexpr Index kCoeffBasedOutputArea = sme_has_gebp_kernel<LhsScalar, RhsScalar>::value
929 ? Index(EIGEN_SME_GEMM_TO_COEFFBASED_OUTPUT_AREA_THRESHOLD(Scalar))
932 template <
typename Dst>
933 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool outputAreaBelowThreshold(
const Dst& dst) {
935 return dst.rows() > 1 && dst.cols() > 1 && dst.size() <= kCoeffBasedOutputArea;
939#ifdef EIGEN_VECTORIZE_SME
942 template <
typename Dst>
943 static EIGEN_STRONG_INLINE
bool tinyKernelWins(
const Dst& dst,
const Rhs& rhs) {
944 constexpr bool row_major = (Dst::Flags &
RowMajorBit) != 0;
945 return sme_tiny_gemm_pair<LhsScalar, RhsScalar>::value &&
946 sme_tiny_gemm_wins<Scalar>(row_major ? dst.cols() : dst.rows(), row_major ? dst.rows() : dst.cols(),
951 template <
typename Dst>
952 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool useRuntimeCoeffBasedProduct(
const Dst& dst,
const Rhs& rhs) {
953 if (rhs.rows() <= 0)
return false;
954#ifdef EIGEN_VECTORIZE_SME
955 if ((rhs.rows() + dst.rows() + dst.cols()) < kCoeffBasedThreshold || outputAreaBelowThreshold(dst))
956 return !tinyKernelWins(dst, rhs);
958 if ((rhs.rows() + dst.rows() + dst.cols()) < kCoeffBasedThreshold)
return true;
969 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool scalarFactorIsZero(
const Lhs& lhs,
const Rhs& rhs) {
970 return scalarFactorIsZero(gemm_accumulates_in_float<LhsScalar, RhsScalar>(), lhs, rhs);
972 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool scalarFactorIsZero(std::false_type,
const Lhs& lhs,
974 return numext::is_exactly_zero(combine_scalar_factors<Scalar>(lhs, rhs));
978 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool scalarFactorIsZero(std::true_type,
const Lhs& lhs,
const Rhs& rhs) {
979 return numext::is_exactly_zero(
float(LhsBlasTraits::extractScalarFactor(lhs)) *
980 float(RhsBlasTraits::extractScalarFactor(rhs)));
984 template <
typename Dst>
985 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool useCoeffBasedPath(
const Dst& dst,
const Lhs& lhs,
const Rhs& rhs) {
986 return useRuntimeCoeffBasedProduct(dst, rhs) && !numext::is_exactly_zero(combine_scalar_factors<Scalar>(lhs, rhs));
989 template <
typename Dst>
990 static void evalTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs) {
991 if (scalarFactorIsZero(lhs, rhs)) {
995 if (useCoeffBasedPath(dst, lhs, rhs))
996 lazyproduct::eval_dynamic(dst, lhs, rhs, internal::assign_op<typename Dst::Scalar, Scalar>());
998 scaleAndEvalTo(dst, lhs, rhs, Scalar(1));
1001 template <
typename Dst>
1002 static void addTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs) {
1003 if (scalarFactorIsZero(lhs, rhs))
return;
1004 if (useCoeffBasedPath(dst, lhs, rhs))
1005 lazyproduct::eval_dynamic(dst, lhs, rhs, internal::add_assign_op<typename Dst::Scalar, Scalar>());
1007 scaleAndAddTo(dst, lhs, rhs, Scalar(1));
1010 template <
typename Dst>
1011 static void subTo(Dst& dst,
const Lhs& lhs,
const Rhs& rhs) {
1012 if (scalarFactorIsZero(lhs, rhs))
return;
1013 if (useCoeffBasedPath(dst, lhs, rhs))
1014 lazyproduct::eval_dynamic(dst, lhs, rhs, internal::sub_assign_op<typename Dst::Scalar, Scalar>());
1016 scaleAndAddTo(dst, lhs, rhs, Scalar(-1));
1019 template <
typename Dest>
1020 static void scaleAndAddTo(Dest& dst,
const Lhs& a_lhs,
const Rhs& a_rhs,
const Scalar& alpha) {
1021 scaleAndAddOrEvalTo(dst, a_lhs, a_rhs, alpha,
false);
1026 template <
typename Dest>
1027 static void scaleAndEvalTo(Dest& dst,
const Lhs& a_lhs,
const Rhs& a_rhs,
const Scalar& alpha) {
1028 scaleAndAddOrEvalTo(dst, a_lhs, a_rhs, alpha,
true);
1034 template <
typename Dest>
1035 static void scaleAndAddOrEvalTo(Dest& dst,
const Lhs& a_lhs,
const Rhs& a_rhs,
const Scalar& alpha,
bool overwrite) {
1036 scaleAndAddOrEvalTo(gemm_accumulates_in_float<LhsScalar, RhsScalar>(), dst, a_lhs, a_rhs, alpha, overwrite);
1042 template <
typename Dest>
1043 static bool evalWithoutGemm(Dest& dst,
const Lhs& a_lhs,
const Rhs& a_rhs,
const Scalar& alpha,
bool overwrite) {
1044 if (a_lhs.rows() == 0 || a_rhs.cols() == 0)
return true;
1045 if (a_lhs.cols() != 0 && dst.cols() != 1 && dst.rows() != 1)
return false;
1047 if (overwrite) dst.setZero();
1048 if (a_lhs.cols() == 0)
return true;
1049 if (dst.cols() == 1) {
1050 typename Dest::ColXpr dst_vec(dst.col(0));
1051 internal::generic_product_impl<Lhs,
typename Rhs::ConstColXpr, DenseShape, DenseShape,
1052 GemvProduct>::scaleAndAddTo(dst_vec, a_lhs, a_rhs.col(0), alpha);
1054 typename Dest::RowXpr dst_vec(dst.row(0));
1055 internal::generic_product_impl<
typename Lhs::ConstRowXpr, Rhs, DenseShape, DenseShape,
1056 GemvProduct>::scaleAndAddTo(dst_vec, a_lhs.row(0), a_rhs, alpha);
1064 template <
typename Dest>
1065 static void scaleAndAddOrEvalTo(std::true_type, Dest& dst,
const Lhs& a_lhs,
const Rhs& a_rhs,
const Scalar& alpha,
1067 EIGEN_INTERNAL_ASSERT_SAME_ROWS(dst, a_lhs);
1068 EIGEN_INTERNAL_ASSERT_SAME_COLS(dst, a_rhs);
1069 if (evalWithoutGemm(dst, a_lhs, a_rhs, alpha, overwrite))
return;
1071 add_const_on_value_type_t<ActualLhsType> lhs = LhsBlasTraits::extract(a_lhs);
1072 add_const_on_value_type_t<ActualRhsType> rhs = RhsBlasTraits::extract(a_rhs);
1075 using CastLhsXpr = std::remove_const_t<
decltype(lhs.template cast<Real>())>;
1076 using CastRhsXpr = std::remove_const_t<
decltype(rhs.template cast<Real>())>;
1077 using CastLhs = std::conditional_t<
1078 std::is_same<typename gemm_operand_traits<CastLhsXpr>::StorageScalar, Real>::value,
1080 using CastRhs = std::conditional_t<
1081 std::is_same<typename gemm_operand_traits<CastRhsXpr>::StorageScalar, Real>::value,
1083 CastLhs cast_lhs = lhs.template cast<Real>();
1084 CastRhs cast_rhs = rhs.template cast<Real>();
1087 Real(alpha) * Real(LhsBlasTraits::extractScalarFactor(a_lhs)) * Real(RhsBlasTraits::extractScalarFactor(a_rhs));
1089 using FloatGemm = generic_product_impl<CastLhs, CastRhs, DenseShape, DenseShape, GemmProduct>;
1090 ei_declare_aligned_stack_constructed_variable(Real, acc_data, dst.size(),
nullptr);
1094 FloatGemm::scaleAndEvalTo(acc, cast_lhs, cast_rhs, actualAlpha);
1096 acc = dst.template cast<Real>();
1097 FloatGemm::scaleAndAddTo(acc, cast_lhs, cast_rhs, actualAlpha);
1099 dst = acc.template cast<typename Dest::Scalar>();
1102 template <
typename Dest>
1103 static void scaleAndAddOrEvalTo(std::false_type, Dest& dst,
const Lhs& a_lhs,
const Rhs& a_rhs,
const Scalar& alpha,
1105 EIGEN_INTERNAL_ASSERT_SAME_ROWS(dst, a_lhs);
1106 EIGEN_INTERNAL_ASSERT_SAME_COLS(dst, a_rhs);
1107 using Gemm = internal::general_matrix_matrix_product<
1109 bool(LhsBlasTraits::NeedToConjugate), RhsScalar,
1112 typename LhsBlasTraits::StorageScalar, typename RhsBlasTraits::StorageScalar>;
1113 if (evalWithoutGemm(dst, a_lhs, a_rhs, alpha, overwrite))
return;
1115 if (overwrite && !internal::gemm_can_overwrite<Gemm>::value) {
1120 add_const_on_value_type_t<ActualLhsType> lhs = LhsBlasTraits::extract(a_lhs);
1121 add_const_on_value_type_t<ActualRhsType> rhs = RhsBlasTraits::extract(a_rhs);
1123 Scalar actualAlpha = combine_scalar_factors(alpha, a_lhs, a_rhs);
1125 using BlockingType =
1127 Dest::MaxRowsAtCompileTime, Dest::MaxColsAtCompileTime, MaxDepthAtCompileTime>;
1130 internal::gemm_functor<Scalar, Index, Gemm, ActualLhsTypeCleaned, ActualRhsTypeCleaned, Dest, BlockingType>;
1132 BlockingType blocking(dst.rows(), dst.cols(), lhs.cols(), 1,
true);
1133#ifdef EIGEN_VECTORIZE_SME
1135 if (tinyKernelWins(dst, a_rhs))
1136 return internal::parallelize_gemm<false>(GemmFunctor(lhs, rhs, dst, actualAlpha, blocking, overwrite),
1137 a_lhs.rows(), a_rhs.cols(), a_lhs.cols(), Dest::Flags &
RowMajorBit);
1139 internal::parallelize_gemm<(Dest::MaxRowsAtCompileTime > 32 || Dest::MaxRowsAtCompileTime == Dynamic)>(
1140 GemmFunctor(lhs, rhs, dst, actualAlpha, blocking, overwrite), a_lhs.rows(), a_rhs.cols(), a_lhs.cols(),
1145template <
typename Lhs,
typename Rhs>
1146struct generic_product_impl<Lhs, Rhs, DenseShape, DenseShape, GemmProduct> : gemm_product_impl<Lhs, Rhs> {};
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
constexpr unsigned int RowMajorBit
Definition Constants.h:71