10#ifndef EIGEN_SME_GENERALBLOCKPANELKERNEL_H
11#define EIGEN_SME_GENERALBLOCKPANELKERNEL_H
14#include "../../InternalHeaderCheck.h"
64static constexpr int kSmeDesignVectorBytes = 64;
73template <
typename Scalar>
75 static constexpr int kGridRows = 2;
76 static constexpr int kGridCols = 2;
77 static constexpr int mr = kGridRows * kSmeDesignVectorBytes / int(
sizeof(Scalar));
78 static constexpr int nr = kGridCols * kSmeDesignVectorBytes / int(
sizeof(Scalar));
81template <
typename RealScalar>
82struct sme_block<std::complex<RealScalar>> {
83 static constexpr int kGridCols = 2;
84 static constexpr int kGridRows = sme_tile_count<RealScalar>::value / (2 * kGridCols);
85 static constexpr int mr = kGridRows * kSmeDesignVectorBytes / int(
sizeof(RealScalar));
86 static constexpr int nr = kGridCols * kSmeDesignVectorBytes / int(
sizeof(RealScalar));
89static constexpr int kSmeMr = sme_block<float>::mr;
90static constexpr int kSmeNr = sme_block<float>::nr;
91static constexpr int kSmeMrC = sme_block<std::complex<float>>::mr;
92static constexpr int kSmeNrC = sme_block<std::complex<float>>::nr;
93#ifdef EIGEN_VECTORIZE_SME_F64F64
94static constexpr int kSmeMrD = sme_block<double>::mr;
95static constexpr int kSmeNrD = sme_block<double>::nr;
96static constexpr int kSmeMrCD = sme_block<std::complex<double>>::mr;
97static constexpr int kSmeNrCD = sme_block<std::complex<double>>::nr;
120template <
typename Scalar>
121EIGEN_ALWAYS_INLINE
void sme_append_lanes(
typename sme_packet_traits<Scalar>::type& cur, svbool_t& tail, int64_t& fill,
122 Scalar*& out,
typename sme_packet_traits<Scalar>::type v, int64_t w,
123 int64_t svl) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
124 using Traits = sme_packet_traits<Scalar>;
125 const auto joined = psplice(tail, cur, v);
126 if (fill + w < svl) {
129 tail = Traits::whilelt(0, fill);
132 pstoreu(Traits::ptrue(), out, joined);
134 tail = svnot_b_z(Traits::whilelt(0, w), Traits::whilelt(fill, svl));
151template <
bool Conjugate,
typename Scalar,
typename Index>
152static EIGEN_ALWAYS_INLINE
void sve_copy_panel_range(Scalar* EIGEN_RESTRICT dst,
const Scalar* EIGEN_RESTRICT src,
153 Index src_stride, Index k0, Index k1,
154 int width) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
155 using Traits = sme_packet_traits<Scalar>;
156 const int svl = Traits::size();
157 if (width == 2 * svl) {
159 const svcount_t pn = Traits::ptrue_c();
161 for (; k + 4 <= k1; k += 4) {
162 const auto v0 = ploadu_x2(pn, &src[k * src_stride]);
163 const auto v1 = ploadu_x2(pn, &src[(k + 1) * src_stride]);
164 const auto v2 = ploadu_x2(pn, &src[(k + 2) * src_stride]);
165 const auto v3 = ploadu_x2(pn, &src[(k + 3) * src_stride]);
166 pstoreu_x2(pn, &dst[k * width], v0);
167 pstoreu_x2(pn, &dst[(k + 1) * width], v1);
168 pstoreu_x2(pn, &dst[(k + 2) * width], v2);
169 pstoreu_x2(pn, &dst[(k + 3) * width], v3);
171 for (; k < k1; ++k) pstoreu_x2(pn, &dst[k * width], ploadu_x2(pn, &src[k * src_stride]));
174 if (width < 2 * svl && width != svl) {
176 const int w_lo = sme_min(width, svl), w_hi = width - w_lo;
177 const svbool_t pg_lo = Traits::whilelt(0, w_lo), pg_hi = Traits::whilelt(0, w_hi);
178 typename Traits::type cur = pset1<typename Traits::type>(Scalar(0));
179 svbool_t tail = svpfalse_b();
181 Scalar* out = dst + k0 * width;
182 for (Index k = k0; k < k1; ++k) {
183 const Scalar* s = &src[k * src_stride];
184 sme_append_lanes(cur, tail, fill, out, ploadu(pg_lo, s), w_lo, svl);
185 if (w_hi > 0) sme_append_lanes(cur, tail, fill, out, ploadu(pg_hi, sme_offset(s, svl)), w_hi, svl);
187 if (fill > 0) pstoreu(Traits::whilelt(0, fill), out, psplice(tail, cur, cur));
190 for (
int off = 0; off < width; off += svl) {
191 const svbool_t pred = sme_packet_traits<Scalar>::whilelt(off, width);
192 for (Index k = k0; k < k1; ++k) {
193 pstoreu(pred, &dst[k * width + off], ploadu(pred, &src[k * src_stride + off]));
200template <
typename Scalar,
typename Index>
201static EIGEN_ALWAYS_INLINE
void sve_copy_panel_quad(Scalar* EIGEN_RESTRICT dst, Index panel_stride,
202 const Scalar* EIGEN_RESTRICT src, Index src_stride,
203 Index depth) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
204 using Traits = sme_packet_traits<Scalar>;
205 const int w = 2 * Traits::size();
206 const svcount_t pn = Traits::ptrue_c();
207 const int strip_bytes = 4 * w * int(
sizeof(Scalar));
208 for (Index k = 0; k < depth; ++k) {
209 const Scalar* c = src + k * src_stride;
211 for (
int l = 0; l < strip_bytes; l += 128)
212 __builtin_prefetch(
reinterpret_cast<const char*
>(c + 4 * src_stride) + l, 0, 2);
213 const auto a = ploadu_x4(pn, c), b = ploadu_x4(pn, c + 2 * w);
214 pstoreu_x2(pn, dst + k * w, pcreate(pget<0>(a), pget<1>(a)));
215 pstoreu_x2(pn, dst + panel_stride + k * w, pcreate(pget<2>(a), pget<3>(a)));
216 pstoreu_x2(pn, dst + 2 * panel_stride + k * w, pcreate(pget<0>(b), pget<1>(b)));
217 pstoreu_x2(pn, dst + 3 * panel_stride + k * w, pcreate(pget<2>(b), pget<3>(b)));
223template <
bool PanelMode,
typename Scalar,
typename Index>
224static EIGEN_ALWAYS_INLINE Index sme_pack_quad_panels(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src,
225 Index src_stride, Index depth, Index rows, Index mr,
227 Index dst_offset) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
228 if (mr != 2 * sme_packet_traits<Scalar>::size())
return 0;
229 const Index panel_stride = PanelMode ? mr * dst_stride : mr * depth;
231 for (; i + 4 * mr <= rows; i += 4 * mr) {
232 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * mr : dst_base + i * depth;
233 sve_copy_panel_quad(dst_panel, panel_stride, src + i, src_stride, depth);
237template <
bool PanelMode,
typename RealScalar,
typename Index>
238static EIGEN_ALWAYS_INLINE Index sme_pack_quad_panels(std::complex<RealScalar>*,
const std::complex<RealScalar>*, Index,
239 Index, Index, Index, Index,
240 Index) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
248template <
bool Conjugate,
typename RealScalar,
typename Index>
249static EIGEN_ALWAYS_INLINE
void sve_copy_panel_range(std::complex<RealScalar>* EIGEN_RESTRICT dst,
250 const std::complex<RealScalar>* EIGEN_RESTRICT src,
251 Index src_stride, Index k0, Index k1,
252 int width) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
253 using Traits = sme_packet_traits<RealScalar>;
254 using Vec =
typename Traits::type;
255 RealScalar* EIGEN_RESTRICT rdst =
reinterpret_cast<RealScalar*
>(dst);
256 const RealScalar* EIGEN_RESTRICT rsrc =
reinterpret_cast<const RealScalar*
>(src);
257 const int svl = Traits::size();
258 const Index step = Index(2 * width);
259 if (width % svl != 0 && width < 2 * svl) {
262 const int w0 = sme_min(width, svl), w1 = width - w0;
263 const svbool_t pg_w0 = Traits::whilelt(0, w0), pg_w1 = Traits::whilelt(0, w1);
264 const svbool_t pg_lo0 = Traits::whilelt(0, 2 * w0), pg_hi0 = Traits::whilelt(svl, 2 * w0);
265 const svbool_t pg_lo1 = Traits::whilelt(0, 2 * w1), pg_hi1 = Traits::whilelt(svl, 2 * w1);
266 Vec cur = pset1<Vec>(RealScalar(0));
267 svbool_t tail = svpfalse_b();
269 RealScalar* out = rdst + k0 * step;
270 for (Index k = k0; k < k1; ++k) {
271 const RealScalar* p = rsrc + Index(2) * k * src_stride;
272 const Vec lo0 = ploadu(pg_lo0, p), hi0 = ploadu(pg_hi0, sme_offset(p, svl));
273 Vec im0 = puzp2(lo0, hi0);
274 EIGEN_IF_CONSTEXPR (Conjugate) {
275 im0 = pnegate(pg_w0, im0);
277 sme_append_lanes(cur, tail, fill, out, puzp1(lo0, hi0), w0, svl);
279 const RealScalar* p1 = sme_offset(p, 2 * svl);
280 const Vec lo1 = ploadu(pg_lo1, p1), hi1 = ploadu(pg_hi1, sme_offset(p1, svl));
281 Vec im1 = puzp2(lo1, hi1);
282 EIGEN_IF_CONSTEXPR (Conjugate) {
283 im1 = pnegate(pg_w1, im1);
285 sme_append_lanes(cur, tail, fill, out, puzp1(lo1, hi1), w1, svl);
286 sme_append_lanes(cur, tail, fill, out, im0, w0, svl);
287 sme_append_lanes(cur, tail, fill, out, im1, w1, svl);
289 sme_append_lanes(cur, tail, fill, out, im0, w0, svl);
292 if (fill > 0) pstoreu(Traits::whilelt(0, fill), out, psplice(tail, cur, cur));
295 for (
int off = 0; off < width; off += svl) {
296 const int w = width - off;
297 const int lanes = 2 * w;
298 const svbool_t pg_w = Traits::whilelt(off, width);
299 const svbool_t pg_lo = Traits::whilelt(0, lanes);
300 const svbool_t pg_hi = Traits::whilelt(svl, lanes);
301 for (Index k = k0; k < k1; ++k) {
302 const RealScalar* p = rsrc + Index(2) * (k * src_stride + Index(off));
303 const Vec v_lo = ploadu(pg_lo, p);
307 const Vec v_hi = ploadu(pg_hi, sme_offset(p, svl));
308 Vec im = puzp2(v_lo, v_hi);
309 EIGEN_IF_CONSTEXPR (Conjugate) {
310 im = pnegate(pg_w, im);
312 pstoreu(pg_w, &rdst[k * step + Index(off)], puzp1(v_lo, v_hi));
313 pstoreu(pg_w, &rdst[k * step + Index(width + off)], im);
320template <
bool Conjugate,
typename Scalar,
typename Index>
321static EIGEN_ALWAYS_INLINE
void sve_copy_panel(Scalar* EIGEN_RESTRICT dst,
const Scalar* EIGEN_RESTRICT src,
322 Index src_stride, Index depth,
323 int width) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
324 sve_copy_panel_range<Conjugate>(dst, src, src_stride, Index(0), depth, width);
330template <
int Lo,
int Hi,
typename RealScalar>
331static EIGEN_ALWAYS_INLINE
void sme_store_lines_vg4(RealScalar* d,
int w,
int svl,
332 uint32_t c) __arm_streaming __arm_in(
"za") {
333 const svbool_t all = sme_packet_traits<RealScalar>::ptrue();
334 const auto lo = sme_read_ver_za_vg4<Lo, RealScalar>(c), hi = sme_read_ver_za_vg4<Hi, RealScalar>(c);
335 pstoreu(all, d, pget<0>(lo));
336 pstoreu(all, d + svl, pget<0>(hi));
337 pstoreu(all, d + w, pget<1>(lo));
338 pstoreu(all, d + w + svl, pget<1>(hi));
339 pstoreu(all, d + 2 * w, pget<2>(lo));
340 pstoreu(all, d + 2 * w + svl, pget<2>(hi));
341 pstoreu(all, d + 3 * w, pget<3>(lo));
342 pstoreu(all, d + 3 * w + svl, pget<3>(hi));
347template <
typename RealScalar,
typename Index>
348static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_four(RealScalar*,
const RealScalar*, Index, Index k0,
349 Index) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
353#ifdef EIGEN_VECTORIZE_SME_F64F64
354template <
int Tile,
typename Index>
355static EIGEN_ALWAYS_INLINE
void sme_write_rows_x4(uint32_t r, svcount_t pn,
const double* p,
356 Index stride) __arm_streaming __arm_inout(
"za") {
357 const auto a0 = ploadu_x4(pn, p), a1 = ploadu_x4(pn, p + stride), a2 = ploadu_x4(pn, p + 2 * stride),
358 a3 = ploadu_x4(pn, p + 3 * stride);
359 sme_write_hor_za_vg4<Tile>(r, pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3));
360 sme_write_hor_za_vg4<Tile + 1>(r, pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3));
361 sme_write_hor_za_vg4<Tile + 2>(r, pget<2>(a0), pget<2>(a1), pget<2>(a2), pget<2>(a3));
362 sme_write_hor_za_vg4<Tile + 3>(r, pget<3>(a0), pget<3>(a1), pget<3>(a2), pget<3>(a3));
368template <
typename Index>
369static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_four(
double* EIGEN_RESTRICT dst,
const double* EIGEN_RESTRICT src,
370 Index src_stride, Index k0,
371 Index k1) __arm_streaming __arm_inout(
"za") {
372 const int svl = sme_packet_traits<double>::size(), w = 2 * svl, d = 4 * svl;
373 const svcount_t pn = sme_packet_traits<double>::ptrue_c();
375 for (; k + d <= k1; k += d) {
377 const bool pf = k + 3 * d <= k1;
378 for (
int r = 0; r < svl; r += 4) {
379 const double* p = src + Index(r) * src_stride + k;
380 const double* q = p + Index(svl) * src_stride;
382 for (
int u = 0; u < 4; ++u) {
383 __builtin_prefetch(p + u * src_stride + 2 * d, 0, 2);
384 __builtin_prefetch(p + u * src_stride + 2 * d + w, 0, 2);
385 __builtin_prefetch(q + u * src_stride + 2 * d, 0, 2);
386 __builtin_prefetch(q + u * src_stride + 2 * d + w, 0, 2);
389 sme_write_rows_x4<0>(uint32_t(r), pn, p, src_stride);
390 sme_write_rows_x4<4>(uint32_t(r), pn, q, src_stride);
392 for (
int c = 0; c < svl; c += 4) {
393 double* d0 = dst + (k + c) * w;
394 sme_store_lines_vg4<0, 4>(d0, w, svl, uint32_t(c));
395 sme_store_lines_vg4<1, 5>(d0 + Index(svl) * w, w, svl, uint32_t(c));
396 sme_store_lines_vg4<2, 6>(d0 + Index(2 * svl) * w, w, svl, uint32_t(c));
397 sme_store_lines_vg4<3, 7>(d0 + Index(3 * svl) * w, w, svl, uint32_t(c));
407template <
typename RealScalar,
typename Index>
408static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_pair(RealScalar* EIGEN_RESTRICT dst,
409 const RealScalar* EIGEN_RESTRICT src, Index src_stride,
410 Index k0, Index k1) __arm_streaming __arm_inout(
"za") {
411 using Traits = sme_packet_traits<RealScalar>;
412 const int svl = Traits::size(), w = 2 * svl;
413 const svcount_t pn = Traits::ptrue_c();
414 Index k = sme_transpose_pack_four(dst, src, src_stride, k0, k1);
415 for (; k + w <= k1; k += w) {
416 const bool pf = k + 2 * w <= k1;
417 for (
int r = 0; r < svl; r += 4) {
418 const RealScalar* p = src + Index(r) * src_stride + k;
419 const RealScalar* q = p + Index(svl) * src_stride;
421 for (
int u = 0; u < 4; ++u) {
422 __builtin_prefetch(p + u * src_stride + 2 * w, 0, 2);
423 __builtin_prefetch(q + u * src_stride + 2 * w, 0, 2);
425 const auto a0 = ploadu_x2(pn, p), a1 = ploadu_x2(pn, p + src_stride), a2 = ploadu_x2(pn, p + 2 * src_stride),
426 a3 = ploadu_x2(pn, p + 3 * src_stride);
427 sme_write_hor_za_vg4<0>(uint32_t(r), pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3));
428 sme_write_hor_za_vg4<1>(uint32_t(r), pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3));
429 const auto b0 = ploadu_x2(pn, q), b1 = ploadu_x2(pn, q + src_stride), b2 = ploadu_x2(pn, q + 2 * src_stride),
430 b3 = ploadu_x2(pn, q + 3 * src_stride);
431 sme_write_hor_za_vg4<2>(uint32_t(r), pget<0>(b0), pget<0>(b1), pget<0>(b2), pget<0>(b3));
432 sme_write_hor_za_vg4<3>(uint32_t(r), pget<1>(b0), pget<1>(b1), pget<1>(b2), pget<1>(b3));
434 for (
int c = 0; c < svl; c += 4) {
435 RealScalar* d0 = dst + (k + c) * w;
436 sme_store_lines_vg4<0, 2>(d0, w, svl, uint32_t(c));
437 sme_store_lines_vg4<1, 3>(d0 + Index(svl) * w, w, svl, uint32_t(c));
444template <
typename RealScalar>
445static EIGEN_ALWAYS_INLINE
typename sme_packet_traits<RealScalar>::type_x2 sme_row_or_zero(
446 const RealScalar* p,
bool valid, svcount_t pn,
447 typename sme_packet_traits<RealScalar>::type zero) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
448 return valid ? ploadu_x2(pn, p) : pcreate(zero, zero);
453template <
typename RealScalar,
typename Index>
454static EIGEN_ALWAYS_INLINE Index sme_transpose_pack_partial(RealScalar* EIGEN_RESTRICT dst,
455 const RealScalar* EIGEN_RESTRICT src, Index src_stride,
457 int width) __arm_streaming __arm_inout(
"za") {
458 using Traits = sme_packet_traits<RealScalar>;
459 using Vec =
typename Traits::type;
460 const int svl = Traits::size(), w2 = 2 * svl;
461 const svcount_t pn = Traits::ptrue_c();
462 const Vec zero = pset1<Vec>(RealScalar(0));
463 const int rows_lo = sme_min(width, svl), rows_hi = width - rows_lo;
464 const svbool_t pg_lo = Traits::whilelt(0, rows_lo);
465 const svbool_t pg_two = Traits::whilelt(0, 2 * width);
467 for (; k + w2 <= k1; k += w2) {
468 for (
int r = 0; r < rows_lo; r += 4) {
469 const RealScalar* q = src + Index(r) * src_stride + k;
470 const auto a0 = sme_row_or_zero<RealScalar>(q, r < rows_lo, pn, zero);
471 const auto a1 = sme_row_or_zero<RealScalar>(q + src_stride, r + 1 < rows_lo, pn, zero);
472 const auto a2 = sme_row_or_zero<RealScalar>(q + 2 * src_stride, r + 2 < rows_lo, pn, zero);
473 const auto a3 = sme_row_or_zero<RealScalar>(q + 3 * src_stride, r + 3 < rows_lo, pn, zero);
474 sme_write_hor_za_vg4<0>(uint32_t(r), pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3));
475 sme_write_hor_za_vg4<1>(uint32_t(r), pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3));
477 for (
int r = 0; r < rows_hi; r += 4) {
478 const RealScalar* q = src + Index(svl + r) * src_stride + k;
479 const auto a0 = sme_row_or_zero<RealScalar>(q, r < rows_hi, pn, zero);
480 const auto a1 = sme_row_or_zero<RealScalar>(q + src_stride, r + 1 < rows_hi, pn, zero);
481 const auto a2 = sme_row_or_zero<RealScalar>(q + 2 * src_stride, r + 2 < rows_hi, pn, zero);
482 const auto a3 = sme_row_or_zero<RealScalar>(q + 3 * src_stride, r + 3 < rows_hi, pn, zero);
483 sme_write_hor_za_vg4<2>(uint32_t(r), pget<0>(a0), pget<0>(a1), pget<0>(a2), pget<0>(a3));
484 sme_write_hor_za_vg4<3>(uint32_t(r), pget<1>(a0), pget<1>(a1), pget<1>(a2), pget<1>(a3));
486 if (2 * width <= svl) {
488 for (
int c = 0; c < svl; c += 4) {
489 const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)),
490 t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c));
491 RealScalar* d0 = dst + (k + c) * width;
492 RealScalar* d1 = d0 + Index(svl) * width;
493 pstoreu(pg_two, d0, psplice(pg_lo, pget<0>(t0), pget<1>(t0)));
494 pstoreu(pg_two, d0 + 2 * width, psplice(pg_lo, pget<2>(t0), pget<3>(t0)));
495 pstoreu(pg_two, d1, psplice(pg_lo, pget<0>(t1), pget<1>(t1)));
496 pstoreu(pg_two, d1 + 2 * width, psplice(pg_lo, pget<2>(t1), pget<3>(t1)));
502 const svcount_t pc = Traits::whilelt_c2(0, width);
503 for (
int c = 0; c < svl; c += 4) {
504 const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)),
505 t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c));
506 const auto t2 = sme_read_ver_za_vg4<2, RealScalar>(uint32_t(c)),
507 t3 = sme_read_ver_za_vg4<3, RealScalar>(uint32_t(c));
508 RealScalar* d0 = dst + (k + c) * width;
509 RealScalar* d1 = d0 + Index(svl) * width;
510 pstoreu_x2(pc, d0, pcreate(pget<0>(t0), pget<0>(t2)));
511 pstoreu_x2(pc, d0 + width, pcreate(pget<1>(t0), pget<1>(t2)));
512 pstoreu_x2(pc, d0 + 2 * width, pcreate(pget<2>(t0), pget<2>(t2)));
513 pstoreu_x2(pc, d0 + 3 * width, pcreate(pget<3>(t0), pget<3>(t2)));
514 pstoreu_x2(pc, d1, pcreate(pget<0>(t1), pget<0>(t3)));
515 pstoreu_x2(pc, d1 + width, pcreate(pget<1>(t1), pget<1>(t3)));
516 pstoreu_x2(pc, d1 + 2 * width, pcreate(pget<2>(t1), pget<2>(t3)));
517 pstoreu_x2(pc, d1 + 3 * width, pcreate(pget<3>(t1), pget<3>(t3)));
521 for (
int c = 0; c < svl; c += 4) {
522 const auto t0 = sme_read_ver_za_vg4<0, RealScalar>(uint32_t(c)),
523 t1 = sme_read_ver_za_vg4<1, RealScalar>(uint32_t(c));
524 RealScalar* d0 = dst + (k + c) * width;
525 RealScalar* d1 = d0 + Index(svl) * width;
526 pstoreu(pg_lo, d0, pget<0>(t0));
527 pstoreu(pg_lo, d0 + width, pget<1>(t0));
528 pstoreu(pg_lo, d0 + 2 * width, pget<2>(t0));
529 pstoreu(pg_lo, d0 + 3 * width, pget<3>(t0));
530 pstoreu(pg_lo, d1, pget<0>(t1));
531 pstoreu(pg_lo, d1 + width, pget<1>(t1));
532 pstoreu(pg_lo, d1 + 2 * width, pget<2>(t1));
533 pstoreu(pg_lo, d1 + 3 * width, pget<3>(t1));
559template <
bool NegateOddRows,
typename RealScalar,
typename Index>
560static EIGEN_ALWAYS_INLINE
void sme_transpose_pack_real(RealScalar* EIGEN_RESTRICT dst,
561 const RealScalar* EIGEN_RESTRICT src, Index src_stride,
563 int width) __arm_streaming __arm_inout(
"za") {
564 using Traits = sme_packet_traits<RealScalar>;
565 using Vec =
typename Traits::type;
566 const Vec zero = pset1<Vec>(RealScalar(0));
567 const svbool_t pg_all = Traits::ptrue();
568 const int svl = Traits::size();
571 EIGEN_IF_CONSTEXPR (!NegateOddRows) {
574 if (svl >= 4 && width == 2 * svl && k1 - k0 >= Index(8 * svl))
575 k = sme_transpose_pack_pair(dst, src, src_stride, k0, k1);
576 else if (svl >= 4 && width < 2 * svl && k1 - k0 >= Index(8 * svl))
577 k = sme_transpose_pack_partial(dst, src, src_stride, k0, k1, width);
579 for (; k < k1; k += svl) {
580 const int dk =
static_cast<int>(sme_min(k1 - k, Index(svl)));
581 const svbool_t pg_d = Traits::whilelt(k, k1);
584 for (; r0 + 2 * svl <= width; r0 += 2 * svl) {
585 for (
int r = 0; r < svl; ++r) {
586 sme_ld1_hor_za<0>(uint32_t(r), pg_d, &src[(r0 + r) * src_stride + k]);
587 sme_ld1_hor_za<1>(uint32_t(r), pg_d, &src[(r0 + svl + r) * src_stride + k]);
589 for (
int c = 0; c < dk; ++c) {
590 Vec v0 = sme_read_ver_za<0>(zero, pg_all, uint32_t(c));
591 Vec v1 = sme_read_ver_za<1>(zero, pg_all, uint32_t(c));
592 EIGEN_IF_CONSTEXPR (NegateOddRows) {
593 if (((k + Index(c)) & Index(1)) != Index(0)) {
594 v0 = pnegate(pg_all, v0);
595 v1 = pnegate(pg_all, v1);
598 pstoreu(pg_all, &dst[(k + c) * width + r0], v0);
599 pstoreu(pg_all, &dst[(k + c) * width + r0 + svl], v1);
606 for (; r0 < width; r0 += svl) {
607 const int rg = sme_min(width - r0, svl);
608 const svbool_t pg_r = Traits::whilelt(r0, width);
609 for (
int r = 0; r < rg; ++r) {
610 sme_ld1_hor_za<0>(uint32_t(r), pg_d, &src[(r0 + r) * src_stride + k]);
612 for (
int c = 0; c < dk; ++c) {
613 Vec v0 = sme_read_ver_za<0>(zero, pg_r, uint32_t(c));
614 EIGEN_IF_CONSTEXPR (NegateOddRows) {
615 if (((k + Index(c)) & Index(1)) != Index(0)) v0 = pnegate(pg_r, v0);
617 pstoreu(pg_r, &dst[(k + c) * width + r0], v0);
623template <
bool Conjugate,
typename Scalar,
typename Index>
624static EIGEN_ALWAYS_INLINE
void sme_transpose_pack_range(Scalar* EIGEN_RESTRICT dst,
const Scalar* EIGEN_RESTRICT src,
625 Index src_stride, Index k0, Index k1,
626 int width) __arm_streaming __arm_inout(
"za") {
627 sme_transpose_pack_real<false>(dst, src, src_stride, k0, k1, width);
635template <
bool Conjugate,
typename RealScalar,
typename Index>
636static EIGEN_ALWAYS_INLINE
void sme_transpose_pack_range(std::complex<RealScalar>* EIGEN_RESTRICT dst,
637 const std::complex<RealScalar>* EIGEN_RESTRICT src,
638 Index src_stride, Index k0, Index k1,
639 int width) __arm_streaming __arm_inout(
"za") {
640 sme_transpose_pack_real<Conjugate>(
reinterpret_cast<RealScalar*
>(dst),
reinterpret_cast<const RealScalar*
>(src),
641 Index(2) * src_stride, Index(2) * k0, Index(2) * k1, width);
646template <
bool Conjugate,
typename Scalar,
typename Index>
647static EIGEN_ALWAYS_INLINE
void sme_transpose_pack(Scalar* EIGEN_RESTRICT dst,
const Scalar* EIGEN_RESTRICT src,
648 Index src_stride, Index depth,
649 int width) __arm_streaming __arm_inout(
"za") {
650 sme_transpose_pack_range<Conjugate>(dst, src, src_stride, Index(0), depth, width);
660template <
bool NegateOddRows,
typename RealScalar,
typename Index>
661static void tail_transpose_pack_real(RealScalar* EIGEN_RESTRICT dst_panel,
const RealScalar* EIGEN_RESTRICT src,
662 Index src_stride, Index depth, Index tail) {
663 using Packet =
typename packet_traits<RealScalar>::type;
664 constexpr int PacketSize = int(packet_traits<RealScalar>::size);
665 const Index peeled_tail = (tail / Index(PacketSize)) * Index(PacketSize);
666 const Index peeled_depth = numext::round_down(depth, Index(PacketSize));
669 for (; i < peeled_tail; i += Index(PacketSize)) {
671 for (; k < peeled_depth; k += Index(PacketSize)) {
672 PacketBlock<Packet, PacketSize> block;
673 for (
int p = 0; p < PacketSize; ++p) {
674 block.packet[p] = ploadu<Packet>(src + (i + Index(p)) * src_stride + k);
677 for (
int p = 0; p < PacketSize; ++p) {
678 Packet row = block.packet[p];
679 EIGEN_IF_CONSTEXPR (NegateOddRows) {
680 if (((k + Index(p)) & Index(1)) != Index(0)) row = pnegate(row);
682 pstoreu(dst_panel + (k + Index(p)) * tail + i, row);
685 for (; k < depth; ++k) {
686 const bool negate = NegateOddRows && ((k & Index(1)) != Index(0));
687 for (Index p = 0; p < Index(PacketSize); ++p) {
688 const RealScalar v = src[(i + p) * src_stride + k];
689 dst_panel[k * tail + i + p] = negate ? -v : v;
693 for (; i < tail; ++i) {
694 for (Index k = 0; k < depth; ++k) {
695 const bool negate = NegateOddRows && ((k & Index(1)) != Index(0));
696 const RealScalar v = src[i * src_stride + k];
697 dst_panel[k * tail + i] = negate ? -v : v;
702template <
bool Conjugate,
typename Scalar,
typename Index>
703static void tail_transpose_pack(Scalar* EIGEN_RESTRICT dst_panel,
const Scalar* EIGEN_RESTRICT src, Index src_stride,
704 Index depth, Index tail) {
705 tail_transpose_pack_real<false>(dst_panel, src, src_stride, depth, tail);
708template <
bool Conjugate,
typename RealScalar,
typename Index>
709static void tail_transpose_pack(std::complex<RealScalar>* EIGEN_RESTRICT dst_panel,
710 const std::complex<RealScalar>* EIGEN_RESTRICT src, Index src_stride, Index depth,
712 tail_transpose_pack_real<Conjugate>(
reinterpret_cast<RealScalar*
>(dst_panel),
713 reinterpret_cast<const RealScalar*
>(src), Index(2) * src_stride, Index(2) * depth,
719template <
bool Conjugate,
typename Scalar,
typename Index>
720__arm_locally_streaming __arm_new(
"za") static
void sme_tail_pack_streaming(Scalar* dst_panel, const Scalar* src,
721 Index src_stride, Index depth,
int tail) {
722 sme_transpose_pack<Conjugate>(dst_panel, src, src_stride, depth, tail);
724template <
bool Conjugate,
typename Scalar,
typename Index>
725static EIGEN_ALWAYS_INLINE
void tail_pack(Scalar* dst_panel,
const Scalar* src, Index src_stride, Index depth,
727 EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsComplex) {
728 if (tail > 4 && depth >= Index(4 * sme_block<Scalar>::nr)) {
729 sme_tail_pack_streaming<Conjugate>(dst_panel, src, src_stride, depth,
static_cast<int>(tail));
733 tail_transpose_pack<Conjugate>(dst_panel, src, src_stride, depth, tail);
737static EIGEN_ALWAYS_INLINE
void sme_neon_ld2(
const float* p, Packet4f& re, Packet4f& im) {
738 const float32x4x2_t v = vld2q_f32(p);
742static EIGEN_ALWAYS_INLINE
void sme_neon_st2(
float* p, Packet4f re, Packet4f im) {
748static EIGEN_ALWAYS_INLINE
void sme_neon_ld2(
const double* p, Packet2d& re, Packet2d& im) {
749 const float64x2x2_t v = vld2q_f64(p);
753static EIGEN_ALWAYS_INLINE
void sme_neon_st2(
double* p, Packet2d re, Packet2d im) {
763template <
bool Conjugate,
typename Scalar,
typename Index>
764static void neon_copy_panel(Scalar* EIGEN_RESTRICT dst,
const Scalar* EIGEN_RESTRICT src, Index src_stride, Index depth,
766 using Packet =
typename packet_traits<Scalar>::type;
767 constexpr Index PS = Index(packet_traits<Scalar>::size);
768 const Index peeled4 = numext::round_down(w, 4 * PS);
769 const Index peeled = numext::round_down(w, PS);
770 for (Index k = 0; k < depth; ++k) {
771 const Scalar* s = src + k * src_stride;
772 Scalar* d = dst + k * w;
774 for (; r < peeled4; r += 4 * PS) {
775 const Packet p0 = ploadu<Packet>(s + r), p1 = ploadu<Packet>(s + r + PS);
776 const Packet p2 = ploadu<Packet>(s + r + 2 * PS), p3 = ploadu<Packet>(s + r + 3 * PS);
778 pstoreu(d + r + PS, p1);
779 pstoreu(d + r + 2 * PS, p2);
780 pstoreu(d + r + 3 * PS, p3);
782 for (; r < peeled; r += PS) pstoreu(d + r, ploadu<Packet>(s + r));
783 for (; r < w; ++r) d[r] = s[r];
786template <
bool Conjugate,
typename RealScalar,
typename Index>
787static void neon_copy_panel(std::complex<RealScalar>* EIGEN_RESTRICT dst,
788 const std::complex<RealScalar>* EIGEN_RESTRICT src, Index src_stride, Index depth,
790 using Packet =
typename packet_traits<RealScalar>::type;
791 constexpr Index PS = Index(packet_traits<RealScalar>::size);
792 RealScalar* rd =
reinterpret_cast<RealScalar*
>(dst);
793 const RealScalar* rs =
reinterpret_cast<const RealScalar*
>(src);
794 const Index peeled = numext::round_down(w, PS);
795 for (Index k = 0; k < depth; ++k) {
796 const RealScalar* s = rs + k * 2 * src_stride;
797 RealScalar* d = rd + k * 2 * w;
799 for (; r < peeled; r += PS) {
801 sme_neon_ld2(s + 2 * r, re, im);
803 pstoreu(d + w + r, Conjugate ? pnegate(im) : im);
807 d[w + r] = Conjugate ? -s[2 * r + 1] : s[2 * r + 1];
827template <
typename DataMapper,
typename EnableIf =
void>
828struct sme_has_incr : std::false_type {};
829template <
typename DataMapper>
830struct sme_has_incr<DataMapper, void_t<decltype(std::declval<const DataMapper&>().incr())>> : std::true_type {};
832template <typename Index, typename DataMapper, std::enable_if_t<sme_has_incr<DataMapper>::value,
bool> =
true>
833EIGEN_ALWAYS_INLINE Index sme_mapper_incr(
const DataMapper& m) {
834 return static_cast<Index
>(m.incr());
836template <typename Index, typename DataMapper, std::enable_if_t<!sme_has_incr<DataMapper>::value,
bool> =
true>
837EIGEN_ALWAYS_INLINE Index sme_mapper_incr(
const DataMapper&) {
843template <
typename DataMapper,
typename Index>
844struct sme_mapper_has_direct_access {
845 static constexpr bool value = std::is_lvalue_reference<decltype(std::declval<const DataMapper&>()(
846 std::declval<Index>(), std::declval<Index>()))>::value;
853template <
bool Conjugate,
typename Scalar,
typename Index>
854EIGEN_ALWAYS_INLINE
void sme_pack_store(Scalar* dst_step, Index w, Index r,
const Scalar& v) {
855 EIGEN_UNUSED_VARIABLE(w);
856 dst_step[r] = conj_if<Conjugate>()(v);
858template <
bool Conjugate,
typename RealScalar,
typename Index>
859EIGEN_ALWAYS_INLINE
void sme_pack_store(std::complex<RealScalar>* dst_step, Index w, Index r,
860 const std::complex<RealScalar>& v) {
861 const std::complex<RealScalar> cv = conj_if<Conjugate>()(v);
862 RealScalar* p =
reinterpret_cast<RealScalar*
>(dst_step);
863 p[r] = numext::real(cv);
864 p[w + r] = numext::imag(cv);
883template <
typename Scalar,
int MR,
typename Index,
typename DataMapper,
bool Conjugate,
bool PanelMode>
884void sme_pack_lhs_fallback(Scalar* dst_base,
const DataMapper& lhs, Index depth, Index rows, Index dst_stride,
885 Index dst_offset,
bool vectorise) {
886 using Packet =
typename packet_traits<Scalar>::type;
887 constexpr Index PacketSize = Index(packet_traits<Scalar>::size);
888 constexpr bool HasPacketPath = !NumTraits<Scalar>::IsComplex;
890 for (Index i = 0; i < rows; i += MR) {
891 const Index w = numext::mini(rows - i, Index(MR));
892 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * w : dst_base + i * depth;
893 const Index peeled_w = (vectorise && HasPacketPath) ? numext::round_down(w, Index(PacketSize)) : Index(0);
894 for (Index k = 0; k < depth; ++k) {
895 Scalar* dst_step = dst_panel + k * w;
897 for (; r < peeled_w; r += PacketSize) {
898 pstoreu(dst_step + r, lhs.template loadPacket<Packet>(i + r, k));
901 sme_pack_store<Conjugate>(dst_step, w, r, lhs(i + r, k));
908template <
typename DataMapper,
typename Index, std::size_t... Is>
909EIGEN_ALWAYS_INLINE std::array<
typename DataMapper::LinearMapper,
sizeof...(Is)> sme_column_mappers(
910 const DataMapper& rhs, Index col, std::index_sequence<Is...>) {
911 return {{rhs.getLinearMapper(0, col + Index(Is))...}};
920template <
typename Scalar,
int NR,
typename Index,
typename DataMapper,
bool Conjugate,
bool PanelMode>
921void sme_pack_rhs_fallback(Scalar* dst_base,
const DataMapper& rhs, Index depth, Index cols, Index dst_stride,
922 Index dst_offset,
bool vectorise) {
923 using Packet =
typename packet_traits<Scalar>::type;
924 using LinearMapper =
typename DataMapper::LinearMapper;
925 constexpr int PacketSize = int(packet_traits<Scalar>::size);
926 constexpr bool HasPacketPath = !NumTraits<Scalar>::IsComplex;
927 const Index peeled_depth = (depth / Index(PacketSize)) * Index(PacketSize);
929 for (Index j = 0; j < cols; j += NR) {
930 const Index w = numext::mini(cols - j, Index(NR));
931 Scalar* dst_panel = PanelMode ? dst_base + j * dst_stride + dst_offset * w : dst_base + j * depth;
932 const Index peeled_w = (vectorise && HasPacketPath) ? numext::round_down(w, Index(PacketSize)) : Index(0);
934 for (; c < peeled_w; c += Index(PacketSize)) {
938 const std::array<LinearMapper, PacketSize> dm =
939 sme_column_mappers(rhs, j + c, std::make_index_sequence<PacketSize>{});
941 for (; k < peeled_depth; k += Index(PacketSize)) {
942 PacketBlock<Packet, PacketSize> block;
943 for (
int p = 0; p < PacketSize; ++p) {
944 block.packet[p] = dm[p].template loadPacket<Packet>(k);
947 for (
int p = 0; p < PacketSize; ++p) {
948 pstoreu(dst_panel + (k + Index(p)) * w + c, block.packet[p]);
951 for (; k < depth; ++k) {
952 for (Index p = 0; p < Index(PacketSize); ++p) {
953 sme_pack_store<Conjugate>(dst_panel + k * w, w, c + p, rhs(k, j + c + p));
958 for (Index k = 0; k < depth; ++k) {
959 sme_pack_store<Conjugate>(dst_panel + k * w, w, c, rhs(k, j + c));
970#ifndef EIGEN_SME_NEON_MAX_DEPTH
971template <
typename Scalar>
972struct sme_neon_max_depth : std::integral_constant<int, 16> {};
974struct sme_neon_max_depth<float> : std::integral_constant<int, 24> {};
976struct sme_neon_max_depth<std::complex<double>> : std::integral_constant<int, 8> {};
978template <
typename Scalar>
979struct sme_neon_max_depth : std::integral_constant<int, EIGEN_SME_NEON_MAX_DEPTH> {};
981#ifndef EIGEN_SME_NEON_MAX_WIDTH
982template <
typename Scalar>
983struct sme_neon_max_width : std::integral_constant<int, 48> {};
985struct sme_neon_max_width<double> : std::integral_constant<int, 40> {};
987struct sme_neon_max_width<std::complex<float>> : std::integral_constant<int, 16> {};
989struct sme_neon_max_width<std::complex<double>> : std::integral_constant<int, 24> {};
991template <
typename Scalar>
992struct sme_neon_max_width : std::integral_constant<int, EIGEN_SME_NEON_MAX_WIDTH> {};
997#ifndef EIGEN_SME_NEON_SHALLOW_DEPTH
998template <
typename Scalar>
999struct sme_neon_shallow_depth : std::integral_constant<int, 8> {};
1001template <
typename Scalar>
1002struct sme_neon_shallow_depth : std::integral_constant<int, EIGEN_SME_NEON_SHALLOW_DEPTH> {};
1004#ifndef EIGEN_SME_NEON_SHALLOW_WIDTH
1005template <
typename Scalar>
1006struct sme_neon_shallow_width : std::integral_constant<int, 96> {};
1008struct sme_neon_shallow_width<double> : std::integral_constant<int, 64> {};
1010struct sme_neon_shallow_width<std::complex<float>> : std::integral_constant<int, 32> {};
1012struct sme_neon_shallow_width<std::complex<double>> : std::integral_constant<int, 24> {};
1014template <
typename Scalar>
1015struct sme_neon_shallow_width : std::integral_constant<int, EIGEN_SME_NEON_SHALLOW_WIDTH> {};
1021#ifndef EIGEN_SME_NEON_MAX_PANEL
1022template <
typename Scalar>
1023struct sme_neon_max_panel : std::integral_constant<int, 256> {};
1024template <
typename RealScalar>
1025struct sme_neon_max_panel<std::complex<RealScalar>> : std::integral_constant<int, 128> {};
1027template <
typename Scalar>
1028struct sme_neon_max_panel : std::integral_constant<int, EIGEN_SME_NEON_MAX_PANEL> {};
1032#ifndef EIGEN_SME_NEON_THIN_DIM
1033template <
typename Scalar>
1034struct sme_neon_thin_dim : std::integral_constant<int, 32> {};
1036struct sme_neon_thin_dim<std::complex<float>> : std::integral_constant<int, 8> {};
1038struct sme_neon_thin_dim<std::complex<double>> : std::integral_constant<int, 4> {};
1040template <
typename Scalar>
1041struct sme_neon_thin_dim : std::integral_constant<int, EIGEN_SME_NEON_THIN_DIM> {};
1045template <
typename Scalar,
typename Index>
1046EIGEN_ALWAYS_INLINE
bool sme_pack_with_neon(Index depth, Index width) {
1047#if defined(EIGEN_SME_NO_NEON_SMALL_BLOCKS)
1048 EIGEN_UNUSED_VARIABLE(depth);
1049 EIGEN_UNUSED_VARIABLE(width);
1051#elif defined(EIGEN_SME_FORCE_NEON_SMALL_BLOCKS)
1052 EIGEN_UNUSED_VARIABLE(depth);
1053 EIGEN_UNUSED_VARIABLE(width);
1056 return depth <= Index(sme_neon_max_depth<Scalar>::value) && width <= Index(sme_neon_max_panel<Scalar>::value);
1062template <
typename Scalar,
typename Index>
1063EIGEN_ALWAYS_INLINE
bool sme_kernel_with_neon(Index rows, Index cols, Index depth, Index strideA, Index strideB) {
1064#if defined(EIGEN_SME_NO_NEON_SMALL_BLOCKS) || defined(EIGEN_SME_FORCE_NEON_SMALL_BLOCKS)
1065 EIGEN_UNUSED_VARIABLE(depth);
1066 return sme_pack_with_neon<Scalar>(strideA, rows) && sme_pack_with_neon<Scalar>(strideB, cols);
1068 const Index wide = numext::maxi(rows, cols);
1069 const Index w = depth <= Index(sme_neon_shallow_depth<Scalar>::value) ? Index(sme_neon_shallow_width<Scalar>::value)
1070 : Index(sme_neon_max_width<Scalar>::value);
1071 return sme_pack_with_neon<Scalar>(strideA, rows) && sme_pack_with_neon<Scalar>(strideB, cols) &&
1072 (wide <= w || numext::mini(rows, cols) <= Index(sme_neon_thin_dim<Scalar>::value));
1084template <
bool UsePacketPath,
bool PanelMode,
typename Scalar,
typename Index,
typename DataMapper,
typename DirectFn,
1085 typename NeonFn,
typename FallbackFn>
1086EIGEN_ALWAYS_INLINE
void sme_dispatch_pack(DirectFn direct, NeonFn neon, FallbackFn fallback, Scalar* block,
1087 const DataMapper& m, Index depth, Index n, Index stride, Index offset,
1089 if (sme_mapper_incr<Index>(m) == 1) {
1090 const Scalar* src = (n > 0 && depth > 0) ? &m(0, 0) :
nullptr;
1091 if (sme_pack_with_neon<Scalar>(PanelMode ? stride : depth, n)) {
1092 neon(block, src, m.stride(), depth, n, stride, offset);
1094 sme_fpsr_guard fpsr;
1095 direct(block, src, m.stride(), depth, n, stride, offset);
1098 fallback(block, m, depth, n, stride, offset, UsePacketPath);
1101template <
bool UsePacketPath,
bool PanelMode,
typename Scalar,
typename Index,
typename DataMapper,
typename DirectFn,
1102 typename NeonFn,
typename FallbackFn>
1103EIGEN_ALWAYS_INLINE
void sme_dispatch_pack(DirectFn, NeonFn, FallbackFn fallback, Scalar* block,
const DataMapper& m,
1104 Index depth, Index n, Index stride, Index offset,
1106 fallback(block, m, depth, n, stride, offset, UsePacketPath);
1139class gebp_traits<float, float, false, false, Architecture::Target, GEBPPacketFull>
1140 :
public gebp_traits<float, float, false, false, Architecture::Target, GEBPPacketHalf> {
1153static_assert(int(gebp_traits<
float, std::complex<float>>::mr) != kSmeMr,
1154 "gebp_traits<float, complex<float>>::mr collides with kSmeMr: the SME gemm_pack_lhs would silently "
1155 "emit SME panel layout for the generic gebp_kernel");
1156static_assert(int(gebp_traits<std::complex<float>,
float>::nr) != kSmeNr,
1157 "gebp_traits<complex<float>, float>::nr collides with kSmeNr: the SME gemm_pack_rhs would silently "
1158 "emit SME panel layout for the generic gebp_kernel");
1160#ifdef EIGEN_VECTORIZE_SME_F64F64
1162class gebp_traits<double, double, false, false, Architecture::Target, GEBPPacketFull>
1163 :
public gebp_traits<double, double, false, false, Architecture::Target, GEBPPacketHalf> {
1166 static constexpr int mr = kSmeMrD;
1167 static constexpr int nr = kSmeNrD;
1170static_assert(int(gebp_traits<
double, std::complex<double>>::mr) != kSmeMrD,
1171 "gebp_traits<double, complex<double>>::mr collides with kSmeMrD: the SME gemm_pack_lhs would silently "
1172 "emit SME panel layout for the generic gebp_kernel");
1173static_assert(int(gebp_traits<std::complex<double>,
double>::nr) != kSmeNrD,
1174 "gebp_traits<complex<double>, double>::nr collides with kSmeNrD: the SME gemm_pack_rhs would silently "
1175 "emit SME panel layout for the generic gebp_kernel");
1183template <
bool ConjLhs_,
bool ConjRhs_>
1184class gebp_traits<std::complex<float>, std::complex<float>, ConjLhs_, ConjRhs_, Architecture::Target, GEBPPacketFull>
1185 :
public gebp_traits<std::complex<float>, std::complex<float>, ConjLhs_, ConjRhs_, Architecture::Target,
1188 static constexpr int mr = kSmeMrC;
1189 static constexpr int nr = kSmeNrC;
1195static_assert(int(gebp_traits<std::complex<float>,
float>::mr) != kSmeMrC,
1196 "gebp_traits<complex<float>, float>::mr collides with kSmeMrC: the SME gemm_pack_lhs would silently "
1197 "emit SME panel layout for the generic gebp_kernel");
1198static_assert(int(gebp_traits<
float, std::complex<float>>::nr) != kSmeNrC,
1199 "gebp_traits<float, complex<float>>::nr collides with kSmeNrC: the SME gemm_pack_rhs would silently "
1200 "emit SME panel layout for the generic gebp_kernel");
1202#ifdef EIGEN_VECTORIZE_SME_F64F64
1203template <
bool ConjLhs_,
bool ConjRhs_>
1204class gebp_traits<std::complex<double>, std::complex<double>, ConjLhs_, ConjRhs_, Architecture::Target, GEBPPacketFull>
1205 :
public gebp_traits<std::complex<double>, std::complex<double>, ConjLhs_, ConjRhs_, Architecture::Target,
1208 static constexpr int mr = kSmeMrCD;
1209 static constexpr int nr = kSmeNrCD;
1212static_assert(int(gebp_traits<std::complex<double>,
double>::mr) != kSmeMrCD,
1213 "gebp_traits<complex<double>, double>::mr collides with kSmeMrCD: the SME gemm_pack_lhs would silently "
1214 "emit SME panel layout for the generic gebp_kernel");
1215static_assert(int(gebp_traits<
double, std::complex<double>>::nr) != kSmeNrCD,
1216 "gebp_traits<double, complex<double>>::nr collides with kSmeNrCD: the SME gemm_pack_rhs would silently "
1217 "emit SME panel layout for the generic gebp_kernel");
1227template <
typename Scalar,
int MR,
typename Index,
typename DataMapper,
bool Conjugate,
bool PanelMode>
1228struct sme_pack_lhs_colmajor {
1231 static EIGEN_ALWAYS_INLINE
void pack_neon(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src, Index src_stride,
1232 Index depth, Index rows, Index dst_stride, Index dst_offset) {
1233 for (Index i = 0; i < rows; i += MR) {
1234 const Index w = numext::mini(Index(MR), rows - i);
1235 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * w : dst_base + i * depth;
1236 neon_copy_panel<Conjugate>(dst_panel, src + i, src_stride, depth, w);
1243 __arm_locally_streaming EIGEN_DONT_INLINE
static void pack_direct(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src,
1244 Index src_stride, Index depth, Index rows,
1245 Index dst_stride, Index dst_offset) {
1246 const Index peeled_rows = (rows / MR) * MR;
1247 const Index i0 = sme_pack_quad_panels<PanelMode>(dst_base, src, src_stride, depth, peeled_rows, Index(MR),
1248 dst_stride, dst_offset);
1251 for (Index i = i0; i < peeled_rows; i += MR) {
1252 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * MR : dst_base + i * depth;
1253 sve_copy_panel<Conjugate>(dst_panel, src + i, src_stride, depth, MR);
1257 if (peeled_rows < rows) {
1258 const Index tail = rows - peeled_rows;
1260 PanelMode ? dst_base + peeled_rows * dst_stride + dst_offset * tail : dst_base + peeled_rows * depth;
1261 sve_copy_panel<Conjugate>(dst_panel, src + peeled_rows, src_stride, depth,
static_cast<int>(tail));
1265 EIGEN_DONT_INLINE
void operator()(Scalar* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
1268 eigen_assert(stride >= depth && offset <= stride);
1272 sme_dispatch_pack<true, PanelMode>(
1273 &pack_direct, &pack_neon, &sme_pack_lhs_fallback<Scalar, MR, Index, DataMapper, Conjugate, PanelMode>, blockA,
1274 lhs, depth, rows, stride, offset, bool_constant<sme_mapper_has_direct_access<DataMapper, Index>::value>{});
1286template <
typename Scalar,
int MR,
typename Index,
typename DataMapper,
bool Conjugate,
bool PanelMode>
1287struct sme_pack_lhs_rowmajor {
1290 static EIGEN_ALWAYS_INLINE
void pack_neon(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src, Index src_stride,
1291 Index depth, Index rows, Index dst_stride, Index dst_offset) {
1292 for (Index i = 0; i < rows; i += MR) {
1293 const Index w = numext::mini(Index(MR), rows - i);
1294 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * w : dst_base + i * depth;
1295 tail_transpose_pack<Conjugate>(dst_panel, src + i * src_stride, src_stride, depth, w);
1299 __arm_locally_streaming __arm_new(
"za") static
void pack_full_panels(Scalar* dst_base,
1300 const Scalar* EIGEN_RESTRICT src,
1301 Index src_stride, Index depth, Index peeled_rows,
1302 Index dst_stride, Index dst_offset) {
1303 for (Index i = 0; i < peeled_rows; i += MR) {
1304 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * MR : dst_base + i * depth;
1305 sme_transpose_pack<Conjugate>(dst_panel, src + i * src_stride, src_stride, depth, MR);
1309 static void pack_direct(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src, Index src_stride, Index depth, Index rows,
1310 Index dst_stride, Index dst_offset) {
1311 const Index peeled_rows = numext::round_down(rows, MR);
1313 if (peeled_rows > 0) {
1314 pack_full_panels(dst_base, src, src_stride, depth, peeled_rows, dst_stride, dst_offset);
1318 if (peeled_rows < rows) {
1319 const Index tail = rows - peeled_rows;
1321 PanelMode ? dst_base + peeled_rows * dst_stride + dst_offset * tail : dst_base + peeled_rows * depth;
1322 tail_pack<Conjugate>(dst_panel, src + peeled_rows * src_stride, src_stride, depth, tail);
1326 EIGEN_DONT_INLINE
void operator()(Scalar* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
1329 eigen_assert(stride >= depth && offset <= stride);
1334 sme_dispatch_pack<false, PanelMode>(
1335 &pack_direct, &pack_neon, &sme_pack_lhs_fallback<Scalar, MR, Index, DataMapper, Conjugate, PanelMode>, blockA,
1336 lhs, depth, rows, stride, offset, bool_constant<sme_mapper_has_direct_access<DataMapper, Index>::value>{});
1349template <
typename Scalar,
int NR,
typename Index,
typename DataMapper,
bool Conjugate,
bool PanelMode>
1350struct sme_pack_rhs_colmajor {
1353 static EIGEN_ALWAYS_INLINE
void pack_neon(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src, Index src_stride,
1354 Index depth, Index cols, Index dst_stride, Index dst_offset) {
1355 for (Index i = 0; i < cols; i += NR) {
1356 const Index w = numext::mini(Index(NR), cols - i);
1357 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * w : dst_base + i * depth;
1358 tail_transpose_pack<Conjugate>(dst_panel, src + i * src_stride, src_stride, depth, w);
1362 __arm_locally_streaming __arm_new(
"za") static
void pack_full_panels(Scalar* dst_base,
1363 const Scalar* EIGEN_RESTRICT src,
1364 Index src_stride, Index depth, Index peeled_cols,
1365 Index dst_stride, Index dst_offset) {
1366 for (Index j = 0; j < peeled_cols; j += NR) {
1367 Scalar* dst_panel = PanelMode ? dst_base + j * dst_stride + dst_offset * NR : dst_base + j * depth;
1368 sme_transpose_pack<Conjugate>(dst_panel, src + j * src_stride, src_stride, depth, NR);
1372 static void pack_direct(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src, Index src_stride, Index depth, Index cols,
1373 Index dst_stride, Index dst_offset) {
1374 const Index peeled_cols = numext::round_down(cols, NR);
1376 if (peeled_cols > 0) {
1377 pack_full_panels(dst_base, src, src_stride, depth, peeled_cols, dst_stride, dst_offset);
1381 if (peeled_cols < cols) {
1382 const Index tail = cols - peeled_cols;
1384 PanelMode ? dst_base + peeled_cols * dst_stride + dst_offset * tail : dst_base + peeled_cols * depth;
1385 tail_pack<Conjugate>(dst_panel, src + peeled_cols * src_stride, src_stride, depth, tail);
1389 EIGEN_DONT_INLINE
void operator()(Scalar* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
1392 eigen_assert(stride >= depth && offset <= stride);
1396 sme_dispatch_pack<true, PanelMode>(
1397 &pack_direct, &pack_neon, &sme_pack_rhs_fallback<Scalar, NR, Index, DataMapper, Conjugate, PanelMode>, blockB,
1398 rhs, depth, cols, stride, offset, bool_constant<sme_mapper_has_direct_access<DataMapper, Index>::value>{});
1404template <
typename Scalar,
int NR,
typename Index,
typename DataMapper,
bool Conjugate,
bool PanelMode>
1405struct sme_pack_rhs_rowmajor {
1408 static EIGEN_ALWAYS_INLINE
void pack_neon(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src, Index src_stride,
1409 Index depth, Index cols, Index dst_stride, Index dst_offset) {
1410 for (Index i = 0; i < cols; i += NR) {
1411 const Index w = numext::mini(Index(NR), cols - i);
1412 Scalar* dst_panel = PanelMode ? dst_base + i * dst_stride + dst_offset * w : dst_base + i * depth;
1413 neon_copy_panel<Conjugate>(dst_panel, src + i, src_stride, depth, w);
1418 __arm_locally_streaming EIGEN_DONT_INLINE
static void pack_direct(Scalar* dst_base,
const Scalar* EIGEN_RESTRICT src,
1419 Index src_stride, Index depth, Index cols,
1420 Index dst_stride, Index dst_offset) {
1421 const Index peeled_cols = (cols / NR) * NR;
1423 for (Index j = 0; j < peeled_cols; j += NR) {
1424 Scalar* dst_panel = PanelMode ? dst_base + j * dst_stride + dst_offset * NR : dst_base + j * depth;
1425 sve_copy_panel<Conjugate>(dst_panel, src + j, src_stride, depth, NR);
1428 if (peeled_cols < cols) {
1429 const Index tail = cols - peeled_cols;
1431 PanelMode ? dst_base + peeled_cols * dst_stride + dst_offset * tail : dst_base + peeled_cols * depth;
1432 sve_copy_panel<Conjugate>(dst_panel, src + peeled_cols, src_stride, depth,
static_cast<int>(tail));
1436 EIGEN_DONT_INLINE
void operator()(Scalar* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
1439 eigen_assert(stride >= depth && offset <= stride);
1444 sme_dispatch_pack<false, PanelMode>(
1445 &pack_direct, &pack_neon, &sme_pack_rhs_fallback<Scalar, NR, Index, DataMapper, Conjugate, PanelMode>, blockB,
1446 rhs, depth, cols, stride, offset, bool_constant<sme_mapper_has_direct_access<DataMapper, Index>::value>{});
1453#define EIGEN_SME_DECLARE_GEMM_PACKERS(SCALAR, MR, NR) \
1454 template <typename Index, typename DataMapper, int Pack2, typename Packet, bool Conjugate, bool PanelMode> \
1455 struct gemm_pack_lhs<SCALAR, Index, DataMapper, MR, Pack2, Packet, ColMajor, Conjugate, PanelMode> \
1456 : sme_pack_lhs_colmajor<SCALAR, MR, Index, DataMapper, Conjugate, PanelMode> {}; \
1458 template <typename Index, typename DataMapper, int Pack2, typename Packet, bool Conjugate, bool PanelMode> \
1459 struct gemm_pack_lhs<SCALAR, Index, DataMapper, MR, Pack2, Packet, RowMajor, Conjugate, PanelMode> \
1460 : sme_pack_lhs_rowmajor<SCALAR, MR, Index, DataMapper, Conjugate, PanelMode> {}; \
1462 template <typename Index, typename DataMapper, bool Conjugate, bool PanelMode> \
1463 struct gemm_pack_rhs<SCALAR, Index, DataMapper, NR, ColMajor, Conjugate, PanelMode> \
1464 : sme_pack_rhs_colmajor<SCALAR, NR, Index, DataMapper, Conjugate, PanelMode> {}; \
1466 template <typename Index, typename DataMapper, bool Conjugate, bool PanelMode> \
1467 struct gemm_pack_rhs<SCALAR, Index, DataMapper, NR, RowMajor, Conjugate, PanelMode> \
1468 : sme_pack_rhs_rowmajor<SCALAR, NR, Index, DataMapper, Conjugate, PanelMode> {};
1470EIGEN_SME_DECLARE_GEMM_PACKERS(
float, kSmeMr, kSmeNr)
1471EIGEN_SME_DECLARE_GEMM_PACKERS(std::complex<float>, kSmeMrC, kSmeNrC)
1472#ifdef EIGEN_VECTORIZE_SME_F64F64
1473EIGEN_SME_DECLARE_GEMM_PACKERS(
double, kSmeMrD, kSmeNrD)
1474EIGEN_SME_DECLARE_GEMM_PACKERS(std::complex<double>, kSmeMrCD, kSmeNrCD)
1477#undef EIGEN_SME_DECLARE_GEMM_PACKERS
1484template <
bool Overwrite>
1485EIGEN_ALWAYS_INLINE svbool_t sme_c_mask() __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
1487 svbool_t m = Overwrite ? svptrue_b32() : svpfalse_b();
1488 EIGEN_IF_CONSTEXPR (Overwrite) __asm__(
"" :
"+Upa"(m));
1491template <
bool Overwrite>
1492EIGEN_ALWAYS_INLINE svfloat32_t sme_c_value(svbool_t m, svfloat32_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
1493 return Overwrite ? svsel_f32(m, svdup_f32(0.0f), v) : v;
1495template <
bool Overwrite>
1496EIGEN_ALWAYS_INLINE svfloat64_t sme_c_value(svbool_t m, svfloat64_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
1497 return Overwrite ? svsel_f64(m, svdup_f64(0.0), v) : v;
1502template <
typename Scalar,
typename Index>
1503EIGEN_DONT_INLINE __arm_locally_streaming
void sme_zero_c(Scalar* C, Index rs, Index cs, Index rows, Index cols) {
1504 if (rs == 1 || cs == 1) {
1505 const Index inner = rs == 1 ? rows : cols, outer = rs == 1 ? cols : rows, stride = rs == 1 ? cs : rs;
1506 const int64_t bytes = int64_t(inner) * int64_t(
sizeof(Scalar)), vl = int64_t(svcntb());
1507 const svuint8_t z = svdup_u8(0);
1508 for (Index o = 0; o < outer; ++o) {
1509 uint8_t* p =
reinterpret_cast<uint8_t*
>(C + o * stride);
1510 for (int64_t b = 0; b < bytes; b += vl) svst1_u8(svwhilelt_b8_s64(b, bytes), p + b, z);
1513 for (Index c = 0; c < cols; ++c)
1514 for (Index r = 0; r < rows; ++r) C[r * rs + c * cs] = Scalar(0);
1525template <
typename Scalar,
int TileId,
bool Overwrite = false,
typename Index>
1526EIGEN_ALWAYS_INLINE
void sme_store_za_tile(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col,
1527 Scalar alpha, Index row_start,
int pw, Index col_start,
1528 int cw) __arm_streaming __arm_inout(
"za") {
1529 using Traits = sme_packet_traits<Scalar>;
1530 using Vec =
typename Traits::type;
1531 const svbool_t cmask = sme_c_mask<Overwrite>();
1532 const svbool_t pg_m = Traits::whilelt(0, pw);
1533 const svbool_t pg_n = Traits::whilelt(0, cw);
1540 const Vec vzero = pset1<Vec>(Scalar(0));
1541 const Vec valpha = pset1<Vec>(alpha);
1546 if (C_stride_row == 1) {
1549 for (; ci + 2 <= cw; ci += 2) {
1550 Scalar* p0 = C + row_start + (col_start + ci) * C_stride_col;
1551 Scalar* p1 = p0 + C_stride_col;
1552 Vec c0 = sme_c_value<Overwrite>(cmask, ploadu(pg_m, p0));
1553 Vec c1 = sme_c_value<Overwrite>(cmask, ploadu(pg_m, p1));
1554 pstoreu(pg_m, p0, pmadd(pg_m, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)ci), valpha, c0));
1555 pstoreu(pg_m, p1, pmadd(pg_m, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)(ci + 1)), valpha, c1));
1558 Scalar* pC = C + row_start + (col_start + ci) * C_stride_col;
1559 Vec vc = sme_c_value<Overwrite>(cmask, ploadu(pg_m, pC));
1560 pstoreu(pg_m, pC, pmadd(pg_m, sme_read_ver_za<TileId>(vzero, pg_m, (uint32_t)ci), valpha, vc));
1562 }
else if (C_stride_col == 1) {
1565 for (; ri + 2 <= pw; ri += 2) {
1566 Scalar* p0 = C + (row_start + ri) * C_stride_row + col_start;
1567 Scalar* p1 = p0 + C_stride_row;
1568 Vec c0 = sme_c_value<Overwrite>(cmask, ploadu(pg_n, p0));
1569 Vec c1 = sme_c_value<Overwrite>(cmask, ploadu(pg_n, p1));
1570 pstoreu(pg_n, p0, pmadd(pg_n, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri), valpha, c0));
1571 pstoreu(pg_n, p1, pmadd(pg_n, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)(ri + 1)), valpha, c1));
1574 Scalar* pC = C + (row_start + ri) * C_stride_row + col_start;
1575 Vec vc = sme_c_value<Overwrite>(cmask, ploadu(pg_n, pC));
1576 pstoreu(pg_n, pC, pmadd(pg_n, sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri), valpha, vc));
1583 Scalar scratch[sme_block<Scalar>::nr];
1584 for (
int ri = 0; ri < pw; ++ri) {
1585 Vec vres = sme_read_hor_za<TileId>(vzero, pg_n, (uint32_t)ri);
1586 vres = pmul(pg_n, vres, valpha);
1587 pstoreu(pg_n, scratch, vres);
1588 for (
int ci = 0; ci < cw; ++ci) {
1589 Scalar& c = C[(row_start + ri) * C_stride_row + (col_start + ci) * C_stride_col];
1590 c = (Overwrite ? Scalar(0) : c) + scratch[ci];
1610template <
int TileLo,
int TileHi,
bool Overwrite = false,
typename Scalar,
typename Index>
1611EIGEN_ALWAYS_INLINE
void sme_store_tile_pair_colmajor(Scalar* EIGEN_RESTRICT C, Index ldc, Scalar alpha,
1612 int svl) __arm_streaming __arm_inout(
"za") {
1613 using Traits = sme_packet_traits<Scalar>;
1614 const svcount_t pn = Traits::ptrue_c();
1615 const svbool_t pg = Traits::ptrue();
1616 const typename Traits::type valpha = pset1<typename Traits::type>(alpha);
1617 const svbool_t m = sme_c_mask<Overwrite>();
1618 for (
int c = 0; c < svl; c += 4) {
1620#if !EIGEN_COMP_GNUC_STRICT
1621 const auto lo = sme_read_ver_za_vg4<TileLo, Scalar>(uint32_t(c));
1622 const auto hi = sme_read_ver_za_vg4<TileHi, Scalar>(uint32_t(c));
1624 Scalar* p = C + Index(c) * ldc;
1625 const auto c0 = ploadu_x2(pn, p), c1 = ploadu_x2(pn, p + ldc), c2 = ploadu_x2(pn, p + 2 * ldc),
1626 c3 = ploadu_x2(pn, p + 3 * ldc);
1627#if EIGEN_COMP_GNUC_STRICT
1628 const auto lo = sme_read_ver_za_vg4<TileLo, Scalar>(uint32_t(c));
1629 const auto hi = sme_read_ver_za_vg4<TileHi, Scalar>(uint32_t(c));
1632 pcreate(pmadd(pg, pget<0>(lo), valpha, sme_c_value<Overwrite>(m, pget<0>(c0))),
1633 pmadd(pg, pget<0>(hi), valpha, sme_c_value<Overwrite>(m, pget<1>(c0)))));
1634 pstoreu_x2(pn, p + ldc,
1635 pcreate(pmadd(pg, pget<1>(lo), valpha, sme_c_value<Overwrite>(m, pget<0>(c1))),
1636 pmadd(pg, pget<1>(hi), valpha, sme_c_value<Overwrite>(m, pget<1>(c1)))));
1637 pstoreu_x2(pn, p + 2 * ldc,
1638 pcreate(pmadd(pg, pget<2>(lo), valpha, sme_c_value<Overwrite>(m, pget<0>(c2))),
1639 pmadd(pg, pget<2>(hi), valpha, sme_c_value<Overwrite>(m, pget<1>(c2)))));
1640 pstoreu_x2(pn, p + 3 * ldc,
1641 pcreate(pmadd(pg, pget<3>(lo), valpha, sme_c_value<Overwrite>(m, pget<0>(c3))),
1642 pmadd(pg, pget<3>(hi), valpha, sme_c_value<Overwrite>(m, pget<1>(c3)))));
1646template <
bool Overwrite = false,
typename Scalar,
typename Index>
1647EIGEN_ALWAYS_INLINE
void sme_store_2x2_grid(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col,
1648 Scalar alpha, Index row_start,
int rlo,
int rhi, Index col_start,
int clo,
1649 int chi) __arm_streaming __arm_inout(
"za") {
1650 const int svl = sme_packet_traits<Scalar>::size();
1651 if (svl >= 4 && C_stride_row == 1 && rlo == svl && rhi == svl && clo == svl && chi == svl) {
1652 Scalar* c0 = C + row_start + col_start * C_stride_col;
1653 sme_store_tile_pair_colmajor<0, 2, Overwrite>(c0, C_stride_col, alpha, svl);
1654 sme_store_tile_pair_colmajor<1, 3, Overwrite>(c0 + Index(svl) * C_stride_col, C_stride_col, alpha, svl);
1657 sme_store_za_tile<Scalar, 0, Overwrite>(C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start, clo);
1659 sme_store_za_tile<Scalar, 1, Overwrite>(C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start + svl, chi);
1662 sme_store_za_tile<Scalar, 2, Overwrite>(C, C_stride_row, C_stride_col, alpha, row_start + svl, rhi, col_start, clo);
1664 sme_store_za_tile<Scalar, 3, Overwrite>(C, C_stride_row, C_stride_col, alpha, row_start + svl, rhi,
1665 col_start + svl, chi);
1675template <
typename Scalar>
1676static EIGEN_ALWAYS_INLINE
void outer_product_2x2(
1677 typename sme_packet_traits<Scalar>::type a_lo,
typename sme_packet_traits<Scalar>::type a_hi,
1678 typename sme_packet_traits<Scalar>::type b_lo,
1679 typename sme_packet_traits<Scalar>::type b_hi) __arm_streaming __arm_inout(
"za") {
1680 const svbool_t all = sme_packet_traits<Scalar>::ptrue();
1681 sme_mopa<0>(all, all, a_lo, b_lo);
1682 sme_mopa<1>(all, all, a_lo, b_hi);
1683 sme_mopa<2>(all, all, a_hi, b_lo);
1684 sme_mopa<3>(all, all, a_hi, b_hi);
1717template <
typename RealScalar,
int TileRe,
int TileIm,
bool Vertical,
bool ScaleByAlpha>
1718EIGEN_ALWAYS_INLINE
void sme_read_slice(
1719 svbool_t pg,
typename sme_packet_traits<RealScalar>::type valpha_re,
1720 typename sme_packet_traits<RealScalar>::type valpha_im,
typename sme_packet_traits<RealScalar>::type vzero,
1721 uint32_t slice,
typename sme_packet_traits<RealScalar>::type& lo,
1722 typename sme_packet_traits<RealScalar>::type& hi) __arm_streaming __arm_inout(
"za") {
1723 using Vec =
typename sme_packet_traits<RealScalar>::type;
1726 re = sme_read_ver_za<TileRe>(vzero, pg, slice);
1727 im = sme_read_ver_za<TileIm>(vzero, pg, slice);
1729 re = sme_read_hor_za<TileRe>(vzero, pg, slice);
1730 im = sme_read_hor_za<TileIm>(vzero, pg, slice);
1732 EIGEN_IF_CONSTEXPR (ScaleByAlpha) {
1733 const Vec out_re = pnmadd(pg, im, valpha_im, pmul(pg, re, valpha_re));
1734 const Vec out_im = pmadd(pg, re, valpha_im, pmul(pg, im, valpha_re));
1735 lo = pzip1(out_re, out_im);
1736 hi = pzip2(out_re, out_im);
1747template <
typename RealScalar,
int TileRe,
int TileIm,
bool Vertical,
bool ScaleByAlpha,
typename Index>
1748EIGEN_ALWAYS_INLINE
void sme_accumulate_pair_impl(
1749 RealScalar* EIGEN_RESTRICT p, Index step,
int slices,
int lanes, svbool_t pg,
1750 typename sme_packet_traits<RealScalar>::type valpha_re,
typename sme_packet_traits<RealScalar>::type valpha_im,
1751 typename sme_packet_traits<RealScalar>::type vzero) __arm_streaming __arm_inout(
"za") {
1752 using Traits = sme_packet_traits<RealScalar>;
1753 using Vec =
typename Traits::type;
1754 const int svl = Traits::size();
1755 const svbool_t pl0 = Traits::whilelt(0, lanes);
1756 const svbool_t pl1 = Traits::whilelt(svl, lanes);
1757 for (
int s = 0; s < slices; ++s, p += step) {
1759 sme_read_slice<RealScalar, TileRe, TileIm, Vertical, ScaleByAlpha>(pg, valpha_re, valpha_im, vzero, uint32_t(s), lo,
1761 pstoreu(pl0, p, padd(pl0, ploadu(pl0, p), lo));
1765 RealScalar* EIGEN_RESTRICT phi = sme_offset(p, Index(svl));
1766 pstoreu(pl1, phi, padd(pl1, ploadu(pl1, phi), hi));
1789template <
typename RealScalar,
int TileRe,
int TileIm,
bool Vertical,
typename Index>
1790EIGEN_ALWAYS_INLINE
void sme_accumulate_pair_real_alpha(
1791 RealScalar* EIGEN_RESTRICT p, Index step,
int slices, Index limit, svbool_t pg,
1792 typename sme_packet_traits<RealScalar>::type valpha,
1793 typename sme_packet_traits<RealScalar>::type vzero) __arm_streaming __arm_inout(
"za") {
1794 using Traits = sme_packet_traits<RealScalar>;
1795 using Vec =
typename Traits::type;
1796 const int svl = Traits::size();
1797 constexpr int kPrefetchSlices = 8;
1798 for (
int s = 0; s < slices; ++s, p += step) {
1799 if (s + kPrefetchSlices < limit) {
1801 EIGEN_OPTIMIZATION_BARRIER(pf)
1802 pf = sme_offset(pf, Index(kPrefetchSlices) * step);
1803 __builtin_prefetch(pf, 1, 3);
1804 __builtin_prefetch(sme_offset(pf, Index(svl)), 1, 3);
1808 re = sme_read_ver_za<TileRe>(vzero, pg, uint32_t(s));
1809 im = sme_read_ver_za<TileIm>(vzero, pg, uint32_t(s));
1811 re = sme_read_hor_za<TileRe>(vzero, pg, uint32_t(s));
1812 im = sme_read_hor_za<TileIm>(vzero, pg, uint32_t(s));
1814 const typename Traits::type_x2 c = pld2(pg, p);
1815 pst2(pg, p, pcreate(pmadd(pg, re, valpha, pget<0>(c)), pmadd(pg, im, valpha, pget<1>(c))));
1823template <
typename RealScalar,
int TileRe,
int TileIm,
bool Vertical,
bool FoldRealAlpha,
typename Index>
1824EIGEN_ALWAYS_INLINE
void sme_accumulate_pair(
1825 bool scale_by_alpha, RealScalar* EIGEN_RESTRICT p, Index step,
int slices, Index limit,
int lanes, svbool_t pg,
1826 typename sme_packet_traits<RealScalar>::type valpha_re,
typename sme_packet_traits<RealScalar>::type valpha_im,
1827 typename sme_packet_traits<RealScalar>::type vzero) __arm_streaming __arm_inout(
"za") {
1828 if (FoldRealAlpha && lanes > sme_packet_traits<RealScalar>::size()) {
1829 sme_accumulate_pair_real_alpha<RealScalar, TileRe, TileIm, Vertical>(p, step, slices, limit, pg, valpha_re, vzero);
1830 }
else if (scale_by_alpha) {
1831 sme_accumulate_pair_impl<RealScalar, TileRe, TileIm, Vertical, true>(p, step, slices, lanes, pg, valpha_re,
1834 sme_accumulate_pair_impl<RealScalar, TileRe, TileIm, Vertical, false>(p, step, slices, lanes, pg, valpha_re,
1841template <
typename RealScalar,
int TileRe,
int TileIm,
bool FoldRealAlpha,
typename Index>
1842EIGEN_ALWAYS_INLINE
void sme_store_za_pair(std::complex<RealScalar>* EIGEN_RESTRICT C, Index C_stride_row,
1843 Index C_stride_col, std::complex<RealScalar> alpha, Index row_start,
int pw,
1844 Index col_start,
int cw, Index rows,
1845 Index cols) __arm_streaming __arm_inout(
"za") {
1846 using Scalar = std::complex<RealScalar>;
1847 using Traits = sme_packet_traits<RealScalar>;
1848 using Vec =
typename Traits::type;
1849 const int svl = Traits::size();
1850 const svbool_t pg_m = Traits::whilelt(0, pw);
1851 const svbool_t pg_n = Traits::whilelt(0, cw);
1852 const Vec vzero = pset1<Vec>(RealScalar(0));
1857 const RealScalar* alpha_parts =
reinterpret_cast<const RealScalar*
>(&alpha);
1858 const Vec valpha_re = pset1<Vec>(alpha_parts[0]);
1859 const Vec valpha_im = pset1<Vec>(alpha_parts[1]);
1864 const bool scale = !(alpha_parts[0] == RealScalar(1) && alpha_parts[1] == RealScalar(0));
1865 RealScalar* EIGEN_RESTRICT rC =
reinterpret_cast<RealScalar*
>(C);
1867 if (C_stride_row == 1) {
1870 RealScalar* p = rC + Index(2) * (row_start + col_start * C_stride_col);
1871 sme_accumulate_pair<RealScalar, TileRe, TileIm, true, FoldRealAlpha>(
1872 scale, p, Index(2) * C_stride_col, cw, cols - col_start, 2 * pw, pg_m, valpha_re, valpha_im, vzero);
1873 }
else if (C_stride_col == 1) {
1875 RealScalar* p = rC + Index(2) * (row_start * C_stride_row + col_start);
1876 sme_accumulate_pair<RealScalar, TileRe, TileIm, false, FoldRealAlpha>(
1877 scale, p, Index(2) * C_stride_row, pw, rows - row_start, 2 * cw, pg_n, valpha_re, valpha_im, vzero);
1883 Scalar scratch[sme_block<Scalar>::nr];
1884 RealScalar* rscratch =
reinterpret_cast<RealScalar*
>(scratch);
1885 const int lanes = 2 * cw;
1886 const svbool_t pl0 = Traits::whilelt(0, lanes);
1887 const svbool_t pl1 = Traits::whilelt(svl, lanes);
1888 for (
int ri = 0; ri < pw; ++ri) {
1890 sme_read_slice<RealScalar, TileRe, TileIm, false, true>(pg_n, valpha_re, valpha_im, vzero, uint32_t(ri), lo, hi);
1891 pstoreu(pl0, rscratch, lo);
1894 pstoreu(pl1, rscratch + svl, hi);
1895 for (
int ci = 0; ci < cw; ++ci) {
1896 C[(row_start + ri) * C_stride_row + (col_start + ci) * C_stride_col] += scratch[ci];
1904template <
typename Scalar,
int R,
int C,
bool ConjLhs,
bool ConjRhs,
1905 bool InGrid = (R < sme_block<Scalar>::kGridRows && C < sme_block<Scalar>::kGridCols)>
1906struct sme_complex_cell {
1907 using RealScalar =
typename NumTraits<Scalar>::Real;
1908 using Vec =
typename sme_packet_traits<RealScalar>::type;
1909 static constexpr int kTileRe = 2 * (R * sme_block<Scalar>::kGridCols + C);
1910 static constexpr int kTileIm = kTileRe + 1;
1912 static EIGEN_ALWAYS_INLINE
void accumulate(svbool_t pm, svbool_t pn, Vec a_re, Vec a_im, Vec b_re,
1913 Vec b_im) __arm_streaming __arm_inout(
"za") {
1914 sme_mopa_signed<kTileRe, false>(pm, pn, a_re, b_re);
1915 sme_mopa_signed<kTileRe, ConjLhs == ConjRhs>(pm, pn, a_im, b_im);
1916 sme_mopa_signed<kTileIm, ConjRhs>(pm, pn, a_re, b_im);
1917 sme_mopa_signed<kTileIm, ConjLhs>(pm, pn, a_im, b_re);
1920 template <
bool FoldRealAlpha,
typename Index>
1921 static EIGEN_ALWAYS_INLINE
void store(Scalar* EIGEN_RESTRICT dst, Index C_stride_row, Index C_stride_col,
1922 Scalar alpha, Index row_start,
int pw, Index col_start,
int cw, Index rows,
1923 Index cols) __arm_streaming __arm_inout(
"za") {
1924 sme_store_za_pair<RealScalar, kTileRe, kTileIm, FoldRealAlpha>(dst, C_stride_row, C_stride_col, alpha, row_start,
1925 pw, col_start, cw, rows, cols);
1929template <
typename Scalar,
int R,
int C,
bool ConjLhs,
bool ConjRhs>
1930struct sme_complex_cell<Scalar, R, C, ConjLhs, ConjRhs, false> {
1931 using Vec =
typename sme_packet_traits<typename NumTraits<Scalar>::Real>::type;
1932 static EIGEN_ALWAYS_INLINE
void accumulate(svbool_t, svbool_t, Vec, Vec, Vec, Vec) __arm_streaming __arm_inout(
"za") {
1934 template <
bool FoldRealAlpha,
typename Index>
1935 static EIGEN_ALWAYS_INLINE
void store(Scalar*, Index, Index, Scalar, Index,
int, Index,
int, Index,
1936 Index) __arm_streaming __arm_inout(
"za") {}
1956template <
bool ConjLhs,
bool ConjRhs,
typename Scalar,
typename Index>
1957EIGEN_ALWAYS_INLINE
void sme_process(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col,
1958 const Scalar* EIGEN_RESTRICT blA,
const Scalar* EIGEN_RESTRICT blB, Index depth,
1959 Scalar alpha, Index row_start,
int pw, Index col_start,
int cw, Index a_step,
1960 bool overwrite) __arm_streaming __arm_inout(
"za") {
1964 using Traits = sme_packet_traits<Scalar>;
1965 using Vec =
typename Traits::type;
1966 const int svl = Traits::size();
1968 for (
int rt = 0; rt < pw; rt += 2 * svl) {
1969 const int rpw = sme_min(pw - rt, 2 * svl);
1970 const int rlo = sme_min(rpw, svl);
1971 const int rhi = rpw - rlo;
1972 const svbool_t pg_rlo = Traits::whilelt(rt, pw);
1973 const svbool_t pg_rhi = Traits::whilelt(rt + svl, pw);
1975 for (
int ct = 0; ct < cw; ct += 2 * svl) {
1976 const int cpw = sme_min(cw - ct, 2 * svl);
1977 const int clo = sme_min(cpw, svl);
1978 const int chi = cpw - clo;
1979 const svbool_t pg_clo = Traits::whilelt(ct, cw);
1980 const svbool_t pg_chi = Traits::whilelt(ct + svl, cw);
1983 if (pw == 2 * svl && cw == 2 * svl) {
1988 const svcount_t pn = Traits::ptrue_c();
1989 const Index depth_4 = (depth / 4) * 4;
1991 if (a_step == Index(pw)) {
1992 for (; k < depth_4; k += 4) {
1993 typename Traits::type_x4 va_01 = ploadu_x4(pn, &blA[k * pw]);
1994 typename Traits::type_x4 vb_01 = ploadu_x4(pn, &blB[k * cw]);
1997 outer_product_2x2<Scalar>(pget<0>(va_01), pget<1>(va_01), pget<0>(vb_01), pget<1>(vb_01));
1999 outer_product_2x2<Scalar>(pget<2>(va_01), pget<3>(va_01), pget<2>(vb_01), pget<3>(vb_01));
2001 typename Traits::type_x4 va_23 = ploadu_x4(pn, &blA[(k + 2) * pw]);
2002 typename Traits::type_x4 vb_23 = ploadu_x4(pn, &blB[(k + 2) * cw]);
2005 outer_product_2x2<Scalar>(pget<0>(va_23), pget<1>(va_23), pget<0>(vb_23), pget<1>(vb_23));
2007 outer_product_2x2<Scalar>(pget<2>(va_23), pget<3>(va_23), pget<2>(vb_23), pget<3>(vb_23));
2011 for (; k < depth_4; k += 4) {
2012 typename Traits::type_x2 va_0 = ploadu_x2(pn, &blA[k * a_step]);
2013 typename Traits::type_x2 va_1 = ploadu_x2(pn, &blA[(k + 1) * a_step]);
2014 typename Traits::type_x4 vb_01 = ploadu_x4(pn, &blB[k * cw]);
2015 outer_product_2x2<Scalar>(pget<0>(va_0), pget<1>(va_0), pget<0>(vb_01), pget<1>(vb_01));
2016 outer_product_2x2<Scalar>(pget<0>(va_1), pget<1>(va_1), pget<2>(vb_01), pget<3>(vb_01));
2017 typename Traits::type_x2 va_2 = ploadu_x2(pn, &blA[(k + 2) * a_step]);
2018 typename Traits::type_x2 va_3 = ploadu_x2(pn, &blA[(k + 3) * a_step]);
2019 typename Traits::type_x4 vb_23 = ploadu_x4(pn, &blB[(k + 2) * cw]);
2020 outer_product_2x2<Scalar>(pget<0>(va_2), pget<1>(va_2), pget<0>(vb_23), pget<1>(vb_23));
2021 outer_product_2x2<Scalar>(pget<0>(va_3), pget<1>(va_3), pget<2>(vb_23), pget<3>(vb_23));
2025 for (; k < depth; ++k) {
2026 typename Traits::type_x2 va = ploadu_x2(pn, &blA[k * a_step]);
2027 typename Traits::type_x2 vb = ploadu_x2(pn, &blB[k * cw]);
2028 outer_product_2x2<Scalar>(pget<0>(va), pget<1>(va), pget<0>(vb), pget<1>(vb));
2031 for (Index k = 0; k < depth; ++k) {
2032 Vec a_lo = ploadu(pg_rlo, &blA[k * a_step + rt]);
2033 Vec b_lo = ploadu(pg_clo, &blB[k * cw + ct]);
2035 Vec a_hi = ploadu(pg_rhi, sme_offset(blA, k * a_step + rt + svl));
2036 Vec b_hi = ploadu(pg_chi, sme_offset(blB, k * cw + ct + svl));
2038 sme_mopa<0>(pg_rlo, pg_clo, a_lo, b_lo);
2039 if (svptest_any(pg_chi, pg_chi)) sme_mopa<1>(pg_rlo, pg_chi, a_lo, b_hi);
2040 if (svptest_any(pg_rhi, pg_rhi)) {
2041 sme_mopa<2>(pg_rhi, pg_clo, a_hi, b_lo);
2042 if (svptest_any(pg_chi, pg_chi)) sme_mopa<3>(pg_rhi, pg_chi, a_hi, b_hi);
2049 sme_store_2x2_grid<true>(C, C_stride_row, C_stride_col, alpha, row_start + rt, rlo, rhi, col_start + ct, clo,
2052 sme_store_2x2_grid<false>(C, C_stride_row, C_stride_col, alpha, row_start + rt, rlo, rhi, col_start + ct, clo,
2064template <
typename Scalar,
bool ConjLhs,
bool ConjRhs,
bool FoldRealAlpha,
typename Index>
2065EIGEN_ALWAYS_INLINE
void sme_store_complex_grid(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col,
2066 Scalar alpha, Index row_start,
int rlo,
int rhi, Index col_start,
2067 int clo,
int chi, Index rows,
2068 Index cols) __arm_streaming __arm_inout(
"za") {
2069 const int svl = sme_packet_traits<typename NumTraits<Scalar>::Real>::size();
2070 sme_complex_cell<Scalar, 0, 0, ConjLhs, ConjRhs>::template store<FoldRealAlpha>(
2071 C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start, clo, rows, cols);
2073 sme_complex_cell<Scalar, 0, 1, ConjLhs, ConjRhs>::template store<FoldRealAlpha>(
2074 C, C_stride_row, C_stride_col, alpha, row_start, rlo, col_start + svl, chi, rows, cols);
2077 sme_complex_cell<Scalar, 1, 0, ConjLhs, ConjRhs>::template store<FoldRealAlpha>(
2078 C, C_stride_row, C_stride_col, alpha, row_start + svl, rhi, col_start, clo, rows, cols);
2080 sme_complex_cell<Scalar, 1, 1, ConjLhs, ConjRhs>::template store<FoldRealAlpha>(
2081 C, C_stride_row, C_stride_col, alpha, row_start + svl, rhi, col_start + svl, chi, rows, cols);
2095template <
bool ConjLhs,
bool ConjRhs,
bool FoldRealAlpha,
typename RealScalar,
typename Index>
2096EIGEN_ALWAYS_INLINE
void sme_process(std::complex<RealScalar>* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col,
2097 const std::complex<RealScalar>* EIGEN_RESTRICT blA,
2098 const std::complex<RealScalar>* EIGEN_RESTRICT blB, Index depth,
2099 std::complex<RealScalar> alpha, Index row_start,
int pw, Index col_start,
int cw,
2100 Index lhs_step, Index rows, Index cols) __arm_streaming __arm_inout(
"za") {
2102 EIGEN_UNUSED_VARIABLE(lhs_step);
2103 using Scalar = std::complex<RealScalar>;
2104 using Traits = sme_packet_traits<RealScalar>;
2105 using Vec =
typename Traits::type;
2106 const int svl = Traits::size();
2107 constexpr int GridRows = sme_block<Scalar>::kGridRows;
2108 constexpr int GridCols = sme_block<Scalar>::kGridCols;
2109 using Cell00 = sme_complex_cell<Scalar, 0, 0, ConjLhs, ConjRhs>;
2110 using Cell01 = sme_complex_cell<Scalar, 0, 1, ConjLhs, ConjRhs>;
2111 using Cell10 = sme_complex_cell<Scalar, 1, 0, ConjLhs, ConjRhs>;
2112 using Cell11 = sme_complex_cell<Scalar, 1, 1, ConjLhs, ConjRhs>;
2114 const RealScalar* EIGEN_RESTRICT rA =
reinterpret_cast<const RealScalar*
>(blA);
2115 const RealScalar* EIGEN_RESTRICT rB =
reinterpret_cast<const RealScalar*
>(blB);
2116 const Index a_step = Index(2 * pw);
2117 const Index b_step = Index(2 * cw);
2119 for (
int rt = 0; rt < pw; rt += GridRows * svl) {
2120 const int rpw = sme_min(pw - rt, GridRows * svl);
2121 const int r0 = sme_min(rpw, svl);
2122 const int r1 = rpw - r0;
2123 const svbool_t pg_r0 = Traits::whilelt(0, rpw);
2124 const svbool_t pg_r1 = Traits::whilelt(svl, rpw);
2126 for (
int ct = 0; ct < cw; ct += GridCols * svl) {
2127 const int cpw = sme_min(cw - ct, GridCols * svl);
2128 const int c0 = sme_min(cpw, svl);
2129 const int c1 = cpw - c0;
2130 const svbool_t pg_c0 = Traits::whilelt(0, cpw);
2131 const svbool_t pg_c1 = Traits::whilelt(svl, cpw);
2134 for (Index k = 0; k < depth; ++k) {
2135 const RealScalar* pa = rA + k * a_step + Index(rt);
2136 const RealScalar* pb = rB + k * b_step + Index(ct);
2142 const Vec a0_re = ploadu(pg_r0, pa);
2143 const Vec a0_im = ploadu(pg_r0, pa + pw);
2144 const Vec b0_re = ploadu(pg_c0, pb);
2145 const Vec b0_im = ploadu(pg_c0, pb + cw);
2146 const Vec a1_re = ploadu(pg_r1, sme_offset(pa, Index(svl)));
2147 const Vec a1_im = ploadu(pg_r1, sme_offset(pa, Index(pw) + Index(svl)));
2148 const Vec b1_re = ploadu(pg_c1, sme_offset(pb, Index(svl)));
2149 const Vec b1_im = ploadu(pg_c1, sme_offset(pb, Index(cw) + Index(svl)));
2150 Cell00::accumulate(pg_r0, pg_c0, a0_re, a0_im, b0_re, b0_im);
2151 if (svptest_any(pg_c1, pg_c1)) {
2152 Cell01::accumulate(pg_r0, pg_c1, a0_re, a0_im, b1_re, b1_im);
2154 if (svptest_any(pg_r1, pg_r1)) {
2155 Cell10::accumulate(pg_r1, pg_c0, a1_re, a1_im, b0_re, b0_im);
2156 if (svptest_any(pg_c1, pg_c1)) {
2157 Cell11::accumulate(pg_r1, pg_c1, a1_re, a1_im, b1_re, b1_im);
2162 sme_store_complex_grid<Scalar, ConjLhs, ConjRhs, FoldRealAlpha>(
2163 C, C_stride_row, C_stride_col, alpha, row_start + rt, r0, r1, col_start + ct, c0, c1, rows, cols);
2170template <
typename Scalar,
typename Index>
2171EIGEN_ALWAYS_INLINE
void sme_process_narrow(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col,
2172 const Scalar* EIGEN_RESTRICT blA0, Index step0,
2173 const Scalar* EIGEN_RESTRICT blA1, Index step1,
int pw1,
2174 const Scalar* EIGEN_RESTRICT blB,
int cw, Index depth, Scalar alpha,
2175 Index row_start, Index col_start,
2176 bool overwrite) __arm_streaming __arm_inout(
"za") {
2177 using Traits = sme_packet_traits<Scalar>;
2178 using Vec =
typename Traits::type;
2179 const int svl = Traits::size();
2180 const svcount_t pc = Traits::ptrue_c();
2181 const svbool_t all = Traits::ptrue();
2182 const svbool_t pn = Traits::whilelt(0, cw);
2183 const svbool_t p1lo = Traits::whilelt(0, pw1);
2184 const svbool_t p1hi = Traits::whilelt(svl, pw1);
2186 for (Index k = 0; k < depth; ++k) {
2187 const auto a0 = ploadu_x2(pc, blA0 + k * step0);
2188 const Vec a1lo = ploadu(p1lo, blA1 + k * step1);
2189 const Vec a1hi = ploadu(p1hi, sme_offset(blA1, k * step1 + svl));
2190 const Vec b = ploadu(pn, blB + k * cw);
2191 sme_mopa<0>(all, pn, pget<0>(a0), b);
2192 sme_mopa<1>(all, pn, pget<1>(a0), b);
2193 sme_mopa<2>(p1lo, pn, a1lo, b);
2194 sme_mopa<3>(p1hi, pn, a1hi, b);
2197 sme_store_za_tile<Scalar, 0, true>(C, C_stride_row, C_stride_col, alpha, row_start, svl, col_start, cw);
2199 sme_store_za_tile<Scalar, 0, false>(C, C_stride_row, C_stride_col, alpha, row_start, svl, col_start, cw);
2201 sme_store_za_tile<Scalar, 1, true>(C, C_stride_row, C_stride_col, alpha, row_start + svl, svl, col_start, cw);
2203 sme_store_za_tile<Scalar, 1, false>(C, C_stride_row, C_stride_col, alpha, row_start + svl, svl, col_start, cw);
2205 sme_store_za_tile<Scalar, 2, true>(C, C_stride_row, C_stride_col, alpha, row_start + 2 * svl, sme_min(pw1, svl),
2208 sme_store_za_tile<Scalar, 2, false>(C, C_stride_row, C_stride_col, alpha, row_start + 2 * svl, sme_min(pw1, svl),
2212 sme_store_za_tile<Scalar, 3, true>(C, C_stride_row, C_stride_col, alpha, row_start + 3 * svl, pw1 - svl,
2215 sme_store_za_tile<Scalar, 3, false>(C, C_stride_row, C_stride_col, alpha, row_start + 3 * svl, pw1 - svl,
2221template <
int Dst,
int Src,
typename Scalar>
2222EIGEN_ALWAYS_INLINE
void sme_fold_tile() __arm_streaming __arm_inout(
"za") {
2223 const svbool_t all = sme_packet_traits<Scalar>::ptrue();
2224 for (
int c = 0; c < sme_packet_traits<Scalar>::size(); c += 4) {
2225 const auto d = sme_read_ver_za_vg4<Dst, Scalar>(uint32_t(c)), s = sme_read_ver_za_vg4<Src, Scalar>(uint32_t(c));
2226 sme_write_ver_za_vg4<Dst>(uint32_t(c),
2227 pcreate(padd(all, pget<0>(d), pget<0>(s)), padd(all, pget<1>(d), pget<1>(s)),
2228 padd(all, pget<2>(d), pget<2>(s)), padd(all, pget<3>(d), pget<3>(s))));
2234template <
typename Scalar,
typename Index>
2235EIGEN_ALWAYS_INLINE
void sme_process_split(Scalar* EIGEN_RESTRICT C, Index C_stride_row, Index C_stride_col,
2236 const Scalar* EIGEN_RESTRICT blA,
const Scalar* EIGEN_RESTRICT blB,
2237 Index depth, Scalar alpha, Index row_start,
int pw, Index col_start,
int cw,
2238 Index a_step,
bool overwrite) __arm_streaming __arm_inout(
"za") {
2239 using Traits = sme_packet_traits<Scalar>;
2240 using Vec =
typename Traits::type;
2241 const int svl = Traits::size();
2242 const svbool_t pr0 = Traits::whilelt(0, pw), pr1 = Traits::whilelt(svl, pw);
2243 const svbool_t pc0 = Traits::whilelt(0, cw), pc1 = Traits::whilelt(svl, cw);
2246 if (pw <= svl && cw <= svl) {
2247 for (; k + 4 <= depth; k += 4) {
2248 const Vec a0 = ploadu(pr0, blA + k * a_step), a1 = ploadu(pr0, blA + (k + 1) * a_step);
2249 const Vec a2 = ploadu(pr0, blA + (k + 2) * a_step), a3 = ploadu(pr0, blA + (k + 3) * a_step);
2250 const Vec b0 = ploadu(pc0, blB + k * cw), b1 = ploadu(pc0, blB + (k + 1) * cw);
2251 const Vec b2 = ploadu(pc0, blB + (k + 2) * cw), b3 = ploadu(pc0, blB + (k + 3) * cw);
2252 sme_mopa<0>(pr0, pc0, a0, b0);
2253 sme_mopa<1>(pr0, pc0, a1, b1);
2254 sme_mopa<2>(pr0, pc0, a2, b2);
2255 sme_mopa<3>(pr0, pc0, a3, b3);
2257 for (; k < depth; ++k) sme_mopa<0>(pr0, pc0, ploadu(pr0, blA + k * a_step), ploadu(pc0, blB + k * cw));
2258 sme_fold_tile<0, 1, Scalar>();
2259 sme_fold_tile<2, 3, Scalar>();
2260 sme_fold_tile<0, 2, Scalar>();
2262 sme_store_za_tile<Scalar, 0, true>(C, C_stride_row, C_stride_col, alpha, row_start, pw, col_start, cw);
2264 sme_store_za_tile<Scalar, 0, false>(C, C_stride_row, C_stride_col, alpha, row_start, pw, col_start, cw);
2265 }
else if (pw <= svl) {
2267 for (; k + 2 <= depth; k += 2) {
2268 const Vec a0 = ploadu(pr0, blA + k * a_step), a1 = ploadu(pr0, blA + (k + 1) * a_step);
2269 const Vec b0 = ploadu(pc0, blB + k * cw), b0h = ploadu(pc1, blB + k * cw + svl);
2270 const Vec b1 = ploadu(pc0, blB + (k + 1) * cw), b1h = ploadu(pc1, blB + (k + 1) * cw + svl);
2271 sme_mopa<0>(pr0, pc0, a0, b0);
2272 sme_mopa<1>(pr0, pc1, a0, b0h);
2273 sme_mopa<2>(pr0, pc0, a1, b1);
2274 sme_mopa<3>(pr0, pc1, a1, b1h);
2277 const Vec a0 = ploadu(pr0, blA + k * a_step);
2278 sme_mopa<0>(pr0, pc0, a0, ploadu(pc0, blB + k * cw));
2279 sme_mopa<1>(pr0, pc1, a0, ploadu(pc1, blB + k * cw + svl));
2281 sme_fold_tile<0, 2, Scalar>();
2282 sme_fold_tile<1, 3, Scalar>();
2284 sme_store_2x2_grid<true>(C, C_stride_row, C_stride_col, alpha, row_start, pw, 0, col_start, svl, cw - svl);
2286 sme_store_2x2_grid<false>(C, C_stride_row, C_stride_col, alpha, row_start, pw, 0, col_start, svl, cw - svl);
2289 for (; k + 2 <= depth; k += 2) {
2290 const Vec a0 = ploadu(pr0, blA + k * a_step), a0h = ploadu(pr1, blA + k * a_step + svl);
2291 const Vec a1 = ploadu(pr0, blA + (k + 1) * a_step), a1h = ploadu(pr1, blA + (k + 1) * a_step + svl);
2292 const Vec b0 = ploadu(pc0, blB + k * cw), b1 = ploadu(pc0, blB + (k + 1) * cw);
2293 sme_mopa<0>(pr0, pc0, a0, b0);
2294 sme_mopa<2>(pr1, pc0, a0h, b0);
2295 sme_mopa<1>(pr0, pc0, a1, b1);
2296 sme_mopa<3>(pr1, pc0, a1h, b1);
2299 const Vec b0 = ploadu(pc0, blB + k * cw);
2300 sme_mopa<0>(pr0, pc0, ploadu(pr0, blA + k * a_step), b0);
2301 sme_mopa<2>(pr1, pc0, ploadu(pr1, blA + k * a_step + svl), b0);
2303 sme_fold_tile<0, 1, Scalar>();
2304 sme_fold_tile<2, 3, Scalar>();
2306 sme_store_2x2_grid<true>(C, C_stride_row, C_stride_col, alpha, row_start, svl, pw - svl, col_start, cw, 0);
2308 sme_store_2x2_grid<false>(C, C_stride_row, C_stride_col, alpha, row_start, svl, pw - svl, col_start, cw, 0);
2317template <
bool ConjLhs,
bool ConjRhs,
bool FoldRealAlpha,
typename Scalar,
typename Index>
2318EIGEN_ALWAYS_INLINE
void sme_process_block(Scalar* C, Index rs, Index cs,
const Scalar* blA,
const Scalar* blB,
2319 Index depth, Scalar alpha, Index row_start,
int pw, Index col_start,
int cw,
2320 Index a_step,
bool split_ok, Index, Index,
2321 bool overwrite) __arm_streaming __arm_inout(
"za") {
2322 const int svl = sme_packet_traits<Scalar>::size();
2323 if (split_ok && (pw <= svl || cw <= svl))
2324 sme_process_split(C, rs, cs, blA, blB, depth, alpha, row_start, pw, col_start, cw, a_step, overwrite);
2326 sme_process<ConjLhs, ConjRhs>(C, rs, cs, blA, blB, depth, alpha, row_start, pw, col_start, cw, a_step, overwrite);
2328template <
bool ConjLhs,
bool ConjRhs,
bool FoldRealAlpha,
typename RealScalar,
typename Index>
2329EIGEN_ALWAYS_INLINE
void sme_process_block(std::complex<RealScalar>* C, Index rs, Index cs,
2330 const std::complex<RealScalar>* blA,
const std::complex<RealScalar>* blB,
2331 Index depth, std::complex<RealScalar> alpha, Index row_start,
int pw,
2332 Index col_start,
int cw, Index a_step,
bool, Index rows, Index cols,
2333 bool) __arm_streaming __arm_inout(
"za") {
2334 sme_process<ConjLhs, ConjRhs, FoldRealAlpha>(C, rs, cs, blA, blB, depth, alpha, row_start, pw, col_start, cw, a_step,
2340template <
typename Scalar>
2341EIGEN_ALWAYS_INLINE
bool sme_split_ok() __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
2342 const int svl = sme_packet_traits<typename NumTraits<Scalar>::Real>::size();
2343 return svl >= 4 && sme_block<Scalar>::mr <= 2 * svl && sme_block<Scalar>::nr <= 2 * svl;
2345template <
typename Scalar>
2346EIGEN_ALWAYS_INLINE
bool sme_narrow_ok() __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
2347 return sme_block<Scalar>::mr == 2 * sme_packet_traits<typename NumTraits<Scalar>::Real>::size();
2351template <
typename Scalar,
typename Index>
2352EIGEN_ALWAYS_INLINE
void sme_process_narrow_dispatch(Scalar* C, Index rs, Index cs,
const Scalar* blA0, Index step0,
2353 const Scalar* blA1, Index step1,
int pw1,
const Scalar* blB,
2354 int cw, Index depth, Scalar alpha, Index row_start,
2356 bool overwrite) __arm_streaming __arm_inout(
"za") {
2357 sme_process_narrow(C, rs, cs, blA0, step0, blA1, step1, pw1, blB, cw, depth, alpha, row_start, col_start, overwrite);
2359template <
typename RealScalar,
typename Index>
2360EIGEN_ALWAYS_INLINE
void sme_process_narrow_dispatch(std::complex<RealScalar>*, Index, Index,
2361 const std::complex<RealScalar>*, Index,
2362 const std::complex<RealScalar>*, Index,
int,
2363 const std::complex<RealScalar>*,
int, Index,
2364 std::complex<RealScalar>, Index, Index,
2365 bool) __arm_streaming __arm_inout(
"za") {}
2368template <
typename Scalar,
typename Index>
2369static EIGEN_ALWAYS_INLINE
void sme_prefetch_next_c(
const Scalar* C, Index C_stride_row, Index C_stride_col, Index i,
2370 Index j, Index rows, Index cols,
int mr,
2371 int nr) __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
2372 if (C_stride_row != 1 || j >= cols)
return;
2373 const Index h = sme_min(rows - i, Index(mr)), w = sme_min(cols - j, Index(nr));
2374 const Index bytes = h * Index(
sizeof(Scalar));
2375 for (Index c = 0; c < w; ++c)
2376 for (Index b = 0; b < bytes; b += 128)
2377 __builtin_prefetch(
reinterpret_cast<const char*
>(C + i + (j + c) * C_stride_col) + b, 1, 2);
2380template <
typename Scalar,
bool ConjLhs,
bool ConjRhs,
bool FoldRealAlpha = false,
typename Index>
2381EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE __arm_locally_streaming __arm_new(
"za") void sme_gebp_impl(
2382 Scalar* C, Index C_stride_row, Index C_stride_col, const Scalar* blockA, const Scalar* blockB, Index rows,
2383 Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB,
bool overwrite) {
2384 constexpr int MR = sme_block<Scalar>::mr;
2385 constexpr int NR = sme_block<Scalar>::nr;
2395 const bool prefetch_c = rows * cols * Index(
sizeof(Scalar)) > Index(256 * 1024);
2396 const bool split_ok = sme_split_ok<Scalar>(), narrow_ok = sme_narrow_ok<Scalar>();
2397 for (Index j = 0; j < cols; j += NR) {
2398 const int cw =
static_cast<int>(sme_min(cols - j, Index(NR)));
2399 const Scalar* blB = blockB + j * strideB + offsetB * cw;
2402 EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsComplex) {
2403 if (narrow_ok && cw <= sme_packet_traits<
typename NumTraits<Scalar>::Real>::size())
2404 for (; i + MR < rows; i += 2 * MR) {
2405 const int pw1 =
static_cast<int>(sme_min(rows - i - MR, Index(MR)));
2406 sme_process_narrow_dispatch(C, C_stride_row, C_stride_col, blockA + i * strideA + offsetA * MR, Index(MR),
2407 blockA + (i + MR) * strideA + offsetA * pw1, Index(pw1), pw1, blB, cw, depth,
2408 alpha, i, j, overwrite);
2411 for (; i < rows; i += MR) {
2412 const int pw =
static_cast<int>(sme_min(rows - i, Index(MR)));
2413 const Scalar* blA = blockA + i * strideA + offsetA * pw;
2415 sme_prefetch_next_c(C, C_stride_row, C_stride_col, i + MR < rows ? i + MR : Index(0),
2416 i + MR < rows ? j : j + NR, rows, cols, MR, NR);
2417 sme_process_block<ConjLhs, ConjRhs, FoldRealAlpha>(C, C_stride_row, C_stride_col, blA, blB, depth, alpha, i, pw,
2418 j, cw, Index(pw), split_ok, rows, cols, overwrite);
2427template <typename Scalar, bool ConjLhs, bool ConjRhs, bool IsComplex = NumTraits<Scalar>::IsComplex>
2428struct sme_gebp_dispatch {
2429 template <
typename Index>
2430 static void run(Scalar* C, Index C_stride_row, Index C_stride_col,
const Scalar* blockA,
const Scalar* blockB,
2431 Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA,
2432 Index offsetB,
bool overwrite =
false) {
2433 sme_gebp_impl<Scalar, ConjLhs, ConjRhs, false>(C, C_stride_row, C_stride_col, blockA, blockB, rows, depth, cols,
2434 alpha, strideA, strideB, offsetA, offsetB, overwrite);
2438template <
typename Scalar,
bool ConjLhs,
bool ConjRhs>
2439struct sme_gebp_dispatch<Scalar, ConjLhs, ConjRhs, true> {
2440 template <
typename Index>
2441 static void run(Scalar* C, Index C_stride_row, Index C_stride_col,
const Scalar* blockA,
const Scalar* blockB,
2442 Index rows, Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA,
2443 Index offsetB,
bool overwrite =
false) {
2444 using RealScalar =
typename NumTraits<Scalar>::Real;
2445 if (numext::imag(alpha) == RealScalar(0) && numext::real(alpha) != RealScalar(1)) {
2446 sme_gebp_impl<Scalar, ConjLhs, ConjRhs, true>(C, C_stride_row, C_stride_col, blockA, blockB, rows, depth, cols,
2447 alpha, strideA, strideB, offsetA, offsetB, overwrite);
2449 sme_gebp_impl<Scalar, ConjLhs, ConjRhs, false>(C, C_stride_row, C_stride_col, blockA, blockB, rows, depth, cols,
2450 alpha, strideA, strideB, offsetA, offsetB, overwrite);
2457template <
typename Scalar,
typename Index>
2458EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE __arm_locally_streaming __arm_new(
"za") void sme_gebp_impl_direct_lhs(
2459 Scalar* C, Index C_stride_row, Index C_stride_col, const Scalar* lhs, Index lda, const Scalar* blockB, Index rows,
2460 Index depth, Index cols, Scalar alpha, Index strideB, Index offsetB,
bool overwrite) {
2461 constexpr int MR = sme_block<Scalar>::mr;
2462 constexpr int NR = sme_block<Scalar>::nr;
2463 const bool split_ok = sme_split_ok<Scalar>(), narrow_ok = sme_narrow_ok<Scalar>();
2464 for (Index j = 0; j < cols; j += NR) {
2465 const int cw =
static_cast<int>(sme_min(cols - j, Index(NR)));
2466 const Scalar* blB = blockB + j * strideB + offsetB * cw;
2468 if (narrow_ok && cw <= sme_packet_traits<Scalar>::size())
2469 for (; i + MR < rows; i += 2 * MR) {
2470 const int pw1 =
static_cast<int>(sme_min(rows - i - MR, Index(MR)));
2471 sme_process_narrow(C, C_stride_row, C_stride_col, lhs + i, lda, lhs + i + MR, lda, pw1, blB, cw, depth, alpha,
2474 for (; i < rows; i += MR) {
2475 const int pw =
static_cast<int>(sme_min(rows - i, Index(MR)));
2476 sme_process_block<false, false, false>(C, C_stride_row, C_stride_col, lhs + i, blB, depth, alpha, i, pw, j, cw,
2477 lda, split_ok, rows, cols, overwrite);
2484#ifndef EIGEN_SME_DIRECT_LHS_MAX_STRIDE_BYTES
2485#define EIGEN_SME_DIRECT_LHS_MAX_STRIDE_BYTES 16384
2487#ifndef EIGEN_SME_DIRECT_LHS_MAX_BLOCK_BYTES
2488#define EIGEN_SME_DIRECT_LHS_MAX_BLOCK_BYTES (4 << 20)
2490#ifndef EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES
2491#define EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES (16 << 20)
2493template <
typename Scalar,
typename Index>
2494bool sme_direct_lhs_ok(Index lhsStride, Index rows, Index depth, Index cols) {
2495#ifdef EIGEN_SME_FORCE_NEON_SMALL_BLOCKS
2496 EIGEN_UNUSED_VARIABLE(lhsStride);
2497 EIGEN_UNUSED_VARIABLE(rows);
2498 EIGEN_UNUSED_VARIABLE(depth);
2499 EIGEN_UNUSED_VARIABLE(cols);
2502 if (NumTraits<Scalar>::IsComplex || depth <= Index(sme_neon_max_depth<Scalar>::value))
return false;
2505 const std::size_t stride_bytes = std::size_t(lhsStride) *
sizeof(Scalar);
2506 if (cols <= Index(sme_block<Scalar>::nr)) {
2507 const std::size_t span_limit = std::size_t(EIGEN_SME_DIRECT_LHS_MAX_SPAN_BYTES);
2508 if (span_limit == 0)
return false;
2509 return cols <= Index(sme_block<Scalar>::nr / 2) || std::size_t(depth) * stride_bytes <= span_limit;
2511 return stride_bytes % 4096 != 0 && stride_bytes <= std::size_t(EIGEN_SME_DIRECT_LHS_MAX_STRIDE_BYTES) &&
2512 std::size_t(rows) * std::size_t(depth) *
sizeof(Scalar) <= std::size_t(EIGEN_SME_DIRECT_LHS_MAX_BLOCK_BYTES);
2521template <
typename Scalar,
int NCol>
2522struct sme_neon_cols;
2524struct sme_neon_cols<float, 4> {
2525 using Vec = float32x4_t;
2526 static EIGEN_ALWAYS_INLINE Vec load(
const float* b) {
return vld1q_f32(b); }
2528 static EIGEN_ALWAYS_INLINE Packet4f madd(Packet4f acc, Packet4f a, Vec b) {
2529 return vfmaq_laneq_f32(acc, a, b, C);
2532 static EIGEN_ALWAYS_INLINE Packet4f nmadd(Packet4f acc, Packet4f a, Vec b) {
2533 return vfmsq_laneq_f32(acc, a, b, C);
2537struct sme_neon_cols<float, 2> {
2538 using Vec = float32x2_t;
2539 static EIGEN_ALWAYS_INLINE Vec load(
const float* b) {
return vld1_f32(b); }
2541 static EIGEN_ALWAYS_INLINE Packet4f madd(Packet4f acc, Packet4f a, Vec b) {
2542 return vfmaq_lane_f32(acc, a, b, C);
2545 static EIGEN_ALWAYS_INLINE Packet4f nmadd(Packet4f acc, Packet4f a, Vec b) {
2546 return vfmsq_lane_f32(acc, a, b, C);
2550struct sme_neon_cols<float, 1> {
2551 using Vec = float32x4_t;
2552 static EIGEN_ALWAYS_INLINE Vec load(
const float* b) {
return vld1q_dup_f32(b); }
2554 static EIGEN_ALWAYS_INLINE Packet4f madd(Packet4f acc, Packet4f a, Vec b) {
2555 return vfmaq_f32(acc, a, b);
2558 static EIGEN_ALWAYS_INLINE Packet4f nmadd(Packet4f acc, Packet4f a, Vec b) {
2559 return vfmsq_f32(acc, a, b);
2563struct sme_neon_cols<double, 4> {
2567 static EIGEN_ALWAYS_INLINE Vec load(
const double* b) {
return {vld1q_f64(b), vld1q_f64(b + 2)}; }
2569 static EIGEN_ALWAYS_INLINE Packet2d madd(Packet2d acc, Packet2d a, Vec b) {
2570 return vfmaq_laneq_f64(acc, a, C < 2 ? b.lo : b.hi, C & 1);
2573 static EIGEN_ALWAYS_INLINE Packet2d nmadd(Packet2d acc, Packet2d a, Vec b) {
2574 return vfmsq_laneq_f64(acc, a, C < 2 ? b.lo : b.hi, C & 1);
2578struct sme_neon_cols<double, 2> {
2579 using Vec = float64x2_t;
2580 static EIGEN_ALWAYS_INLINE Vec load(
const double* b) {
return vld1q_f64(b); }
2582 static EIGEN_ALWAYS_INLINE Packet2d madd(Packet2d acc, Packet2d a, Vec b) {
2583 return vfmaq_laneq_f64(acc, a, b, C);
2586 static EIGEN_ALWAYS_INLINE Packet2d nmadd(Packet2d acc, Packet2d a, Vec b) {
2587 return vfmsq_laneq_f64(acc, a, b, C);
2591struct sme_neon_cols<double, 1> {
2592 using Vec = float64x2_t;
2593 static EIGEN_ALWAYS_INLINE Vec load(
const double* b) {
return vld1q_dup_f64(b); }
2595 static EIGEN_ALWAYS_INLINE Packet2d madd(Packet2d acc, Packet2d a, Vec b) {
2596 return vfmaq_f64(acc, a, b);
2599 static EIGEN_ALWAYS_INLINE Packet2d nmadd(Packet2d acc, Packet2d a, Vec b) {
2600 return vfmsq_f64(acc, a, b);
2605struct sme_neon_cols<float, 8> {
2609 static EIGEN_ALWAYS_INLINE Vec load(
const float* b) {
return {vld1q_f32(b), vld1q_f32(b + 4)}; }
2611 static EIGEN_ALWAYS_INLINE Packet4f madd(Packet4f acc, Packet4f a, Vec b) {
2612 return vfmaq_laneq_f32(acc, a, C < 4 ? b.lo : b.hi, C & 3);
2615 static EIGEN_ALWAYS_INLINE Packet4f nmadd(Packet4f acc, Packet4f a, Vec b) {
2616 return vfmsq_laneq_f32(acc, a, C < 4 ? b.lo : b.hi, C & 3);
2620struct sme_neon_cols<double, 8> {
2624 static EIGEN_ALWAYS_INLINE Vec load(
const double* b) {
2625 return {{vld1q_f64(b), vld1q_f64(b + 2), vld1q_f64(b + 4), vld1q_f64(b + 6)}};
2628 static EIGEN_ALWAYS_INLINE Packet2d madd(Packet2d acc, Packet2d a, Vec b) {
2629 return vfmaq_laneq_f64(acc, a, b.v[C >> 1], C & 1);
2632 static EIGEN_ALWAYS_INLINE Packet2d nmadd(Packet2d acc, Packet2d a, Vec b) {
2633 return vfmsq_laneq_f64(acc, a, b.v[C >> 1], C & 1);
2638template <
int C,
int NCol>
2639struct sme_neon_col_loop {
2640 template <
typename Cols,
typename Packet,
int NPack>
2641 static EIGEN_ALWAYS_INLINE
void real(Packet (&acc)[NPack][NCol],
const Packet (&av)[NPack],
typename Cols::Vec b) {
2642 for (
int p = 0; p < NPack; ++p) acc[p][C] = Cols::template madd<C>(acc[p][C], av[p], b);
2643 sme_neon_col_loop<C + 1, NCol>::template real<Cols, Packet, NPack>(acc, av, b);
2645 template <
bool ConjLhs,
bool ConjRhs,
typename Cols,
typename Packet,
int NPack>
2646 static EIGEN_ALWAYS_INLINE
void cplx(Packet (&acc_re)[NPack][NCol], Packet (&acc_im)[NPack][NCol],
2647 const Packet (&are)[NPack],
const Packet (&aim)[NPack],
typename Cols::Vec bre,
2648 typename Cols::Vec bim) {
2649 for (
int p = 0; p < NPack; ++p) {
2650 acc_re[p][C] = Cols::template madd<C>(acc_re[p][C], are[p], bre);
2651 acc_re[p][C] = (ConjLhs != ConjRhs) ? Cols::template madd<C>(acc_re[p][C], aim[p], bim)
2652 : Cols::template nmadd<C>(acc_re[p][C], aim[p], bim);
2653 acc_im[p][C] = ConjLhs ? Cols::template nmadd<C>(acc_im[p][C], aim[p], bre)
2654 : Cols::template madd<C>(acc_im[p][C], aim[p], bre);
2655 acc_im[p][C] = ConjRhs ? Cols::template nmadd<C>(acc_im[p][C], are[p], bim)
2656 : Cols::template madd<C>(acc_im[p][C], are[p], bim);
2658 sme_neon_col_loop<C + 1, NCol>::template cplx<ConjLhs, ConjRhs, Cols, Packet, NPack>(acc_re, acc_im, are, aim, bre,
2663struct sme_neon_col_loop<NCol, NCol> {
2664 template <
typename Cols,
typename Packet,
int NPack>
2665 static EIGEN_ALWAYS_INLINE
void real(Packet (&)[NPack][NCol],
const Packet (&)[NPack],
typename Cols::Vec) {}
2666 template <
bool ConjLhs,
bool ConjRhs,
typename Cols,
typename Packet,
int NPack>
2667 static EIGEN_ALWAYS_INLINE
void cplx(Packet (&)[NPack][NCol], Packet (&)[NPack][NCol],
const Packet (&)[NPack],
2668 const Packet (&)[NPack],
typename Cols::Vec,
typename Cols::Vec) {}
2674template <
typename Scalar,
typename Index,
int NPack,
int NCol>
2675struct sme_neon_tile {
2676 using Packet =
typename packet_traits<Scalar>::type;
2677 using Cols = sme_neon_cols<Scalar, NCol>;
2678 static constexpr int PS = packet_traits<Scalar>::size;
2680 static constexpr int KU = (NPack * NCol >= 8) ? 1 : 8 / (NPack * NCol);
2681 static EIGEN_ALWAYS_INLINE
void step(Packet (&acc)[NPack][NCol],
const Scalar* a,
const Scalar* b) {
2683 for (
int p = 0; p < NPack; ++p) av[p] = ploadu<Packet>(a + p * PS);
2684 sme_neon_col_loop<0, NCol>::template real<Cols, Packet, NPack>(acc, av, Cols::load(b));
2686 static EIGEN_ALWAYS_INLINE
void run(Scalar* C, Index rs, Index cs,
const Scalar* blA,
const Scalar* blB, Index depth,
2687 Scalar alpha, Index as, Index bs) {
2688 Packet acc[KU][NPack][NCol];
2689 for (
int u = 0; u < KU; ++u)
2690 for (
int p = 0; p < NPack; ++p)
2691 for (
int c = 0; c < NCol; ++c) acc[u][p][c] = pset1<Packet>(Scalar(0));
2693 for (; k + KU <= depth; k += KU) {
2694 for (
int u = 0; u < KU; ++u) step(acc[u], blA + (k + u) * as, blB + (k + u) * bs);
2696 for (; k < depth; ++k) step(acc[0], blA + k * as, blB + k * bs);
2697 for (
int u = 1; u < KU; ++u)
2698 for (
int p = 0; p < NPack; ++p)
2699 for (
int c = 0; c < NCol; ++c) acc[0][p][c] = padd(acc[0][p][c], acc[u][p][c]);
2700 const Packet valpha = pset1<Packet>(alpha);
2701 for (
int c = 0; c < NCol; ++c) {
2702 for (
int p = 0; p < NPack; ++p) {
2703 Scalar* pc = C + Index(p * PS) * rs + Index(c) * cs;
2705 pstoreu(pc, pmadd(acc[0][p][c], valpha, ploadu<Packet>(pc)));
2707 pscatter<Scalar, Packet>(pc, pmadd(acc[0][p][c], valpha, pgather<Scalar, Packet>(pc, rs)), rs);
2714template <
typename Scalar,
typename Index,
int NCol>
2715struct sme_neon_tile<Scalar, Index, 0, NCol> {
2716 static EIGEN_ALWAYS_INLINE
void run(Scalar* C, Index rs, Index cs,
const Scalar* blA,
const Scalar* blB, Index depth,
2717 Scalar alpha, Index as, Index bs) {
2718 EIGEN_UNUSED_VARIABLE(rs);
2719 constexpr int KU = (NCol >= 4) ? 1 : 4 / NCol;
2720 Scalar acc[KU][NCol];
2721 for (
int u = 0; u < KU; ++u)
2722 for (
int c = 0; c < NCol; ++c) acc[u][c] = Scalar(0);
2724 for (; k + KU <= depth; k += KU) {
2725 for (
int u = 0; u < KU; ++u) {
2726 const Scalar a = blA[(k + u) * as];
2727 const Scalar* b = blB + (k + u) * bs;
2728 for (
int c = 0; c < NCol; ++c) acc[u][c] += a * b[c];
2731 for (; k < depth; ++k) {
2732 const Scalar a = blA[k * as];
2733 const Scalar* b = blB + k * bs;
2734 for (
int c = 0; c < NCol; ++c) acc[0][c] += a * b[c];
2736 for (
int u = 1; u < KU; ++u)
2737 for (
int c = 0; c < NCol; ++c) acc[0][c] += acc[u][c];
2738 for (
int c = 0; c < NCol; ++c) C[Index(c) * cs] += alpha * acc[0][c];
2744template <
typename RealScalar>
2745EIGEN_ALWAYS_INLINE
void sme_neon_cstore_scalar(RealScalar* pc, RealScalar re, RealScalar im, RealScalar ar,
2747 if (ai == RealScalar(0)) {
2751 pc[0] += ar * re - ai * im;
2752 pc[1] += ar * im + ai * re;
2758template <
typename RealScalar,
typename Index,
bool ConjLhs,
bool ConjRhs,
int NPack,
int NCol>
2759struct sme_neon_ctile {
2760 using Scalar = std::complex<RealScalar>;
2761 using Packet =
typename packet_traits<RealScalar>::type;
2762 using Cols = sme_neon_cols<RealScalar, NCol>;
2763 static constexpr int PS = packet_traits<RealScalar>::size;
2769 template <
bool RealAlpha>
2770 static EIGEN_ALWAYS_INLINE
void store_unit_stride(Scalar* C, Index cs,
const Packet (&acc_re)[NPack][NCol],
2771 const Packet (&acc_im)[NPack][NCol], RealScalar ar, RealScalar ai) {
2772 const Packet valpha_re = pset1<Packet>(ar), valpha_im = pset1<Packet>(ai);
2773 for (
int c = 0; c < NCol; ++c) {
2774 for (
int p = 0; p < NPack; ++p) {
2775 RealScalar* pc =
reinterpret_cast<RealScalar*
>(C + Index(p * PS) + Index(c) * cs);
2777 sme_neon_ld2(pc, cre, cim);
2778 EIGEN_IF_CONSTEXPR (RealAlpha) {
2779 cre = pmadd(valpha_re, acc_re[p][c], cre);
2780 cim = pmadd(valpha_re, acc_im[p][c], cim);
2782 cre = padd(cre, pnmadd(valpha_im, acc_im[p][c], pmul(valpha_re, acc_re[p][c])));
2783 cim = padd(cim, pmadd(valpha_im, acc_re[p][c], pmul(valpha_re, acc_im[p][c])));
2785 sme_neon_st2(pc, cre, cim);
2790 static constexpr int KU = (NPack * NCol >= 8) ? 1 : 8 / (NPack * NCol);
2791 static EIGEN_ALWAYS_INLINE
void step(Packet (&acc_re)[NPack][NCol], Packet (&acc_im)[NPack][NCol],
2792 const RealScalar* a,
const RealScalar* b, Index pw, Index cw) {
2793 Packet are[NPack], aim[NPack];
2794 for (
int p = 0; p < NPack; ++p) {
2795 are[p] = ploadu<Packet>(a + p * PS);
2796 aim[p] = ploadu<Packet>(a + pw + p * PS);
2798 sme_neon_col_loop<0, NCol>::template cplx<ConjLhs, ConjRhs, Cols, Packet, NPack>(acc_re, acc_im, are, aim,
2799 Cols::load(b), Cols::load(b + cw));
2801 static EIGEN_ALWAYS_INLINE
void run(Scalar* C, Index rs, Index cs,
const RealScalar* rA,
const RealScalar* rB,
2802 Index depth, Scalar alpha, Index pw, Index cw) {
2803 Packet acc_re[KU][NPack][NCol], acc_im[KU][NPack][NCol];
2804 for (
int u = 0; u < KU; ++u) {
2805 for (
int p = 0; p < NPack; ++p) {
2806 for (
int c = 0; c < NCol; ++c) {
2807 acc_re[u][p][c] = pset1<Packet>(RealScalar(0));
2808 acc_im[u][p][c] = pset1<Packet>(RealScalar(0));
2813 for (; k + KU <= depth; k += KU) {
2814 for (
int u = 0; u < KU; ++u) step(acc_re[u], acc_im[u], rA + (k + u) * 2 * pw, rB + (k + u) * 2 * cw, pw, cw);
2816 for (; k < depth; ++k) step(acc_re[0], acc_im[0], rA + k * 2 * pw, rB + k * 2 * cw, pw, cw);
2817 for (
int u = 1; u < KU; ++u) {
2818 for (
int p = 0; p < NPack; ++p) {
2819 for (
int c = 0; c < NCol; ++c) {
2820 acc_re[0][p][c] = padd(acc_re[0][p][c], acc_re[u][p][c]);
2821 acc_im[0][p][c] = padd(acc_im[0][p][c], acc_im[u][p][c]);
2825 const RealScalar ar = numext::real(alpha), ai = numext::imag(alpha);
2827 if (ai == RealScalar(0)) {
2828 store_unit_stride<true>(C, cs, acc_re[0], acc_im[0], ar, ai);
2830 store_unit_stride<false>(C, cs, acc_re[0], acc_im[0], ar, ai);
2834 for (
int c = 0; c < NCol; ++c) {
2835 for (
int p = 0; p < NPack; ++p) {
2836 RealScalar* pc =
reinterpret_cast<RealScalar*
>(C + Index(p * PS) * rs + Index(c) * cs);
2837 RealScalar re[PS], im[PS];
2838 pstoreu(re, acc_re[0][p][c]);
2839 pstoreu(im, acc_im[0][p][c]);
2840 for (
int l = 0; l < PS; ++l) sme_neon_cstore_scalar(pc + Index(l) * 2 * rs, re[l], im[l], ar, ai);
2846template <
typename RealScalar,
typename Index,
bool ConjLhs,
bool ConjRhs,
int NCol>
2847struct sme_neon_ctile<RealScalar, Index, ConjLhs, ConjRhs, 0, NCol> {
2848 using Scalar = std::complex<RealScalar>;
2849 static EIGEN_ALWAYS_INLINE
void run(Scalar* C, Index rs, Index cs,
const RealScalar* rA,
const RealScalar* rB,
2850 Index depth, Scalar alpha, Index pw, Index cw) {
2851 EIGEN_UNUSED_VARIABLE(rs);
2852 constexpr int KU = (NCol >= 4) ? 1 : 4 / NCol;
2853 RealScalar acc_re[KU][NCol], acc_im[KU][NCol];
2854 for (
int u = 0; u < KU; ++u)
2855 for (
int c = 0; c < NCol; ++c) acc_re[u][c] = acc_im[u][c] = RealScalar(0);
2856 for (Index k = 0; k < depth; ++k) {
2857 const int u = int(k % KU);
2858 const RealScalar* a = rA + k * 2 * pw;
2859 const RealScalar* b = rB + k * 2 * cw;
2860 const RealScalar are = a[0], aim = ConjLhs ? -a[pw] : a[pw];
2861 for (
int c = 0; c < NCol; ++c) {
2862 const RealScalar bre = b[c], bim = ConjRhs ? -b[cw + c] : b[cw + c];
2863 acc_re[u][c] += are * bre - aim * bim;
2864 acc_im[u][c] += are * bim + aim * bre;
2867 for (
int u = 1; u < KU; ++u)
2868 for (
int c = 0; c < NCol; ++c) {
2869 acc_re[0][c] += acc_re[u][c];
2870 acc_im[0][c] += acc_im[u][c];
2872 const RealScalar ar = numext::real(alpha), ai = numext::imag(alpha);
2873 for (
int c = 0; c < NCol; ++c)
2874 sme_neon_cstore_scalar(
reinterpret_cast<RealScalar*
>(C + Index(c) * cs), acc_re[0][c], acc_im[0][c], ar, ai);
2880template <
int NCol,
typename Scalar,
typename Index>
2881EIGEN_ALWAYS_INLINE
void sme_neon_row_tail(std::true_type, Scalar* C, Index rs, Index cs,
const Scalar* A, Index as,
2882 const Scalar* B, Index bs, Index depth, Scalar alpha, Index r, Index rows) {
2883 constexpr int NV = NCol / packet_traits<Scalar>::size;
2884 for (; r + 2 <= rows; r += 2)
2885 sme_neon_tile<Scalar, Index, NV, 2>::run(C + r * rs, cs, rs, B, A + r, depth, alpha, bs, as);
2886 if (r < rows) sme_neon_tile<Scalar, Index, NV, 1>::run(C + r * rs, cs, rs, B, A + r, depth, alpha, bs, as);
2888template <
int NCol,
typename Scalar,
typename Index>
2889EIGEN_ALWAYS_INLINE
void sme_neon_row_tail(std::false_type, Scalar* C, Index rs, Index cs,
const Scalar* A, Index as,
2890 const Scalar* B, Index bs, Index depth, Scalar alpha, Index r, Index rows) {
2891 for (; r < rows; ++r) sme_neon_tile<Scalar, Index, 0, NCol>::run(C + r * rs, rs, cs, A + r, B, depth, alpha, as, bs);
2896template <
int NCol,
typename Scalar,
typename Index>
2897EIGEN_ALWAYS_INLINE
void sme_neon_column_group(Scalar* C, Index rs, Index cs,
const Scalar* A, Index as,
2898 const Scalar* B, Index bs, Index depth, Scalar alpha, Index rows) {
2899 constexpr Index PS = Index(packet_traits<Scalar>::size);
2901 for (; r + 3 * PS <= rows; r += 3 * PS)
2902 sme_neon_tile<Scalar, Index, 3, NCol>::run(C + r * rs, rs, cs, A + r, B, depth, alpha, as, bs);
2903 for (; r + 2 * PS <= rows; r += 2 * PS)
2904 sme_neon_tile<Scalar, Index, 2, NCol>::run(C + r * rs, rs, cs, A + r, B, depth, alpha, as, bs);
2905 for (; r + PS <= rows; r += PS)
2906 sme_neon_tile<Scalar, Index, 1, NCol>::run(C + r * rs, rs, cs, A + r, B, depth, alpha, as, bs);
2907 sme_neon_row_tail<NCol>(bool_constant<NCol % PS == 0>(), C, rs, cs, A, as, B, bs, depth, alpha, r, rows);
2910template <
bool ConjLhs,
bool ConjRhs,
int NCol,
typename RealScalar,
typename Index>
2911EIGEN_ALWAYS_INLINE
void sme_neon_crow_tail(std::true_type, std::complex<RealScalar>* C, Index rs, Index cs,
2912 const RealScalar* rA,
const RealScalar* rB, Index depth,
2913 std::complex<RealScalar> alpha, Index r, Index pw, Index cw) {
2914 constexpr int NV = NCol / packet_traits<RealScalar>::size;
2916 for (; r + 2 <= pw; r += 2)
2917 sme_neon_ctile<RealScalar, Index, ConjRhs, ConjLhs, NV, 2>::run(C + r * rs, cs, rs, rB, rA + r, depth, alpha, cw,
2920 sme_neon_ctile<RealScalar, Index, ConjRhs, ConjLhs, NV, 1>::run(C + r * rs, cs, rs, rB, rA + r, depth, alpha, cw,
2923template <
bool ConjLhs,
bool ConjRhs,
int NCol,
typename RealScalar,
typename Index>
2924EIGEN_ALWAYS_INLINE
void sme_neon_crow_tail(std::false_type, std::complex<RealScalar>* C, Index rs, Index cs,
2925 const RealScalar* rA,
const RealScalar* rB, Index depth,
2926 std::complex<RealScalar> alpha, Index r, Index pw, Index cw) {
2928 sme_neon_ctile<RealScalar, Index, ConjLhs, ConjRhs, 0, NCol>::run(C + r * rs, rs, cs, rA + r, rB, depth, alpha, pw,
2932template <
bool ConjLhs,
bool ConjRhs,
int NCol,
typename RealScalar,
typename Index>
2933EIGEN_ALWAYS_INLINE
void sme_neon_column_group(std::complex<RealScalar>* C, Index rs, Index cs,
const RealScalar* rA,
2934 const RealScalar* rB, Index depth, std::complex<RealScalar> alpha,
2935 Index pw, Index cw) {
2937 constexpr Index PS = Index(packet_traits<RealScalar>::size);
2939 for (; r + 2 * PS <= pw; r += 2 * PS)
2940 sme_neon_ctile<RealScalar, Index, ConjLhs, ConjRhs, 2, NCol>::run(C + r * rs, rs, cs, rA + r, rB, depth, alpha, pw,
2942 for (; r + PS <= pw; r += PS)
2943 sme_neon_ctile<RealScalar, Index, ConjLhs, ConjRhs, 1, NCol>::run(C + r * rs, rs, cs, rA + r, rB, depth, alpha, pw,
2945 sme_neon_crow_tail<ConjLhs, ConjRhs, NCol>(bool_constant<NCol % PS == 0>(), C, rs, cs, rA, rB, depth, alpha, r, pw,
2950template <
typename Scalar,
typename Index>
2951EIGEN_ALWAYS_INLINE
void sme_neon_block_strided(Scalar* C, Index rs, Index cs,
const Scalar* A, Index as,
2952 const Scalar* B, Index bs, Index depth, Scalar alpha, Index rows,
2955 for (; c + 8 <= cols; c += 8) sme_neon_column_group<8>(C + c * cs, rs, cs, A, as, B + c, bs, depth, alpha, rows);
2956 for (; c + 4 <= cols; c += 4) sme_neon_column_group<4>(C + c * cs, rs, cs, A, as, B + c, bs, depth, alpha, rows);
2957 for (; c + 2 <= cols; c += 2) sme_neon_column_group<2>(C + c * cs, rs, cs, A, as, B + c, bs, depth, alpha, rows);
2958 for (; c < cols; ++c) sme_neon_column_group<1>(C + c * cs, rs, cs, A, as, B + c, bs, depth, alpha, rows);
2962template <
bool ConjLhs,
bool ConjRhs,
typename Scalar,
typename Index>
2963EIGEN_ALWAYS_INLINE
void sme_neon_block(Scalar* C, Index rs, Index cs,
const Scalar* blA,
const Scalar* blB,
2964 Index depth, Scalar alpha, Index pw, Index cw) {
2965 sme_neon_block_strided(C, rs, cs, blA, pw, blB, cw, depth, alpha, pw, cw);
2968template <
bool ConjLhs,
bool ConjRhs,
typename RealScalar,
typename Index>
2969EIGEN_ALWAYS_INLINE
void sme_neon_block(std::complex<RealScalar>* C, Index rs, Index cs,
2970 const std::complex<RealScalar>* blA,
const std::complex<RealScalar>* blB,
2971 Index depth, std::complex<RealScalar> alpha, Index pw, Index cw) {
2972 const RealScalar* rA =
reinterpret_cast<const RealScalar*
>(blA);
2973 const RealScalar* rB =
reinterpret_cast<const RealScalar*
>(blB);
2975 for (; c + 4 <= cw; c += 4)
2976 sme_neon_column_group<ConjLhs, ConjRhs, 4>(C + c * cs, rs, cs, rA, rB + c, depth, alpha, pw, cw);
2977 for (; c + 2 <= cw; c += 2)
2978 sme_neon_column_group<ConjLhs, ConjRhs, 2>(C + c * cs, rs, cs, rA, rB + c, depth, alpha, pw, cw);
2979 for (; c < cw; ++c) sme_neon_column_group<ConjLhs, ConjRhs, 1>(C + c * cs, rs, cs, rA, rB + c, depth, alpha, pw, cw);
2983template <
typename Scalar,
bool ConjLhs,
bool ConjRhs,
typename Index>
2984EIGEN_ALWAYS_INLINE
void sme_gebp_neon(Scalar* C, Index C_stride_row, Index C_stride_col,
const Scalar* blockA,
2985 const Scalar* blockB, Index rows, Index depth, Index cols, Scalar alpha,
2986 Index strideA, Index strideB, Index offsetA, Index offsetB) {
2987 constexpr Index MR = Index(sme_block<Scalar>::mr);
2988 constexpr Index NR = Index(sme_block<Scalar>::nr);
2989 for (Index j = 0; j < cols; j += NR) {
2990 const Index cw = numext::mini(cols - j, NR);
2991 const Scalar* blB = blockB + j * strideB + offsetB * cw;
2992 for (Index i = 0; i < rows; i += MR) {
2993 const Index pw = numext::mini(rows - i, MR);
2994 const Scalar* blA = blockA + i * strideA + offsetA * pw;
2995 sme_neon_block<ConjLhs, ConjRhs>(C + i * C_stride_row + j * C_stride_col, C_stride_row, C_stride_col, blA, blB,
2996 depth, alpha, pw, cw);
3001template <
typename Scalar,
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3002struct sme_gebp_kernel {
3003 using ResScalar = Scalar;
3006 static constexpr bool CanOverwrite =
true;
3011 static bool overwrite_pays(Index rows, Index cols, Index depth) {
3012 if (rows <= 0 || cols <= 0 || sme_kernel_with_neon<Scalar>(rows, cols, depth, depth, depth))
return false;
3014 return sizeof(Scalar) == 4 || rows <= Index((128 * 1024) /
sizeof(Scalar)) / cols;
3017 EIGEN_DONT_INLINE
void operator()(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB, Index rows,
3018 Index depth, Index cols, ResScalar alpha, Index strideA = -1, Index strideB = -1,
3019 Index offsetA = 0, Index offsetB = 0) {
3020 run(
false, res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3023 EIGEN_DONT_INLINE
void operator()(gebp_overwrite_t,
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB,
3024 Index rows, Index depth, Index cols, ResScalar alpha, Index strideA = -1,
3025 Index strideB = -1, Index offsetA = 0, Index offsetB = 0) {
3026 run(
true, res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3031 EIGEN_DONT_INLINE
void run_direct_lhs(
const DataMapper& res,
const Scalar* lhs, Index lhsStride,
const Scalar* blockB,
3032 Index rows, Index depth, Index cols, ResScalar alpha, Index strideB = -1,
3033 Index offsetB = 0) {
3034 run_direct_lhs_impl(
false, res, lhs, lhsStride, blockB, rows, depth, cols, alpha, strideB, offsetB);
3036 EIGEN_DONT_INLINE
void run_direct_lhs(gebp_overwrite_t,
const DataMapper& res,
const Scalar* lhs, Index lhsStride,
3037 const Scalar* blockB, Index rows, Index depth, Index cols, ResScalar alpha,
3038 Index strideB = -1, Index offsetB = 0) {
3039 run_direct_lhs_impl(
true, res, lhs, lhsStride, blockB, rows, depth, cols, alpha, strideB, offsetB);
3044 EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE
void run(
bool overwrite,
const DataMapper& res,
const Scalar* blockA,
3045 const Scalar* blockB, Index rows, Index depth, Index cols,
3046 ResScalar alpha, Index strideA, Index strideB, Index offsetA,
3050 static_assert(NumTraits<Scalar>::IsComplex || (!ConjugateLhs && !ConjugateRhs),
3051 "the SME kernel does not support conjugation of real scalars");
3052 static_assert(mr == sme_block<Scalar>::mr && nr == sme_block<Scalar>::nr,
3053 "the SME kernel expects packed panels of the SME block width");
3055 if (strideA == -1) strideA = depth;
3056 if (strideB == -1) strideB = depth;
3058 if (rows <= 0 || cols <= 0)
return;
3060 Scalar* C_base =
const_cast<Scalar*
>(&res(0, 0));
3061 const Index C_stride_row = &res(1, 0) - &res(0, 0);
3062 const Index C_stride_col = &res(0, 1) - &res(0, 0);
3065 if (overwrite) gemm_zero_result(C_base, rows, cols, C_stride_row, C_stride_col);
3068 if (sme_kernel_with_neon<Scalar>(rows, cols, depth, strideA, strideB)) {
3069 if (overwrite) gemm_zero_result(C_base, rows, cols, C_stride_row, C_stride_col);
3070 sme_gebp_neon<Scalar, ConjugateLhs, ConjugateRhs>(C_base, C_stride_row, C_stride_col, blockA, blockB, rows, depth,
3071 cols, alpha, strideA, strideB, offsetA, offsetB);
3075 sme_fpsr_guard fpsr;
3076 const bool zero_first = overwrite && NumTraits<Scalar>::IsComplex;
3077 if (zero_first) sme_zero_c(C_base, C_stride_row, C_stride_col, rows, cols);
3078 sme_gebp_dispatch<Scalar, ConjugateLhs, ConjugateRhs>::run(C_base, C_stride_row, C_stride_col, blockA, blockB, rows,
3079 depth, cols, alpha, strideA, strideB, offsetA, offsetB,
3080 overwrite && !zero_first);
3083 EIGEN_DONT_INLINE EIGEN_GEMM_NO_CLONE
void run_direct_lhs_impl(
bool overwrite,
const DataMapper& res,
3084 const Scalar* lhs, Index lhsStride,
3085 const Scalar* blockB, Index rows, Index depth,
3086 Index cols, ResScalar alpha, Index strideB,
3088 if (strideB == -1) strideB = depth;
3089 if (rows <= 0 || cols <= 0)
return;
3090 Scalar* C_base =
const_cast<Scalar*
>(&res(0, 0));
3091 const Index C_stride_row = &res(1, 0) - &res(0, 0);
3092 const Index C_stride_col = &res(0, 1) - &res(0, 0);
3094 if (overwrite) gemm_zero_result(C_base, rows, cols, C_stride_row, C_stride_col);
3097 sme_fpsr_guard fpsr;
3098 const bool zero_first = overwrite && NumTraits<Scalar>::IsComplex;
3099 if (zero_first) sme_zero_c(C_base, C_stride_row, C_stride_col, rows, cols);
3100 sme_gebp_impl_direct_lhs<Scalar, Index>(C_base, C_stride_row, C_stride_col, lhs, lhsStride, blockB, rows, depth,
3101 cols, alpha, strideB, offsetB, overwrite && !zero_first);
3105#define EIGEN_SME_DECLARE_GEBP_KERNEL(SCALAR) \
3106 template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs> \
3107 struct gebp_kernel<SCALAR, SCALAR, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> \
3108 : sme_gebp_kernel<SCALAR, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {};
3110EIGEN_SME_DECLARE_GEBP_KERNEL(
float)
3111EIGEN_SME_DECLARE_GEBP_KERNEL(std::complex<float>)
3112#ifdef EIGEN_VECTORIZE_SME_F64F64
3113EIGEN_SME_DECLARE_GEBP_KERNEL(
double)
3114EIGEN_SME_DECLARE_GEBP_KERNEL(std::complex<double>)
3117#undef EIGEN_SME_DECLARE_GEBP_KERNEL
3123static_assert(sme_has_gebp_kernel<float, float>::value,
"the SME float kernel is not advertised to the GEMM driver");
3124static_assert(sme_has_gebp_kernel<std::complex<float>, std::complex<float>>::value,
3125 "the SME complex<float> kernel is not advertised to the GEMM driver");
3126#ifdef EIGEN_VECTORIZE_SME_F64F64
3127static_assert(sme_has_gebp_kernel<double, double>::value,
"the SME double kernel is not advertised to the GEMM driver");
3128static_assert(sme_has_gebp_kernel<std::complex<double>, std::complex<double>>::value,
3129 "the SME complex<double> kernel is not advertised to the GEMM driver");
3131static_assert(!sme_has_gebp_kernel<double, double>::value,
3132 "double is advertised to the GEMM driver without FEAT_SME_F64F64 to implement it");
3133static_assert(!sme_has_gebp_kernel<std::complex<double>, std::complex<double>>::value,
3134 "complex<double> is advertised to the GEMM driver without FEAT_SME_F64F64 to implement it");
3176template <
typename Index>
3177static EIGEN_ALWAYS_INLINE
void sme_symm_panel_regions(Index j,
int w, Index depth, Index k2, Index& t_end,
3178 Index& s_end) __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
3179 const Index raw_t = j - k2, raw_s = j + Index(w) - k2;
3180 t_end = raw_t <= 0 ? Index(0) : sme_min(raw_t, depth);
3181 s_end = raw_s <= 0 ? Index(0) : sme_min(raw_s, depth);
3186template <
typename Scalar,
int StorageOrder,
bool IsLhs,
typename Index>
3187EIGEN_DONT_INLINE __arm_locally_streaming __arm_new(
"za") void sme_symm_pack_dense_regions(
3188 Scalar* block, const Scalar* EIGEN_RESTRICT base, Index stride, Index depth, Index outer, Index k2) {
3189 constexpr int PACK = IsLhs ? sme_block<Scalar>::mr : sme_block<Scalar>::nr;
3190 constexpr bool ColM = (StorageOrder ==
ColMajor);
3193 constexpr bool ConjTransposed = !IsLhs;
3194 constexpr bool ConjDirect = IsLhs;
3196 for (Index j = 0; j < outer; j += PACK) {
3197 const int w =
static_cast<int>(sme_min(outer - j, Index(PACK)));
3198 Scalar* dst = block + j * depth;
3200 sme_symm_panel_regions(j, w, depth, k2, t_end, s_end);
3204 EIGEN_IF_CONSTEXPR (ColM) {
3205 sve_copy_panel_range<ConjTransposed>(dst, base + j + k2 * stride, stride, Index(0), t_end, w);
3207 sme_transpose_pack_range<ConjTransposed>(dst, base + j * stride + k2, stride, Index(0), t_end, w);
3211 if (s_end < depth) {
3212 EIGEN_IF_CONSTEXPR (ColM) {
3213 sme_transpose_pack_range<ConjDirect>(dst, base + k2 + j * stride, stride, s_end, depth, w);
3215 sve_copy_panel_range<ConjDirect>(dst, base + k2 * stride + j, stride, s_end, depth, w);
3229template <
typename Scalar,
int StorageOrder,
bool IsLhs,
typename Index>
3230EIGEN_DONT_INLINE
void sme_symm_pack_straddle(Scalar* block,
const Scalar* EIGEN_RESTRICT base, Index stride,
3231 Index depth, Index outer, Index k2) {
3232 constexpr int PACK = IsLhs ? sme_block<Scalar>::mr : sme_block<Scalar>::nr;
3233 constexpr bool ColM = (StorageOrder ==
ColMajor);
3236 constexpr bool ConjHead = IsLhs;
3237 constexpr bool ConjTail = !IsLhs;
3239 for (Index j = 0; j < outer; j += PACK) {
3240 const int w =
static_cast<int>(numext::mini(outer - j, Index(PACK)));
3241 Scalar* dst = block + j * depth;
3243 sme_symm_panel_regions(j, w, depth, k2, t_end, s_end);
3245 for (Index k = t_end; k < s_end; ++k) {
3246 const Index row = k2 + k;
3247 const int cs =
static_cast<int>(row - j);
3248 Scalar* dst_row = dst + k * w;
3249 const Index wi = Index(w);
3250 EIGEN_IF_CONSTEXPR (ColM) {
3251 const Scalar* head = base + row + j * stride;
3252 for (
int c = 0; c < cs; ++c, head += stride) sme_pack_store<ConjHead>(dst_row, wi, Index(c), *head);
3253 const Scalar* tail = base + j + row * stride;
3257 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
3258 sme_pack_store<false>(dst_row, wi, Index(cs), Scalar(numext::real(tail[cs])));
3261 for (; c < w; ++c) sme_pack_store<ConjTail>(dst_row, wi, Index(c), tail[c]);
3263 const Scalar* head = base + row * stride + j;
3264 for (
int c = 0; c < cs; ++c) sme_pack_store<ConjHead>(dst_row, wi, Index(c), head[c]);
3265 const Scalar* tail = base + (j + Index(cs)) * stride + row;
3267 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex) {
3268 sme_pack_store<false>(dst_row, wi, Index(cs), Scalar(numext::real(*tail)));
3272 for (; c < w; ++c, tail += stride) sme_pack_store<ConjTail>(dst_row, wi, Index(c), *tail);
3279template <
typename Scalar,
int StorageOrder,
bool IsLhs,
typename Index>
3280EIGEN_DONT_INLINE
void sme_symm_pack_panels(Scalar* block,
const Scalar* EIGEN_RESTRICT base, Index stride, Index depth,
3281 Index outer, Index k2) {
3283 sme_fpsr_guard fpsr;
3284 sme_symm_pack_dense_regions<Scalar, StorageOrder, IsLhs, Index>(block, base, stride, depth, outer, k2);
3286 sme_symm_pack_straddle<Scalar, StorageOrder, IsLhs, Index>(block, base, stride, depth, outer, k2);
3291template <
typename Scalar,
int StorageOrder,
typename Index>
3292struct sme_symm_pack_lhs {
3295 EIGEN_DONT_INLINE
void operator()(Scalar* blockA,
const Scalar* lhs_, Index lhsStride, Index cols, Index rows)
const {
3296 sme_symm_pack_panels<Scalar, StorageOrder, true, Index>(blockA, lhs_, lhsStride, cols, rows, Index(0));
3300template <
typename Scalar,
int StorageOrder,
typename Index>
3301struct sme_symm_pack_rhs {
3303 EIGEN_DONT_INLINE
void operator()(Scalar* blockB,
const Scalar* rhs_, Index rhsStride, Index rows, Index cols,
3305 sme_symm_pack_panels<Scalar, StorageOrder, false, Index>(blockB, rhs_, rhsStride, rows, cols, k2);
3309#define EIGEN_SME_DECLARE_SYMM_PACKERS(SCALAR, MR, NR) \
3310 template <typename Index, int Pack2_dummy, int StorageOrder> \
3311 struct symm_pack_lhs<SCALAR, Index, MR, Pack2_dummy, StorageOrder> \
3312 : sme_symm_pack_lhs<SCALAR, StorageOrder, Index> {}; \
3314 template <typename Index, int StorageOrder> \
3315 struct symm_pack_rhs<SCALAR, Index, NR, StorageOrder> : sme_symm_pack_rhs<SCALAR, StorageOrder, Index> {};
3317EIGEN_SME_DECLARE_SYMM_PACKERS(
float, kSmeMr, kSmeNr)
3318EIGEN_SME_DECLARE_SYMM_PACKERS(std::complex<float>, kSmeMrC, kSmeNrC)
3319#ifdef EIGEN_VECTORIZE_SME_F64F64
3320EIGEN_SME_DECLARE_SYMM_PACKERS(
double, kSmeMrD, kSmeNrD)
3321EIGEN_SME_DECLARE_SYMM_PACKERS(std::complex<double>, kSmeMrCD, kSmeNrCD)
3324#undef EIGEN_SME_DECLARE_SYMM_PACKERS
3329template <
typename Scalar>
3330struct sme_tiny_neon;
3332struct sme_tiny_neon<float> {
3333 using V = float32x4_t;
3334 static constexpr int PS = 4;
3335 static EIGEN_ALWAYS_INLINE V ld(
const float* p) {
return vld1q_f32(p); }
3336 static EIGEN_ALWAYS_INLINE V zero() {
return vdupq_n_f32(0.f); }
3338 static EIGEN_ALWAYS_INLINE V ld_part(
const float* p,
int n) {
3339 if (n <= 0)
return zero();
3340 if (n == 1)
return vld1q_lane_f32(p, zero(), 0);
3341 const V v = vcombine_f32(vld1_f32(p), vdup_n_f32(0.f));
3342 return n == 2 ? v : vld1q_lane_f32(p + 2, v, 2);
3345 static EIGEN_ALWAYS_INLINE V ld_tbl(
const float* p, uint8x16_t idx) {
3346 return vreinterpretq_f32_u8(vqtbl1q_u8(vreinterpretq_u8_f32(vld1q_f32(p)), idx));
3349 static EIGEN_ALWAYS_INLINE V fma_lane(V c, V a, V b) {
3350 return vfmaq_laneq_f32(c, a, b, L);
3352 static EIGEN_ALWAYS_INLINE V fma_n(V c, V a,
float b) {
return vfmaq_n_f32(c, a, b); }
3353 static EIGEN_ALWAYS_INLINE
void st(
float* p, V v) { vst1q_f32(p, v); }
3354 static EIGEN_ALWAYS_INLINE V add(V a, V b) {
return vaddq_f32(a, b); }
3356 static EIGEN_ALWAYS_INLINE
void transpose(V (&r)[PS]) {
3357 const float64x2_t t0 = vreinterpretq_f64_f32(vtrn1q_f32(r[0], r[1]));
3358 const float64x2_t t1 = vreinterpretq_f64_f32(vtrn2q_f32(r[0], r[1]));
3359 const float64x2_t t2 = vreinterpretq_f64_f32(vtrn1q_f32(r[2], r[3]));
3360 const float64x2_t t3 = vreinterpretq_f64_f32(vtrn2q_f32(r[2], r[3]));
3361 r[0] = vreinterpretq_f32_f64(vtrn1q_f64(t0, t2));
3362 r[1] = vreinterpretq_f32_f64(vtrn1q_f64(t1, t3));
3363 r[2] = vreinterpretq_f32_f64(vtrn2q_f64(t0, t2));
3364 r[3] = vreinterpretq_f32_f64(vtrn2q_f64(t1, t3));
3368struct sme_tiny_neon<double> {
3369 using V = float64x2_t;
3370 static constexpr int PS = 2;
3371 static EIGEN_ALWAYS_INLINE V ld(
const double* p) {
return vld1q_f64(p); }
3372 static EIGEN_ALWAYS_INLINE V zero() {
return vdupq_n_f64(0.); }
3373 static EIGEN_ALWAYS_INLINE V ld_part(
const double* p,
int n) {
return n > 0 ? vld1q_lane_f64(p, zero(), 0) : zero(); }
3374 static EIGEN_ALWAYS_INLINE V ld_tbl(
const double* p, uint8x16_t idx) {
3375 return vreinterpretq_f64_u8(vqtbl1q_u8(vreinterpretq_u8_f64(vld1q_f64(p)), idx));
3378 static EIGEN_ALWAYS_INLINE V fma_lane(V c, V a, V b) {
3379 return vfmaq_laneq_f64(c, a, b, L);
3381 static EIGEN_ALWAYS_INLINE V fma_n(V c, V a,
double b) {
return vfmaq_n_f64(c, a, b); }
3382 static EIGEN_ALWAYS_INLINE
void st(
double* p, V v) { vst1q_f64(p, v); }
3383 static EIGEN_ALWAYS_INLINE V add(V a, V b) {
return vaddq_f64(a, b); }
3384 static EIGEN_ALWAYS_INLINE
void transpose(V (&r)[PS]) {
3385 const V t0 = vtrn1q_f64(r[0], r[1]);
3386 r[1] = vtrn2q_f64(r[0], r[1]);
3393template <
typename T,
int RV,
int NC,
int S>
3394struct sme_tiny_step {
3395 using V =
typename T::V;
3396 static constexpr int PS = T::PS;
3399 static EIGEN_ALWAYS_INLINE
void col(V (&acc)[S][NC][RV],
const V (&a)[PS][RV],
const V (&b)[NC],
3400 std::integral_constant<int, U>) {
3402 for (
int j = 0; j < NC; ++j) {
3404 for (
int r = 0; r < RV; ++r) acc[U % S][j][r] = T::template fma_lane<U>(acc[U % S][j][r], a[U][r], b[j]);
3406 col(acc, a, b, std::integral_constant<int, U + 1>());
3408 static EIGEN_ALWAYS_INLINE
void col(V (&)[S][NC][RV],
const V (&)[PS][RV],
const V (&)[NC],
3409 std::integral_constant<int, PS>) {}
3412 static EIGEN_ALWAYS_INLINE
void row(V (&acc)[NC][RV],
const V (&a)[RV],
const V (&b)[NC / PS],
3413 std::integral_constant<int, J>) {
3415 for (
int r = 0; r < RV; ++r) acc[J][r] = T::template fma_lane<J % PS>(acc[J][r], a[r], b[J / PS]);
3416 row(acc, a, b, std::integral_constant<int, J + 1>());
3418 static EIGEN_ALWAYS_INLINE
void row(V (&)[NC][RV],
const V (&)[RV],
const V (&)[NC / PS],
3419 std::integral_constant<int, NC>) {}
3424template <
typename Scalar,
int RV,
int NC,
int LhsOrder,
int RhsOrder,
typename Index>
3425EIGEN_DONT_INLINE
void sme_tiny_gemm_kernel(Index m, Index n, Index depth,
const Scalar* A, Index lda,
const Scalar* B,
3426 Index ldb, Scalar* C, Index incr, Index ldc, Scalar alpha) {
3427 using T = sme_tiny_neon<Scalar>;
3428 using V =
typename T::V;
3429 constexpr int PS = T::PS, MR = RV * PS, S = 16 / (RV * NC) < PS ? 16 / (RV * NC) : PS;
3430 using Step = sme_tiny_step<T, RV, NC, S>;
3433 for (
int t = 0; t < S; ++t) {
3435 for (
int j = 0; j < NC; ++j) {
3437 for (
int r = 0; r < RV; ++r) acc[t][j][r] = T::zero();
3440 const Scalar* a_row[MR];
3441 for (
int i = 0; i < MR; ++i) a_row[i] = A + numext::mini(Index(i), m - 1) * lda;
3442 const Scalar* b_col[NC];
3443 for (
int j = 0; j < NC; ++j) b_col[j] = B + numext::mini(Index(j), n - 1) * ldb;
3446 int a_cnt[RV], b_cnt[NC / PS];
3447 Index a_off[RV], b_off[NC / PS];
3448 uint8x16_t a_idx[RV], b_idx[NC / PS];
3449 const auto shift_table = [](
int cnt) {
3450 EIGEN_ALIGN16 uint8_t t[16];
3451 for (
int b = 0; b < 16; ++b)
3452 t[b] = b < cnt *
int(
sizeof(Scalar)) ? uint8_t(b + (PS - cnt) *
int(
sizeof(Scalar))) : 0xff;
3455 for (
int r = 0; r < RV; ++r) {
3456 a_cnt[r] = int(numext::mini(Index(PS), m - r * PS));
3457 a_off[r] = r * PS + a_cnt[r] - PS;
3458 a_idx[r] = shift_table(a_cnt[r]);
3460 for (
int c = 0; c < NC / PS; ++c) {
3461 b_cnt[c] = int(numext::maxi(Index(0), numext::mini(Index(PS), n - c * PS)));
3462 b_off[c] = numext::maxi(Index(0), Index(c * PS + b_cnt[c] - PS));
3463 b_idx[c] = shift_table(b_cnt[c]);
3465 const bool a_full = m >= PS, b_full = n >= PS;
3466 const Index kmain = (depth / PS) * PS;
3467 for (Index k = 0; k < kmain; k += PS) {
3469 EIGEN_IF_CONSTEXPR (LhsOrder ==
ColMajor) {
3471 for (
int u = 0; u < PS; ++u) {
3473 for (
int r = 0; r < RV; ++r)
3474 a[u][r] = a_cnt[r] == PS ? T::ld(A + (k + u) * lda + r * PS)
3475 : a_full ? T::ld_tbl(A + (k + u) * lda + a_off[r], a_idx[r])
3476 : T::ld_part(A + (k + u) * lda + r * PS, a_cnt[r]);
3480 for (
int r = 0; r < RV; ++r) {
3483 for (
int q = 0; q < PS; ++q) blk[q] = T::ld(a_row[r * PS + q] + k);
3486 for (
int u = 0; u < PS; ++u) a[u][r] = blk[u];
3489 EIGEN_IF_CONSTEXPR (RhsOrder ==
ColMajor) {
3492 for (
int j = 0; j < NC; ++j) b[j] = T::ld(b_col[j] + k);
3493 Step::col(acc, a, b, std::integral_constant<int, 0>());
3496 for (
int u = 0; u < PS; ++u) {
3499 for (
int c = 0; c < NC / PS; ++c)
3500 b[c] = b_cnt[c] == PS ? T::ld(B + (k + u) * ldb + c * PS)
3501 : b_cnt[c] == 0 ? T::zero()
3502 : b_full ? T::ld_tbl(B + (k + u) * ldb + b_off[c], b_idx[c])
3503 : T::ld_part(B + (k + u) * ldb + c * PS, b_cnt[c]);
3504 Step::row(acc[u % S], a[u], b, std::integral_constant<int, 0>());
3508 for (Index k = kmain; k < depth; ++k) {
3509 EIGEN_ALIGN16 Scalar at[MR];
3510 for (
int i = 0; i < MR; ++i)
3511 at[i] = LhsOrder ==
ColMajor ? A[k * lda + numext::mini(Index(i), m - 1)] : a_row[i][k];
3514 for (
int r = 0; r < RV; ++r) a[r] = T::ld(at + r * PS);
3516 for (
int j = 0; j < NC; ++j) {
3517 const Scalar b = RhsOrder ==
ColMajor ? b_col[j][k] : B[k * ldb + numext::mini(Index(j), n - 1)];
3519 for (
int r = 0; r < RV; ++r) acc[0][j][r] = T::fma_n(acc[0][j][r], a[r], b);
3523 for (
int t = 1; t < S; ++t) {
3525 for (
int j = 0; j < NC; ++j) {
3527 for (
int r = 0; r < RV; ++r) acc[0][j][r] = T::add(acc[0][j][r], acc[t][j][r]);
3530 EIGEN_ALIGN16 Scalar out[NC][MR];
3532 for (
int j = 0; j < NC; ++j) {
3534 for (
int r = 0; r < RV; ++r) T::st(&out[j][r * PS], acc[0][j][r]);
3536 for (Index j = 0; j < n; ++j)
3537 for (Index i = 0; i < m; ++i) C[i * incr + j * ldc] += alpha * out[j][i];
3540template <
typename Scalar,
int LhsOrder,
int RhsOrder,
typename Index>
3541bool sme_tiny_gemm(Index rows, Index cols, Index depth,
const Scalar* lhs, Index lhsStride,
const Scalar* rhs,
3542 Index rhsStride, Scalar* res, Index resIncr, Index resStride, Scalar alpha) {
3543 constexpr int PS = sme_tiny_neon<Scalar>::PS;
3544 if (!sme_tiny_gemm_wins<Scalar>(rows, cols, depth))
return false;
3545 const bool one_vec = rows <= PS, four_cols = cols <= 4;
3546 if (one_vec && four_cols)
3547 sme_tiny_gemm_kernel<Scalar, 1, 4, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res,
3548 resIncr, resStride, alpha);
3550 sme_tiny_gemm_kernel<Scalar, 1, 8, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res,
3551 resIncr, resStride, alpha);
3553 sme_tiny_gemm_kernel<Scalar, 2, 4, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res,
3554 resIncr, resStride, alpha);
3556 sme_tiny_gemm_kernel<Scalar, 2, 8, LhsOrder, RhsOrder>(rows, cols, depth, lhs, lhsStride, rhs, rhsStride, res,
3557 resIncr, resStride, alpha);
3562#ifndef EIGEN_SME_NEON_SMALL_PRODUCT
3563template <
typename Scalar>
3564struct sme_neon_small_product : std::integral_constant<int, 10240> {};
3566struct sme_neon_small_product<float> : std::integral_constant<int, 28672> {};
3568struct sme_neon_small_product<std::complex<double>> : std::integral_constant<int, 5120> {};
3570template <
typename Scalar>
3571struct sme_neon_small_product : std::integral_constant<int, EIGEN_SME_NEON_SMALL_PRODUCT> {};
3574template <
typename Scalar>
3575struct sme_neon_small_cost {
3576 static constexpr bool fitted =
false;
3577 static constexpr int tile = 0, result = 0, rhs = 0, bias = 0;
3580struct sme_neon_small_cost<float> {
3581 static constexpr bool fitted =
true;
3582 static constexpr int tile = 0, result = 8, rhs = 8, bias = 24576;
3585struct sme_neon_small_cost<double> {
3586 static constexpr bool fitted =
true;
3587 static constexpr int tile = 8, result = 25, rhs = 4, bias = 11264;
3590template <
typename Scalar,
typename Index>
3591EIGEN_ALWAYS_INLINE
bool sme_small_gemm_wins(Index rows, Index cols, Index depth,
bool packed_rhs) {
3592#if defined(EIGEN_SME_NO_NEON_SMALL_BLOCKS) || defined(EIGEN_SME_FORCE_NEON_SMALL_BLOCKS) || defined(EIGEN_USE_BLAS)
3593 EIGEN_UNUSED_VARIABLE(rows);
3594 EIGEN_UNUSED_VARIABLE(cols);
3595 EIGEN_UNUSED_VARIABLE(depth);
3596 EIGEN_UNUSED_VARIABLE(packed_rhs);
3599 constexpr bool kReal = !NumTraits<Scalar>::IsComplex;
3600 constexpr Index kRhs = Index(65536 /
sizeof(Scalar));
3601 constexpr Index kThin = Index(32 /
sizeof(Scalar));
3602 constexpr int kSize = sme_neon_small_product<Scalar>::value;
3603 using Cost = sme_neon_small_cost<Scalar>;
3604#ifdef EIGEN_SME_NEON_SMALL_PRODUCT
3605 constexpr bool kFitted =
false;
3607 constexpr bool kFitted = Cost::fitted;
3609 if (rows <= 0 || cols <= 0 || depth <= 0)
return false;
3612 if (rows <= kThin && cols <= kRhs && depth <= (kReal ? Index(128) : kRhs) && cols * depth <= kRhs)
return true;
3613 if (kFitted && rows <= Index(128) && cols <= Index(128) && depth <= Index(256)) {
3614 constexpr int64_t kW = kSmeDesignVectorBytes / int(
sizeof(
typename NumTraits<Scalar>::Real));
3615 const int64_t m = int64_t(rows), n = int64_t(cols), k = int64_t(depth);
3616 const int64_t tiles = numext::div_ceil(m, kW) * numext::div_ceil(n, kW) * k;
3619 const int64_t column_bytes = k * int64_t(
sizeof(Scalar));
3620 if (!(packed_rhs && rows <= kThin && n * column_bytes * column_bytes >= (int64_t(3) << 25)))
3621 return 4 * m * n * k < Cost::tile * tiles + Cost::result * m * n + Cost::rhs * n * k + 4 * int64_t(Cost::bias);
3624 if (kReal && cols <= 3 && rows >= 3 * Index(sme_block<Scalar>::mr))
return false;
3626 return rows <= kSize && cols <= kSize && int64_t(rows) * int64_t(cols) <= kSize && depth <= kSize &&
3627 int64_t(rows) * int64_t(cols) * int64_t(depth) <= kSize;
3632template <
bool ConjLhs,
bool ConjRhs,
bool LhsInPlace,
bool RhsInPlace,
typename Scalar,
typename Index>
3633EIGEN_ALWAYS_INLINE
void sme_small_gemm_kernel(std::false_type, Scalar* C, Index incr, Index ldc,
const Scalar* A,
3634 Index,
const Scalar* B, Index, Index rows, Index depth, Index cols,
3636 sme_gebp_neon<Scalar, ConjLhs, ConjRhs>(C, incr, ldc, A, B, rows, depth, cols, alpha, depth, depth, Index(0),
3640template <
bool ConjLhs,
bool ConjRhs,
bool LhsInPlace,
bool RhsInPlace,
typename Scalar,
typename Index>
3641EIGEN_ALWAYS_INLINE
void sme_small_gemm_kernel(std::true_type, Scalar* C, Index incr, Index ldc,
const Scalar* A,
3642 Index lda,
const Scalar* B, Index ldb, Index rows, Index depth,
3643 Index cols, Scalar alpha) {
3644 static_assert(!ConjLhs && !ConjRhs,
"the SME kernel does not support conjugation of real scalars");
3645 const Index mr = LhsInPlace ? rows : Index(sme_block<Scalar>::mr),
3646 nr = RhsInPlace ? cols : Index(sme_block<Scalar>::nr);
3647 for (Index j = 0; j < cols; j += nr) {
3648 const Index cw = numext::mini(nr, cols - j);
3649 for (Index i = 0; i < rows; i += mr) {
3650 const Index pw = numext::mini(mr, rows - i);
3651 sme_neon_block_strided(C + i * incr + j * ldc, incr, ldc, LhsInPlace ? A + i : A + i * depth,
3652 LhsInPlace ? lda : pw, RhsInPlace ? B + j : B + j * depth, RhsInPlace ? ldb : cw, depth,
3661template <
typename Scalar,
int LhsOrder,
int RhsOrder,
bool ConjLhs,
bool ConjRhs,
int ResInnerStr
ide,
typename Index>
3662EIGEN_DONT_INLINE
void sme_small_gemm(Index rows, Index cols, Index depth,
const Scalar* lhs, Index lhsStride,
3663 const Scalar* rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
3664 Scalar alpha, Scalar* blockA_, Scalar* blockB_) {
3665 constexpr int MR = sme_block<Scalar>::mr;
3666 constexpr int NR = sme_block<Scalar>::nr;
3667 constexpr bool kReal = !NumTraits<Scalar>::IsComplex;
3668 constexpr bool kLhsInPlace = kReal && LhsOrder ==
ColMajor, kRhsInPlace = kReal && RhsOrder ==
RowMajor;
3669 using LhsMapper = const_blas_data_mapper<Scalar, Index, LhsOrder>;
3670 using RhsMapper = const_blas_data_mapper<Scalar, Index, RhsOrder>;
3672 std::conditional_t<LhsOrder == ColMajor, sme_pack_lhs_colmajor<Scalar, MR, Index, LhsMapper, false, false>,
3673 sme_pack_lhs_rowmajor<Scalar, MR, Index, LhsMapper, false, false>>;
3675 std::conditional_t<RhsOrder == ColMajor, sme_pack_rhs_colmajor<Scalar, NR, Index, RhsMapper, false, false>,
3676 sme_pack_rhs_rowmajor<Scalar, NR, Index, RhsMapper, false, false>>;
3677 ei_declare_aligned_stack_constructed_variable(Scalar, blockA,
3678 kLhsInPlace ? 0 : std::size_t(rows) * std::size_t(depth), blockA_);
3679 ei_declare_aligned_stack_constructed_variable(Scalar, blockB,
3680 kRhsInPlace ? 0 : std::size_t(depth) * std::size_t(cols), blockB_);
3681 if (!kLhsInPlace) PackLhs::pack_neon(blockA, lhs, lhsStride, depth, rows, 0, 0);
3682 if (!kRhsInPlace) PackRhs::pack_neon(blockB, rhs, rhsStride, depth, cols, 0, 0);
3684 const Index incr = ResInnerStride == Dynamic ? resIncr : Index(ResInnerStride);
3685 sme_small_gemm_kernel<ConjLhs, ConjRhs, kLhsInPlace, kRhsInPlace>(
3686 bool_constant<kReal>(), res, incr, resStride, kLhsInPlace ? lhs : blockA, lhsStride, kRhsInPlace ? rhs : blockB,
3687 rhsStride, rows, depth, cols, alpha);
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
@ Vertical
Definition Constants.h:267