10#ifndef EIGEN_PACKET_MATH_SME_H
11#define EIGEN_PACKET_MATH_SME_H
14#include "../../InternalHeaderCheck.h"
47template <
typename RealScalar>
50struct sme_tile_count<float> {
51 static constexpr int value = 4;
54struct sme_tile_count<double> {
55 static constexpr int value = 8;
65#if !__is_identifier(__arm_agnostic)
66#define EIGEN_SME_ZA_AGNOSTIC __arm_agnostic("sme_za_state")
69#ifndef EIGEN_SME_ZA_AGNOSTIC
70#define EIGEN_SME_ZA_AGNOSTIC
78template <
typename Scalar>
79struct sme_packet_traits {};
82struct sme_packet_traits<float> {
83 using type = svfloat32_t;
84 using type_x2 = svfloat32x2_t;
85 using type_x4 = svfloat32x4_t;
86 static EIGEN_ALWAYS_INLINE
int size() __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
87 return static_cast<int>(svcntsw());
89 static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
return svptrue_b32(); }
90 static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
return svptrue_c32(); }
91 static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
92 return svwhilelt_b32(begin, end);
94 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c2(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
95#if EIGEN_COMP_GNUC_STRICT
98 EIGEN_OPTIMIZATION_BARRIER(end)
100 return svwhilelt_c32_s64(begin, end, 2);
102 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c4(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
103 return svwhilelt_c32_s64(begin, end, 4);
107#ifdef EIGEN_VECTORIZE_SME_F64F64
109struct sme_packet_traits<double> {
110 using type = svfloat64_t;
111 using type_x2 = svfloat64x2_t;
112 using type_x4 = svfloat64x4_t;
113 static EIGEN_ALWAYS_INLINE
int size() __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
114 return static_cast<int>(svcntsd());
116 static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
return svptrue_b64(); }
117 static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
return svptrue_c64(); }
118 static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
119 return svwhilelt_b64(begin, end);
121 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c2(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
122#if EIGEN_COMP_GNUC_STRICT
124 EIGEN_OPTIMIZATION_BARRIER(end)
126 return svwhilelt_c64_s64(begin, end, 2);
128 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c4(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
129 return svwhilelt_c64_s64(begin, end, 4);
136template <
typename Packet>
137struct sme_unpacket_traits {};
139struct sme_unpacket_traits<svfloat32_t> {
141 static EIGEN_ALWAYS_INLINE svfloat32_t dup(
float from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
142 return svdup_f32(from);
145#ifdef EIGEN_VECTORIZE_SME_F64F64
147struct sme_unpacket_traits<svfloat64_t> {
149 static EIGEN_ALWAYS_INLINE svfloat64_t dup(
double from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
150 return svdup_f64(from);
161struct unpacket_traits<svfloat32_t> {};
162#ifdef EIGEN_VECTORIZE_SME_F64F64
164struct unpacket_traits<svfloat64_t> {};
169template <
typename Packet>
170EIGEN_ALWAYS_INLINE Packet
171pset1(
typename sme_unpacket_traits<Packet>::type from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
172 return sme_unpacket_traits<Packet>::dup(from);
182template <
typename Scalar>
183EIGEN_ALWAYS_INLINE
typename sme_packet_traits<Scalar>::type ploadu(
184 svbool_t pg,
const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
185 return svld1(pg, from);
187template <
typename Scalar>
188EIGEN_ALWAYS_INLINE
typename sme_packet_traits<Scalar>::type_x2 ploadu_x2(
189 svcount_t pn,
const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
190 return svld1_x2(pn, from);
192template <
typename Scalar>
193EIGEN_ALWAYS_INLINE
typename sme_packet_traits<Scalar>::type_x4 ploadu_x4(
194 svcount_t pn,
const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
195 return svld1_x4(pn, from);
197template <
typename Scalar>
198EIGEN_ALWAYS_INLINE
void pstoreu(svbool_t pg, Scalar* to,
199 typename sme_packet_traits<Scalar>::type from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
202template <
typename Scalar>
203EIGEN_ALWAYS_INLINE
void pstoreu_x2(
204 svcount_t pn, Scalar* to,
typename sme_packet_traits<Scalar>::type_x2 from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
207template <
typename Scalar>
208EIGEN_ALWAYS_INLINE
void pstoreu_x4(
209 svcount_t pn, Scalar* to,
typename sme_packet_traits<Scalar>::type_x4 from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
214template <
typename Scalar>
215EIGEN_ALWAYS_INLINE
typename sme_packet_traits<Scalar>::type_x2 pld2(
216 svbool_t pg,
const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
217 return svld2(pg, from);
219template <
typename Scalar>
220EIGEN_ALWAYS_INLINE
void pst2(svbool_t pg, Scalar* to,
221 typename sme_packet_traits<Scalar>::type_x2 from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
228template <
typename Packet>
229EIGEN_ALWAYS_INLINE Packet padd(svbool_t pg, Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
230 return svadd_x(pg, a, b);
232template <
typename Packet>
233EIGEN_ALWAYS_INLINE Packet pmul(svbool_t pg, Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
234 return svmul_x(pg, a, b);
236template <
typename Packet>
237EIGEN_ALWAYS_INLINE Packet pnegate(svbool_t pg, Packet a) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
238 return svneg_x(pg, a);
240template <
typename Packet>
241EIGEN_ALWAYS_INLINE Packet pmadd(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
242 return svmla_x(pg, c, a, b);
244template <
typename Packet>
245EIGEN_ALWAYS_INLINE Packet pnmadd(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
246 return svmls_x(pg, c, a, b);
249template <
typename Packet>
250EIGEN_ALWAYS_INLINE Packet pmadd_m(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
251 return svmla_m(pg, c, a, b);
254template <
typename Packet>
255EIGEN_ALWAYS_INLINE
typename sme_unpacket_traits<Packet>::type predux(svbool_t pg,
256 Packet a) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
257 return svaddv(pg, a);
262EIGEN_ALWAYS_INLINE svfloat32_t pget(svfloat32x2_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
263 return svget2_f32(v, Lane);
266EIGEN_ALWAYS_INLINE svfloat32_t pget(svfloat32x4_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
267 return svget4_f32(v, Lane);
269#ifdef EIGEN_VECTORIZE_SME_F64F64
271EIGEN_ALWAYS_INLINE svfloat64_t pget(svfloat64x2_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
272 return svget2_f64(v, Lane);
275EIGEN_ALWAYS_INLINE svfloat64_t pget(svfloat64x4_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
276 return svget4_f64(v, Lane);
279template <
typename Packet>
280EIGEN_ALWAYS_INLINE
auto pcreate(Packet a, Packet b) EIGEN_SME_ZA_AGNOSTIC __arm_streaming
281 ->
decltype(svcreate2(a, b)) {
282 return svcreate2(a, b);
284template <
typename Packet>
285EIGEN_ALWAYS_INLINE
auto pcreate(Packet a, Packet b, Packet c, Packet d) EIGEN_SME_ZA_AGNOSTIC __arm_streaming
286 ->
decltype(svcreate4(a, b, c, d)) {
287 return svcreate4(a, b, c, d);
293template <
typename Packet>
294EIGEN_ALWAYS_INLINE Packet puzp1(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
297template <
typename Packet>
298EIGEN_ALWAYS_INLINE Packet puzp2(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
301template <
typename Packet>
302EIGEN_ALWAYS_INLINE Packet pzip1(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
305template <
typename Packet>
306EIGEN_ALWAYS_INLINE Packet pzip2(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
309template <
typename Packet>
310EIGEN_ALWAYS_INLINE Packet psplice(svbool_t pg, Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
311 return svsplice(pg, a, b);
317struct sme_fpsr_guard {
318 EIGEN_ALWAYS_INLINE sme_fpsr_guard() {
asm volatile(
"mrs %0, fpsr" :
"=r"(value) : :
"memory"); }
319 EIGEN_ALWAYS_INLINE ~sme_fpsr_guard() {
asm volatile(
"msr fpsr, %0" : :
"r"(value) :
"memory"); }
320 sme_fpsr_guard(
const sme_fpsr_guard&) =
delete;
321 sme_fpsr_guard& operator=(
const sme_fpsr_guard&) =
delete;
329EIGEN_ALWAYS_INLINE T sme_min(T a, T b) __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
330 return a < b ? a : b;
338EIGEN_ALWAYS_INLINE T* sme_offset(T* p, Index n) __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
339 return reinterpret_cast<T*
>(uintptr_t(p) + ptrdiff_t(n) *
sizeof(T));
346EIGEN_ALWAYS_INLINE
void sme_ld1_hor_za(uint32_t slice, svbool_t pg,
const float* p) __arm_streaming __arm_inout(
"za") {
347 svld1_hor_za32(Tile, slice, pg, p);
350EIGEN_ALWAYS_INLINE svfloat32_t sme_read_hor_za(svfloat32_t zero, svbool_t pg,
351 uint32_t slice) __arm_streaming __arm_in(
"za") {
352 return svread_hor_za32_f32_m(zero, pg, Tile, slice);
355EIGEN_ALWAYS_INLINE svfloat32_t sme_read_ver_za(svfloat32_t zero, svbool_t pg,
356 uint32_t slice) __arm_streaming __arm_in(
"za") {
357 return svread_ver_za32_f32_m(zero, pg, Tile, slice);
361EIGEN_ALWAYS_INLINE
void sme_write_hor_za_vg4(uint32_t slice, svfloat32_t a, svfloat32_t b, svfloat32_t c,
362 svfloat32_t d) __arm_streaming __arm_inout(
"za") {
363 svwrite_hor_za32_f32_vg4(Tile, slice, svcreate4_f32(a, b, c, d));
366EIGEN_ALWAYS_INLINE
void sme_write_ver_za_vg4(uint32_t slice, svfloat32x4_t v) __arm_streaming __arm_inout(
"za") {
367 svwrite_ver_za32_f32_vg4(Tile, slice, v);
371EIGEN_ALWAYS_INLINE
void sme_mopa(svbool_t pm, svbool_t pn, svfloat32_t a,
372 svfloat32_t b) __arm_streaming __arm_inout(
"za") {
373 svmopa_za32_f32_m(Tile, pm, pn, a, b);
376EIGEN_ALWAYS_INLINE
void sme_mops(svbool_t pm, svbool_t pn, svfloat32_t a,
377 svfloat32_t b) __arm_streaming __arm_inout(
"za") {
378 svmops_za32_f32_m(Tile, pm, pn, a, b);
382EIGEN_ALWAYS_INLINE
void sme_madd_za_vg1x4(uint32_t slice, svfloat32x4_t x,
383 svfloat32_t y) __arm_streaming __arm_inout(
"za") {
384 svmla_single_za32_f32_vg1x4(slice, x, y);
386EIGEN_ALWAYS_INLINE
void sme_madd_za_vg1x4(uint32_t slice, svfloat32x4_t x,
387 svfloat32x4_t y) __arm_streaming __arm_inout(
"za") {
388 svmla_za32_f32_vg1x4(slice, x, y);
390EIGEN_ALWAYS_INLINE
void sme_write_za_vg1x4(uint32_t slice, svfloat32x4_t x) __arm_streaming __arm_inout(
"za") {
391 svwrite_za32_f32_vg1x4(slice, x);
394#ifdef EIGEN_VECTORIZE_SME_F64F64
396EIGEN_ALWAYS_INLINE
void sme_ld1_hor_za(uint32_t slice, svbool_t pg,
397 const double* p) __arm_streaming __arm_inout(
"za") {
398 svld1_hor_za64(Tile, slice, pg, p);
401EIGEN_ALWAYS_INLINE svfloat64_t sme_read_hor_za(svfloat64_t zero, svbool_t pg,
402 uint32_t slice) __arm_streaming __arm_in(
"za") {
403 return svread_hor_za64_f64_m(zero, pg, Tile, slice);
406EIGEN_ALWAYS_INLINE svfloat64_t sme_read_ver_za(svfloat64_t zero, svbool_t pg,
407 uint32_t slice) __arm_streaming __arm_in(
"za") {
408 return svread_ver_za64_f64_m(zero, pg, Tile, slice);
411EIGEN_ALWAYS_INLINE
void sme_write_hor_za_vg4(uint32_t slice, svfloat64_t a, svfloat64_t b, svfloat64_t c,
412 svfloat64_t d) __arm_streaming __arm_inout(
"za") {
413 svwrite_hor_za64_f64_vg4(Tile, slice, svcreate4_f64(a, b, c, d));
416EIGEN_ALWAYS_INLINE
void sme_write_ver_za_vg4(uint32_t slice, svfloat64x4_t v) __arm_streaming __arm_inout(
"za") {
417 svwrite_ver_za64_f64_vg4(Tile, slice, v);
420EIGEN_ALWAYS_INLINE
void sme_mopa(svbool_t pm, svbool_t pn, svfloat64_t a,
421 svfloat64_t b) __arm_streaming __arm_inout(
"za") {
422 svmopa_za64_f64_m(Tile, pm, pn, a, b);
425EIGEN_ALWAYS_INLINE
void sme_mops(svbool_t pm, svbool_t pn, svfloat64_t a,
426 svfloat64_t b) __arm_streaming __arm_inout(
"za") {
427 svmops_za64_f64_m(Tile, pm, pn, a, b);
429EIGEN_ALWAYS_INLINE
void sme_madd_za_vg1x4(uint32_t slice, svfloat64x4_t x,
430 svfloat64_t y) __arm_streaming __arm_inout(
"za") {
431 svmla_single_za64_f64_vg1x4(slice, x, y);
433EIGEN_ALWAYS_INLINE
void sme_madd_za_vg1x4(uint32_t slice, svfloat64x4_t x,
434 svfloat64x4_t y) __arm_streaming __arm_inout(
"za") {
435 svmla_za64_f64_vg1x4(slice, x, y);
437EIGEN_ALWAYS_INLINE
void sme_write_za_vg1x4(uint32_t slice, svfloat64x4_t x) __arm_streaming __arm_inout(
"za") {
438 svwrite_za64_f64_vg1x4(slice, x);
444template <
typename Scalar>
447struct sme_za_read<float> {
449 static EIGEN_ALWAYS_INLINE svfloat32x4_t ver_vg4(uint32_t slice) __arm_streaming __arm_in(
"za") {
450 return svread_ver_za32_f32_vg4(Tile, slice);
452 static EIGEN_ALWAYS_INLINE svfloat32x4_t vg1x4(uint32_t slice) __arm_streaming __arm_in(
"za") {
453 return svread_za32_f32_vg1x4(slice);
456#ifdef EIGEN_VECTORIZE_SME_F64F64
458struct sme_za_read<double> {
460 static EIGEN_ALWAYS_INLINE svfloat64x4_t ver_vg4(uint32_t slice) __arm_streaming __arm_in(
"za") {
461 return svread_ver_za64_f64_vg4(Tile, slice);
463 static EIGEN_ALWAYS_INLINE svfloat64x4_t vg1x4(uint32_t slice) __arm_streaming __arm_in(
"za") {
464 return svread_za64_f64_vg1x4(slice);
468template <
int Tile,
typename Scalar>
469EIGEN_ALWAYS_INLINE
typename sme_packet_traits<Scalar>::type_x4 sme_read_ver_za_vg4(
470 uint32_t slice) __arm_streaming __arm_in(
"za") {
471 return sme_za_read<Scalar>::template ver_vg4<Tile>(slice);
473template <
typename Scalar>
474EIGEN_ALWAYS_INLINE
typename sme_packet_traits<Scalar>::type_x4 sme_read_za_vg1x4(
475 uint32_t slice) __arm_streaming __arm_in(
"za") {
476 return sme_za_read<Scalar>::vg1x4(slice);
481template <
int Tile,
bool Subtract,
typename Packet>
482EIGEN_ALWAYS_INLINE
void sme_mopa_signed(svbool_t pm, svbool_t pn, Packet a,
483 Packet b) __arm_streaming __arm_inout(
"za") {
484 EIGEN_IF_CONSTEXPR (Subtract) {
485 sme_mops<Tile>(pm, pn, a, b);
487 sme_mopa<Tile>(pm, pn, a, b);