Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// This Source Code Form is subject to the terms of the Mozilla
5// Public License v. 2.0. If a copy of the MPL was not distributed
6// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
7// SPDX-FileCopyrightText: The Eigen Authors
8// SPDX-License-Identifier: MPL-2.0
9
10#ifndef EIGEN_PACKET_MATH_SME_H
11#define EIGEN_PACKET_MATH_SME_H
12
13// IWYU pragma: private
14#include "../../InternalHeaderCheck.h"
15
16namespace Eigen {
17namespace internal {
18
19// ---------------------------------------------------------------------------
20// Streaming-mode packet layer of the SME backend.
21//
22// The evaluators keep NEON packets; the streaming SVE registers and the ZA array
23// are reached only from the product kernels, through the operations below, which
24// cannot be the packet_traits type or p* specializations:
25// - a streaming vector (svfloat32_t, ...) is sizeless -- no sizeof, not a
26// member or array element type, lane count known at run time -- so neither
27// unpacket_traits<>::size nor PacketBlock can describe it;
28// - code touching one must carry __arm_streaming (or _compatible), which is
29// part of the function type: a p* specialization cannot add it, and an
30// evaluator cannot call it without a mode switch, nor at all where the
31// target has no non-streaming SVE (-mcpu=apple-m4);
32// - streaming mode runs scalar FP 30-80x slower and NEON not at all on Apple M4.
33// So these are __arm_streaming overloads on the streaming vector types: Eigen's
34// names and argument order where a counterpart exists (pset1, ploadu, pstoreu,
35// padd, pmul, pnegate, pmadd, pnmadd, predux), the arithmetic and memory
36// operations taking the SVE governing predicate first; the SVE name with Eigen's
37// p prefix where none does (pget, pcreate, pld2, pst2, puzp1/2, pzip1/2, psplice); an sme_
38// prefix for the ZA tile operations; and the predicates and lane counts as
39// members of sme_packet_traits. Eigen's templates of the same names drop out of
40// overload resolution, by arity or by substitution failure on the empty
41// unpacket_traits below.
42// ---------------------------------------------------------------------------
43
44// ZA tile count at a given element width: 4 ZA.S tiles, 8 ZA.D tiles. Architectural rather than
45// feature-dependent, so it stays outside sme_packet_traits, which has no double specialization
46// without FEAT_SME_F64F64.
47template <typename RealScalar>
48struct sme_tile_count;
49template <>
50struct sme_tile_count<float> {
51 static constexpr int value = 4;
52};
53template <>
54struct sme_tile_count<double> {
55 static constexpr int value = 8;
56};
57
58// Clang 23 and later never inline a private-ZA function (one with neither a shared-ZA attribute nor
59// __arm_agnostic("sme_za_state")) into a caller with ZA state, always_inline or not: the call stays
60// out of line behind a TPIDR2 lazy-save setup. So every function reachable from __arm_inout("za") or
61// __arm_new("za") code needs a shared-ZA attribute or EIGEN_SME_ZA_AGNOSTIC, whether or not it uses
62// intrinsics; agnostic rather than shared because private-ZA streaming entry points such as
63// pack_direct call the same helpers. GCC and Clang before 20 lack the keyword and inline these anyway.
64#if EIGEN_COMP_CLANG
65#if !__is_identifier(__arm_agnostic)
66#define EIGEN_SME_ZA_AGNOSTIC __arm_agnostic("sme_za_state")
67#endif
68#endif
69#ifndef EIGEN_SME_ZA_AGNOSTIC
70#define EIGEN_SME_ZA_AGNOSTIC
71#endif
72
73// Scalar -> streaming vector, its two- and four-vector tuples, and the predicates of its element
74// width; size() is the lane count of one streaming vector, a runtime value. whilelt takes int64_t:
75// svwhilelt_b* is overloaded on the four fixed-width types only, so Index -- `long` where int64_t
76// is `long long` -- matches none of them exactly; every bound the kernels pass is a non-negative
77// Index.
78template <typename Scalar>
79struct sme_packet_traits {};
80
81template <>
82struct sme_packet_traits<float> {
83 using type = svfloat32_t;
84 using type_x2 = svfloat32x2_t;
85 using type_x4 = svfloat32x4_t;
86 static EIGEN_ALWAYS_INLINE int size() __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
87 return static_cast<int>(svcntsw());
88 }
89 static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming EIGEN_SME_ZA_AGNOSTIC { return svptrue_b32(); }
90 static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming EIGEN_SME_ZA_AGNOSTIC { return svptrue_c32(); }
91 static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
92 return svwhilelt_b32(begin, end);
93 }
94 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c2(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
95#if EIGEN_COMP_GNUC_STRICT
96 // GCC 14 and 15 crash folding a counter whilelt whose bounds are small constants, as a full panel's width
97 // inlined into a copy's tail branch is; keep `end` opaque to it.
98 EIGEN_OPTIMIZATION_BARRIER(end)
99#endif
100 return svwhilelt_c32_s64(begin, end, 2);
101 }
102 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c4(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
103 return svwhilelt_c32_s64(begin, end, 4);
104 }
105};
106
107#ifdef EIGEN_VECTORIZE_SME_F64F64
108template <>
109struct sme_packet_traits<double> {
110 using type = svfloat64_t;
111 using type_x2 = svfloat64x2_t;
112 using type_x4 = svfloat64x4_t;
113 static EIGEN_ALWAYS_INLINE int size() __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
114 return static_cast<int>(svcntsd());
115 }
116 static EIGEN_ALWAYS_INLINE svbool_t ptrue() __arm_streaming EIGEN_SME_ZA_AGNOSTIC { return svptrue_b64(); }
117 static EIGEN_ALWAYS_INLINE svcount_t ptrue_c() __arm_streaming EIGEN_SME_ZA_AGNOSTIC { return svptrue_c64(); }
118 static EIGEN_ALWAYS_INLINE svbool_t whilelt(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
119 return svwhilelt_b64(begin, end);
120 }
121 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c2(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
122#if EIGEN_COMP_GNUC_STRICT
123 // See the float specialization.
124 EIGEN_OPTIMIZATION_BARRIER(end)
125#endif
126 return svwhilelt_c64_s64(begin, end, 2);
127 }
128 static EIGEN_ALWAYS_INLINE svcount_t whilelt_c4(int64_t begin, int64_t end) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
129 return svwhilelt_c64_s64(begin, end, 4);
130 }
131};
132#endif
133
134// Streaming vector -> scalar. The primary template has no members, so pset1's parameter type
135// below fails substitution for every other packet type and Eigen's own pset1 is selected there.
136template <typename Packet>
137struct sme_unpacket_traits {};
138template <>
139struct sme_unpacket_traits<svfloat32_t> {
140 using type = float;
141 static EIGEN_ALWAYS_INLINE svfloat32_t dup(float from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
142 return svdup_f32(from);
143 }
144};
145#ifdef EIGEN_VECTORIZE_SME_F64F64
146template <>
147struct sme_unpacket_traits<svfloat64_t> {
148 using type = double;
149 static EIGEN_ALWAYS_INLINE svfloat64_t dup(double from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
150 return svdup_f64(from);
151 }
152};
153#endif
154
155// A streaming vector is neither a packet nor a scalar, so unpacket_traits has no members for it: a
156// signature naming unpacket_traits<Packet>::type, such as Eigen's pset1, fails substitution, and a
157// generic operation called without its predicate (predux(v), pfirst(v)) does not compile, where
158// mapping the vector to itself would make it a one-lane "scalar" and resolve those to the identity.
159// The primary template cannot serve: it applies sizeof, which a sizeless type does not have.
160template <>
161struct unpacket_traits<svfloat32_t> {};
162#ifdef EIGEN_VECTORIZE_SME_F64F64
163template <>
164struct unpacket_traits<svfloat64_t> {};
165#endif
166
167// An overload of Eigen's pset1 rather than a specialization (see the file comment); the two never
168// compete, since each fails substitution on the other's vector types.
169template <typename Packet>
170EIGEN_ALWAYS_INLINE Packet
171pset1(typename sme_unpacket_traits<Packet>::type from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
172 return sme_unpacket_traits<Packet>::dup(from);
173}
174
175// Every operation takes its governing predicate, as the ACLE intrinsics do: the kernels keep the
176// predicate of the enclosing loop live, and an all-true one materialized per operation instead
177// costs a PTRUE and a P register inside the ZA store loops (48 PTRUEs and 10 predicate spills
178// against 10 and none in the float GEMM kernel). The vector type follows the scalar pointer or the
179// vector arguments, so one template per operation covers every element width through the
180// type-generic ACLE overloads. SVE loads have no alignment requirement, hence only the unaligned
181// spellings; _x2/_x4 move two and four consecutive vectors under one SME2 predicate-as-counter.
182template <typename Scalar>
183EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type ploadu(
184 svbool_t pg, const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
185 return svld1(pg, from);
186}
187template <typename Scalar>
188EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x2 ploadu_x2(
189 svcount_t pn, const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
190 return svld1_x2(pn, from);
191}
192template <typename Scalar>
193EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x4 ploadu_x4(
194 svcount_t pn, const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
195 return svld1_x4(pn, from);
196}
197template <typename Scalar>
198EIGEN_ALWAYS_INLINE void pstoreu(svbool_t pg, Scalar* to,
199 typename sme_packet_traits<Scalar>::type from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
200 svst1(pg, to, from);
201}
202template <typename Scalar>
203EIGEN_ALWAYS_INLINE void pstoreu_x2(
204 svcount_t pn, Scalar* to, typename sme_packet_traits<Scalar>::type_x2 from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
205 svst1(pn, to, from);
206}
207template <typename Scalar>
208EIGEN_ALWAYS_INLINE void pstoreu_x4(
209 svcount_t pn, Scalar* to, typename sme_packet_traits<Scalar>::type_x4 from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
210 svst1(pn, to, from);
211}
212// Two-element structure load and store (LD2/ST2): pld2 splits interleaved pairs into the even and odd
213// lanes, the real and imaginary parts of a complex array, and pst2 interleaves them back.
214template <typename Scalar>
215EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x2 pld2(
216 svbool_t pg, const Scalar* from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
217 return svld2(pg, from);
218}
219template <typename Scalar>
220EIGEN_ALWAYS_INLINE void pst2(svbool_t pg, Scalar* to,
221 typename sme_packet_traits<Scalar>::type_x2 from) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
222 svst2(pg, to, from);
223}
224
225// Arithmetic in Eigen's argument order after the predicate: pmadd(pg, a, b, c) = a * b + c and
226// pnmadd(pg, a, b, c) = c - a * b, each one fused operation. Inactive lanes are unspecified (the
227// _x forms); every consumer stores through a predicate of its own.
228template <typename Packet>
229EIGEN_ALWAYS_INLINE Packet padd(svbool_t pg, Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
230 return svadd_x(pg, a, b);
231}
232template <typename Packet>
233EIGEN_ALWAYS_INLINE Packet pmul(svbool_t pg, Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
234 return svmul_x(pg, a, b);
235}
236template <typename Packet>
237EIGEN_ALWAYS_INLINE Packet pnegate(svbool_t pg, Packet a) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
238 return svneg_x(pg, a);
239}
240template <typename Packet>
241EIGEN_ALWAYS_INLINE Packet pmadd(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
242 return svmla_x(pg, c, a, b);
243}
244template <typename Packet>
245EIGEN_ALWAYS_INLINE Packet pnmadd(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
246 return svmls_x(pg, c, a, b);
247}
248// Merging form: inactive lanes keep c, in the one FMLA that a select after the _x form does not fold to.
249template <typename Packet>
250EIGEN_ALWAYS_INLINE Packet pmadd_m(svbool_t pg, Packet a, Packet b, Packet c) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
251 return svmla_m(pg, c, a, b);
252}
253// Sum of the active lanes.
254template <typename Packet>
255EIGEN_ALWAYS_INLINE typename sme_unpacket_traits<Packet>::type predux(svbool_t pg,
256 Packet a) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
257 return svaddv(pg, a);
258}
259
260// Vector tuples: the lane of a tuple is an instruction immediate, hence a template parameter.
261template <int Lane>
262EIGEN_ALWAYS_INLINE svfloat32_t pget(svfloat32x2_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
263 return svget2_f32(v, Lane);
264}
265template <int Lane>
266EIGEN_ALWAYS_INLINE svfloat32_t pget(svfloat32x4_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
267 return svget4_f32(v, Lane);
268}
269#ifdef EIGEN_VECTORIZE_SME_F64F64
270template <int Lane>
271EIGEN_ALWAYS_INLINE svfloat64_t pget(svfloat64x2_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
272 return svget2_f64(v, Lane);
273}
274template <int Lane>
275EIGEN_ALWAYS_INLINE svfloat64_t pget(svfloat64x4_t v) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
276 return svget4_f64(v, Lane);
277}
278#endif
279template <typename Packet>
280EIGEN_ALWAYS_INLINE auto pcreate(Packet a, Packet b) EIGEN_SME_ZA_AGNOSTIC __arm_streaming
281 -> decltype(svcreate2(a, b)) {
282 return svcreate2(a, b);
283}
284template <typename Packet>
285EIGEN_ALWAYS_INLINE auto pcreate(Packet a, Packet b, Packet c, Packet d) EIGEN_SME_ZA_AGNOSTIC __arm_streaming
286 -> decltype(svcreate4(a, b, c, d)) {
287 return svcreate4(a, b, c, d);
288}
289
290// Permutes: puzp1/puzp2 gather the even/odd lanes of the concatenation (a, b), pzip1/pzip2
291// interleave its low/high halves, psplice(pg, a, b) is the active lanes of a followed by the leading
292// lanes of b.
293template <typename Packet>
294EIGEN_ALWAYS_INLINE Packet puzp1(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
295 return svuzp1(a, b);
296}
297template <typename Packet>
298EIGEN_ALWAYS_INLINE Packet puzp2(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
299 return svuzp2(a, b);
300}
301template <typename Packet>
302EIGEN_ALWAYS_INLINE Packet pzip1(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
303 return svzip1(a, b);
304}
305template <typename Packet>
306EIGEN_ALWAYS_INLINE Packet pzip2(Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
307 return svzip2(a, b);
308}
309template <typename Packet>
310EIGEN_ALWAYS_INLINE Packet psplice(svbool_t pg, Packet a, Packet b) __arm_streaming EIGEN_SME_ZA_AGNOSTIC {
311 return svsplice(pg, a, b);
312}
313
314// Entering and leaving streaming mode sets FPSR's cumulative exception flags (Arm DDI0616, RMHTLZ), also when the OS
315// resumes a thread in streaming mode, and FP arithmetic into ZA raises none. So a call into streaming code keeps the
316// caller's flags and reports none of its own.
317struct sme_fpsr_guard {
318 EIGEN_ALWAYS_INLINE sme_fpsr_guard() { asm volatile("mrs %0, fpsr" : "=r"(value) : : "memory"); }
319 EIGEN_ALWAYS_INLINE ~sme_fpsr_guard() { asm volatile("msr fpsr, %0" : : "r"(value) : "memory"); }
320 sme_fpsr_guard(const sme_fpsr_guard&) = delete;
321 sme_fpsr_guard& operator=(const sme_fpsr_guard&) = delete;
322
323 private:
324 std::uint64_t value;
325};
326
327// min() usable from streaming functions (numext::mini lacks the __arm_streaming_compatible attribute).
328template <typename T>
329EIGEN_ALWAYS_INLINE T sme_min(T a, T b) __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
330 return a < b ? a : b;
331}
332
333// Offset a pointer by n elements without forming the pointer value: an access whose predicate is
334// empty makes no memory reference, but computing an address more than one past the end of the
335// object is undefined regardless, so the second-vector accesses of the kernels reach their address
336// through uintptr_t.
337template <typename T>
338EIGEN_ALWAYS_INLINE T* sme_offset(T* p, Index n) __arm_streaming_compatible EIGEN_SME_ZA_AGNOSTIC {
339 return reinterpret_cast<T*>(uintptr_t(p) + ptrdiff_t(n) * sizeof(T));
340}
341
342// ZA tile access. A tile number is an instruction immediate, hence a template parameter; a slice
343// number is a register operand and stays a value. The element width follows the vector or pointer
344// argument, or is given as the scalar type where there is neither (sme_za_read).
345template <int Tile>
346EIGEN_ALWAYS_INLINE void sme_ld1_hor_za(uint32_t slice, svbool_t pg, const float* p) __arm_streaming __arm_inout("za") {
347 svld1_hor_za32(Tile, slice, pg, p);
348}
349template <int Tile>
350EIGEN_ALWAYS_INLINE svfloat32_t sme_read_hor_za(svfloat32_t zero, svbool_t pg,
351 uint32_t slice) __arm_streaming __arm_in("za") {
352 return svread_hor_za32_f32_m(zero, pg, Tile, slice);
353}
354template <int Tile>
355EIGEN_ALWAYS_INLINE svfloat32_t sme_read_ver_za(svfloat32_t zero, svbool_t pg,
356 uint32_t slice) __arm_streaming __arm_in("za") {
357 return svread_ver_za32_f32_m(zero, pg, Tile, slice);
358}
359// Four slices at once (SME2 MOVA ... vg4).
360template <int Tile>
361EIGEN_ALWAYS_INLINE void sme_write_hor_za_vg4(uint32_t slice, svfloat32_t a, svfloat32_t b, svfloat32_t c,
362 svfloat32_t d) __arm_streaming __arm_inout("za") {
363 svwrite_hor_za32_f32_vg4(Tile, slice, svcreate4_f32(a, b, c, d));
364}
365template <int Tile>
366EIGEN_ALWAYS_INLINE void sme_write_ver_za_vg4(uint32_t slice, svfloat32x4_t v) __arm_streaming __arm_inout("za") {
367 svwrite_ver_za32_f32_vg4(Tile, slice, v);
368}
369// Outer product accumulate (FMOPA) and subtract (FMOPS) into a tile.
370template <int Tile>
371EIGEN_ALWAYS_INLINE void sme_mopa(svbool_t pm, svbool_t pn, svfloat32_t a,
372 svfloat32_t b) __arm_streaming __arm_inout("za") {
373 svmopa_za32_f32_m(Tile, pm, pn, a, b);
374}
375template <int Tile>
376EIGEN_ALWAYS_INLINE void sme_mops(svbool_t pm, svbool_t pn, svfloat32_t a,
377 svfloat32_t b) __arm_streaming __arm_inout("za") {
378 svmops_za32_f32_m(Tile, pm, pn, a, b);
379}
380// SME2 multi-vector FMLA into four ZA array vectors, the vector-length accumulators of the vector
381// kernels; four independent groups hide the accumulator latency.
382EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat32x4_t x,
383 svfloat32_t y) __arm_streaming __arm_inout("za") {
384 svmla_single_za32_f32_vg1x4(slice, x, y);
385}
386EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat32x4_t x,
387 svfloat32x4_t y) __arm_streaming __arm_inout("za") {
388 svmla_za32_f32_vg1x4(slice, x, y);
389}
390EIGEN_ALWAYS_INLINE void sme_write_za_vg1x4(uint32_t slice, svfloat32x4_t x) __arm_streaming __arm_inout("za") {
391 svwrite_za32_f32_vg1x4(slice, x);
392}
393
394#ifdef EIGEN_VECTORIZE_SME_F64F64
395template <int Tile>
396EIGEN_ALWAYS_INLINE void sme_ld1_hor_za(uint32_t slice, svbool_t pg,
397 const double* p) __arm_streaming __arm_inout("za") {
398 svld1_hor_za64(Tile, slice, pg, p);
399}
400template <int Tile>
401EIGEN_ALWAYS_INLINE svfloat64_t sme_read_hor_za(svfloat64_t zero, svbool_t pg,
402 uint32_t slice) __arm_streaming __arm_in("za") {
403 return svread_hor_za64_f64_m(zero, pg, Tile, slice);
404}
405template <int Tile>
406EIGEN_ALWAYS_INLINE svfloat64_t sme_read_ver_za(svfloat64_t zero, svbool_t pg,
407 uint32_t slice) __arm_streaming __arm_in("za") {
408 return svread_ver_za64_f64_m(zero, pg, Tile, slice);
409}
410template <int Tile>
411EIGEN_ALWAYS_INLINE void sme_write_hor_za_vg4(uint32_t slice, svfloat64_t a, svfloat64_t b, svfloat64_t c,
412 svfloat64_t d) __arm_streaming __arm_inout("za") {
413 svwrite_hor_za64_f64_vg4(Tile, slice, svcreate4_f64(a, b, c, d));
414}
415template <int Tile>
416EIGEN_ALWAYS_INLINE void sme_write_ver_za_vg4(uint32_t slice, svfloat64x4_t v) __arm_streaming __arm_inout("za") {
417 svwrite_ver_za64_f64_vg4(Tile, slice, v);
418}
419template <int Tile>
420EIGEN_ALWAYS_INLINE void sme_mopa(svbool_t pm, svbool_t pn, svfloat64_t a,
421 svfloat64_t b) __arm_streaming __arm_inout("za") {
422 svmopa_za64_f64_m(Tile, pm, pn, a, b);
423}
424template <int Tile>
425EIGEN_ALWAYS_INLINE void sme_mops(svbool_t pm, svbool_t pn, svfloat64_t a,
426 svfloat64_t b) __arm_streaming __arm_inout("za") {
427 svmops_za64_f64_m(Tile, pm, pn, a, b);
428}
429EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat64x4_t x,
430 svfloat64_t y) __arm_streaming __arm_inout("za") {
431 svmla_single_za64_f64_vg1x4(slice, x, y);
432}
433EIGEN_ALWAYS_INLINE void sme_madd_za_vg1x4(uint32_t slice, svfloat64x4_t x,
434 svfloat64x4_t y) __arm_streaming __arm_inout("za") {
435 svmla_za64_f64_vg1x4(slice, x, y);
436}
437EIGEN_ALWAYS_INLINE void sme_write_za_vg1x4(uint32_t slice, svfloat64x4_t x) __arm_streaming __arm_inout("za") {
438 svwrite_za64_f64_vg1x4(slice, x);
439}
440#endif // EIGEN_VECTORIZE_SME_F64F64
441
442// The two reads with no width-carrying argument: four vertical slices of a tile, and four ZA array
443// vectors.
444template <typename Scalar>
445struct sme_za_read;
446template <>
447struct sme_za_read<float> {
448 template <int Tile>
449 static EIGEN_ALWAYS_INLINE svfloat32x4_t ver_vg4(uint32_t slice) __arm_streaming __arm_in("za") {
450 return svread_ver_za32_f32_vg4(Tile, slice);
451 }
452 static EIGEN_ALWAYS_INLINE svfloat32x4_t vg1x4(uint32_t slice) __arm_streaming __arm_in("za") {
453 return svread_za32_f32_vg1x4(slice);
454 }
455};
456#ifdef EIGEN_VECTORIZE_SME_F64F64
457template <>
458struct sme_za_read<double> {
459 template <int Tile>
460 static EIGEN_ALWAYS_INLINE svfloat64x4_t ver_vg4(uint32_t slice) __arm_streaming __arm_in("za") {
461 return svread_ver_za64_f64_vg4(Tile, slice);
462 }
463 static EIGEN_ALWAYS_INLINE svfloat64x4_t vg1x4(uint32_t slice) __arm_streaming __arm_in("za") {
464 return svread_za64_f64_vg1x4(slice);
465 }
466};
467#endif
468template <int Tile, typename Scalar>
469EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x4 sme_read_ver_za_vg4(
470 uint32_t slice) __arm_streaming __arm_in("za") {
471 return sme_za_read<Scalar>::template ver_vg4<Tile>(slice);
472}
473template <typename Scalar>
474EIGEN_ALWAYS_INLINE typename sme_packet_traits<Scalar>::type_x4 sme_read_za_vg1x4(
475 uint32_t slice) __arm_streaming __arm_in("za") {
476 return sme_za_read<Scalar>::vg1x4(slice);
477}
478
479// Outer-product accumulate with a compile-time sign: the complex kernel's four real products differ
480// only in whether they add or subtract into the tile.
481template <int Tile, bool Subtract, typename Packet>
482EIGEN_ALWAYS_INLINE void sme_mopa_signed(svbool_t pm, svbool_t pn, Packet a,
483 Packet b) __arm_streaming __arm_inout("za") {
484 EIGEN_IF_CONSTEXPR (Subtract) {
485 sme_mops<Tile>(pm, pn, a, b);
486 } else {
487 sme_mopa<Tile>(pm, pn, a, b);
488 }
489}
490
491} // namespace internal
492} // namespace Eigen
493
494#endif // EIGEN_PACKET_MATH_SME_H