Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2020, Arm Limited and Contributors
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_PACKET_MATH_SVE_H
12#define EIGEN_PACKET_MATH_SVE_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18namespace internal {
19#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
21#endif
22
23#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
24#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
25#endif
26
27#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
28
29template <typename Scalar, int SVEVectorLength>
30struct sve_packet_size_selector {
31 enum { size = SVEVectorLength / (sizeof(Scalar) * CHAR_BIT) };
32};
33
34// A fixed-length SVE packet needs exactly its own size in alignment, no more --
35// hard-coding Aligned64 over-constrains every vector length below 512 and, since
36// the requirement then exceeds what a fixed-size object can offer, disables
37// vectorization for those types outright. The Alignment enum tops out at
38// Aligned128 (AlignedMask is 255), so clamp there for VL >= 2048.
39template <int SVEVectorLength>
40struct sve_packet_alignment_selector {
41 enum { alignment = plain_enum_min(SVEVectorLength / CHAR_BIT, Aligned128) };
42};
43
44/********************************* int32 **************************************/
45// Keep typedef: Doxygen 1.13.2 misparses an attributed using declaration.
46typedef svint32_t PacketXi __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
47
48template <>
49struct packet_traits<numext::int32_t> : default_packet_traits {
50 using type = PacketXi;
51 using half = PacketXi; // Half not implemented yet
52 enum {
53 Vectorizable = 1,
54 AlignedOnScalar = 1,
55 size = sve_packet_size_selector<numext::int32_t, EIGEN_ARM64_SVE_VL>::size,
56
57 HasAdd = 1,
58 HasSub = 1,
59 HasShift = 1,
60 HasMul = 1,
61 HasNegate = 1,
62 HasAbs = 1,
63 HasArg = 0,
64 HasMin = 1,
65 HasMax = 1,
66 HasConj = 1,
67 // HasSetLinear and HasCmp stay 0 even though plset and pcmp_{eq,lt,le} are
68 // implemented below: both flags route work onto a packet path that measures
69 // far slower than the scalar loop GCC autovectorizes. On Neoverse V2 at
70 // VL=128, HasSetLinear costs LinSpaced 20.7 -> 60.6 us for float and
71 // 20.7 -> 121.6 us for double, and HasCmp costs (a < b).select(a, b)
72 // 6.2 -> 50.6 us. Neither is explained by pselect: these numbers are with
73 // the svsel specialization below in place, and it made the LinSpaced case
74 // worse rather than better. The cause is higher up, in how those evaluators
75 // drive the packet path, and should be found before either flag is set.
76 HasSetLinear = 0,
77 HasReduxp = 0 // Not implemented in SVE
78 };
79};
80
81template <>
82struct unpacket_traits<PacketXi> {
83 using type = numext::int32_t;
84 using half = PacketXi; // Half not yet implemented
85 enum {
86 size = sve_packet_size_selector<numext::int32_t, EIGEN_ARM64_SVE_VL>::size,
87 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
88 vectorizable = true,
89 masked_load_available = false,
90 masked_store_available = false
91 };
92};
93
94template <>
95EIGEN_STRONG_INLINE void prefetch<numext::int32_t>(const numext::int32_t* addr) {
96 svprfw(svptrue_b32(), addr, SV_PLDL1KEEP);
97}
98
99template <>
100EIGEN_STRONG_INLINE PacketXi pset1<PacketXi>(const numext::int32_t& from) {
101 return svdup_n_s32(from);
102}
103
104template <>
105EIGEN_STRONG_INLINE PacketXi plset<PacketXi>(const numext::int32_t& a) {
106 return svindex_s32(a, 1);
107}
108
109template <>
110EIGEN_STRONG_INLINE PacketXi padd<PacketXi>(const PacketXi& a, const PacketXi& b) {
111 return svadd_s32_x(svptrue_b32(), a, b);
112}
113
114template <>
115EIGEN_STRONG_INLINE PacketXi psub<PacketXi>(const PacketXi& a, const PacketXi& b) {
116 return svsub_s32_x(svptrue_b32(), a, b);
117}
118
119template <>
120EIGEN_STRONG_INLINE PacketXi pnegate(const PacketXi& a) {
121 return svneg_s32_x(svptrue_b32(), a);
122}
123
124template <>
125EIGEN_STRONG_INLINE PacketXi pmul<PacketXi>(const PacketXi& a, const PacketXi& b) {
126 return svmul_s32_x(svptrue_b32(), a, b);
127}
128
129template <>
130EIGEN_STRONG_INLINE PacketXi pdiv<PacketXi>(const PacketXi& a, const PacketXi& b) {
131 return svdiv_s32_x(svptrue_b32(), a, b);
132}
133
134template <>
135EIGEN_STRONG_INLINE PacketXi pmadd(const PacketXi& a, const PacketXi& b, const PacketXi& c) {
136 return svmla_s32_x(svptrue_b32(), c, a, b);
137}
138
139template <>
140EIGEN_STRONG_INLINE PacketXi pmin<PacketXi>(const PacketXi& a, const PacketXi& b) {
141 return svmin_s32_x(svptrue_b32(), a, b);
142}
143
144template <>
145EIGEN_STRONG_INLINE PacketXi pmax<PacketXi>(const PacketXi& a, const PacketXi& b) {
146 return svmax_s32_x(svptrue_b32(), a, b);
147}
148
149template <>
150EIGEN_STRONG_INLINE PacketXi pcmp_le<PacketXi>(const PacketXi& a, const PacketXi& b) {
151 return svdup_n_s32_z(svcmple_s32(svptrue_b32(), a, b), 0xffffffffu);
152}
153
154template <>
155EIGEN_STRONG_INLINE PacketXi pcmp_lt<PacketXi>(const PacketXi& a, const PacketXi& b) {
156 return svdup_n_s32_z(svcmplt_s32(svptrue_b32(), a, b), 0xffffffffu);
157}
158
159template <>
160EIGEN_STRONG_INLINE PacketXi pcmp_eq<PacketXi>(const PacketXi& a, const PacketXi& b) {
161 return svdup_n_s32_z(svcmpeq_s32(svptrue_b32(), a, b), 0xffffffffu);
162}
163
164template <>
165EIGEN_STRONG_INLINE PacketXi ptrue<PacketXi>(const PacketXi& /*a*/) {
166 return svdup_n_s32_x(svptrue_b32(), 0xffffffffu);
167}
168
169template <>
170EIGEN_STRONG_INLINE PacketXi pzero<PacketXi>(const PacketXi& /*a*/) {
171 return svdup_n_s32_x(svptrue_b32(), 0);
172}
173
174template <>
175EIGEN_STRONG_INLINE PacketXi pand<PacketXi>(const PacketXi& a, const PacketXi& b) {
176 return svand_s32_x(svptrue_b32(), a, b);
177}
178
179template <>
180EIGEN_STRONG_INLINE PacketXi por<PacketXi>(const PacketXi& a, const PacketXi& b) {
181 return svorr_s32_x(svptrue_b32(), a, b);
182}
183
184template <>
185EIGEN_STRONG_INLINE PacketXi pxor<PacketXi>(const PacketXi& a, const PacketXi& b) {
186 return sveor_s32_x(svptrue_b32(), a, b);
187}
188
189template <>
190EIGEN_STRONG_INLINE PacketXi pandnot<PacketXi>(const PacketXi& a, const PacketXi& b) {
191 return svbic_s32_x(svptrue_b32(), a, b);
192}
193
194// SVE selects on a predicate, so turn Eigen's all-ones/all-zeros value mask back
195// into one and use svsel rather than the generic por(pand, pandnot). Measured on
196// Neoverse V2 at VL=128: latency 3.18 -> 2.55 ns, throughput 0.80 -> 0.64 ns/op.
197// SVE2's single-instruction svbsl is no faster than this, so it is not worth an
198// ISA-conditional path. The mask is compared as an integer: as a float it would
199// be a NaN bit pattern, and relying on NaN != 0 is needlessly subtle.
200template <>
201EIGEN_STRONG_INLINE PacketXi pselect<PacketXi>(const PacketXi& mask, const PacketXi& a, const PacketXi& b) {
202 return svsel_s32(svcmpne_n_s32(svptrue_b32(), mask, 0), a, b);
203}
204
205template <int N>
206EIGEN_STRONG_INLINE PacketXi parithmetic_shift_right(const PacketXi& a) {
207 // ASR, not ASRD: ASRD is the shift-for-divide form, which rounds toward zero,
208 // whereas a C++ arithmetic right shift rounds toward negative infinity.
209 return svasr_n_s32_x(svptrue_b32(), a, N);
210}
211
212template <int N>
213EIGEN_STRONG_INLINE PacketXi plogical_shift_right(const PacketXi& a) {
214 return svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), svreinterpret_u32_s32(a), N));
215}
216
217template <int N>
218EIGEN_STRONG_INLINE PacketXi plogical_shift_left(const PacketXi& a) {
219 return svlsl_n_s32_x(svptrue_b32(), a, N);
220}
221
222template <>
223EIGEN_STRONG_INLINE PacketXi pload<PacketXi>(const numext::int32_t* from) {
224 EIGEN_DEBUG_ALIGNED_LOAD return svld1_s32(svptrue_b32(), from);
225}
226
227template <>
228EIGEN_STRONG_INLINE PacketXi ploadu<PacketXi>(const numext::int32_t* from) {
229 EIGEN_DEBUG_UNALIGNED_LOAD return svld1_s32(svptrue_b32(), from);
230}
231
232template <>
233EIGEN_STRONG_INLINE PacketXi ploaddup<PacketXi>(const numext::int32_t* from) {
234 // Load the size/2 values this reads into the low half and interleave them
235 // with themselves: svzip1 only consumes the low halves of its operands.
236 // The predicate is exact rather than svptrue -- ploaddup may only touch
237 // size/2 elements, and a wider one would read past the end of the input.
238 constexpr uint64_t kHalf = uint64_t(packet_traits<numext::int32_t>::size) / 2;
239 svint32_t lo = svld1_s32(svwhilelt_b32(uint64_t(0), kHalf), from);
240 return svzip1_s32(lo, lo);
241}
242
243template <>
244EIGEN_STRONG_INLINE PacketXi ploadquad<PacketXi>(const numext::int32_t* from) {
245 // As ploaddup, one zip further: size/4 values, each repeated four times.
246 // At the smallest vector length size/4 rounds to zero, where one element
247 // still has to be read.
248 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<numext::int32_t>::size) / 4, uint64_t(1));
249 svint32_t lo = svld1_s32(svwhilelt_b32(uint64_t(0), kQuarter), from);
250 lo = svzip1_s32(lo, lo);
251 return svzip1_s32(lo, lo);
252}
253
254template <>
255EIGEN_STRONG_INLINE void pstore<numext::int32_t>(numext::int32_t* to, const PacketXi& from) {
256 EIGEN_DEBUG_ALIGNED_STORE svst1_s32(svptrue_b32(), to, from);
257}
258
259template <>
260EIGEN_STRONG_INLINE void pstoreu<numext::int32_t>(numext::int32_t* to, const PacketXi& from) {
261 EIGEN_DEBUG_UNALIGNED_STORE svst1_s32(svptrue_b32(), to, from);
262}
263
264// The 32-bit gathers and scatters address lane i through the int32 index i * stride. Where that would overflow, the
265// lanes are addressed one at a time.
266EIGEN_STRONG_INLINE bool sve_s32index_fits(Index stride) {
267 const Index max_stride = Index(NumTraits<numext::int32_t>::highest()) / Index(EIGEN_ARM64_SVE_VL / 32 - 1);
268 return stride <= max_stride && stride >= -max_stride;
269}
270
271template <>
272EIGEN_DEVICE_FUNC inline PacketXi pgather<numext::int32_t, PacketXi>(const numext::int32_t* from, Index stride) {
273 if (!sve_s32index_fits(stride))
274 return pgather_partial<numext::int32_t, PacketXi>(from, stride, unpacket_traits<PacketXi>::size);
275 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
276 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
277 return svld1_gather_s32index_s32(svptrue_b32(), from, indices);
278}
279
280template <>
281EIGEN_DEVICE_FUNC inline void pscatter<numext::int32_t, PacketXi>(numext::int32_t* to, const PacketXi& from,
282 Index stride) {
283 if (!sve_s32index_fits(stride))
284 return pscatter_partial<numext::int32_t, PacketXi>(to, from, stride, unpacket_traits<PacketXi>::size);
285 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
286 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
287 svst1_scatter_s32index_s32(svptrue_b32(), to, indices, from);
288}
289
290template <>
291EIGEN_STRONG_INLINE numext::int32_t pfirst<PacketXi>(const PacketXi& a) {
292 // svlasta returns the first element if all predicate bits are 0
293 return svlasta_s32(svpfalse_b(), a);
294}
295
296template <>
297EIGEN_STRONG_INLINE PacketXi preverse(const PacketXi& a) {
298 return svrev_s32(a);
299}
300
301template <>
302EIGEN_STRONG_INLINE PacketXi pabs(const PacketXi& a) {
303 return svabs_s32_x(svptrue_b32(), a);
304}
305
306template <>
307EIGEN_STRONG_INLINE numext::int32_t predux<PacketXi>(const PacketXi& a) {
308 return static_cast<numext::int32_t>(svaddv_s32(svptrue_b32(), a));
309}
310
311template <>
312EIGEN_STRONG_INLINE bool predux_any(const PacketXi& a) {
313 return svptest_any(svptrue_b32(), svcmpne_n_s32(svptrue_b32(), a, 0));
314}
315
316template <>
317EIGEN_STRONG_INLINE numext::int32_t predux_mul<PacketXi>(const PacketXi& a) {
318 // Multiply the vector by its reverse.
319 svint32_t prod = svmul_s32_x(svptrue_b32(), a, svrev_s32(a));
320
321 // Reduce with interleave-and-multiply.
322 // NOTE: Skip the final reduction since it is already handled by `rev` above.
323 for (int n = unpacket_traits<PacketXi>::size; n > 2; n >>= 1)
324 prod = svmul_s32_x(svptrue_b32(), svzip1_s32(prod, prod), svzip2_s32(prod, prod));
325
326 // The reduction is done to the first element.
327 return pfirst<PacketXi>(prod);
328}
329
330template <>
331EIGEN_STRONG_INLINE numext::int32_t predux_min<PacketXi>(const PacketXi& a) {
332 return svminv_s32(svptrue_b32(), a);
333}
334
335template <>
336EIGEN_STRONG_INLINE numext::int32_t predux_max<PacketXi>(const PacketXi& a) {
337 return svmaxv_s32(svptrue_b32(), a);
338}
339
340template <int N>
341EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<PacketXi, N>& kernel) {
342 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
343 for (int stride = N / 2; stride > 0; stride >>= 1) {
344 for (int block = 0; block < N; block += 2 * stride) {
345 for (int k = 0; k < stride; ++k) {
346 PacketXi lo = svzip1_s32(kernel.packet[block + k], kernel.packet[block + k + stride]);
347 PacketXi hi = svzip2_s32(kernel.packet[block + k], kernel.packet[block + k + stride]);
348 kernel.packet[block + k] = lo;
349 kernel.packet[block + k + stride] = hi;
350 }
351 }
352 }
353}
354
355/********************************* int64 **************************************/
356// Keep typedef: Doxygen 1.13.2 misparses an attributed using declaration.
357typedef svint64_t PacketXl __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
358
359template <>
360struct packet_traits<numext::int64_t> : default_packet_traits {
361 using type = PacketXl;
362 using half = PacketXl; // Half not implemented yet
363 enum {
364 Vectorizable = 1,
365 AlignedOnScalar = 1,
366 size = sve_packet_size_selector<numext::int64_t, EIGEN_ARM64_SVE_VL>::size,
367
368 HasAdd = 1,
369 HasSub = 1,
370 HasShift = 1,
371 HasMul = 1,
372 HasNegate = 1,
373 HasAbs = 1,
374 HasArg = 0,
375 HasMin = 1,
376 HasMax = 1,
377 HasConj = 1,
378 // See the int32 traits above for why HasSetLinear stays 0.
379 HasSetLinear = 0,
380 HasReduxp = 0 // Not implemented in SVE
381 };
382};
383
384template <>
385struct unpacket_traits<PacketXl> {
386 using type = numext::int64_t;
387 using half = PacketXl; // Half not yet implemented
388 enum {
389 size = sve_packet_size_selector<numext::int64_t, EIGEN_ARM64_SVE_VL>::size,
390 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
391 vectorizable = true,
392 masked_load_available = false,
393 masked_store_available = false
394 };
395};
396
397template <>
398EIGEN_STRONG_INLINE void prefetch<numext::int64_t>(const numext::int64_t* addr) {
399 svprfd(svptrue_b64(), addr, SV_PLDL1KEEP);
400}
401
402template <>
403EIGEN_STRONG_INLINE PacketXl pset1<PacketXl>(const numext::int64_t& from) {
404 return svdup_n_s64(from);
405}
406
407template <>
408EIGEN_STRONG_INLINE PacketXl plset<PacketXl>(const numext::int64_t& a) {
409 return svindex_s64(a, 1);
410}
411
412template <>
413EIGEN_STRONG_INLINE PacketXl padd<PacketXl>(const PacketXl& a, const PacketXl& b) {
414 return svadd_s64_x(svptrue_b64(), a, b);
415}
416
417template <>
418EIGEN_STRONG_INLINE PacketXl psub<PacketXl>(const PacketXl& a, const PacketXl& b) {
419 return svsub_s64_x(svptrue_b64(), a, b);
420}
421
422template <>
423EIGEN_STRONG_INLINE PacketXl pnegate(const PacketXl& a) {
424 return svneg_s64_x(svptrue_b64(), a);
425}
426
427template <>
428EIGEN_STRONG_INLINE PacketXl pmul<PacketXl>(const PacketXl& a, const PacketXl& b) {
429 return svmul_s64_x(svptrue_b64(), a, b);
430}
431
432template <>
433EIGEN_STRONG_INLINE PacketXl pdiv<PacketXl>(const PacketXl& a, const PacketXl& b) {
434 return svdiv_s64_x(svptrue_b64(), a, b);
435}
436
437template <>
438EIGEN_STRONG_INLINE PacketXl pmadd(const PacketXl& a, const PacketXl& b, const PacketXl& c) {
439 return svmla_s64_x(svptrue_b64(), c, a, b);
440}
441
442template <>
443EIGEN_STRONG_INLINE PacketXl pmin<PacketXl>(const PacketXl& a, const PacketXl& b) {
444 return svmin_s64_x(svptrue_b64(), a, b);
445}
446
447template <>
448EIGEN_STRONG_INLINE PacketXl pmax<PacketXl>(const PacketXl& a, const PacketXl& b) {
449 return svmax_s64_x(svptrue_b64(), a, b);
450}
451
452template <>
453EIGEN_STRONG_INLINE PacketXl pcmp_le<PacketXl>(const PacketXl& a, const PacketXl& b) {
454 return svdup_n_s64_z(svcmple_s64(svptrue_b64(), a, b), numext::int64_t(-1));
455}
456
457template <>
458EIGEN_STRONG_INLINE PacketXl pcmp_lt<PacketXl>(const PacketXl& a, const PacketXl& b) {
459 return svdup_n_s64_z(svcmplt_s64(svptrue_b64(), a, b), numext::int64_t(-1));
460}
461
462template <>
463EIGEN_STRONG_INLINE PacketXl pcmp_eq<PacketXl>(const PacketXl& a, const PacketXl& b) {
464 return svdup_n_s64_z(svcmpeq_s64(svptrue_b64(), a, b), numext::int64_t(-1));
465}
466
467template <>
468EIGEN_STRONG_INLINE PacketXl ptrue<PacketXl>(const PacketXl& /*a*/) {
469 return svdup_n_s64_x(svptrue_b64(), numext::int64_t(-1));
470}
471
472template <>
473EIGEN_STRONG_INLINE PacketXl pzero<PacketXl>(const PacketXl& /*a*/) {
474 return svdup_n_s64_x(svptrue_b64(), 0);
475}
476
477template <>
478EIGEN_STRONG_INLINE PacketXl pand<PacketXl>(const PacketXl& a, const PacketXl& b) {
479 return svand_s64_x(svptrue_b64(), a, b);
480}
481
482template <>
483EIGEN_STRONG_INLINE PacketXl por<PacketXl>(const PacketXl& a, const PacketXl& b) {
484 return svorr_s64_x(svptrue_b64(), a, b);
485}
486
487template <>
488EIGEN_STRONG_INLINE PacketXl pxor<PacketXl>(const PacketXl& a, const PacketXl& b) {
489 return sveor_s64_x(svptrue_b64(), a, b);
490}
491
492template <>
493EIGEN_STRONG_INLINE PacketXl pandnot<PacketXl>(const PacketXl& a, const PacketXl& b) {
494 return svbic_s64_x(svptrue_b64(), a, b);
495}
496
497// See pselect<PacketXi>.
498template <>
499EIGEN_STRONG_INLINE PacketXl pselect<PacketXl>(const PacketXl& mask, const PacketXl& a, const PacketXl& b) {
500 return svsel_s64(svcmpne_n_s64(svptrue_b64(), mask, 0), a, b);
501}
502
503template <int N>
504EIGEN_STRONG_INLINE PacketXl parithmetic_shift_right(PacketXl a) {
505 // ASR, not ASRD: see parithmetic_shift_right<PacketXi>.
506 return svasr_n_s64_x(svptrue_b64(), a, N);
507}
508
509template <int N>
510EIGEN_STRONG_INLINE PacketXl plogical_shift_right(PacketXl a) {
511 return svreinterpret_s64_u64(svlsr_n_u64_x(svptrue_b64(), svreinterpret_u64_s64(a), N));
512}
513
514template <int N>
515EIGEN_STRONG_INLINE PacketXl plogical_shift_left(PacketXl a) {
516 return svlsl_n_s64_x(svptrue_b64(), a, N);
517}
518
519template <>
520EIGEN_STRONG_INLINE PacketXl pload<PacketXl>(const numext::int64_t* from) {
521 EIGEN_DEBUG_ALIGNED_LOAD return svld1_s64(svptrue_b64(), from);
522}
523
524template <>
525EIGEN_STRONG_INLINE PacketXl ploadu<PacketXl>(const numext::int64_t* from) {
526 EIGEN_DEBUG_UNALIGNED_LOAD return svld1_s64(svptrue_b64(), from);
527}
528
529template <>
530EIGEN_STRONG_INLINE PacketXl ploaddup<PacketXl>(const numext::int64_t* from) {
531 // Load the size/2 values this reads into the low half and interleave them
532 // with themselves: svzip1 only consumes the low halves of its operands.
533 // The predicate is exact rather than svptrue -- ploaddup may only touch
534 // size/2 elements, and a wider one would read past the end of the input.
535 constexpr uint64_t kHalf = uint64_t(packet_traits<numext::int64_t>::size) / 2;
536 svint64_t lo = svld1_s64(svwhilelt_b64(uint64_t(0), kHalf), from);
537 return svzip1_s64(lo, lo);
538}
539
540template <>
541EIGEN_STRONG_INLINE PacketXl ploadquad<PacketXl>(const numext::int64_t* from) {
542 // As ploaddup, one zip further: size/4 values, each repeated four times.
543 // At the smallest vector length size/4 rounds to zero, where one element
544 // still has to be read.
545 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<numext::int64_t>::size) / 4, uint64_t(1));
546 svint64_t lo = svld1_s64(svwhilelt_b64(uint64_t(0), kQuarter), from);
547 lo = svzip1_s64(lo, lo);
548 return svzip1_s64(lo, lo);
549}
550
551template <>
552EIGEN_STRONG_INLINE void pstore<numext::int64_t>(numext::int64_t* to, const PacketXl& from) {
553 EIGEN_DEBUG_ALIGNED_STORE svst1_s64(svptrue_b64(), to, from);
554}
555
556template <>
557EIGEN_STRONG_INLINE void pstoreu<numext::int64_t>(numext::int64_t* to, const PacketXl& from) {
558 EIGEN_DEBUG_UNALIGNED_STORE svst1_s64(svptrue_b64(), to, from);
559}
560
561template <>
562EIGEN_DEVICE_FUNC inline PacketXl pgather<numext::int64_t, PacketXl>(const numext::int64_t* from, Index stride) {
563 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
564 svint64_t indices = svindex_s64(0, stride);
565 return svld1_gather_s64index_s64(svptrue_b64(), from, indices);
566}
567
568template <>
569EIGEN_DEVICE_FUNC inline void pscatter<numext::int64_t, PacketXl>(numext::int64_t* to, const PacketXl& from,
570 Index stride) {
571 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
572 svint64_t indices = svindex_s64(0, stride);
573 svst1_scatter_s64index_s64(svptrue_b64(), to, indices, from);
574}
575
576template <>
577EIGEN_STRONG_INLINE numext::int64_t pfirst<PacketXl>(const PacketXl& a) {
578 // svlasta returns the first element if all predicate bits are 0
579 return svlasta_s64(svpfalse_b(), a);
580}
581
582template <>
583EIGEN_STRONG_INLINE PacketXl preverse(const PacketXl& a) {
584 return svrev_s64(a);
585}
586
587template <>
588EIGEN_STRONG_INLINE PacketXl pabs(const PacketXl& a) {
589 return svabs_s64_x(svptrue_b64(), a);
590}
591
592template <>
593EIGEN_STRONG_INLINE numext::int64_t predux<PacketXl>(const PacketXl& a) {
594 return static_cast<numext::int64_t>(svaddv_s64(svptrue_b64(), a));
595}
596
597template <>
598EIGEN_STRONG_INLINE numext::int64_t predux_mul<PacketXl>(const PacketXl& a) {
599 // Multiplying by the reverse pairs lane i with lane n-1-i, so every product of
600 // a pair appears in both halves; interleaving the halves then folds them
601 // together, halving the live span each round. At VL = 128 there are two lanes
602 // and the first multiply has already combined them.
603 PacketXl prod = svmul_s64_x(svptrue_b64(), a, svrev_s64(a));
604 EIGEN_UNROLL_LOOP
605 for (int n = unpacket_traits<PacketXl>::size; n > 2; n >>= 1) {
606 prod = svmul_s64_x(svptrue_b64(), svzip1_s64(prod, prod), svzip2_s64(prod, prod));
607 }
608 return pfirst<PacketXl>(prod);
609}
610
611template <>
612EIGEN_STRONG_INLINE numext::int64_t predux_min<PacketXl>(const PacketXl& a) {
613 return svminv_s64(svptrue_b64(), a);
614}
615
616template <>
617EIGEN_STRONG_INLINE numext::int64_t predux_max<PacketXl>(const PacketXl& a) {
618 return svmaxv_s64(svptrue_b64(), a);
619}
620
621template <int N>
622EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<PacketXl, N>& kernel) {
623 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
624 for (int stride = N / 2; stride > 0; stride >>= 1) {
625 for (int block = 0; block < N; block += 2 * stride) {
626 for (int k = 0; k < stride; ++k) {
627 PacketXl lo = svzip1_s64(kernel.packet[block + k], kernel.packet[block + k + stride]);
628 PacketXl hi = svzip2_s64(kernel.packet[block + k], kernel.packet[block + k + stride]);
629 kernel.packet[block + k] = lo;
630 kernel.packet[block + k + stride] = hi;
631 }
632 }
633 }
634}
635
636/********************************* float32 ************************************/
637
638// Keep typedef: Doxygen 1.13.2 misparses an attributed using declaration.
639typedef svfloat32_t PacketXf __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
640
641template <>
642struct packet_traits<float> : default_packet_traits {
643 using type = PacketXf;
644 using half = PacketXf;
645
646 enum {
647 Vectorizable = 1,
648 AlignedOnScalar = 1,
649 size = sve_packet_size_selector<float, EIGEN_ARM64_SVE_VL>::size,
650
651 HasAdd = 1,
652 HasSub = 1,
653 HasShift = 1,
654 HasMul = 1,
655 HasNegate = 1,
656 HasAbs = 1,
657 HasArg = 0,
658 HasMin = 1,
659 HasMax = 1,
660 HasConj = 1,
661 // See the int32 traits above for why HasSetLinear stays 0.
662 HasSetLinear = 0,
663 HasReduxp = 0, // Not implemented in SVE
664
665 HasDiv = 1,
666
667 HasCmp = 1,
668 HasSin = EIGEN_FAST_MATH,
669 HasCos = EIGEN_FAST_MATH,
670 HasTan = EIGEN_FAST_MATH,
671 HasACos = 1,
672 HasASin = 1,
673 HasATan = 1,
674 HasATanh = 1,
675 HasLog = 1,
676 HasLog1p = 1,
677 HasExpm1 = 1,
678 HasExp = 1,
679 HasPow = 1,
680 HasSqrt = 1,
681 HasRsqrt = 1,
682 HasCbrt = 1,
683 HasTanh = EIGEN_FAST_MATH,
684 HasErf = EIGEN_FAST_MATH,
685 HasErfc = EIGEN_FAST_MATH
686 };
687};
688
689template <>
690struct unpacket_traits<PacketXf> {
691 using type = float;
692 using half = PacketXf; // Half not yet implemented
693 using integer_packet = PacketXi;
694
695 enum {
696 size = sve_packet_size_selector<float, EIGEN_ARM64_SVE_VL>::size,
697 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
698 vectorizable = true,
699 masked_load_available = false,
700 masked_store_available = false
701 };
702};
703
704template <>
705EIGEN_STRONG_INLINE PacketXf pset1<PacketXf>(const float& from) {
706 return svdup_n_f32(from);
707}
708
709template <>
710EIGEN_STRONG_INLINE PacketXf pset1frombits<PacketXf>(numext::uint32_t from) {
711 return svreinterpret_f32_u32(svdup_n_u32_x(svptrue_b32(), from));
712}
713
714template <>
715EIGEN_STRONG_INLINE PacketXf plset<PacketXf>(const float& a) {
716 // As the double overload: svindex + a convert, no constant array.
717 return svadd_f32_x(svptrue_b32(), pset1<PacketXf>(a), svcvt_f32_s32_x(svptrue_b32(), svindex_s32(0, 1)));
718}
719
720template <>
721EIGEN_STRONG_INLINE PacketXf padd<PacketXf>(const PacketXf& a, const PacketXf& b) {
722 return svadd_f32_x(svptrue_b32(), a, b);
723}
724
725template <>
726EIGEN_STRONG_INLINE PacketXf psub<PacketXf>(const PacketXf& a, const PacketXf& b) {
727 return svsub_f32_x(svptrue_b32(), a, b);
728}
729
730template <>
731EIGEN_STRONG_INLINE PacketXf pnegate(const PacketXf& a) {
732 return svneg_f32_x(svptrue_b32(), a);
733}
734
735template <>
736EIGEN_STRONG_INLINE PacketXf pmul<PacketXf>(const PacketXf& a, const PacketXf& b) {
737 return svmul_f32_x(svptrue_b32(), a, b);
738}
739
740template <>
741EIGEN_STRONG_INLINE PacketXf pdiv<PacketXf>(const PacketXf& a, const PacketXf& b) {
742 return svdiv_f32_x(svptrue_b32(), a, b);
743}
744
745template <>
746struct is_fused_madd<PacketXf> : true_type {};
747template <>
748EIGEN_STRONG_INLINE PacketXf pmadd(const PacketXf& a, const PacketXf& b, const PacketXf& c) {
749 return svmla_f32_x(svptrue_b32(), c, a, b);
750}
751
752template <>
753struct pminmax_propagates_nan<PacketXf> : bool_constant<true> {};
754
755template <>
756EIGEN_STRONG_INLINE PacketXf pmin<PacketXf>(const PacketXf& a, const PacketXf& b) {
757 return svmin_f32_x(svptrue_b32(), a, b);
758}
759
760template <>
761EIGEN_STRONG_INLINE PacketXf pmin<PropagateNumbers, PacketXf>(const PacketXf& a, const PacketXf& b) {
762 return svminnm_f32_x(svptrue_b32(), a, b);
763}
764
765template <>
766EIGEN_STRONG_INLINE PacketXf pmax<PacketXf>(const PacketXf& a, const PacketXf& b) {
767 return svmax_f32_x(svptrue_b32(), a, b);
768}
769
770template <>
771EIGEN_STRONG_INLINE PacketXf pmax<PropagateNumbers, PacketXf>(const PacketXf& a, const PacketXf& b) {
772 return svmaxnm_f32_x(svptrue_b32(), a, b);
773}
774
775// Float comparisons in SVE return svbool (predicate). Use svdup to set active
776// lanes to 1 (0xffffffffu) and inactive lanes to 0.
777template <>
778EIGEN_STRONG_INLINE PacketXf pcmp_le<PacketXf>(const PacketXf& a, const PacketXf& b) {
779 return svreinterpret_f32_u32(svdup_n_u32_z(svcmple_f32(svptrue_b32(), a, b), 0xffffffffu));
780}
781
782template <>
783EIGEN_STRONG_INLINE PacketXf pcmp_lt<PacketXf>(const PacketXf& a, const PacketXf& b) {
784 return svreinterpret_f32_u32(svdup_n_u32_z(svcmplt_f32(svptrue_b32(), a, b), 0xffffffffu));
785}
786
787template <>
788EIGEN_STRONG_INLINE PacketXf pcmp_eq<PacketXf>(const PacketXf& a, const PacketXf& b) {
789 return svreinterpret_f32_u32(svdup_n_u32_z(svcmpeq_f32(svptrue_b32(), a, b), 0xffffffffu));
790}
791
792// Do a predicate inverse (svnot_b_z) on the predicate resulted from the
793// greater/equal comparison (svcmpge_f32). Then fill a float vector with the
794// active elements.
795template <>
796EIGEN_STRONG_INLINE PacketXf pcmp_lt_or_nan<PacketXf>(const PacketXf& a, const PacketXf& b) {
797 return svreinterpret_f32_u32(svdup_n_u32_z(svnot_b_z(svptrue_b32(), svcmpge_f32(svptrue_b32(), a, b)), 0xffffffffu));
798}
799
800template <>
801EIGEN_STRONG_INLINE PacketXf pfloor<PacketXf>(const PacketXf& a) {
802 return svrintm_f32_x(svptrue_b32(), a);
803}
804template <>
805EIGEN_STRONG_INLINE PacketXf pceil<PacketXf>(const PacketXf& a) {
806 return svrintp_f32_x(svptrue_b32(), a);
807}
808template <>
809EIGEN_STRONG_INLINE PacketXf print<PacketXf>(const PacketXf& a) {
810 return svrintn_f32_x(svptrue_b32(), a);
811}
812template <>
813EIGEN_STRONG_INLINE PacketXf ptrunc<PacketXf>(const PacketXf& a) {
814 return svrintz_f32_x(svptrue_b32(), a);
815}
816template <>
817EIGEN_STRONG_INLINE PacketXf pround<PacketXf>(const PacketXf& a) {
818 return svrinta_f32_x(svptrue_b32(), a);
819}
820
821template <>
822EIGEN_STRONG_INLINE PacketXf ptrue<PacketXf>(const PacketXf& /*a*/) {
823 PacketXf r = svreinterpret_f32_u32(svdup_n_u32_x(svptrue_b32(), 0xffffffffu));
824 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
825 return r;
826}
827
828// Logical Operations are not supported for float, so reinterpret casts
829template <>
830EIGEN_STRONG_INLINE PacketXf pand<PacketXf>(const PacketXf& a, const PacketXf& b) {
831 return svreinterpret_f32_u32(svand_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
832}
833
834template <>
835EIGEN_STRONG_INLINE PacketXf por<PacketXf>(const PacketXf& a, const PacketXf& b) {
836 return svreinterpret_f32_u32(svorr_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
837}
838
839template <>
840EIGEN_STRONG_INLINE PacketXf pxor<PacketXf>(const PacketXf& a, const PacketXf& b) {
841 return svreinterpret_f32_u32(sveor_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
842}
843
844template <>
845EIGEN_STRONG_INLINE PacketXf pandnot<PacketXf>(const PacketXf& a, const PacketXf& b) {
846 return svreinterpret_f32_u32(svbic_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
847}
848
849// See pselect<PacketXi>.
850template <>
851EIGEN_STRONG_INLINE PacketXf pselect<PacketXf>(const PacketXf& mask, const PacketXf& a, const PacketXf& b) {
852 return svsel_f32(svcmpne_n_s32(svptrue_b32(), svreinterpret_s32_f32(mask), 0), a, b);
853}
854
855template <>
856EIGEN_STRONG_INLINE PacketXf pload<PacketXf>(const float* from) {
857 EIGEN_DEBUG_ALIGNED_LOAD return svld1_f32(svptrue_b32(), from);
858}
859
860template <>
861EIGEN_STRONG_INLINE PacketXf ploadu<PacketXf>(const float* from) {
862 EIGEN_DEBUG_UNALIGNED_LOAD return svld1_f32(svptrue_b32(), from);
863}
864
865template <>
866EIGEN_STRONG_INLINE PacketXf ploaddup<PacketXf>(const float* from) {
867 // Load the size/2 values this reads into the low half and interleave them
868 // with themselves: svzip1 only consumes the low halves of its operands.
869 // The predicate is exact rather than svptrue -- ploaddup may only touch
870 // size/2 elements, and a wider one would read past the end of the input.
871 constexpr uint64_t kHalf = uint64_t(packet_traits<float>::size) / 2;
872 svfloat32_t lo = svld1_f32(svwhilelt_b32(uint64_t(0), kHalf), from);
873 return svzip1_f32(lo, lo);
874}
875
876template <>
877EIGEN_STRONG_INLINE PacketXf ploadquad<PacketXf>(const float* from) {
878 // As ploaddup, one zip further: size/4 values, each repeated four times.
879 // At the smallest vector length size/4 rounds to zero, where one element
880 // still has to be read.
881 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<float>::size) / 4, uint64_t(1));
882 svfloat32_t lo = svld1_f32(svwhilelt_b32(uint64_t(0), kQuarter), from);
883 lo = svzip1_f32(lo, lo);
884 return svzip1_f32(lo, lo);
885}
886
887template <>
888EIGEN_STRONG_INLINE void pstore<float>(float* to, const PacketXf& from) {
889 EIGEN_DEBUG_ALIGNED_STORE svst1_f32(svptrue_b32(), to, from);
890}
891
892template <>
893EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const PacketXf& from) {
894 EIGEN_DEBUG_UNALIGNED_STORE svst1_f32(svptrue_b32(), to, from);
895}
896
897template <>
898EIGEN_DEVICE_FUNC inline PacketXf pgather<float, PacketXf>(const float* from, Index stride) {
899 if (!sve_s32index_fits(stride))
900 return pgather_partial<float, PacketXf>(from, stride, unpacket_traits<PacketXf>::size);
901 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
902 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
903 return svld1_gather_s32index_f32(svptrue_b32(), from, indices);
904}
905
906template <>
907EIGEN_DEVICE_FUNC inline void pscatter<float, PacketXf>(float* to, const PacketXf& from, Index stride) {
908 if (!sve_s32index_fits(stride))
909 return pscatter_partial<float, PacketXf>(to, from, stride, unpacket_traits<PacketXf>::size);
910 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
911 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
912 svst1_scatter_s32index_f32(svptrue_b32(), to, indices, from);
913}
914
915template <>
916EIGEN_STRONG_INLINE float pfirst<PacketXf>(const PacketXf& a) {
917 // svlasta returns the first element if all predicate bits are 0
918 return svlasta_f32(svpfalse_b(), a);
919}
920
921template <>
922EIGEN_STRONG_INLINE PacketXf preverse(const PacketXf& a) {
923 return svrev_f32(a);
924}
925
926template <>
927EIGEN_STRONG_INLINE PacketXf pabs(const PacketXf& a) {
928 return svabs_f32_x(svptrue_b32(), a);
929}
930
931// TODO(tellenbach): Should this go into MathFunctions.h? If so, change for
932// all vector extensions and the generic version.
933template <>
934EIGEN_STRONG_INLINE PacketXf pfrexp<PacketXf>(const PacketXf& a, PacketXf& exponent) {
935 return pfrexp_generic(a, exponent);
936}
937
938template <>
939EIGEN_STRONG_INLINE float predux<PacketXf>(const PacketXf& a) {
940 return svaddv_f32(svptrue_b32(), a);
941}
942
943template <>
944EIGEN_STRONG_INLINE bool predux_any(const PacketXf& a) {
945 const svuint32_t bits = svreinterpret_u32_f32(a);
946 return svptest_any(svptrue_b32(), svcmpne_n_u32(svptrue_b32(), bits, 0));
947}
948
949template <>
950EIGEN_STRONG_INLINE bool predux_all(const PacketXf& a) {
951 const svbool_t all = svptrue_b32();
952 return !svptest_any(all, svcmpeq_n_f32(all, a, 0.0f));
953}
954
955template <>
956EIGEN_STRONG_INLINE Index predux_count(const PacketXf& a) {
957 const svbool_t all = svptrue_b32();
958 return static_cast<Index>(svcntp_b32(all, svcmpne_n_f32(all, a, 0.0f)));
959}
960
961// Other reduction functions:
962// mul
963template <>
964EIGEN_STRONG_INLINE float predux_mul<PacketXf>(const PacketXf& a) {
965 // Multiply the vector by its reverse.
966 svfloat32_t prod = svmul_f32_x(svptrue_b32(), a, svrev_f32(a));
967
968 // Reduce with interleave-and-multiply.
969 // NOTE: Skip the final reduction since it is already handled by `rev` above.
970 for (int n = unpacket_traits<PacketXf>::size; n > 2; n >>= 1)
971 prod = svmul_f32_x(svptrue_b32(), svzip1_f32(prod, prod), svzip2_f32(prod, prod));
972
973 // The reduction is done to the first element.
974 return pfirst<PacketXf>(prod);
975}
976
977template <>
978EIGEN_STRONG_INLINE float predux_min<PacketXf>(const PacketXf& a) {
979 return svminv_f32(svptrue_b32(), a);
980}
981
982template <>
983EIGEN_STRONG_INLINE float predux_max<PacketXf>(const PacketXf& a) {
984 return svmaxv_f32(svptrue_b32(), a);
985}
986
987template <int N>
988EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<PacketXf, N>& kernel) {
989 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
990 for (int stride = N / 2; stride > 0; stride >>= 1) {
991 for (int block = 0; block < N; block += 2 * stride) {
992 for (int k = 0; k < stride; ++k) {
993 PacketXf lo = svzip1_f32(kernel.packet[block + k], kernel.packet[block + k + stride]);
994 PacketXf hi = svzip2_f32(kernel.packet[block + k], kernel.packet[block + k + stride]);
995 kernel.packet[block + k] = lo;
996 kernel.packet[block + k + stride] = hi;
997 }
998 }
999 }
1000}
1001
1002template <>
1003EIGEN_STRONG_INLINE PacketXf pldexp<PacketXf>(const PacketXf& a, const PacketXf& exponent) {
1004 // fscale is a * 2^n rounded once; the conversion saturates, and a saturated n over- or underflows every finite a.
1005 return svscale_f32_x(svptrue_b32(), a, svcvt_s32_f32_x(svptrue_b32(), exponent));
1006}
1007
1008template <>
1009EIGEN_STRONG_INLINE PacketXf psqrt<PacketXf>(const PacketXf& a) {
1010 return svsqrt_f32_x(svptrue_b32(), a);
1011}
1012
1013/********************************* float64 ************************************/
1014// Double was missing from this backend: packet_traits<double> fell through to
1015// default_packet_traits, so every double operation under EIGEN_ARM64_USE_SVE was
1016// scalar.
1017// Keep typedef: Doxygen 1.13.2 misparses an attributed using declaration.
1018typedef svfloat64_t PacketXd __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
1019
1020template <>
1021struct packet_traits<double> : default_packet_traits {
1022 using type = PacketXd;
1023 using half = PacketXd; // Half not implemented yet
1024
1025 enum {
1026 Vectorizable = 1,
1027 AlignedOnScalar = 1,
1028 size = sve_packet_size_selector<double, EIGEN_ARM64_SVE_VL>::size,
1029
1030 HasAdd = 1,
1031 HasSub = 1,
1032 HasShift = 1,
1033 HasMul = 1,
1034 HasNegate = 1,
1035 HasAbs = 1,
1036 HasArg = 0,
1037 HasMin = 1,
1038 HasMax = 1,
1039 HasConj = 1,
1040 // See the int32 traits above for why HasSetLinear stays 0.
1041 HasSetLinear = 0,
1042 HasReduxp = 0, // Not implemented in SVE
1043
1044 HasDiv = 1,
1045 HasCmp = 1,
1046 HasSqrt = 1,
1047 HasRsqrt = 1,
1048 // asin and acos are absent from EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE,
1049 // so they stay scalar.
1050 HasSin = EIGEN_FAST_MATH,
1051 HasCos = EIGEN_FAST_MATH,
1052 HasTan = EIGEN_FAST_MATH,
1053 HasATan = 1,
1054 HasATanh = 1,
1055 HasLog = 1,
1056 HasLog1p = 1,
1057 HasExpm1 = 1,
1058 HasExp = 1,
1059 HasPow = 1,
1060 HasCbrt = 1,
1061 HasTanh = EIGEN_FAST_MATH
1062 };
1063};
1064
1065template <>
1066struct unpacket_traits<PacketXd> {
1067 using type = double;
1068 using half = PacketXd; // Half not yet implemented
1069 using integer_packet = PacketXl;
1070
1071 enum {
1072 size = sve_packet_size_selector<double, EIGEN_ARM64_SVE_VL>::size,
1073 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
1074 vectorizable = true,
1075 masked_load_available = false,
1076 masked_store_available = false
1077 };
1078};
1079
1080template <>
1081EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1082 svprfd(svptrue_b64(), addr, SV_PLDL1KEEP);
1083}
1084
1085template <>
1086EIGEN_STRONG_INLINE PacketXd pset1<PacketXd>(const double& from) {
1087 return svdup_n_f64(from);
1088}
1089
1090template <>
1091EIGEN_STRONG_INLINE PacketXd pset1frombits<PacketXd>(numext::uint64_t from) {
1092 return svreinterpret_f64_u64(svdup_n_u64_x(svptrue_b64(), from));
1093}
1094
1095template <>
1096EIGEN_STRONG_INLINE PacketXd plset<PacketXd>(const double& a) {
1097 // The ramp comes from svindex + a widening convert rather than a constant
1098 // array: no static storage, no load, and no alignment question.
1099 return svadd_f64_x(svptrue_b64(), pset1<PacketXd>(a), svcvt_f64_s64_x(svptrue_b64(), svindex_s64(0, 1)));
1100}
1101
1102template <>
1103EIGEN_STRONG_INLINE PacketXd padd<PacketXd>(const PacketXd& a, const PacketXd& b) {
1104 return svadd_f64_x(svptrue_b64(), a, b);
1105}
1106
1107template <>
1108EIGEN_STRONG_INLINE PacketXd psub<PacketXd>(const PacketXd& a, const PacketXd& b) {
1109 return svsub_f64_x(svptrue_b64(), a, b);
1110}
1111
1112template <>
1113EIGEN_STRONG_INLINE PacketXd pnegate(const PacketXd& a) {
1114 return svneg_f64_x(svptrue_b64(), a);
1115}
1116
1117template <>
1118EIGEN_STRONG_INLINE PacketXd pmul<PacketXd>(const PacketXd& a, const PacketXd& b) {
1119 return svmul_f64_x(svptrue_b64(), a, b);
1120}
1121
1122template <>
1123EIGEN_STRONG_INLINE PacketXd pdiv<PacketXd>(const PacketXd& a, const PacketXd& b) {
1124 return svdiv_f64_x(svptrue_b64(), a, b);
1125}
1126
1127template <>
1128struct is_fused_madd<PacketXd> : true_type {};
1129template <>
1130EIGEN_STRONG_INLINE PacketXd pmadd(const PacketXd& a, const PacketXd& b, const PacketXd& c) {
1131 return svmla_f64_x(svptrue_b64(), c, a, b);
1132}
1133
1134template <>
1135struct pminmax_propagates_nan<PacketXd> : bool_constant<true> {};
1136
1137template <>
1138EIGEN_STRONG_INLINE PacketXd pmin<PacketXd>(const PacketXd& a, const PacketXd& b) {
1139 return svmin_f64_x(svptrue_b64(), a, b);
1140}
1141
1142template <>
1143EIGEN_STRONG_INLINE PacketXd pmin<PropagateNumbers, PacketXd>(const PacketXd& a, const PacketXd& b) {
1144 return svminnm_f64_x(svptrue_b64(), a, b);
1145}
1146
1147template <>
1148EIGEN_STRONG_INLINE PacketXd pmax<PacketXd>(const PacketXd& a, const PacketXd& b) {
1149 return svmax_f64_x(svptrue_b64(), a, b);
1150}
1151
1152template <>
1153EIGEN_STRONG_INLINE PacketXd pmax<PropagateNumbers, PacketXd>(const PacketXd& a, const PacketXd& b) {
1154 return svmaxnm_f64_x(svptrue_b64(), a, b);
1155}
1156
1157// Comparisons in SVE return a predicate. Use svdup to set active lanes to all
1158// ones and inactive lanes to zero, matching the float path above.
1159template <>
1160EIGEN_STRONG_INLINE PacketXd pcmp_le<PacketXd>(const PacketXd& a, const PacketXd& b) {
1161 return svreinterpret_f64_u64(svdup_n_u64_z(svcmple_f64(svptrue_b64(), a, b), 0xffffffffffffffffull));
1162}
1163
1164template <>
1165EIGEN_STRONG_INLINE PacketXd pcmp_lt<PacketXd>(const PacketXd& a, const PacketXd& b) {
1166 return svreinterpret_f64_u64(svdup_n_u64_z(svcmplt_f64(svptrue_b64(), a, b), 0xffffffffffffffffull));
1167}
1168
1169template <>
1170EIGEN_STRONG_INLINE PacketXd pcmp_eq<PacketXd>(const PacketXd& a, const PacketXd& b) {
1171 return svreinterpret_f64_u64(svdup_n_u64_z(svcmpeq_f64(svptrue_b64(), a, b), 0xffffffffffffffffull));
1172}
1173
1174template <>
1175EIGEN_STRONG_INLINE PacketXd pcmp_lt_or_nan<PacketXd>(const PacketXd& a, const PacketXd& b) {
1176 return svreinterpret_f64_u64(
1177 svdup_n_u64_z(svnot_b_z(svptrue_b64(), svcmpge_f64(svptrue_b64(), a, b)), 0xffffffffffffffffull));
1178}
1179
1180template <>
1181EIGEN_STRONG_INLINE PacketXd pfloor<PacketXd>(const PacketXd& a) {
1182 return svrintm_f64_x(svptrue_b64(), a);
1183}
1184template <>
1185EIGEN_STRONG_INLINE PacketXd pceil<PacketXd>(const PacketXd& a) {
1186 return svrintp_f64_x(svptrue_b64(), a);
1187}
1188template <>
1189EIGEN_STRONG_INLINE PacketXd print<PacketXd>(const PacketXd& a) {
1190 return svrintn_f64_x(svptrue_b64(), a);
1191}
1192template <>
1193EIGEN_STRONG_INLINE PacketXd ptrunc<PacketXd>(const PacketXd& a) {
1194 return svrintz_f64_x(svptrue_b64(), a);
1195}
1196template <>
1197EIGEN_STRONG_INLINE PacketXd pround<PacketXd>(const PacketXd& a) {
1198 return svrinta_f64_x(svptrue_b64(), a);
1199}
1200
1201template <>
1202EIGEN_STRONG_INLINE PacketXd ptrue<PacketXd>(const PacketXd& /*a*/) {
1203 PacketXd r = svreinterpret_f64_u64(svdup_n_u64_x(svptrue_b64(), 0xffffffffffffffffull));
1204 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1205 return r;
1206}
1207
1208// Logical operations are not supported for float64, so reinterpret casts.
1209template <>
1210EIGEN_STRONG_INLINE PacketXd pand<PacketXd>(const PacketXd& a, const PacketXd& b) {
1211 return svreinterpret_f64_u64(svand_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1212}
1213
1214template <>
1215EIGEN_STRONG_INLINE PacketXd por<PacketXd>(const PacketXd& a, const PacketXd& b) {
1216 return svreinterpret_f64_u64(svorr_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1217}
1218
1219template <>
1220EIGEN_STRONG_INLINE PacketXd pxor<PacketXd>(const PacketXd& a, const PacketXd& b) {
1221 return svreinterpret_f64_u64(sveor_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1222}
1223
1224template <>
1225EIGEN_STRONG_INLINE PacketXd pandnot<PacketXd>(const PacketXd& a, const PacketXd& b) {
1226 return svreinterpret_f64_u64(svbic_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1227}
1228
1229// See pselect<PacketXi>.
1230template <>
1231EIGEN_STRONG_INLINE PacketXd pselect<PacketXd>(const PacketXd& mask, const PacketXd& a, const PacketXd& b) {
1232 return svsel_f64(svcmpne_n_s64(svptrue_b64(), svreinterpret_s64_f64(mask), 0), a, b);
1233}
1234
1235template <>
1236EIGEN_STRONG_INLINE PacketXd pload<PacketXd>(const double* from) {
1237 EIGEN_DEBUG_ALIGNED_LOAD return svld1_f64(svptrue_b64(), from);
1238}
1239
1240template <>
1241EIGEN_STRONG_INLINE PacketXd ploadu<PacketXd>(const double* from) {
1242 EIGEN_DEBUG_UNALIGNED_LOAD return svld1_f64(svptrue_b64(), from);
1243}
1244
1245template <>
1246EIGEN_STRONG_INLINE PacketXd ploaddup<PacketXd>(const double* from) {
1247 // Load the size/2 values this reads into the low half and interleave them
1248 // with themselves: svzip1 only consumes the low halves of its operands.
1249 // The predicate is exact rather than svptrue -- ploaddup may only touch
1250 // size/2 elements, and a wider one would read past the end of the input.
1251 constexpr uint64_t kHalf = uint64_t(packet_traits<double>::size) / 2;
1252 svfloat64_t lo = svld1_f64(svwhilelt_b64(uint64_t(0), kHalf), from);
1253 return svzip1_f64(lo, lo);
1254}
1255
1256template <>
1257EIGEN_STRONG_INLINE PacketXd ploadquad<PacketXd>(const double* from) {
1258 // As ploaddup, one zip further: size/4 values, each repeated four times.
1259 // At the smallest vector length size/4 rounds to zero, where one element
1260 // still has to be read.
1261 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<double>::size) / 4, uint64_t(1));
1262 svfloat64_t lo = svld1_f64(svwhilelt_b64(uint64_t(0), kQuarter), from);
1263 lo = svzip1_f64(lo, lo);
1264 return svzip1_f64(lo, lo);
1265}
1266
1267template <>
1268EIGEN_STRONG_INLINE void pstore<double>(double* to, const PacketXd& from) {
1269 EIGEN_DEBUG_ALIGNED_STORE svst1_f64(svptrue_b64(), to, from);
1270}
1271
1272template <>
1273EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const PacketXd& from) {
1274 EIGEN_DEBUG_UNALIGNED_STORE svst1_f64(svptrue_b64(), to, from);
1275}
1276
1277template <>
1278EIGEN_DEVICE_FUNC inline PacketXd pgather<double, PacketXd>(const double* from, Index stride) {
1279 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
1280 svint64_t indices = svindex_s64(0, stride);
1281 return svld1_gather_s64index_f64(svptrue_b64(), from, indices);
1282}
1283
1284template <>
1285EIGEN_DEVICE_FUNC inline void pscatter<double, PacketXd>(double* to, const PacketXd& from, Index stride) {
1286 // Index format: {base=0, base+stride, base+stride*2, base+stride*3, ...}
1287 svint64_t indices = svindex_s64(0, stride);
1288 svst1_scatter_s64index_f64(svptrue_b64(), to, indices, from);
1289}
1290
1291template <>
1292EIGEN_STRONG_INLINE double pfirst<PacketXd>(const PacketXd& a) {
1293 // svlasta returns the first element if all predicate bits are 0
1294 return svlasta_f64(svpfalse_b(), a);
1295}
1296
1297template <>
1298EIGEN_STRONG_INLINE PacketXd preverse(const PacketXd& a) {
1299 return svrev_f64(a);
1300}
1301
1302template <>
1303EIGEN_STRONG_INLINE PacketXd pabs(const PacketXd& a) {
1304 return svabs_f64_x(svptrue_b64(), a);
1305}
1306
1307template <>
1308EIGEN_STRONG_INLINE double predux<PacketXd>(const PacketXd& a) {
1309 return svaddv_f64(svptrue_b64(), a);
1310}
1311
1312template <>
1313EIGEN_STRONG_INLINE bool predux_any(const PacketXd& a) {
1314 const svuint64_t bits = svreinterpret_u64_f64(a);
1315 return svptest_any(svptrue_b64(), svcmpne_n_u64(svptrue_b64(), bits, 0));
1316}
1317
1318template <>
1319EIGEN_STRONG_INLINE bool predux_all(const PacketXd& a) {
1320 const svbool_t all = svptrue_b64();
1321 return !svptest_any(all, svcmpeq_n_f64(all, a, 0.0));
1322}
1323
1324template <>
1325EIGEN_STRONG_INLINE Index predux_count(const PacketXd& a) {
1326 const svbool_t all = svptrue_b64();
1327 return static_cast<Index>(svcntp_b64(all, svcmpne_n_f64(all, a, 0.0)));
1328}
1329
1330template <>
1331EIGEN_STRONG_INLINE double predux_mul<PacketXd>(const PacketXd& a) {
1332 // Multiply the vector by its reverse.
1333 svfloat64_t prod = svmul_f64_x(svptrue_b64(), a, svrev_f64(a));
1334
1335 // Reduce with interleave-and-multiply.
1336 // NOTE: Skip the final reduction since it is already handled by `rev` above.
1337 for (int n = unpacket_traits<PacketXd>::size; n > 2; n >>= 1)
1338 prod = svmul_f64_x(svptrue_b64(), svzip1_f64(prod, prod), svzip2_f64(prod, prod));
1339
1340 // The reduction is done to the first element.
1341 return pfirst<PacketXd>(prod);
1342}
1343
1344template <>
1345EIGEN_STRONG_INLINE double predux_min<PacketXd>(const PacketXd& a) {
1346 return svminv_f64(svptrue_b64(), a);
1347}
1348
1349template <>
1350EIGEN_STRONG_INLINE double predux_max<PacketXd>(const PacketXd& a) {
1351 return svmaxv_f64(svptrue_b64(), a);
1352}
1353
1354template <int N>
1355EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<PacketXd, N>& kernel) {
1356 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
1357 for (int stride = N / 2; stride > 0; stride >>= 1) {
1358 for (int block = 0; block < N; block += 2 * stride) {
1359 for (int k = 0; k < stride; ++k) {
1360 PacketXd lo = svzip1_f64(kernel.packet[block + k], kernel.packet[block + k + stride]);
1361 PacketXd hi = svzip2_f64(kernel.packet[block + k], kernel.packet[block + k + stride]);
1362 kernel.packet[block + k] = lo;
1363 kernel.packet[block + k + stride] = hi;
1364 }
1365 }
1366 }
1367}
1368
1369template <>
1370EIGEN_STRONG_INLINE PacketXd pfrexp<PacketXd>(const PacketXd& a, PacketXd& exponent) {
1371 return pfrexp_generic(a, exponent);
1372}
1373
1374template <>
1375EIGEN_STRONG_INLINE PacketXd pldexp<PacketXd>(const PacketXd& a, const PacketXd& exponent) {
1376 return svscale_f64_x(svptrue_b64(), a, svcvt_s64_f64_x(svptrue_b64(), exponent)); // see pldexp<PacketXf>
1377}
1378
1379template <>
1380EIGEN_STRONG_INLINE PacketXd psqrt<PacketXd>(const PacketXd& a) {
1381 return svsqrt_f64_x(svptrue_b64(), a);
1382}
1383
1384template <>
1385EIGEN_STRONG_INLINE PacketXd prsqrt<PacketXd>(const PacketXd& a) {
1386 // Newton off the FRSQRTE seed, as NEON's Packet2d does. The generic
1387 // preciprocal(psqrt(x)) form is correct but pays a double-precision FDIV,
1388 // which is slow enough here to lose to the scalar loop.
1389 return generic_rsqrt_newton_step<PacketXd, /*Steps=*/3>::run(a, svrsqrte_f64(a));
1390}
1391
1392/*---------------- load/store segment support ----------------*/
1393
1394// Predicates with lanes [begin, begin + count) active. begin is a literal 0 at the assignment
1395// call sites, so that branch folds and the common case is a single whilelt.
1396EIGEN_STRONG_INLINE svbool_t sve_segment_predicate_b32(Index begin, Index count) {
1397 eigen_assert(begin >= 0 && count >= 0 && begin + count <= EIGEN_ARM64_SVE_VL / 32);
1398 if (begin == 0) return svwhilelt_b32(uint64_t(0), uint64_t(count));
1399 return svnot_b_z(svwhilelt_b32(uint64_t(0), uint64_t(begin + count)), svwhilelt_b32(uint64_t(0), uint64_t(begin)));
1400}
1401
1402EIGEN_STRONG_INLINE svbool_t sve_segment_predicate_b64(Index begin, Index count) {
1403 eigen_assert(begin >= 0 && count >= 0 && begin + count <= EIGEN_ARM64_SVE_VL / 64);
1404 if (begin == 0) return svwhilelt_b64(uint64_t(0), uint64_t(count));
1405 return svnot_b_z(svwhilelt_b64(uint64_t(0), uint64_t(begin + count)), svwhilelt_b64(uint64_t(0), uint64_t(begin)));
1406}
1407
1408/*---------------- int32 ----------------*/
1409
1410template <>
1411struct has_packet_segment<PacketXi> : std::true_type {};
1412
1413template <>
1414inline PacketXi ploaduSegment<PacketXi>(const numext::int32_t* from, Index begin, Index count) {
1415 return svld1_s32(sve_segment_predicate_b32(begin, count), from);
1416}
1417
1418template <>
1419inline void pstoreuSegment<numext::int32_t, PacketXi>(numext::int32_t* to, const PacketXi& from, Index begin,
1420 Index count) {
1421 svst1_s32(sve_segment_predicate_b32(begin, count), to, from);
1422}
1423
1424/*---------------- int64 ----------------*/
1425
1426template <>
1427struct has_packet_segment<PacketXl> : std::true_type {};
1428
1429template <>
1430inline PacketXl ploaduSegment<PacketXl>(const numext::int64_t* from, Index begin, Index count) {
1431 return svld1_s64(sve_segment_predicate_b64(begin, count), from);
1432}
1433
1434template <>
1435inline void pstoreuSegment<numext::int64_t, PacketXl>(numext::int64_t* to, const PacketXl& from, Index begin,
1436 Index count) {
1437 svst1_s64(sve_segment_predicate_b64(begin, count), to, from);
1438}
1439
1440/*---------------- float ----------------*/
1441
1442template <>
1443struct has_packet_segment<PacketXf> : std::true_type {};
1444
1445template <>
1446inline PacketXf ploaduSegment<PacketXf>(const float* from, Index begin, Index count) {
1447 return svld1_f32(sve_segment_predicate_b32(begin, count), from);
1448}
1449
1450template <>
1451inline void pstoreuSegment<float, PacketXf>(float* to, const PacketXf& from, Index begin, Index count) {
1452 svst1_f32(sve_segment_predicate_b32(begin, count), to, from);
1453}
1454
1455/*---------------- double ----------------*/
1456
1457template <>
1458struct has_packet_segment<PacketXd> : std::true_type {};
1459
1460template <>
1461inline PacketXd ploaduSegment<PacketXd>(const double* from, Index begin, Index count) {
1462 return svld1_f64(sve_segment_predicate_b64(begin, count), from);
1463}
1464
1465template <>
1466inline void pstoreuSegment<double, PacketXd>(double* to, const PacketXd& from, Index begin, Index count) {
1467 svst1_f64(sve_segment_predicate_b64(begin, count), to, from);
1468}
1469
1470/*---------------- end load/store segment support ----------------*/
1471
1472} // namespace internal
1473} // namespace Eigen
1474
1475#endif // EIGEN_PACKET_MATH_SVE_H
@ Aligned128
Definition Constants.h:241