11#ifndef EIGEN_PACKET_MATH_SVE_H
12#define EIGEN_PACKET_MATH_SVE_H
15#include "../../InternalHeaderCheck.h"
19#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
23#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
24#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
27#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
29template <
typename Scalar,
int SVEVectorLength>
30struct sve_packet_size_selector {
31 enum { size = SVEVectorLength / (
sizeof(Scalar) * CHAR_BIT) };
39template <
int SVEVectorLength>
40struct sve_packet_alignment_selector {
41 enum { alignment = plain_enum_min(SVEVectorLength / CHAR_BIT,
Aligned128) };
46typedef svint32_t PacketXi __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
49struct packet_traits<numext::int32_t> : default_packet_traits {
50 using type = PacketXi;
51 using half = PacketXi;
55 size = sve_packet_size_selector<numext::int32_t, EIGEN_ARM64_SVE_VL>::size,
82struct unpacket_traits<PacketXi> {
83 using type = numext::int32_t;
84 using half = PacketXi;
86 size = sve_packet_size_selector<numext::int32_t, EIGEN_ARM64_SVE_VL>::size,
87 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
89 masked_load_available =
false,
90 masked_store_available =
false
95EIGEN_STRONG_INLINE
void prefetch<numext::int32_t>(
const numext::int32_t* addr) {
96 svprfw(svptrue_b32(), addr, SV_PLDL1KEEP);
100EIGEN_STRONG_INLINE PacketXi pset1<PacketXi>(
const numext::int32_t& from) {
101 return svdup_n_s32(from);
105EIGEN_STRONG_INLINE PacketXi plset<PacketXi>(
const numext::int32_t& a) {
106 return svindex_s32(a, 1);
110EIGEN_STRONG_INLINE PacketXi padd<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
111 return svadd_s32_x(svptrue_b32(), a, b);
115EIGEN_STRONG_INLINE PacketXi psub<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
116 return svsub_s32_x(svptrue_b32(), a, b);
120EIGEN_STRONG_INLINE PacketXi pnegate(
const PacketXi& a) {
121 return svneg_s32_x(svptrue_b32(), a);
125EIGEN_STRONG_INLINE PacketXi pmul<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
126 return svmul_s32_x(svptrue_b32(), a, b);
130EIGEN_STRONG_INLINE PacketXi pdiv<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
131 return svdiv_s32_x(svptrue_b32(), a, b);
135EIGEN_STRONG_INLINE PacketXi pmadd(
const PacketXi& a,
const PacketXi& b,
const PacketXi& c) {
136 return svmla_s32_x(svptrue_b32(), c, a, b);
140EIGEN_STRONG_INLINE PacketXi pmin<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
141 return svmin_s32_x(svptrue_b32(), a, b);
145EIGEN_STRONG_INLINE PacketXi pmax<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
146 return svmax_s32_x(svptrue_b32(), a, b);
150EIGEN_STRONG_INLINE PacketXi pcmp_le<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
151 return svdup_n_s32_z(svcmple_s32(svptrue_b32(), a, b), 0xffffffffu);
155EIGEN_STRONG_INLINE PacketXi pcmp_lt<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
156 return svdup_n_s32_z(svcmplt_s32(svptrue_b32(), a, b), 0xffffffffu);
160EIGEN_STRONG_INLINE PacketXi pcmp_eq<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
161 return svdup_n_s32_z(svcmpeq_s32(svptrue_b32(), a, b), 0xffffffffu);
165EIGEN_STRONG_INLINE PacketXi ptrue<PacketXi>(
const PacketXi& ) {
166 return svdup_n_s32_x(svptrue_b32(), 0xffffffffu);
170EIGEN_STRONG_INLINE PacketXi pzero<PacketXi>(
const PacketXi& ) {
171 return svdup_n_s32_x(svptrue_b32(), 0);
175EIGEN_STRONG_INLINE PacketXi pand<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
176 return svand_s32_x(svptrue_b32(), a, b);
180EIGEN_STRONG_INLINE PacketXi por<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
181 return svorr_s32_x(svptrue_b32(), a, b);
185EIGEN_STRONG_INLINE PacketXi pxor<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
186 return sveor_s32_x(svptrue_b32(), a, b);
190EIGEN_STRONG_INLINE PacketXi pandnot<PacketXi>(
const PacketXi& a,
const PacketXi& b) {
191 return svbic_s32_x(svptrue_b32(), a, b);
201EIGEN_STRONG_INLINE PacketXi pselect<PacketXi>(
const PacketXi& mask,
const PacketXi& a,
const PacketXi& b) {
202 return svsel_s32(svcmpne_n_s32(svptrue_b32(), mask, 0), a, b);
206EIGEN_STRONG_INLINE PacketXi parithmetic_shift_right(
const PacketXi& a) {
209 return svasr_n_s32_x(svptrue_b32(), a, N);
213EIGEN_STRONG_INLINE PacketXi plogical_shift_right(
const PacketXi& a) {
214 return svreinterpret_s32_u32(svlsr_n_u32_x(svptrue_b32(), svreinterpret_u32_s32(a), N));
218EIGEN_STRONG_INLINE PacketXi plogical_shift_left(
const PacketXi& a) {
219 return svlsl_n_s32_x(svptrue_b32(), a, N);
223EIGEN_STRONG_INLINE PacketXi pload<PacketXi>(
const numext::int32_t* from) {
224 EIGEN_DEBUG_ALIGNED_LOAD
return svld1_s32(svptrue_b32(), from);
228EIGEN_STRONG_INLINE PacketXi ploadu<PacketXi>(
const numext::int32_t* from) {
229 EIGEN_DEBUG_UNALIGNED_LOAD
return svld1_s32(svptrue_b32(), from);
233EIGEN_STRONG_INLINE PacketXi ploaddup<PacketXi>(
const numext::int32_t* from) {
238 constexpr uint64_t kHalf = uint64_t(packet_traits<numext::int32_t>::size) / 2;
239 svint32_t lo = svld1_s32(svwhilelt_b32(uint64_t(0), kHalf), from);
240 return svzip1_s32(lo, lo);
244EIGEN_STRONG_INLINE PacketXi ploadquad<PacketXi>(
const numext::int32_t* from) {
248 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<numext::int32_t>::size) / 4, uint64_t(1));
249 svint32_t lo = svld1_s32(svwhilelt_b32(uint64_t(0), kQuarter), from);
250 lo = svzip1_s32(lo, lo);
251 return svzip1_s32(lo, lo);
255EIGEN_STRONG_INLINE
void pstore<numext::int32_t>(numext::int32_t* to,
const PacketXi& from) {
256 EIGEN_DEBUG_ALIGNED_STORE svst1_s32(svptrue_b32(), to, from);
260EIGEN_STRONG_INLINE
void pstoreu<numext::int32_t>(numext::int32_t* to,
const PacketXi& from) {
261 EIGEN_DEBUG_UNALIGNED_STORE svst1_s32(svptrue_b32(), to, from);
266EIGEN_STRONG_INLINE
bool sve_s32index_fits(Index stride) {
267 const Index max_stride = Index(NumTraits<numext::int32_t>::highest()) / Index(EIGEN_ARM64_SVE_VL / 32 - 1);
268 return stride <= max_stride && stride >= -max_stride;
272EIGEN_DEVICE_FUNC
inline PacketXi pgather<numext::int32_t, PacketXi>(
const numext::int32_t* from, Index stride) {
273 if (!sve_s32index_fits(stride))
274 return pgather_partial<numext::int32_t, PacketXi>(from, stride, unpacket_traits<PacketXi>::size);
276 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
277 return svld1_gather_s32index_s32(svptrue_b32(), from, indices);
281EIGEN_DEVICE_FUNC
inline void pscatter<numext::int32_t, PacketXi>(numext::int32_t* to,
const PacketXi& from,
283 if (!sve_s32index_fits(stride))
284 return pscatter_partial<numext::int32_t, PacketXi>(to, from, stride, unpacket_traits<PacketXi>::size);
286 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
287 svst1_scatter_s32index_s32(svptrue_b32(), to, indices, from);
291EIGEN_STRONG_INLINE numext::int32_t pfirst<PacketXi>(
const PacketXi& a) {
293 return svlasta_s32(svpfalse_b(), a);
297EIGEN_STRONG_INLINE PacketXi preverse(
const PacketXi& a) {
302EIGEN_STRONG_INLINE PacketXi pabs(
const PacketXi& a) {
303 return svabs_s32_x(svptrue_b32(), a);
307EIGEN_STRONG_INLINE numext::int32_t predux<PacketXi>(
const PacketXi& a) {
308 return static_cast<numext::int32_t
>(svaddv_s32(svptrue_b32(), a));
312EIGEN_STRONG_INLINE
bool predux_any(
const PacketXi& a) {
313 return svptest_any(svptrue_b32(), svcmpne_n_s32(svptrue_b32(), a, 0));
317EIGEN_STRONG_INLINE numext::int32_t predux_mul<PacketXi>(
const PacketXi& a) {
319 svint32_t prod = svmul_s32_x(svptrue_b32(), a, svrev_s32(a));
323 for (
int n = unpacket_traits<PacketXi>::size; n > 2; n >>= 1)
324 prod = svmul_s32_x(svptrue_b32(), svzip1_s32(prod, prod), svzip2_s32(prod, prod));
327 return pfirst<PacketXi>(prod);
331EIGEN_STRONG_INLINE numext::int32_t predux_min<PacketXi>(
const PacketXi& a) {
332 return svminv_s32(svptrue_b32(), a);
336EIGEN_STRONG_INLINE numext::int32_t predux_max<PacketXi>(
const PacketXi& a) {
337 return svmaxv_s32(svptrue_b32(), a);
341EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<PacketXi, N>& kernel) {
342 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
343 for (
int stride = N / 2; stride > 0; stride >>= 1) {
344 for (
int block = 0; block < N; block += 2 * stride) {
345 for (
int k = 0; k < stride; ++k) {
346 PacketXi lo = svzip1_s32(kernel.packet[block + k], kernel.packet[block + k + stride]);
347 PacketXi hi = svzip2_s32(kernel.packet[block + k], kernel.packet[block + k + stride]);
348 kernel.packet[block + k] = lo;
349 kernel.packet[block + k + stride] = hi;
357typedef svint64_t PacketXl __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
360struct packet_traits<numext::int64_t> : default_packet_traits {
361 using type = PacketXl;
362 using half = PacketXl;
366 size = sve_packet_size_selector<numext::int64_t, EIGEN_ARM64_SVE_VL>::size,
385struct unpacket_traits<PacketXl> {
386 using type = numext::int64_t;
387 using half = PacketXl;
389 size = sve_packet_size_selector<numext::int64_t, EIGEN_ARM64_SVE_VL>::size,
390 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
392 masked_load_available =
false,
393 masked_store_available =
false
398EIGEN_STRONG_INLINE
void prefetch<numext::int64_t>(
const numext::int64_t* addr) {
399 svprfd(svptrue_b64(), addr, SV_PLDL1KEEP);
403EIGEN_STRONG_INLINE PacketXl pset1<PacketXl>(
const numext::int64_t& from) {
404 return svdup_n_s64(from);
408EIGEN_STRONG_INLINE PacketXl plset<PacketXl>(
const numext::int64_t& a) {
409 return svindex_s64(a, 1);
413EIGEN_STRONG_INLINE PacketXl padd<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
414 return svadd_s64_x(svptrue_b64(), a, b);
418EIGEN_STRONG_INLINE PacketXl psub<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
419 return svsub_s64_x(svptrue_b64(), a, b);
423EIGEN_STRONG_INLINE PacketXl pnegate(
const PacketXl& a) {
424 return svneg_s64_x(svptrue_b64(), a);
428EIGEN_STRONG_INLINE PacketXl pmul<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
429 return svmul_s64_x(svptrue_b64(), a, b);
433EIGEN_STRONG_INLINE PacketXl pdiv<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
434 return svdiv_s64_x(svptrue_b64(), a, b);
438EIGEN_STRONG_INLINE PacketXl pmadd(
const PacketXl& a,
const PacketXl& b,
const PacketXl& c) {
439 return svmla_s64_x(svptrue_b64(), c, a, b);
443EIGEN_STRONG_INLINE PacketXl pmin<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
444 return svmin_s64_x(svptrue_b64(), a, b);
448EIGEN_STRONG_INLINE PacketXl pmax<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
449 return svmax_s64_x(svptrue_b64(), a, b);
453EIGEN_STRONG_INLINE PacketXl pcmp_le<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
454 return svdup_n_s64_z(svcmple_s64(svptrue_b64(), a, b), numext::int64_t(-1));
458EIGEN_STRONG_INLINE PacketXl pcmp_lt<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
459 return svdup_n_s64_z(svcmplt_s64(svptrue_b64(), a, b), numext::int64_t(-1));
463EIGEN_STRONG_INLINE PacketXl pcmp_eq<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
464 return svdup_n_s64_z(svcmpeq_s64(svptrue_b64(), a, b), numext::int64_t(-1));
468EIGEN_STRONG_INLINE PacketXl ptrue<PacketXl>(
const PacketXl& ) {
469 return svdup_n_s64_x(svptrue_b64(), numext::int64_t(-1));
473EIGEN_STRONG_INLINE PacketXl pzero<PacketXl>(
const PacketXl& ) {
474 return svdup_n_s64_x(svptrue_b64(), 0);
478EIGEN_STRONG_INLINE PacketXl pand<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
479 return svand_s64_x(svptrue_b64(), a, b);
483EIGEN_STRONG_INLINE PacketXl por<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
484 return svorr_s64_x(svptrue_b64(), a, b);
488EIGEN_STRONG_INLINE PacketXl pxor<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
489 return sveor_s64_x(svptrue_b64(), a, b);
493EIGEN_STRONG_INLINE PacketXl pandnot<PacketXl>(
const PacketXl& a,
const PacketXl& b) {
494 return svbic_s64_x(svptrue_b64(), a, b);
499EIGEN_STRONG_INLINE PacketXl pselect<PacketXl>(
const PacketXl& mask,
const PacketXl& a,
const PacketXl& b) {
500 return svsel_s64(svcmpne_n_s64(svptrue_b64(), mask, 0), a, b);
504EIGEN_STRONG_INLINE PacketXl parithmetic_shift_right(PacketXl a) {
506 return svasr_n_s64_x(svptrue_b64(), a, N);
510EIGEN_STRONG_INLINE PacketXl plogical_shift_right(PacketXl a) {
511 return svreinterpret_s64_u64(svlsr_n_u64_x(svptrue_b64(), svreinterpret_u64_s64(a), N));
515EIGEN_STRONG_INLINE PacketXl plogical_shift_left(PacketXl a) {
516 return svlsl_n_s64_x(svptrue_b64(), a, N);
520EIGEN_STRONG_INLINE PacketXl pload<PacketXl>(
const numext::int64_t* from) {
521 EIGEN_DEBUG_ALIGNED_LOAD
return svld1_s64(svptrue_b64(), from);
525EIGEN_STRONG_INLINE PacketXl ploadu<PacketXl>(
const numext::int64_t* from) {
526 EIGEN_DEBUG_UNALIGNED_LOAD
return svld1_s64(svptrue_b64(), from);
530EIGEN_STRONG_INLINE PacketXl ploaddup<PacketXl>(
const numext::int64_t* from) {
535 constexpr uint64_t kHalf = uint64_t(packet_traits<numext::int64_t>::size) / 2;
536 svint64_t lo = svld1_s64(svwhilelt_b64(uint64_t(0), kHalf), from);
537 return svzip1_s64(lo, lo);
541EIGEN_STRONG_INLINE PacketXl ploadquad<PacketXl>(
const numext::int64_t* from) {
545 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<numext::int64_t>::size) / 4, uint64_t(1));
546 svint64_t lo = svld1_s64(svwhilelt_b64(uint64_t(0), kQuarter), from);
547 lo = svzip1_s64(lo, lo);
548 return svzip1_s64(lo, lo);
552EIGEN_STRONG_INLINE
void pstore<numext::int64_t>(numext::int64_t* to,
const PacketXl& from) {
553 EIGEN_DEBUG_ALIGNED_STORE svst1_s64(svptrue_b64(), to, from);
557EIGEN_STRONG_INLINE
void pstoreu<numext::int64_t>(numext::int64_t* to,
const PacketXl& from) {
558 EIGEN_DEBUG_UNALIGNED_STORE svst1_s64(svptrue_b64(), to, from);
562EIGEN_DEVICE_FUNC
inline PacketXl pgather<numext::int64_t, PacketXl>(
const numext::int64_t* from, Index stride) {
564 svint64_t indices = svindex_s64(0, stride);
565 return svld1_gather_s64index_s64(svptrue_b64(), from, indices);
569EIGEN_DEVICE_FUNC
inline void pscatter<numext::int64_t, PacketXl>(numext::int64_t* to,
const PacketXl& from,
572 svint64_t indices = svindex_s64(0, stride);
573 svst1_scatter_s64index_s64(svptrue_b64(), to, indices, from);
577EIGEN_STRONG_INLINE numext::int64_t pfirst<PacketXl>(
const PacketXl& a) {
579 return svlasta_s64(svpfalse_b(), a);
583EIGEN_STRONG_INLINE PacketXl preverse(
const PacketXl& a) {
588EIGEN_STRONG_INLINE PacketXl pabs(
const PacketXl& a) {
589 return svabs_s64_x(svptrue_b64(), a);
593EIGEN_STRONG_INLINE numext::int64_t predux<PacketXl>(
const PacketXl& a) {
594 return static_cast<numext::int64_t
>(svaddv_s64(svptrue_b64(), a));
598EIGEN_STRONG_INLINE numext::int64_t predux_mul<PacketXl>(
const PacketXl& a) {
603 PacketXl prod = svmul_s64_x(svptrue_b64(), a, svrev_s64(a));
605 for (
int n = unpacket_traits<PacketXl>::size; n > 2; n >>= 1) {
606 prod = svmul_s64_x(svptrue_b64(), svzip1_s64(prod, prod), svzip2_s64(prod, prod));
608 return pfirst<PacketXl>(prod);
612EIGEN_STRONG_INLINE numext::int64_t predux_min<PacketXl>(
const PacketXl& a) {
613 return svminv_s64(svptrue_b64(), a);
617EIGEN_STRONG_INLINE numext::int64_t predux_max<PacketXl>(
const PacketXl& a) {
618 return svmaxv_s64(svptrue_b64(), a);
622EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<PacketXl, N>& kernel) {
623 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
624 for (
int stride = N / 2; stride > 0; stride >>= 1) {
625 for (
int block = 0; block < N; block += 2 * stride) {
626 for (
int k = 0; k < stride; ++k) {
627 PacketXl lo = svzip1_s64(kernel.packet[block + k], kernel.packet[block + k + stride]);
628 PacketXl hi = svzip2_s64(kernel.packet[block + k], kernel.packet[block + k + stride]);
629 kernel.packet[block + k] = lo;
630 kernel.packet[block + k + stride] = hi;
639typedef svfloat32_t PacketXf __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
642struct packet_traits<float> : default_packet_traits {
643 using type = PacketXf;
644 using half = PacketXf;
649 size = sve_packet_size_selector<float, EIGEN_ARM64_SVE_VL>::size,
668 HasSin = EIGEN_FAST_MATH,
669 HasCos = EIGEN_FAST_MATH,
670 HasTan = EIGEN_FAST_MATH,
683 HasTanh = EIGEN_FAST_MATH,
684 HasErf = EIGEN_FAST_MATH,
685 HasErfc = EIGEN_FAST_MATH
690struct unpacket_traits<PacketXf> {
692 using half = PacketXf;
693 using integer_packet = PacketXi;
696 size = sve_packet_size_selector<float, EIGEN_ARM64_SVE_VL>::size,
697 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
699 masked_load_available =
false,
700 masked_store_available =
false
705EIGEN_STRONG_INLINE PacketXf pset1<PacketXf>(
const float& from) {
706 return svdup_n_f32(from);
710EIGEN_STRONG_INLINE PacketXf pset1frombits<PacketXf>(numext::uint32_t from) {
711 return svreinterpret_f32_u32(svdup_n_u32_x(svptrue_b32(), from));
715EIGEN_STRONG_INLINE PacketXf plset<PacketXf>(
const float& a) {
717 return svadd_f32_x(svptrue_b32(), pset1<PacketXf>(a), svcvt_f32_s32_x(svptrue_b32(), svindex_s32(0, 1)));
721EIGEN_STRONG_INLINE PacketXf padd<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
722 return svadd_f32_x(svptrue_b32(), a, b);
726EIGEN_STRONG_INLINE PacketXf psub<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
727 return svsub_f32_x(svptrue_b32(), a, b);
731EIGEN_STRONG_INLINE PacketXf pnegate(
const PacketXf& a) {
732 return svneg_f32_x(svptrue_b32(), a);
736EIGEN_STRONG_INLINE PacketXf pmul<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
737 return svmul_f32_x(svptrue_b32(), a, b);
741EIGEN_STRONG_INLINE PacketXf pdiv<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
742 return svdiv_f32_x(svptrue_b32(), a, b);
746struct is_fused_madd<PacketXf> : true_type {};
748EIGEN_STRONG_INLINE PacketXf pmadd(
const PacketXf& a,
const PacketXf& b,
const PacketXf& c) {
749 return svmla_f32_x(svptrue_b32(), c, a, b);
753struct pminmax_propagates_nan<PacketXf> : bool_constant<true> {};
756EIGEN_STRONG_INLINE PacketXf pmin<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
757 return svmin_f32_x(svptrue_b32(), a, b);
761EIGEN_STRONG_INLINE PacketXf pmin<PropagateNumbers, PacketXf>(
const PacketXf& a,
const PacketXf& b) {
762 return svminnm_f32_x(svptrue_b32(), a, b);
766EIGEN_STRONG_INLINE PacketXf pmax<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
767 return svmax_f32_x(svptrue_b32(), a, b);
771EIGEN_STRONG_INLINE PacketXf pmax<PropagateNumbers, PacketXf>(
const PacketXf& a,
const PacketXf& b) {
772 return svmaxnm_f32_x(svptrue_b32(), a, b);
778EIGEN_STRONG_INLINE PacketXf pcmp_le<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
779 return svreinterpret_f32_u32(svdup_n_u32_z(svcmple_f32(svptrue_b32(), a, b), 0xffffffffu));
783EIGEN_STRONG_INLINE PacketXf pcmp_lt<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
784 return svreinterpret_f32_u32(svdup_n_u32_z(svcmplt_f32(svptrue_b32(), a, b), 0xffffffffu));
788EIGEN_STRONG_INLINE PacketXf pcmp_eq<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
789 return svreinterpret_f32_u32(svdup_n_u32_z(svcmpeq_f32(svptrue_b32(), a, b), 0xffffffffu));
796EIGEN_STRONG_INLINE PacketXf pcmp_lt_or_nan<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
797 return svreinterpret_f32_u32(svdup_n_u32_z(svnot_b_z(svptrue_b32(), svcmpge_f32(svptrue_b32(), a, b)), 0xffffffffu));
801EIGEN_STRONG_INLINE PacketXf pfloor<PacketXf>(
const PacketXf& a) {
802 return svrintm_f32_x(svptrue_b32(), a);
805EIGEN_STRONG_INLINE PacketXf pceil<PacketXf>(
const PacketXf& a) {
806 return svrintp_f32_x(svptrue_b32(), a);
809EIGEN_STRONG_INLINE PacketXf print<PacketXf>(
const PacketXf& a) {
810 return svrintn_f32_x(svptrue_b32(), a);
813EIGEN_STRONG_INLINE PacketXf ptrunc<PacketXf>(
const PacketXf& a) {
814 return svrintz_f32_x(svptrue_b32(), a);
817EIGEN_STRONG_INLINE PacketXf pround<PacketXf>(
const PacketXf& a) {
818 return svrinta_f32_x(svptrue_b32(), a);
822EIGEN_STRONG_INLINE PacketXf ptrue<PacketXf>(
const PacketXf& ) {
823 PacketXf r = svreinterpret_f32_u32(svdup_n_u32_x(svptrue_b32(), 0xffffffffu));
824 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
830EIGEN_STRONG_INLINE PacketXf pand<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
831 return svreinterpret_f32_u32(svand_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
835EIGEN_STRONG_INLINE PacketXf por<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
836 return svreinterpret_f32_u32(svorr_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
840EIGEN_STRONG_INLINE PacketXf pxor<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
841 return svreinterpret_f32_u32(sveor_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
845EIGEN_STRONG_INLINE PacketXf pandnot<PacketXf>(
const PacketXf& a,
const PacketXf& b) {
846 return svreinterpret_f32_u32(svbic_u32_x(svptrue_b32(), svreinterpret_u32_f32(a), svreinterpret_u32_f32(b)));
851EIGEN_STRONG_INLINE PacketXf pselect<PacketXf>(
const PacketXf& mask,
const PacketXf& a,
const PacketXf& b) {
852 return svsel_f32(svcmpne_n_s32(svptrue_b32(), svreinterpret_s32_f32(mask), 0), a, b);
856EIGEN_STRONG_INLINE PacketXf pload<PacketXf>(
const float* from) {
857 EIGEN_DEBUG_ALIGNED_LOAD
return svld1_f32(svptrue_b32(), from);
861EIGEN_STRONG_INLINE PacketXf ploadu<PacketXf>(
const float* from) {
862 EIGEN_DEBUG_UNALIGNED_LOAD
return svld1_f32(svptrue_b32(), from);
866EIGEN_STRONG_INLINE PacketXf ploaddup<PacketXf>(
const float* from) {
871 constexpr uint64_t kHalf = uint64_t(packet_traits<float>::size) / 2;
872 svfloat32_t lo = svld1_f32(svwhilelt_b32(uint64_t(0), kHalf), from);
873 return svzip1_f32(lo, lo);
877EIGEN_STRONG_INLINE PacketXf ploadquad<PacketXf>(
const float* from) {
881 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<float>::size) / 4, uint64_t(1));
882 svfloat32_t lo = svld1_f32(svwhilelt_b32(uint64_t(0), kQuarter), from);
883 lo = svzip1_f32(lo, lo);
884 return svzip1_f32(lo, lo);
888EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const PacketXf& from) {
889 EIGEN_DEBUG_ALIGNED_STORE svst1_f32(svptrue_b32(), to, from);
893EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const PacketXf& from) {
894 EIGEN_DEBUG_UNALIGNED_STORE svst1_f32(svptrue_b32(), to, from);
898EIGEN_DEVICE_FUNC
inline PacketXf pgather<float, PacketXf>(
const float* from, Index stride) {
899 if (!sve_s32index_fits(stride))
900 return pgather_partial<float, PacketXf>(from, stride, unpacket_traits<PacketXf>::size);
902 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
903 return svld1_gather_s32index_f32(svptrue_b32(), from, indices);
907EIGEN_DEVICE_FUNC
inline void pscatter<float, PacketXf>(
float* to,
const PacketXf& from, Index stride) {
908 if (!sve_s32index_fits(stride))
909 return pscatter_partial<float, PacketXf>(to, from, stride, unpacket_traits<PacketXf>::size);
911 svint32_t indices = svindex_s32(0, numext::int32_t(stride));
912 svst1_scatter_s32index_f32(svptrue_b32(), to, indices, from);
916EIGEN_STRONG_INLINE
float pfirst<PacketXf>(
const PacketXf& a) {
918 return svlasta_f32(svpfalse_b(), a);
922EIGEN_STRONG_INLINE PacketXf preverse(
const PacketXf& a) {
927EIGEN_STRONG_INLINE PacketXf pabs(
const PacketXf& a) {
928 return svabs_f32_x(svptrue_b32(), a);
934EIGEN_STRONG_INLINE PacketXf pfrexp<PacketXf>(
const PacketXf& a, PacketXf& exponent) {
935 return pfrexp_generic(a, exponent);
939EIGEN_STRONG_INLINE
float predux<PacketXf>(
const PacketXf& a) {
940 return svaddv_f32(svptrue_b32(), a);
944EIGEN_STRONG_INLINE
bool predux_any(
const PacketXf& a) {
945 const svuint32_t bits = svreinterpret_u32_f32(a);
946 return svptest_any(svptrue_b32(), svcmpne_n_u32(svptrue_b32(), bits, 0));
950EIGEN_STRONG_INLINE
bool predux_all(
const PacketXf& a) {
951 const svbool_t all = svptrue_b32();
952 return !svptest_any(all, svcmpeq_n_f32(all, a, 0.0f));
956EIGEN_STRONG_INLINE Index predux_count(
const PacketXf& a) {
957 const svbool_t all = svptrue_b32();
958 return static_cast<Index
>(svcntp_b32(all, svcmpne_n_f32(all, a, 0.0f)));
964EIGEN_STRONG_INLINE
float predux_mul<PacketXf>(
const PacketXf& a) {
966 svfloat32_t prod = svmul_f32_x(svptrue_b32(), a, svrev_f32(a));
970 for (
int n = unpacket_traits<PacketXf>::size; n > 2; n >>= 1)
971 prod = svmul_f32_x(svptrue_b32(), svzip1_f32(prod, prod), svzip2_f32(prod, prod));
974 return pfirst<PacketXf>(prod);
978EIGEN_STRONG_INLINE
float predux_min<PacketXf>(
const PacketXf& a) {
979 return svminv_f32(svptrue_b32(), a);
983EIGEN_STRONG_INLINE
float predux_max<PacketXf>(
const PacketXf& a) {
984 return svmaxv_f32(svptrue_b32(), a);
988EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<PacketXf, N>& kernel) {
989 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
990 for (
int stride = N / 2; stride > 0; stride >>= 1) {
991 for (
int block = 0; block < N; block += 2 * stride) {
992 for (
int k = 0; k < stride; ++k) {
993 PacketXf lo = svzip1_f32(kernel.packet[block + k], kernel.packet[block + k + stride]);
994 PacketXf hi = svzip2_f32(kernel.packet[block + k], kernel.packet[block + k + stride]);
995 kernel.packet[block + k] = lo;
996 kernel.packet[block + k + stride] = hi;
1003EIGEN_STRONG_INLINE PacketXf pldexp<PacketXf>(
const PacketXf& a,
const PacketXf& exponent) {
1005 return svscale_f32_x(svptrue_b32(), a, svcvt_s32_f32_x(svptrue_b32(), exponent));
1009EIGEN_STRONG_INLINE PacketXf psqrt<PacketXf>(
const PacketXf& a) {
1010 return svsqrt_f32_x(svptrue_b32(), a);
1018typedef svfloat64_t PacketXd __attribute__((arm_sve_vector_bits(EIGEN_ARM64_SVE_VL)));
1021struct packet_traits<double> : default_packet_traits {
1022 using type = PacketXd;
1023 using half = PacketXd;
1027 AlignedOnScalar = 1,
1028 size = sve_packet_size_selector<double, EIGEN_ARM64_SVE_VL>::size,
1050 HasSin = EIGEN_FAST_MATH,
1051 HasCos = EIGEN_FAST_MATH,
1052 HasTan = EIGEN_FAST_MATH,
1061 HasTanh = EIGEN_FAST_MATH
1066struct unpacket_traits<PacketXd> {
1067 using type = double;
1068 using half = PacketXd;
1069 using integer_packet = PacketXl;
1072 size = sve_packet_size_selector<double, EIGEN_ARM64_SVE_VL>::size,
1073 alignment = sve_packet_alignment_selector<EIGEN_ARM64_SVE_VL>::alignment,
1074 vectorizable =
true,
1075 masked_load_available =
false,
1076 masked_store_available =
false
1081EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) {
1082 svprfd(svptrue_b64(), addr, SV_PLDL1KEEP);
1086EIGEN_STRONG_INLINE PacketXd pset1<PacketXd>(
const double& from) {
1087 return svdup_n_f64(from);
1091EIGEN_STRONG_INLINE PacketXd pset1frombits<PacketXd>(numext::uint64_t from) {
1092 return svreinterpret_f64_u64(svdup_n_u64_x(svptrue_b64(), from));
1096EIGEN_STRONG_INLINE PacketXd plset<PacketXd>(
const double& a) {
1099 return svadd_f64_x(svptrue_b64(), pset1<PacketXd>(a), svcvt_f64_s64_x(svptrue_b64(), svindex_s64(0, 1)));
1103EIGEN_STRONG_INLINE PacketXd padd<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1104 return svadd_f64_x(svptrue_b64(), a, b);
1108EIGEN_STRONG_INLINE PacketXd psub<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1109 return svsub_f64_x(svptrue_b64(), a, b);
1113EIGEN_STRONG_INLINE PacketXd pnegate(
const PacketXd& a) {
1114 return svneg_f64_x(svptrue_b64(), a);
1118EIGEN_STRONG_INLINE PacketXd pmul<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1119 return svmul_f64_x(svptrue_b64(), a, b);
1123EIGEN_STRONG_INLINE PacketXd pdiv<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1124 return svdiv_f64_x(svptrue_b64(), a, b);
1128struct is_fused_madd<PacketXd> : true_type {};
1130EIGEN_STRONG_INLINE PacketXd pmadd(
const PacketXd& a,
const PacketXd& b,
const PacketXd& c) {
1131 return svmla_f64_x(svptrue_b64(), c, a, b);
1135struct pminmax_propagates_nan<PacketXd> : bool_constant<true> {};
1138EIGEN_STRONG_INLINE PacketXd pmin<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1139 return svmin_f64_x(svptrue_b64(), a, b);
1143EIGEN_STRONG_INLINE PacketXd pmin<PropagateNumbers, PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1144 return svminnm_f64_x(svptrue_b64(), a, b);
1148EIGEN_STRONG_INLINE PacketXd pmax<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1149 return svmax_f64_x(svptrue_b64(), a, b);
1153EIGEN_STRONG_INLINE PacketXd pmax<PropagateNumbers, PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1154 return svmaxnm_f64_x(svptrue_b64(), a, b);
1160EIGEN_STRONG_INLINE PacketXd pcmp_le<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1161 return svreinterpret_f64_u64(svdup_n_u64_z(svcmple_f64(svptrue_b64(), a, b), 0xffffffffffffffffull));
1165EIGEN_STRONG_INLINE PacketXd pcmp_lt<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1166 return svreinterpret_f64_u64(svdup_n_u64_z(svcmplt_f64(svptrue_b64(), a, b), 0xffffffffffffffffull));
1170EIGEN_STRONG_INLINE PacketXd pcmp_eq<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1171 return svreinterpret_f64_u64(svdup_n_u64_z(svcmpeq_f64(svptrue_b64(), a, b), 0xffffffffffffffffull));
1175EIGEN_STRONG_INLINE PacketXd pcmp_lt_or_nan<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1176 return svreinterpret_f64_u64(
1177 svdup_n_u64_z(svnot_b_z(svptrue_b64(), svcmpge_f64(svptrue_b64(), a, b)), 0xffffffffffffffffull));
1181EIGEN_STRONG_INLINE PacketXd pfloor<PacketXd>(
const PacketXd& a) {
1182 return svrintm_f64_x(svptrue_b64(), a);
1185EIGEN_STRONG_INLINE PacketXd pceil<PacketXd>(
const PacketXd& a) {
1186 return svrintp_f64_x(svptrue_b64(), a);
1189EIGEN_STRONG_INLINE PacketXd print<PacketXd>(
const PacketXd& a) {
1190 return svrintn_f64_x(svptrue_b64(), a);
1193EIGEN_STRONG_INLINE PacketXd ptrunc<PacketXd>(
const PacketXd& a) {
1194 return svrintz_f64_x(svptrue_b64(), a);
1197EIGEN_STRONG_INLINE PacketXd pround<PacketXd>(
const PacketXd& a) {
1198 return svrinta_f64_x(svptrue_b64(), a);
1202EIGEN_STRONG_INLINE PacketXd ptrue<PacketXd>(
const PacketXd& ) {
1203 PacketXd r = svreinterpret_f64_u64(svdup_n_u64_x(svptrue_b64(), 0xffffffffffffffffull));
1204 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1210EIGEN_STRONG_INLINE PacketXd pand<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1211 return svreinterpret_f64_u64(svand_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1215EIGEN_STRONG_INLINE PacketXd por<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1216 return svreinterpret_f64_u64(svorr_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1220EIGEN_STRONG_INLINE PacketXd pxor<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1221 return svreinterpret_f64_u64(sveor_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1225EIGEN_STRONG_INLINE PacketXd pandnot<PacketXd>(
const PacketXd& a,
const PacketXd& b) {
1226 return svreinterpret_f64_u64(svbic_u64_x(svptrue_b64(), svreinterpret_u64_f64(a), svreinterpret_u64_f64(b)));
1231EIGEN_STRONG_INLINE PacketXd pselect<PacketXd>(
const PacketXd& mask,
const PacketXd& a,
const PacketXd& b) {
1232 return svsel_f64(svcmpne_n_s64(svptrue_b64(), svreinterpret_s64_f64(mask), 0), a, b);
1236EIGEN_STRONG_INLINE PacketXd pload<PacketXd>(
const double* from) {
1237 EIGEN_DEBUG_ALIGNED_LOAD
return svld1_f64(svptrue_b64(), from);
1241EIGEN_STRONG_INLINE PacketXd ploadu<PacketXd>(
const double* from) {
1242 EIGEN_DEBUG_UNALIGNED_LOAD
return svld1_f64(svptrue_b64(), from);
1246EIGEN_STRONG_INLINE PacketXd ploaddup<PacketXd>(
const double* from) {
1251 constexpr uint64_t kHalf = uint64_t(packet_traits<double>::size) / 2;
1252 svfloat64_t lo = svld1_f64(svwhilelt_b64(uint64_t(0), kHalf), from);
1253 return svzip1_f64(lo, lo);
1257EIGEN_STRONG_INLINE PacketXd ploadquad<PacketXd>(
const double* from) {
1261 constexpr uint64_t kQuarter = numext::maxi(uint64_t(packet_traits<double>::size) / 4, uint64_t(1));
1262 svfloat64_t lo = svld1_f64(svwhilelt_b64(uint64_t(0), kQuarter), from);
1263 lo = svzip1_f64(lo, lo);
1264 return svzip1_f64(lo, lo);
1268EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const PacketXd& from) {
1269 EIGEN_DEBUG_ALIGNED_STORE svst1_f64(svptrue_b64(), to, from);
1273EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const PacketXd& from) {
1274 EIGEN_DEBUG_UNALIGNED_STORE svst1_f64(svptrue_b64(), to, from);
1278EIGEN_DEVICE_FUNC
inline PacketXd pgather<double, PacketXd>(
const double* from, Index stride) {
1280 svint64_t indices = svindex_s64(0, stride);
1281 return svld1_gather_s64index_f64(svptrue_b64(), from, indices);
1285EIGEN_DEVICE_FUNC
inline void pscatter<double, PacketXd>(
double* to,
const PacketXd& from, Index stride) {
1287 svint64_t indices = svindex_s64(0, stride);
1288 svst1_scatter_s64index_f64(svptrue_b64(), to, indices, from);
1292EIGEN_STRONG_INLINE
double pfirst<PacketXd>(
const PacketXd& a) {
1294 return svlasta_f64(svpfalse_b(), a);
1298EIGEN_STRONG_INLINE PacketXd preverse(
const PacketXd& a) {
1299 return svrev_f64(a);
1303EIGEN_STRONG_INLINE PacketXd pabs(
const PacketXd& a) {
1304 return svabs_f64_x(svptrue_b64(), a);
1308EIGEN_STRONG_INLINE
double predux<PacketXd>(
const PacketXd& a) {
1309 return svaddv_f64(svptrue_b64(), a);
1313EIGEN_STRONG_INLINE
bool predux_any(
const PacketXd& a) {
1314 const svuint64_t bits = svreinterpret_u64_f64(a);
1315 return svptest_any(svptrue_b64(), svcmpne_n_u64(svptrue_b64(), bits, 0));
1319EIGEN_STRONG_INLINE
bool predux_all(
const PacketXd& a) {
1320 const svbool_t all = svptrue_b64();
1321 return !svptest_any(all, svcmpeq_n_f64(all, a, 0.0));
1325EIGEN_STRONG_INLINE Index predux_count(
const PacketXd& a) {
1326 const svbool_t all = svptrue_b64();
1327 return static_cast<Index
>(svcntp_b64(all, svcmpne_n_f64(all, a, 0.0)));
1331EIGEN_STRONG_INLINE
double predux_mul<PacketXd>(
const PacketXd& a) {
1333 svfloat64_t prod = svmul_f64_x(svptrue_b64(), a, svrev_f64(a));
1337 for (
int n = unpacket_traits<PacketXd>::size; n > 2; n >>= 1)
1338 prod = svmul_f64_x(svptrue_b64(), svzip1_f64(prod, prod), svzip2_f64(prod, prod));
1341 return pfirst<PacketXd>(prod);
1345EIGEN_STRONG_INLINE
double predux_min<PacketXd>(
const PacketXd& a) {
1346 return svminv_f64(svptrue_b64(), a);
1350EIGEN_STRONG_INLINE
double predux_max<PacketXd>(
const PacketXd& a) {
1351 return svmaxv_f64(svptrue_b64(), a);
1355EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<PacketXd, N>& kernel) {
1356 EIGEN_STATIC_ASSERT((N & (N - 1)) == 0, EIGEN_INTERNAL_ERROR_PLEASE_FILE_A_BUG_REPORT);
1357 for (
int stride = N / 2; stride > 0; stride >>= 1) {
1358 for (
int block = 0; block < N; block += 2 * stride) {
1359 for (
int k = 0; k < stride; ++k) {
1360 PacketXd lo = svzip1_f64(kernel.packet[block + k], kernel.packet[block + k + stride]);
1361 PacketXd hi = svzip2_f64(kernel.packet[block + k], kernel.packet[block + k + stride]);
1362 kernel.packet[block + k] = lo;
1363 kernel.packet[block + k + stride] = hi;
1370EIGEN_STRONG_INLINE PacketXd pfrexp<PacketXd>(
const PacketXd& a, PacketXd& exponent) {
1371 return pfrexp_generic(a, exponent);
1375EIGEN_STRONG_INLINE PacketXd pldexp<PacketXd>(
const PacketXd& a,
const PacketXd& exponent) {
1376 return svscale_f64_x(svptrue_b64(), a, svcvt_s64_f64_x(svptrue_b64(), exponent));
1380EIGEN_STRONG_INLINE PacketXd psqrt<PacketXd>(
const PacketXd& a) {
1381 return svsqrt_f64_x(svptrue_b64(), a);
1385EIGEN_STRONG_INLINE PacketXd prsqrt<PacketXd>(
const PacketXd& a) {
1389 return generic_rsqrt_newton_step<PacketXd, 3>::run(a, svrsqrte_f64(a));
1396EIGEN_STRONG_INLINE svbool_t sve_segment_predicate_b32(Index begin, Index count) {
1397 eigen_assert(begin >= 0 && count >= 0 && begin + count <= EIGEN_ARM64_SVE_VL / 32);
1398 if (begin == 0)
return svwhilelt_b32(uint64_t(0), uint64_t(count));
1399 return svnot_b_z(svwhilelt_b32(uint64_t(0), uint64_t(begin + count)), svwhilelt_b32(uint64_t(0), uint64_t(begin)));
1402EIGEN_STRONG_INLINE svbool_t sve_segment_predicate_b64(Index begin, Index count) {
1403 eigen_assert(begin >= 0 && count >= 0 && begin + count <= EIGEN_ARM64_SVE_VL / 64);
1404 if (begin == 0)
return svwhilelt_b64(uint64_t(0), uint64_t(count));
1405 return svnot_b_z(svwhilelt_b64(uint64_t(0), uint64_t(begin + count)), svwhilelt_b64(uint64_t(0), uint64_t(begin)));
1411struct has_packet_segment<PacketXi> : std::true_type {};
1414inline PacketXi ploaduSegment<PacketXi>(
const numext::int32_t* from, Index begin, Index count) {
1415 return svld1_s32(sve_segment_predicate_b32(begin, count), from);
1419inline void pstoreuSegment<numext::int32_t, PacketXi>(numext::int32_t* to,
const PacketXi& from, Index begin,
1421 svst1_s32(sve_segment_predicate_b32(begin, count), to, from);
1427struct has_packet_segment<PacketXl> : std::true_type {};
1430inline PacketXl ploaduSegment<PacketXl>(
const numext::int64_t* from, Index begin, Index count) {
1431 return svld1_s64(sve_segment_predicate_b64(begin, count), from);
1435inline void pstoreuSegment<numext::int64_t, PacketXl>(numext::int64_t* to,
const PacketXl& from, Index begin,
1437 svst1_s64(sve_segment_predicate_b64(begin, count), to, from);
1443struct has_packet_segment<PacketXf> : std::true_type {};
1446inline PacketXf ploaduSegment<PacketXf>(
const float* from, Index begin, Index count) {
1447 return svld1_f32(sve_segment_predicate_b32(begin, count), from);
1451inline void pstoreuSegment<float, PacketXf>(
float* to,
const PacketXf& from, Index begin, Index count) {
1452 svst1_f32(sve_segment_predicate_b32(begin, count), to, from);
1458struct has_packet_segment<PacketXd> : std::true_type {};
1461inline PacketXd ploaduSegment<PacketXd>(
const double* from, Index begin, Index count) {
1462 return svld1_f64(sve_segment_predicate_b64(begin, count), from);
1466inline void pstoreuSegment<double, PacketXd>(
double* to,
const PacketXd& from, Index begin, Index count) {
1467 svst1_f64(sve_segment_predicate_b64(begin, count), to, from);
@ Aligned128
Definition Constants.h:241