13#ifndef EIGEN_PACKET_MATH_NEON_H
14#define EIGEN_PACKET_MATH_NEON_H
17#include "../../InternalHeaderCheck.h"
23#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
24#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
27#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
28#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
31#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
33#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
35#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16
39#if EIGEN_COMP_MSVC_STRICT
45using Packet2f = eigen_packet_wrapper<float32x2_t, 0>;
46using Packet4f = eigen_packet_wrapper<float32x4_t, 1>;
47using Packet4c = eigen_packet_wrapper<int32_t, 2>;
48using Packet8c = eigen_packet_wrapper<int8x8_t, 3>;
49using Packet16c = eigen_packet_wrapper<int8x16_t, 4>;
50using Packet4uc = eigen_packet_wrapper<uint32_t, 5>;
51using Packet8uc = eigen_packet_wrapper<uint8x8_t, 6>;
52using Packet16uc = eigen_packet_wrapper<uint8x16_t, 7>;
53using Packet4s = eigen_packet_wrapper<int16x4_t, 8>;
54using Packet8s = eigen_packet_wrapper<int16x8_t, 9>;
55using Packet4us = eigen_packet_wrapper<uint16x4_t, 10>;
56using Packet8us = eigen_packet_wrapper<uint16x8_t, 11>;
57using Packet2i = eigen_packet_wrapper<int32x2_t, 12>;
58using Packet4i = eigen_packet_wrapper<int32x4_t, 13>;
59using Packet2ui = eigen_packet_wrapper<uint32x2_t, 14>;
60using Packet4ui = eigen_packet_wrapper<uint32x4_t, 15>;
61using Packet2l = eigen_packet_wrapper<int64x2_t, 16>;
62using Packet2ul = eigen_packet_wrapper<uint64x2_t, 17>;
64EIGEN_ALWAYS_INLINE Packet4f make_packet4f(
float a,
float b,
float c,
float d) {
65 float from[4] = {a, b, c, d};
66 return vld1q_f32(from);
69EIGEN_ALWAYS_INLINE Packet2f make_packet2f(
float a,
float b) {
70 float from[2] = {a, b};
71 return vld1_f32(from);
76using Packet2f = float32x2_t;
77using Packet4f = float32x4_t;
78using Packet4c = eigen_packet_wrapper<int32_t, 2>;
79using Packet8c = int8x8_t;
80using Packet16c = int8x16_t;
81using Packet4uc = eigen_packet_wrapper<uint32_t, 5>;
82using Packet8uc = uint8x8_t;
83using Packet16uc = uint8x16_t;
84using Packet4s = int16x4_t;
85using Packet8s = int16x8_t;
86using Packet4us = uint16x4_t;
87using Packet8us = uint16x8_t;
88using Packet2i = int32x2_t;
89using Packet4i = int32x4_t;
90using Packet2ui = uint32x2_t;
91using Packet4ui = uint32x4_t;
92using Packet2l = int64x2_t;
93using Packet2ul = uint64x2_t;
95EIGEN_ALWAYS_INLINE Packet4f make_packet4f(
float a,
float b,
float c,
float d) {
return Packet4f{a, b, c, d}; }
96EIGEN_ALWAYS_INLINE Packet2f make_packet2f(
float a,
float b) {
return Packet2f{a, b}; }
100EIGEN_STRONG_INLINE Packet4f shuffle1(
const Packet4f& m,
int mask) {
101 const float* a =
reinterpret_cast<const float*
>(&m);
103 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3)));
111template <
bool interleave>
112EIGEN_STRONG_INLINE Packet4f shuffle2(
const Packet4f& m,
const Packet4f& n,
int mask) {
113 const float* a =
reinterpret_cast<const float*
>(&m);
114 const float* b =
reinterpret_cast<const float*
>(&n);
116 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
121EIGEN_STRONG_INLINE Packet4f shuffle2<true>(
const Packet4f& m,
const Packet4f& n,
int mask) {
122 const float* a =
reinterpret_cast<const float*
>(&m);
123 const float* b =
reinterpret_cast<const float*
>(&n);
125 make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
129EIGEN_STRONG_INLINE
static int eigen_neon_shuffle_mask(
int p,
int q,
int r,
int s) {
130 return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
133EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(
const Packet4f& a,
int p,
int q,
int r,
int s) {
134 return shuffle1(a, eigen_neon_shuffle_mask(p, q, r, s));
136EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(
const Packet4f& a,
const Packet4f& b,
int p,
int q,
int r,
int s) {
137 return shuffle2<false>(a, b, eigen_neon_shuffle_mask(p, q, r, s));
139EIGEN_STRONG_INLINE Packet4f vec4f_movelh(
const Packet4f& a,
const Packet4f& b) {
140 return shuffle2<false>(a, b, eigen_neon_shuffle_mask(0, 1, 0, 1));
142EIGEN_STRONG_INLINE Packet4f vec4f_movehl(
const Packet4f& a,
const Packet4f& b) {
143 return shuffle2<false>(b, a, eigen_neon_shuffle_mask(2, 3, 2, 3));
145EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(
const Packet4f& a,
const Packet4f& b) {
146 return shuffle2<true>(a, b, eigen_neon_shuffle_mask(0, 0, 1, 1));
148EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(
const Packet4f& a,
const Packet4f& b) {
149 return shuffle2<true>(a, b, eigen_neon_shuffle_mask(2, 2, 3, 3));
151#define vec4f_duplane(a, p) Packet4f(vdupq_lane_f32(vget_low_f32(a), p))
153#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = pset1<Packet4f>(X)
155#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
156 const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X))
158#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = pset1<Packet4i>(X)
160#if EIGEN_ARCH_ARM64 && EIGEN_COMP_GNUC
164#define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) :);
165#elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
166#define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR);
168#define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR)
170#define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("pld [%[addr]]\n" ::[addr] "r"(ADDR) :);
173#define EIGEN_ARM_PREFETCH(ADDR)
177struct packet_traits<float> : default_packet_traits {
178 using type = Packet4f;
179 using half = Packet2f;
199 HasSin = EIGEN_FAST_MATH,
200 HasCos = EIGEN_FAST_MATH,
201 HasTan = EIGEN_FAST_MATH,
219 HasTanh = EIGEN_FAST_MATH,
220 HasErf = EIGEN_FAST_MATH,
221 HasErfc = EIGEN_FAST_MATH,
228struct packet_traits<int8_t> : default_packet_traits {
229 using type = Packet16c;
230 using half = Packet8c;
253struct packet_traits<uint8_t> : default_packet_traits {
254 using type = Packet16uc;
255 using half = Packet8uc;
280struct packet_traits<int16_t> : default_packet_traits {
281 using type = Packet8s;
282 using half = Packet4s;
305struct packet_traits<uint16_t> : default_packet_traits {
306 using type = Packet8us;
307 using half = Packet4us;
331struct packet_traits<int32_t> : default_packet_traits {
332 using type = Packet4i;
333 using half = Packet2i;
357struct packet_traits<uint32_t> : default_packet_traits {
358 using type = Packet4ui;
359 using half = Packet2ui;
385struct packet_traits<int64_t> : default_packet_traits {
386 using type = Packet2l;
387 using half = Packet2l;
410struct packet_traits<uint64_t> : default_packet_traits {
411 using type = Packet2ul;
412 using half = Packet2ul;
434template <
typename Packet,
typename Scalar>
435struct neon_unpacket_default {
438 static constexpr int size =
sizeof(Packet) /
sizeof(Scalar);
439 static constexpr int alignment =
sizeof(Packet);
440 static constexpr bool vectorizable =
true;
441 static constexpr bool masked_load_available =
false;
442 static constexpr bool masked_store_available =
false;
446struct unpacket_traits<Packet2f> : neon_unpacket_default<Packet2f, float> {
447 using integer_packet = Packet2i;
450struct unpacket_traits<Packet4f> : neon_unpacket_default<Packet4f, float> {
451 using half = Packet2f;
452 using integer_packet = Packet4i;
455struct unpacket_traits<Packet4c> : neon_unpacket_default<Packet4c, int8_t> {};
457struct unpacket_traits<Packet8c> : neon_unpacket_default<Packet8c, int8_t> {
458 using half = Packet4c;
461struct unpacket_traits<Packet16c> : neon_unpacket_default<Packet16c, int8_t> {
462 using half = Packet8c;
465struct unpacket_traits<Packet4uc> : neon_unpacket_default<Packet4uc, uint8_t> {};
467struct unpacket_traits<Packet8uc> : neon_unpacket_default<Packet8uc, uint8_t> {
468 using half = Packet4uc;
471struct unpacket_traits<Packet16uc> : neon_unpacket_default<Packet16uc, uint8_t> {
472 using half = Packet8uc;
475struct unpacket_traits<Packet4s> : neon_unpacket_default<Packet4s, int16_t> {};
477struct unpacket_traits<Packet8s> : neon_unpacket_default<Packet8s, int16_t> {
478 using half = Packet4s;
481struct unpacket_traits<Packet4us> : neon_unpacket_default<Packet4us, uint16_t> {};
483struct unpacket_traits<Packet8us> : neon_unpacket_default<Packet8us, uint16_t> {
484 using half = Packet4us;
487struct unpacket_traits<Packet2i> : neon_unpacket_default<Packet2i, int32_t> {};
489struct unpacket_traits<Packet4i> : neon_unpacket_default<Packet4i, int32_t> {
490 using half = Packet2i;
493struct unpacket_traits<Packet2ui> : neon_unpacket_default<Packet2ui, uint32_t> {};
495struct unpacket_traits<Packet4ui> : neon_unpacket_default<Packet4ui, uint32_t> {
496 using half = Packet2ui;
499struct unpacket_traits<Packet2l> : neon_unpacket_default<Packet2l, int64_t> {};
501struct unpacket_traits<Packet2ul> : neon_unpacket_default<Packet2ul, uint64_t> {};
504EIGEN_STRONG_INLINE Packet2f pzero(
const Packet2f& ) {
505 return vdup_n_f32(0.0f);
509EIGEN_STRONG_INLINE Packet4f pzero(
const Packet4f& ) {
510 return vdupq_n_f32(0.0f);
514EIGEN_STRONG_INLINE Packet2f pset1<Packet2f>(
const float& from) {
515 return vdup_n_f32(from);
518EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
519 return vdupq_n_f32(from);
522EIGEN_STRONG_INLINE Packet4c pset1<Packet4c>(
const int8_t& from) {
523 return vget_lane_s32(vreinterpret_s32_s8(vdup_n_s8(from)), 0);
526EIGEN_STRONG_INLINE Packet8c pset1<Packet8c>(
const int8_t& from) {
527 return vdup_n_s8(from);
530EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(
const int8_t& from) {
531 return vdupq_n_s8(from);
534EIGEN_STRONG_INLINE Packet4uc pset1<Packet4uc>(
const uint8_t& from) {
535 return vget_lane_u32(vreinterpret_u32_u8(vdup_n_u8(from)), 0);
538EIGEN_STRONG_INLINE Packet8uc pset1<Packet8uc>(
const uint8_t& from) {
539 return vdup_n_u8(from);
542EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(
const uint8_t& from) {
543 return vdupq_n_u8(from);
546EIGEN_STRONG_INLINE Packet4s pset1<Packet4s>(
const int16_t& from) {
547 return vdup_n_s16(from);
550EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(
const int16_t& from) {
551 return vdupq_n_s16(from);
554EIGEN_STRONG_INLINE Packet4us pset1<Packet4us>(
const uint16_t& from) {
555 return vdup_n_u16(from);
558EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(
const uint16_t& from) {
559 return vdupq_n_u16(from);
562EIGEN_STRONG_INLINE Packet2i pset1<Packet2i>(
const int32_t& from) {
563 return vdup_n_s32(from);
566EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int32_t& from) {
567 return vdupq_n_s32(from);
570EIGEN_STRONG_INLINE Packet2ui pset1<Packet2ui>(
const uint32_t& from) {
571 return vdup_n_u32(from);
574EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(
const uint32_t& from) {
575 return vdupq_n_u32(from);
578EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(
const int64_t& from) {
579 return vdupq_n_s64(from);
582EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(
const uint64_t& from) {
583 return vdupq_n_u64(from);
587EIGEN_STRONG_INLINE Packet2f pset1frombits<Packet2f>(uint32_t from) {
588 return vreinterpret_f32_u32(vdup_n_u32(from));
591EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
592 return vreinterpretq_f32_u32(vdupq_n_u32(from));
596EIGEN_STRONG_INLINE Packet2f plset<Packet2f>(
const float& a) {
597 const float c[] = {0.0f, 1.0f};
598 return vadd_f32(pset1<Packet2f>(a), vld1_f32(c));
601EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a) {
602 const float c[] = {0.0f, 1.0f, 2.0f, 3.0f};
603 return vaddq_f32(pset1<Packet4f>(a), vld1q_f32(c));
606EIGEN_STRONG_INLINE Packet4c plset<Packet4c>(
const int8_t& a) {
607 return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_u32(vdup_n_u32(0x03020100)), vdup_n_s8(a))), 0);
610EIGEN_STRONG_INLINE Packet8c plset<Packet8c>(
const int8_t& a) {
611 const int8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
612 return vadd_s8(pset1<Packet8c>(a), vld1_s8(c));
615EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(
const int8_t& a) {
616 const int8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
617 return vaddq_s8(pset1<Packet16c>(a), vld1q_s8(c));
620EIGEN_STRONG_INLINE Packet4uc plset<Packet4uc>(
const uint8_t& a) {
621 return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(0x03020100)), vdup_n_u8(a))), 0);
624EIGEN_STRONG_INLINE Packet8uc plset<Packet8uc>(
const uint8_t& a) {
625 const uint8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
626 return vadd_u8(pset1<Packet8uc>(a), vld1_u8(c));
629EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(
const uint8_t& a) {
630 const uint8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
631 return vaddq_u8(pset1<Packet16uc>(a), vld1q_u8(c));
634EIGEN_STRONG_INLINE Packet4s plset<Packet4s>(
const int16_t& a) {
635 const int16_t c[] = {0, 1, 2, 3};
636 return vadd_s16(pset1<Packet4s>(a), vld1_s16(c));
639EIGEN_STRONG_INLINE Packet4us plset<Packet4us>(
const uint16_t& a) {
640 const uint16_t c[] = {0, 1, 2, 3};
641 return vadd_u16(pset1<Packet4us>(a), vld1_u16(c));
644EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(
const int16_t& a) {
645 const int16_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
646 return vaddq_s16(pset1<Packet8s>(a), vld1q_s16(c));
649EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(
const uint16_t& a) {
650 const uint16_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
651 return vaddq_u16(pset1<Packet8us>(a), vld1q_u16(c));
654EIGEN_STRONG_INLINE Packet2i plset<Packet2i>(
const int32_t& a) {
655 const int32_t c[] = {0, 1};
656 return vadd_s32(pset1<Packet2i>(a), vld1_s32(c));
659EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int32_t& a) {
660 const int32_t c[] = {0, 1, 2, 3};
661 return vaddq_s32(pset1<Packet4i>(a), vld1q_s32(c));
664EIGEN_STRONG_INLINE Packet2ui plset<Packet2ui>(
const uint32_t& a) {
665 const uint32_t c[] = {0, 1};
666 return vadd_u32(pset1<Packet2ui>(a), vld1_u32(c));
669EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(
const uint32_t& a) {
670 const uint32_t c[] = {0, 1, 2, 3};
671 return vaddq_u32(pset1<Packet4ui>(a), vld1q_u32(c));
674EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(
const int64_t& a) {
675 const int64_t c[] = {0, 1};
676 return vaddq_s64(pset1<Packet2l>(a), vld1q_s64(c));
679EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(
const uint64_t& a) {
680 const uint64_t c[] = {0, 1};
681 return vaddq_u64(pset1<Packet2ul>(a), vld1q_u64(c));
685EIGEN_STRONG_INLINE Packet2f padd<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
686 return vadd_f32(a, b);
689EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
690 return vaddq_f32(a, b);
693EIGEN_STRONG_INLINE Packet4c padd<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
694 return vget_lane_s32(
695 vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
698EIGEN_STRONG_INLINE Packet8c padd<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
699 return vadd_s8(a, b);
702EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
703 return vaddq_s8(a, b);
706EIGEN_STRONG_INLINE Packet4uc padd<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
707 return vget_lane_u32(
708 vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
711EIGEN_STRONG_INLINE Packet8uc padd<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
712 return vadd_u8(a, b);
715EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
716 return vaddq_u8(a, b);
719EIGEN_STRONG_INLINE Packet4s padd<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
720 return vadd_s16(a, b);
723EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
724 return vaddq_s16(a, b);
727EIGEN_STRONG_INLINE Packet4us padd<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
728 return vadd_u16(a, b);
731EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
732 return vaddq_u16(a, b);
735EIGEN_STRONG_INLINE Packet2i padd<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
736 return vadd_s32(a, b);
739EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
740 return vaddq_s32(a, b);
743EIGEN_STRONG_INLINE Packet2ui padd<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
744 return vadd_u32(a, b);
747EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
748 return vaddq_u32(a, b);
751EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
752 return vaddq_s64(a, b);
755EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
756 return vaddq_u64(a, b);
760EIGEN_STRONG_INLINE Packet2f psub<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
761 return vsub_f32(a, b);
764EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
765 return vsubq_f32(a, b);
768EIGEN_STRONG_INLINE Packet4c psub<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
769 return vget_lane_s32(
770 vreinterpret_s32_s8(vsub_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
773EIGEN_STRONG_INLINE Packet8c psub<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
774 return vsub_s8(a, b);
777EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
778 return vsubq_s8(a, b);
781EIGEN_STRONG_INLINE Packet4uc psub<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
782 return vget_lane_u32(
783 vreinterpret_u32_u8(vsub_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
786EIGEN_STRONG_INLINE Packet8uc psub<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
787 return vsub_u8(a, b);
790EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
791 return vsubq_u8(a, b);
794EIGEN_STRONG_INLINE Packet4s psub<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
795 return vsub_s16(a, b);
798EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
799 return vsubq_s16(a, b);
802EIGEN_STRONG_INLINE Packet4us psub<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
803 return vsub_u16(a, b);
806EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
807 return vsubq_u16(a, b);
810EIGEN_STRONG_INLINE Packet2i psub<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
811 return vsub_s32(a, b);
814EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
815 return vsubq_s32(a, b);
818EIGEN_STRONG_INLINE Packet2ui psub<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
819 return vsub_u32(a, b);
822EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
823 return vsubq_u32(a, b);
826EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
827 return vsubq_s64(a, b);
830EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
831 return vsubq_u64(a, b);
835EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(
const Packet2f& a,
const Packet2f& b);
837EIGEN_STRONG_INLINE Packet2f paddsub<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
838 Packet2f mask = make_packet2f(numext::bit_cast<float>(0x80000000u), 0.0f);
839 return padd(a, pxor(mask, b));
842EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b);
844EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
845 Packet4f mask = make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
846 return padd(a, pxor(mask, b));
850EIGEN_STRONG_INLINE Packet2f pnegate(
const Packet2f& a) {
854EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
858EIGEN_STRONG_INLINE Packet4c pnegate(
const Packet4c& a) {
859 return vget_lane_s32(vreinterpret_s32_s8(vneg_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
862EIGEN_STRONG_INLINE Packet8c pnegate(
const Packet8c& a) {
866EIGEN_STRONG_INLINE Packet16c pnegate(
const Packet16c& a) {
870EIGEN_STRONG_INLINE Packet4s pnegate(
const Packet4s& a) {
874EIGEN_STRONG_INLINE Packet8s pnegate(
const Packet8s& a) {
878EIGEN_STRONG_INLINE Packet2i pnegate(
const Packet2i& a) {
882EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
886EIGEN_STRONG_INLINE Packet2l pnegate(
const Packet2l& a) {
889#elif !EIGEN_GNUC_STRICT_AT_LEAST(12, 0, 0)
890 return psub(pzero(a), a);
894 int64x2_t z = vdupq_n_s64(0);
895 asm(
"vsub.i64 %q0, %q1, %q0" :
"+w"(x) :
"w"(z));
901EIGEN_STRONG_INLINE Packet2f pmul<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
902 return vmul_f32(a, b);
905EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
906 return vmulq_f32(a, b);
909EIGEN_STRONG_INLINE Packet4c pmul<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
910 return vget_lane_s32(
911 vreinterpret_s32_s8(vmul_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
914EIGEN_STRONG_INLINE Packet8c pmul<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
915 return vmul_s8(a, b);
918EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
919 return vmulq_s8(a, b);
922EIGEN_STRONG_INLINE Packet4uc pmul<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
923 return vget_lane_u32(
924 vreinterpret_u32_u8(vmul_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
927EIGEN_STRONG_INLINE Packet8uc pmul<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
928 return vmul_u8(a, b);
931EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
932 return vmulq_u8(a, b);
935EIGEN_STRONG_INLINE Packet4s pmul<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
936 return vmul_s16(a, b);
939EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
940 return vmulq_s16(a, b);
943EIGEN_STRONG_INLINE Packet4us pmul<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
944 return vmul_u16(a, b);
947EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
948 return vmulq_u16(a, b);
951EIGEN_STRONG_INLINE Packet2i pmul<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
952 return vmul_s32(a, b);
955EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
956 return vmulq_s32(a, b);
959EIGEN_STRONG_INLINE Packet2ui pmul<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
960 return vmul_u32(a, b);
963EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
964 return vmulq_u32(a, b);
967EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
968 const uint32x2_t al = vmovn_u64(a);
969 const uint32x2_t bl = vmovn_u64(b);
970 const uint64x2_t hi = vpaddlq_u32(vmulq_u32(vreinterpretq_u32_u64(a), vrev64q_u32(vreinterpretq_u32_u64(b))));
971 return vmlal_u32(vshlq_n_u64(hi, 32), al, bl);
974EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
975 return vreinterpretq_s64_u64(pmul(vreinterpretq_u64_s64(a), vreinterpretq_u64_s64(b)));
979EIGEN_STRONG_INLINE Packet4c pdiv<Packet4c>(
const Packet4c& ,
const Packet4c& ) {
980 eigen_assert(
false &&
"packet integer division are not supported by NEON");
981 return pset1<Packet4c>(0);
984EIGEN_STRONG_INLINE Packet8c pdiv<Packet8c>(
const Packet8c& ,
const Packet8c& ) {
985 eigen_assert(
false &&
"packet integer division are not supported by NEON");
986 return pset1<Packet8c>(0);
989EIGEN_STRONG_INLINE Packet16c pdiv<Packet16c>(
const Packet16c& ,
const Packet16c& ) {
990 eigen_assert(
false &&
"packet integer division are not supported by NEON");
991 return pset1<Packet16c>(0);
994EIGEN_STRONG_INLINE Packet4uc pdiv<Packet4uc>(
const Packet4uc& ,
const Packet4uc& ) {
995 eigen_assert(
false &&
"packet integer division are not supported by NEON");
996 return pset1<Packet4uc>(0);
999EIGEN_STRONG_INLINE Packet8uc pdiv<Packet8uc>(
const Packet8uc& ,
const Packet8uc& ) {
1000 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1001 return pset1<Packet8uc>(0);
1004EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(
const Packet16uc& ,
const Packet16uc& ) {
1005 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1006 return pset1<Packet16uc>(0);
1009EIGEN_STRONG_INLINE Packet4s pdiv<Packet4s>(
const Packet4s& ,
const Packet4s& ) {
1010 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1011 return pset1<Packet4s>(0);
1014EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(
const Packet8s& ,
const Packet8s& ) {
1015 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1016 return pset1<Packet8s>(0);
1019EIGEN_STRONG_INLINE Packet4us pdiv<Packet4us>(
const Packet4us& ,
const Packet4us& ) {
1020 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1021 return pset1<Packet4us>(0);
1024EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(
const Packet8us& ,
const Packet8us& ) {
1025 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1026 return pset1<Packet8us>(0);
1029EIGEN_STRONG_INLINE Packet2i pdiv<Packet2i>(
const Packet2i& ,
const Packet2i& ) {
1030 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1031 return pset1<Packet2i>(0);
1034EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& ,
const Packet4i& ) {
1035 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1036 return pset1<Packet4i>(0);
1039EIGEN_STRONG_INLINE Packet2ui pdiv<Packet2ui>(
const Packet2ui& ,
const Packet2ui& ) {
1040 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1041 return pset1<Packet2ui>(0);
1044EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(
const Packet4ui& ,
const Packet4ui& ) {
1045 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1046 return pset1<Packet4ui>(0);
1049EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(
const Packet2l& ,
const Packet2l& ) {
1050 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1051 return pset1<Packet2l>(0LL);
1054EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(
const Packet2ul& ,
const Packet2ul& ) {
1055 eigen_assert(
false &&
"packet integer division are not supported by NEON");
1056 return pset1<Packet2ul>(0ULL);
1059#ifdef EIGEN_VECTORIZE_FMA
1061struct is_fused_madd<Packet4f> : true_type {};
1063EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1064 return vfmaq_f32(c, a, b);
1067struct is_fused_madd<Packet2f> : true_type {};
1069EIGEN_STRONG_INLINE Packet2f pmadd(
const Packet2f& a,
const Packet2f& b,
const Packet2f& c) {
1070 return vfma_f32(c, a, b);
1073EIGEN_STRONG_INLINE Packet4f pnmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1074 return vfmsq_f32(c, a, b);
1077EIGEN_STRONG_INLINE Packet2f pnmadd(
const Packet2f& a,
const Packet2f& b,
const Packet2f& c) {
1078 return vfms_f32(c, a, b);
1082EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1083 return vmlaq_f32(c, a, b);
1086EIGEN_STRONG_INLINE Packet2f pmadd(
const Packet2f& a,
const Packet2f& b,
const Packet2f& c) {
1087 return vmla_f32(c, a, b);
1090EIGEN_STRONG_INLINE Packet4f pnmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1091 return vmlsq_f32(c, a, b);
1094EIGEN_STRONG_INLINE Packet2f pnmadd(
const Packet2f& a,
const Packet2f& b,
const Packet2f& c) {
1095 return vmls_f32(c, a, b);
1100EIGEN_STRONG_INLINE Packet4c pmadd(
const Packet4c& a,
const Packet4c& b,
const Packet4c& c) {
1101 return vget_lane_s32(
1102 vreinterpret_s32_s8(vmla_s8(vreinterpret_s8_s32(vdup_n_s32(c)), vreinterpret_s8_s32(vdup_n_s32(a)),
1103 vreinterpret_s8_s32(vdup_n_s32(b)))),
1107EIGEN_STRONG_INLINE Packet8c pmadd(
const Packet8c& a,
const Packet8c& b,
const Packet8c& c) {
1108 return vmla_s8(c, a, b);
1111EIGEN_STRONG_INLINE Packet16c pmadd(
const Packet16c& a,
const Packet16c& b,
const Packet16c& c) {
1112 return vmlaq_s8(c, a, b);
1115EIGEN_STRONG_INLINE Packet4uc pmadd(
const Packet4uc& a,
const Packet4uc& b,
const Packet4uc& c) {
1116 return vget_lane_u32(
1117 vreinterpret_u32_u8(vmla_u8(vreinterpret_u8_u32(vdup_n_u32(c)), vreinterpret_u8_u32(vdup_n_u32(a)),
1118 vreinterpret_u8_u32(vdup_n_u32(b)))),
1122EIGEN_STRONG_INLINE Packet8uc pmadd(
const Packet8uc& a,
const Packet8uc& b,
const Packet8uc& c) {
1123 return vmla_u8(c, a, b);
1126EIGEN_STRONG_INLINE Packet16uc pmadd(
const Packet16uc& a,
const Packet16uc& b,
const Packet16uc& c) {
1127 return vmlaq_u8(c, a, b);
1130EIGEN_STRONG_INLINE Packet4s pmadd(
const Packet4s& a,
const Packet4s& b,
const Packet4s& c) {
1131 return vmla_s16(c, a, b);
1134EIGEN_STRONG_INLINE Packet8s pmadd(
const Packet8s& a,
const Packet8s& b,
const Packet8s& c) {
1135 return vmlaq_s16(c, a, b);
1138EIGEN_STRONG_INLINE Packet4us pmadd(
const Packet4us& a,
const Packet4us& b,
const Packet4us& c) {
1139 return vmla_u16(c, a, b);
1142EIGEN_STRONG_INLINE Packet8us pmadd(
const Packet8us& a,
const Packet8us& b,
const Packet8us& c) {
1143 return vmlaq_u16(c, a, b);
1146EIGEN_STRONG_INLINE Packet2i pmadd(
const Packet2i& a,
const Packet2i& b,
const Packet2i& c) {
1147 return vmla_s32(c, a, b);
1150EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
1151 return vmlaq_s32(c, a, b);
1154EIGEN_STRONG_INLINE Packet2ui pmadd(
const Packet2ui& a,
const Packet2ui& b,
const Packet2ui& c) {
1155 return vmla_u32(c, a, b);
1158EIGEN_STRONG_INLINE Packet4ui pmadd(
const Packet4ui& a,
const Packet4ui& b,
const Packet4ui& c) {
1159 return vmlaq_u32(c, a, b);
1163EIGEN_STRONG_INLINE Packet2f pabsdiff<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1164 return vabd_f32(a, b);
1167EIGEN_STRONG_INLINE Packet4f pabsdiff<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1168 return vabdq_f32(a, b);
1171EIGEN_STRONG_INLINE Packet4c pabsdiff<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1172 return vget_lane_s32(
1173 vreinterpret_s32_s8(vabd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1176EIGEN_STRONG_INLINE Packet8c pabsdiff<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1177 return vabd_s8(a, b);
1180EIGEN_STRONG_INLINE Packet16c pabsdiff<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1181 return vabdq_s8(a, b);
1184EIGEN_STRONG_INLINE Packet4uc pabsdiff<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1185 return vget_lane_u32(
1186 vreinterpret_u32_u8(vabd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1189EIGEN_STRONG_INLINE Packet8uc pabsdiff<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1190 return vabd_u8(a, b);
1193EIGEN_STRONG_INLINE Packet16uc pabsdiff<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1194 return vabdq_u8(a, b);
1197EIGEN_STRONG_INLINE Packet4s pabsdiff<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1198 return vabd_s16(a, b);
1201EIGEN_STRONG_INLINE Packet8s pabsdiff<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1202 return vabdq_s16(a, b);
1205EIGEN_STRONG_INLINE Packet4us pabsdiff<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1206 return vabd_u16(a, b);
1209EIGEN_STRONG_INLINE Packet8us pabsdiff<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1210 return vabdq_u16(a, b);
1213EIGEN_STRONG_INLINE Packet2i pabsdiff<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1214 return vabd_s32(a, b);
1217EIGEN_STRONG_INLINE Packet4i pabsdiff<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1218 return vabdq_s32(a, b);
1221EIGEN_STRONG_INLINE Packet2ui pabsdiff<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1222 return vabd_u32(a, b);
1225EIGEN_STRONG_INLINE Packet4ui pabsdiff<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1226 return vabdq_u32(a, b);
1230EIGEN_STRONG_INLINE Packet2f pmin<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1231 return vmin_f32(a, b);
1234EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1235 return vminq_f32(a, b);
1238#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1242EIGEN_STRONG_INLINE Packet4f pmin<PropagateNumbers, Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1243 return vminnmq_f32(a, b);
1246EIGEN_STRONG_INLINE Packet2f pmin<PropagateNumbers, Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1247 return vminnm_f32(a, b);
1252struct pminmax_propagates_nan<Packet2f> : bool_constant<true> {};
1254struct pminmax_propagates_nan<Packet4f> : bool_constant<true> {};
1257EIGEN_STRONG_INLINE Packet4c pmin<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1258 return vget_lane_s32(
1259 vreinterpret_s32_s8(vmin_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1262EIGEN_STRONG_INLINE Packet8c pmin<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1263 return vmin_s8(a, b);
1266EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1267 return vminq_s8(a, b);
1270EIGEN_STRONG_INLINE Packet4uc pmin<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1271 return vget_lane_u32(
1272 vreinterpret_u32_u8(vmin_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1275EIGEN_STRONG_INLINE Packet8uc pmin<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1276 return vmin_u8(a, b);
1279EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1280 return vminq_u8(a, b);
1283EIGEN_STRONG_INLINE Packet4s pmin<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1284 return vmin_s16(a, b);
1287EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1288 return vminq_s16(a, b);
1291EIGEN_STRONG_INLINE Packet4us pmin<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1292 return vmin_u16(a, b);
1295EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1296 return vminq_u16(a, b);
1299EIGEN_STRONG_INLINE Packet2i pmin<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1300 return vmin_s32(a, b);
1303EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1304 return vminq_s32(a, b);
1307EIGEN_STRONG_INLINE Packet2ui pmin<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1308 return vmin_u32(a, b);
1311EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1312 return vminq_u32(a, b);
1315EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1316 return vsubq_u64(a, vqsubq_u64(a, b));
1320EIGEN_STRONG_INLINE Packet2f pmax<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1321 return vmax_f32(a, b);
1324EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1325 return vmaxq_f32(a, b);
1328#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1332EIGEN_STRONG_INLINE Packet4f pmax<PropagateNumbers, Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1333 return vmaxnmq_f32(a, b);
1336EIGEN_STRONG_INLINE Packet2f pmax<PropagateNumbers, Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1337 return vmaxnm_f32(a, b);
1342EIGEN_STRONG_INLINE Packet4c pmax<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1343 return vget_lane_s32(
1344 vreinterpret_s32_s8(vmax_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1347EIGEN_STRONG_INLINE Packet8c pmax<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1348 return vmax_s8(a, b);
1351EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1352 return vmaxq_s8(a, b);
1355EIGEN_STRONG_INLINE Packet4uc pmax<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1356 return vget_lane_u32(
1357 vreinterpret_u32_u8(vmax_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1360EIGEN_STRONG_INLINE Packet8uc pmax<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1361 return vmax_u8(a, b);
1364EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1365 return vmaxq_u8(a, b);
1368EIGEN_STRONG_INLINE Packet4s pmax<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1369 return vmax_s16(a, b);
1372EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1373 return vmaxq_s16(a, b);
1376EIGEN_STRONG_INLINE Packet4us pmax<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1377 return vmax_u16(a, b);
1380EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1381 return vmaxq_u16(a, b);
1384EIGEN_STRONG_INLINE Packet2i pmax<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1385 return vmax_s32(a, b);
1388EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1389 return vmaxq_s32(a, b);
1392EIGEN_STRONG_INLINE Packet2ui pmax<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1393 return vmax_u32(a, b);
1396EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1397 return vmaxq_u32(a, b);
1400EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1401 return vaddq_u64(b, vqsubq_u64(a, b));
1407#if EIGEN_COMP_GNUC_STRICT && !EIGEN_ARCH_ARM64
1408EIGEN_STRONG_INLINE uint32x2_t neon_vcge_f32(float32x2_t a, float32x2_t b) {
1410 __asm__(
"vcge.f32 %P0, %P1, %P2" :
"=w"(r) :
"w"(a),
"w"(b));
1413EIGEN_STRONG_INLINE uint32x4_t neon_vcgeq_f32(float32x4_t a, float32x4_t b) {
1415 __asm__(
"vcge.f32 %q0, %q1, %q2" :
"=w"(r) :
"w"(a),
"w"(b));
1418EIGEN_STRONG_INLINE uint32x2_t neon_vcgt_f32(float32x2_t a, float32x2_t b) {
1420 __asm__(
"vcgt.f32 %P0, %P1, %P2" :
"=w"(r) :
"w"(a),
"w"(b));
1423EIGEN_STRONG_INLINE uint32x4_t neon_vcgtq_f32(float32x4_t a, float32x4_t b) {
1425 __asm__(
"vcgt.f32 %q0, %q1, %q2" :
"=w"(r) :
"w"(a),
"w"(b));
1429EIGEN_STRONG_INLINE uint32x2_t neon_vcge_f32(float32x2_t a, float32x2_t b) {
return vcge_f32(a, b); }
1430EIGEN_STRONG_INLINE uint32x4_t neon_vcgeq_f32(float32x4_t a, float32x4_t b) {
return vcgeq_f32(a, b); }
1431EIGEN_STRONG_INLINE uint32x2_t neon_vcgt_f32(float32x2_t a, float32x2_t b) {
return vcgt_f32(a, b); }
1432EIGEN_STRONG_INLINE uint32x4_t neon_vcgtq_f32(float32x4_t a, float32x4_t b) {
return vcgtq_f32(a, b); }
1436EIGEN_STRONG_INLINE Packet2f pcmp_le<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1437 return vreinterpret_f32_u32(neon_vcge_f32(b, a));
1440EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1441 return vreinterpretq_f32_u32(neon_vcgeq_f32(b, a));
1444EIGEN_STRONG_INLINE Packet4c pcmp_le<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1445 return vget_lane_s32(
1446 vreinterpret_s32_u8(vcle_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1449EIGEN_STRONG_INLINE Packet8c pcmp_le<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1450 return vreinterpret_s8_u8(vcle_s8(a, b));
1453EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1454 return vreinterpretq_s8_u8(vcleq_s8(a, b));
1457EIGEN_STRONG_INLINE Packet4uc pcmp_le<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1458 return vget_lane_u32(
1459 vreinterpret_u32_u8(vcle_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1462EIGEN_STRONG_INLINE Packet8uc pcmp_le<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1463 return vcle_u8(a, b);
1466EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1467 return vcleq_u8(a, b);
1470EIGEN_STRONG_INLINE Packet4s pcmp_le<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1471 return vreinterpret_s16_u16(vcle_s16(a, b));
1474EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1475 return vreinterpretq_s16_u16(vcleq_s16(a, b));
1478EIGEN_STRONG_INLINE Packet4us pcmp_le<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1479 return vcle_u16(a, b);
1482EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1483 return vcleq_u16(a, b);
1486EIGEN_STRONG_INLINE Packet2i pcmp_le<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1487 return vreinterpret_s32_u32(vcle_s32(a, b));
1490EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1491 return vreinterpretq_s32_u32(vcleq_s32(a, b));
1494EIGEN_STRONG_INLINE Packet2ui pcmp_le<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1495 return vcle_u32(a, b);
1498EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1499 return vcleq_u32(a, b);
1503EIGEN_STRONG_INLINE Packet2f pcmp_lt<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1504 return vreinterpret_f32_u32(neon_vcgt_f32(b, a));
1507EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1508 return vreinterpretq_f32_u32(neon_vcgtq_f32(b, a));
1511EIGEN_STRONG_INLINE Packet4c pcmp_lt<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1512 return vget_lane_s32(
1513 vreinterpret_s32_u8(vclt_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1516EIGEN_STRONG_INLINE Packet8c pcmp_lt<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1517 return vreinterpret_s8_u8(vclt_s8(a, b));
1520EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1521 return vreinterpretq_s8_u8(vcltq_s8(a, b));
1524EIGEN_STRONG_INLINE Packet4uc pcmp_lt<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1525 return vget_lane_u32(
1526 vreinterpret_u32_u8(vclt_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1529EIGEN_STRONG_INLINE Packet8uc pcmp_lt<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1530 return vclt_u8(a, b);
1533EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1534 return vcltq_u8(a, b);
1537EIGEN_STRONG_INLINE Packet4s pcmp_lt<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1538 return vreinterpret_s16_u16(vclt_s16(a, b));
1541EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1542 return vreinterpretq_s16_u16(vcltq_s16(a, b));
1545EIGEN_STRONG_INLINE Packet4us pcmp_lt<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1546 return vclt_u16(a, b);
1549EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1550 return vcltq_u16(a, b);
1553EIGEN_STRONG_INLINE Packet2i pcmp_lt<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1554 return vreinterpret_s32_u32(vclt_s32(a, b));
1557EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1558 return vreinterpretq_s32_u32(vcltq_s32(a, b));
1561EIGEN_STRONG_INLINE Packet2ui pcmp_lt<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1562 return vclt_u32(a, b);
1565EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1566 return vcltq_u32(a, b);
1570EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1572 return vreinterpretq_s64_u64(vcltq_s64(a, b));
1574 return vshrq_n_s64(vqsubq_s64(a, b), 63);
1578EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1580 return vcltq_u64(a, b);
1582 const uint64x2_t flag = vshrq_n_u64(vreinterpretq_u64_u8(vdupq_n_u8(0xFF)), 1);
1583 return vreinterpretq_u64_s64(vshrq_n_s64(vreinterpretq_s64_u64(vqaddq_u64(vqsubq_u64(b, a), flag)), 63));
1587EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1589 return vreinterpretq_s64_u64(vcleq_s64(a, b));
1591 return vreinterpretq_s64_u8(vmvnq_u8(vreinterpretq_u8_s64(pcmp_lt(b, a))));
1595EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1597 return vcleq_u64(a, b);
1599 return vreinterpretq_u64_u8(vmvnq_u8(vreinterpretq_u8_u64(pcmp_lt(b, a))));
1604EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1605 return vbslq_s64(vreinterpretq_u64_s64(pcmp_lt(a, b)), a, b);
1608EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1609 return vbslq_s64(vreinterpretq_u64_s64(pcmp_lt(b, a)), a, b);
1613EIGEN_STRONG_INLINE Packet2f pcmp_eq<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1614 return vreinterpret_f32_u32(vceq_f32(a, b));
1617EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1618 return vreinterpretq_f32_u32(vceqq_f32(a, b));
1621EIGEN_STRONG_INLINE Packet4c pcmp_eq<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1622 return vget_lane_s32(
1623 vreinterpret_s32_u8(vceq_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1626EIGEN_STRONG_INLINE Packet8c pcmp_eq<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1627 return vreinterpret_s8_u8(vceq_s8(a, b));
1630EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1631 return vreinterpretq_s8_u8(vceqq_s8(a, b));
1634EIGEN_STRONG_INLINE Packet4uc pcmp_eq<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1635 return vget_lane_u32(
1636 vreinterpret_u32_u8(vceq_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1639EIGEN_STRONG_INLINE Packet8uc pcmp_eq<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1640 return vceq_u8(a, b);
1643EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1644 return vceqq_u8(a, b);
1647EIGEN_STRONG_INLINE Packet4s pcmp_eq<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1648 return vreinterpret_s16_u16(vceq_s16(a, b));
1651EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1652 return vreinterpretq_s16_u16(vceqq_s16(a, b));
1655EIGEN_STRONG_INLINE Packet4us pcmp_eq<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1656 return vceq_u16(a, b);
1659EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1660 return vceqq_u16(a, b);
1663EIGEN_STRONG_INLINE Packet2i pcmp_eq<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1664 return vreinterpret_s32_u32(vceq_s32(a, b));
1667EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1668 return vreinterpretq_s32_u32(vceqq_s32(a, b));
1671EIGEN_STRONG_INLINE Packet2ui pcmp_eq<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1672 return vceq_u32(a, b);
1675EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1676 return vceqq_u32(a, b);
1679EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1681 return vceqq_u64(a, b);
1683 const uint32x4_t eq = vceqq_u32(vreinterpretq_u32_u64(a), vreinterpretq_u32_u64(b));
1684 return vreinterpretq_u64_u32(vandq_u32(eq, vrev64q_u32(eq)));
1688EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1690 return vreinterpretq_s64_u64(vceqq_s64(a, b));
1692 return vreinterpretq_s64_u64(pcmp_eq(vreinterpretq_u64_s64(a), vreinterpretq_u64_s64(b)));
1697EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1698 return vreinterpret_f32_u32(vmvn_u32(neon_vcge_f32(a, b)));
1701EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1702 return vreinterpretq_f32_u32(vmvnq_u32(neon_vcgeq_f32(a, b)));
1709EIGEN_STRONG_INLINE Packet2f pisinf<Packet2f>(
const Packet2f& a) {
1710 return vreinterpret_f32_u32(vcage_f32(a, pinf<Packet2f>()));
1713EIGEN_STRONG_INLINE Packet4f pisinf<Packet4f>(
const Packet4f& a) {
1714 return vreinterpretq_f32_u32(vcageq_f32(a, pinf<Packet4f>()));
1717EIGEN_STRONG_INLINE Packet2f pisfinite<Packet2f>(
const Packet2f& a) {
1718 return vreinterpret_f32_u32(vcagt_f32(pinf<Packet2f>(), a));
1721EIGEN_STRONG_INLINE Packet4f pisfinite<Packet4f>(
const Packet4f& a) {
1722 return vreinterpretq_f32_u32(vcagtq_f32(pinf<Packet4f>(), a));
1725EIGEN_STRONG_INLINE Packet2f pisnan<Packet2f>(
const Packet2f& a) {
1726 return vreinterpret_f32_u32(vmvn_u32(vcage_f32(pinf<Packet2f>(), a)));
1729EIGEN_STRONG_INLINE Packet4f pisnan<Packet4f>(
const Packet4f& a) {
1730 return vreinterpretq_f32_u32(vmvnq_u32(vcageq_f32(pinf<Packet4f>(), a)));
1735EIGEN_STRONG_INLINE Packet2f pand<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1736 return vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1739EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1740 return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1743EIGEN_STRONG_INLINE Packet4c pand<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1747EIGEN_STRONG_INLINE Packet8c pand<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1748 return vand_s8(a, b);
1751EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1752 return vandq_s8(a, b);
1755EIGEN_STRONG_INLINE Packet4uc pand<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1759EIGEN_STRONG_INLINE Packet8uc pand<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1760 return vand_u8(a, b);
1763EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1764 return vandq_u8(a, b);
1767EIGEN_STRONG_INLINE Packet4s pand<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1768 return vand_s16(a, b);
1771EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1772 return vandq_s16(a, b);
1775EIGEN_STRONG_INLINE Packet4us pand<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1776 return vand_u16(a, b);
1779EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1780 return vandq_u16(a, b);
1783EIGEN_STRONG_INLINE Packet2i pand<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1784 return vand_s32(a, b);
1787EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1788 return vandq_s32(a, b);
1791EIGEN_STRONG_INLINE Packet2ui pand<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1792 return vand_u32(a, b);
1795EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1796 return vandq_u32(a, b);
1799EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1800 return vandq_s64(a, b);
1803EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1804 return vandq_u64(a, b);
1808EIGEN_STRONG_INLINE Packet2f por<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1809 return vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1812EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1813 return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1816EIGEN_STRONG_INLINE Packet4c por<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1820EIGEN_STRONG_INLINE Packet8c por<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1821 return vorr_s8(a, b);
1824EIGEN_STRONG_INLINE Packet16c por<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1825 return vorrq_s8(a, b);
1828EIGEN_STRONG_INLINE Packet4uc por<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1832EIGEN_STRONG_INLINE Packet8uc por<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1833 return vorr_u8(a, b);
1836EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1837 return vorrq_u8(a, b);
1840EIGEN_STRONG_INLINE Packet4s por<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1841 return vorr_s16(a, b);
1844EIGEN_STRONG_INLINE Packet8s por<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1845 return vorrq_s16(a, b);
1848EIGEN_STRONG_INLINE Packet4us por<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1849 return vorr_u16(a, b);
1852EIGEN_STRONG_INLINE Packet8us por<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1853 return vorrq_u16(a, b);
1856EIGEN_STRONG_INLINE Packet2i por<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1857 return vorr_s32(a, b);
1860EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1861 return vorrq_s32(a, b);
1864EIGEN_STRONG_INLINE Packet2ui por<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1865 return vorr_u32(a, b);
1868EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1869 return vorrq_u32(a, b);
1872EIGEN_STRONG_INLINE Packet2l por<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1873 return vorrq_s64(a, b);
1876EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1877 return vorrq_u64(a, b);
1881EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1882 return vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1885EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1886 return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1889EIGEN_STRONG_INLINE Packet4c pxor<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1893EIGEN_STRONG_INLINE Packet8c pxor<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1894 return veor_s8(a, b);
1897EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1898 return veorq_s8(a, b);
1901EIGEN_STRONG_INLINE Packet4uc pxor<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1905EIGEN_STRONG_INLINE Packet8uc pxor<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1906 return veor_u8(a, b);
1909EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1910 return veorq_u8(a, b);
1913EIGEN_STRONG_INLINE Packet4s pxor<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1914 return veor_s16(a, b);
1917EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1918 return veorq_s16(a, b);
1921EIGEN_STRONG_INLINE Packet4us pxor<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1922 return veor_u16(a, b);
1925EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1926 return veorq_u16(a, b);
1929EIGEN_STRONG_INLINE Packet2i pxor<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
1930 return veor_s32(a, b);
1933EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1934 return veorq_s32(a, b);
1937EIGEN_STRONG_INLINE Packet2ui pxor<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
1938 return veor_u32(a, b);
1941EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1942 return veorq_u32(a, b);
1945EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1946 return veorq_s64(a, b);
1949EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1950 return veorq_u64(a, b);
1954EIGEN_STRONG_INLINE Packet2f pandnot<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
1955 return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1958EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1959 return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1962EIGEN_STRONG_INLINE Packet4c pandnot<Packet4c>(
const Packet4c& a,
const Packet4c& b) {
1966EIGEN_STRONG_INLINE Packet8c pandnot<Packet8c>(
const Packet8c& a,
const Packet8c& b) {
1967 return vbic_s8(a, b);
1970EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1971 return vbicq_s8(a, b);
1974EIGEN_STRONG_INLINE Packet4uc pandnot<Packet4uc>(
const Packet4uc& a,
const Packet4uc& b) {
1978EIGEN_STRONG_INLINE Packet8uc pandnot<Packet8uc>(
const Packet8uc& a,
const Packet8uc& b) {
1979 return vbic_u8(a, b);
1982EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1983 return vbicq_u8(a, b);
1986EIGEN_STRONG_INLINE Packet4s pandnot<Packet4s>(
const Packet4s& a,
const Packet4s& b) {
1987 return vbic_s16(a, b);
1990EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1991 return vbicq_s16(a, b);
1994EIGEN_STRONG_INLINE Packet4us pandnot<Packet4us>(
const Packet4us& a,
const Packet4us& b) {
1995 return vbic_u16(a, b);
1998EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1999 return vbicq_u16(a, b);
2002EIGEN_STRONG_INLINE Packet2i pandnot<Packet2i>(
const Packet2i& a,
const Packet2i& b) {
2003 return vbic_s32(a, b);
2006EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
2007 return vbicq_s32(a, b);
2010EIGEN_STRONG_INLINE Packet2ui pandnot<Packet2ui>(
const Packet2ui& a,
const Packet2ui& b) {
2011 return vbic_u32(a, b);
2014EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
2015 return vbicq_u32(a, b);
2018EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
2019 return vbicq_s64(a, b);
2022EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
2023 return vbicq_u64(a, b);
2027EIGEN_STRONG_INLINE Packet2f pnot<Packet2f>(
const Packet2f& a) {
2028 return vreinterpret_f32_u32(vmvn_u32(vreinterpret_u32_f32(a)));
2031EIGEN_STRONG_INLINE Packet4f pnot<Packet4f>(
const Packet4f& a) {
2032 return vreinterpretq_f32_u32(vmvnq_u32(vreinterpretq_u32_f32(a)));
2035EIGEN_STRONG_INLINE Packet4c pnot<Packet4c>(
const Packet4c& a) {
2039EIGEN_STRONG_INLINE Packet8c pnot<Packet8c>(
const Packet8c& a) {
2043EIGEN_STRONG_INLINE Packet16c pnot<Packet16c>(
const Packet16c& a) {
2047EIGEN_STRONG_INLINE Packet4uc pnot<Packet4uc>(
const Packet4uc& a) {
2051EIGEN_STRONG_INLINE Packet8uc pnot<Packet8uc>(
const Packet8uc& a) {
2055EIGEN_STRONG_INLINE Packet16uc pnot<Packet16uc>(
const Packet16uc& a) {
2059EIGEN_STRONG_INLINE Packet4s pnot<Packet4s>(
const Packet4s& a) {
2063EIGEN_STRONG_INLINE Packet8s pnot<Packet8s>(
const Packet8s& a) {
2064 return vmvnq_s16(a);
2067EIGEN_STRONG_INLINE Packet4us pnot<Packet4us>(
const Packet4us& a) {
2071EIGEN_STRONG_INLINE Packet8us pnot<Packet8us>(
const Packet8us& a) {
2072 return vmvnq_u16(a);
2075EIGEN_STRONG_INLINE Packet2i pnot<Packet2i>(
const Packet2i& a) {
2079EIGEN_STRONG_INLINE Packet4i pnot<Packet4i>(
const Packet4i& a) {
2080 return vmvnq_s32(a);
2083EIGEN_STRONG_INLINE Packet2ui pnot<Packet2ui>(
const Packet2ui& a) {
2087EIGEN_STRONG_INLINE Packet4ui pnot<Packet4ui>(
const Packet4ui& a) {
2088 return vmvnq_u32(a);
2091EIGEN_STRONG_INLINE Packet2l pnot<Packet2l>(
const Packet2l& a) {
2092 return vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a)));
2095EIGEN_STRONG_INLINE Packet2ul pnot<Packet2ul>(
const Packet2ul& a) {
2096 return vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(a)));
2100EIGEN_STRONG_INLINE Packet4c parithmetic_shift_right(
const Packet4c& a) {
2101 return vget_lane_s32(vreinterpret_s32_s8(vshr_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
2104EIGEN_STRONG_INLINE Packet8c parithmetic_shift_right(
const Packet8c& a) {
2105 return vshr_n_s8(a, N);
2108EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(
const Packet16c& a) {
2109 return vshrq_n_s8(a, N);
2112EIGEN_STRONG_INLINE Packet4uc parithmetic_shift_right(
const Packet4uc& a) {
2113 return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
2116EIGEN_STRONG_INLINE Packet8uc parithmetic_shift_right(
const Packet8uc& a) {
2117 return vshr_n_u8(a, N);
2120EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(
const Packet16uc& a) {
2121 return vshrq_n_u8(a, N);
2124EIGEN_STRONG_INLINE Packet4s parithmetic_shift_right(
const Packet4s& a) {
2125 return vshr_n_s16(a, N);
2128EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(
const Packet8s& a) {
2129 return vshrq_n_s16(a, N);
2132EIGEN_STRONG_INLINE Packet4us parithmetic_shift_right(
const Packet4us& a) {
2133 return vshr_n_u16(a, N);
2136EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(
const Packet8us& a) {
2137 return vshrq_n_u16(a, N);
2140EIGEN_STRONG_INLINE Packet2i parithmetic_shift_right(
const Packet2i& a) {
2141 return vshr_n_s32(a, N);
2144EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(
const Packet4i& a) {
2145 return vshrq_n_s32(a, N);
2148EIGEN_STRONG_INLINE Packet2ui parithmetic_shift_right(
const Packet2ui& a) {
2149 return vshr_n_u32(a, N);
2152EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(
const Packet4ui& a) {
2153 return vshrq_n_u32(a, N);
2156EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(
const Packet2l& a) {
2157 return vshrq_n_s64(a, N);
2160EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(
const Packet2ul& a) {
2161 return vshrq_n_u64(a, N);
2165EIGEN_STRONG_INLINE Packet4c plogical_shift_right(
const Packet4c& a) {
2166 return vget_lane_s32(vreinterpret_s32_u8(vshr_n_u8(vreinterpret_u8_s32(vdup_n_s32(a)), N)), 0);
2169EIGEN_STRONG_INLINE Packet8c plogical_shift_right(
const Packet8c& a) {
2170 return vreinterpret_s8_u8(vshr_n_u8(vreinterpret_u8_s8(a), N));
2173EIGEN_STRONG_INLINE Packet16c plogical_shift_right(
const Packet16c& a) {
2174 return vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(a), N));
2177EIGEN_STRONG_INLINE Packet4uc plogical_shift_right(
const Packet4uc& a) {
2178 return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
2181EIGEN_STRONG_INLINE Packet8uc plogical_shift_right(
const Packet8uc& a) {
2182 return vshr_n_u8(a, N);
2185EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(
const Packet16uc& a) {
2186 return vshrq_n_u8(a, N);
2189EIGEN_STRONG_INLINE Packet4s plogical_shift_right(
const Packet4s& a) {
2190 return vreinterpret_s16_u16(vshr_n_u16(vreinterpret_u16_s16(a), N));
2193EIGEN_STRONG_INLINE Packet8s plogical_shift_right(
const Packet8s& a) {
2194 return vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_s16(a), N));
2197EIGEN_STRONG_INLINE Packet4us plogical_shift_right(
const Packet4us& a) {
2198 return vshr_n_u16(a, N);
2201EIGEN_STRONG_INLINE Packet8us plogical_shift_right(
const Packet8us& a) {
2202 return vshrq_n_u16(a, N);
2205EIGEN_STRONG_INLINE Packet2i plogical_shift_right(
const Packet2i& a) {
2206 return vreinterpret_s32_u32(vshr_n_u32(vreinterpret_u32_s32(a), N));
2209EIGEN_STRONG_INLINE Packet4i plogical_shift_right(
const Packet4i& a) {
2210 return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a), N));
2213EIGEN_STRONG_INLINE Packet2ui plogical_shift_right(
const Packet2ui& a) {
2214 return vshr_n_u32(a, N);
2217EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(
const Packet4ui& a) {
2218 return vshrq_n_u32(a, N);
2221EIGEN_STRONG_INLINE Packet2l plogical_shift_right(
const Packet2l& a) {
2222 return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a), N));
2225EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(
const Packet2ul& a) {
2226 return vshrq_n_u64(a, N);
2230EIGEN_STRONG_INLINE Packet4c plogical_shift_left(
const Packet4c& a) {
2231 return vget_lane_s32(vreinterpret_s32_s8(vshl_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
2234EIGEN_STRONG_INLINE Packet8c plogical_shift_left(
const Packet8c& a) {
2235 return vshl_n_s8(a, N);
2238EIGEN_STRONG_INLINE Packet16c plogical_shift_left(
const Packet16c& a) {
2239 return vshlq_n_s8(a, N);
2242EIGEN_STRONG_INLINE Packet4uc plogical_shift_left(
const Packet4uc& a) {
2243 return vget_lane_u32(vreinterpret_u32_u8(vshl_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
2246EIGEN_STRONG_INLINE Packet8uc plogical_shift_left(
const Packet8uc& a) {
2247 return vshl_n_u8(a, N);
2250EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(
const Packet16uc& a) {
2251 return vshlq_n_u8(a, N);
2254EIGEN_STRONG_INLINE Packet4s plogical_shift_left(
const Packet4s& a) {
2255 return vshl_n_s16(a, N);
2258EIGEN_STRONG_INLINE Packet8s plogical_shift_left(
const Packet8s& a) {
2259 return vshlq_n_s16(a, N);
2262EIGEN_STRONG_INLINE Packet4us plogical_shift_left(
const Packet4us& a) {
2263 return vshl_n_u16(a, N);
2266EIGEN_STRONG_INLINE Packet8us plogical_shift_left(
const Packet8us& a) {
2267 return vshlq_n_u16(a, N);
2270EIGEN_STRONG_INLINE Packet2i plogical_shift_left(
const Packet2i& a) {
2271 return vshl_n_s32(a, N);
2274EIGEN_STRONG_INLINE Packet4i plogical_shift_left(
const Packet4i& a) {
2275 return vshlq_n_s32(a, N);
2278EIGEN_STRONG_INLINE Packet2ui plogical_shift_left(
const Packet2ui& a) {
2279 return vshl_n_u32(a, N);
2282EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(
const Packet4ui& a) {
2283 return vshlq_n_u32(a, N);
2286EIGEN_STRONG_INLINE Packet2l plogical_shift_left(
const Packet2l& a) {
2287 return vshlq_n_s64(a, N);
2290EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(
const Packet2ul& a) {
2291 return vshlq_n_u64(a, N);
2295EIGEN_STRONG_INLINE Packet2f pload<Packet2f>(
const float* from) {
2296 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_f32(assume_aligned<unpacket_traits<Packet2f>::alignment>(from));
2299EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from) {
2300 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f32(assume_aligned<unpacket_traits<Packet4f>::alignment>(from));
2303EIGEN_STRONG_INLINE Packet4c pload<Packet4c>(
const int8_t* from) {
2305 memcpy(&res, from,
sizeof(Packet4c));
2309EIGEN_STRONG_INLINE Packet8c pload<Packet8c>(
const int8_t* from) {
2310 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_s8(assume_aligned<unpacket_traits<Packet8c>::alignment>(from));
2313EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(
const int8_t* from) {
2314 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s8(assume_aligned<unpacket_traits<Packet16c>::alignment>(from));
2317EIGEN_STRONG_INLINE Packet4uc pload<Packet4uc>(
const uint8_t* from) {
2319 memcpy(&res, from,
sizeof(Packet4uc));
2323EIGEN_STRONG_INLINE Packet8uc pload<Packet8uc>(
const uint8_t* from) {
2324 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_u8(assume_aligned<unpacket_traits<Packet8uc>::alignment>(from));
2327EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(
const uint8_t* from) {
2328 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u8(assume_aligned<unpacket_traits<Packet16uc>::alignment>(from));
2331EIGEN_STRONG_INLINE Packet4s pload<Packet4s>(
const int16_t* from) {
2332 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_s16(assume_aligned<unpacket_traits<Packet4s>::alignment>(from));
2335EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(
const int16_t* from) {
2336 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s16(assume_aligned<unpacket_traits<Packet8s>::alignment>(from));
2339EIGEN_STRONG_INLINE Packet4us pload<Packet4us>(
const uint16_t* from) {
2340 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_u16(assume_aligned<unpacket_traits<Packet4us>::alignment>(from));
2343EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(
const uint16_t* from) {
2344 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u16(assume_aligned<unpacket_traits<Packet8us>::alignment>(from));
2347EIGEN_STRONG_INLINE Packet2i pload<Packet2i>(
const int32_t* from) {
2348 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_s32(assume_aligned<unpacket_traits<Packet2i>::alignment>(from));
2351EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int32_t* from) {
2352 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s32(assume_aligned<unpacket_traits<Packet4i>::alignment>(from));
2355EIGEN_STRONG_INLINE Packet2ui pload<Packet2ui>(
const uint32_t* from) {
2356 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_u32(assume_aligned<unpacket_traits<Packet2ui>::alignment>(from));
2359EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(
const uint32_t* from) {
2360 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u32(assume_aligned<unpacket_traits<Packet4ui>::alignment>(from));
2363EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(
const int64_t* from) {
2364 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_s64(assume_aligned<unpacket_traits<Packet2l>::alignment>(from));
2367EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(
const uint64_t* from) {
2368 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_u64(assume_aligned<unpacket_traits<Packet2ul>::alignment>(from));
2372EIGEN_STRONG_INLINE Packet2f ploadu<Packet2f>(
const float* from) {
2373 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_f32(from);
2376EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from) {
2377 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f32(from);
2380EIGEN_STRONG_INLINE Packet4c ploadu<Packet4c>(
const int8_t* from) {
2382 memcpy(&res, from,
sizeof(Packet4c));
2386EIGEN_STRONG_INLINE Packet8c ploadu<Packet8c>(
const int8_t* from) {
2387 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_s8(from);
2390EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(
const int8_t* from) {
2391 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s8(from);
2394EIGEN_STRONG_INLINE Packet4uc ploadu<Packet4uc>(
const uint8_t* from) {
2396 memcpy(&res, from,
sizeof(Packet4uc));
2400EIGEN_STRONG_INLINE Packet8uc ploadu<Packet8uc>(
const uint8_t* from) {
2401 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_u8(from);
2404EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(
const uint8_t* from) {
2405 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u8(from);
2408EIGEN_STRONG_INLINE Packet4s ploadu<Packet4s>(
const int16_t* from) {
2409 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_s16(from);
2412EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(
const int16_t* from) {
2413 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s16(from);
2416EIGEN_STRONG_INLINE Packet4us ploadu<Packet4us>(
const uint16_t* from) {
2417 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_u16(from);
2420EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(
const uint16_t* from) {
2421 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u16(from);
2424EIGEN_STRONG_INLINE Packet2i ploadu<Packet2i>(
const int32_t* from) {
2425 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_s32(from);
2428EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int32_t* from) {
2429 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s32(from);
2432EIGEN_STRONG_INLINE Packet2ui ploadu<Packet2ui>(
const uint32_t* from) {
2433 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_u32(from);
2436EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(
const uint32_t* from) {
2437 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u32(from);
2440EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(
const int64_t* from) {
2441 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_s64(from);
2444EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(
const uint64_t* from) {
2445 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_u64(from);
2449EIGEN_STRONG_INLINE Packet2f ploaddup<Packet2f>(
const float* from) {
2450 return vld1_dup_f32(from);
2453EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from) {
2454 return vcombine_f32(vld1_dup_f32(from), vld1_dup_f32(from + 1));
2457EIGEN_STRONG_INLINE Packet4c ploaddup<Packet4c>(
const int8_t* from) {
2458 const int8x8_t a = vreinterpret_s8_s32(vdup_n_s32(pload<Packet4c>(from)));
2459 return vget_lane_s32(vreinterpret_s32_s8(vzip_s8(a, a).val[0]), 0);
2462EIGEN_STRONG_INLINE Packet8c ploaddup<Packet8c>(
const int8_t* from) {
2463 const int8x8_t a = vld1_s8(from);
2464 return vzip_s8(a, a).val[0];
2467EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(
const int8_t* from) {
2468 const int8x8_t a = vld1_s8(from);
2469 const int8x8x2_t b = vzip_s8(a, a);
2470 return vcombine_s8(b.val[0], b.val[1]);
2473EIGEN_STRONG_INLINE Packet4uc ploaddup<Packet4uc>(
const uint8_t* from) {
2474 const uint8x8_t a = vreinterpret_u8_u32(vdup_n_u32(pload<Packet4uc>(from)));
2475 return vget_lane_u32(vreinterpret_u32_u8(vzip_u8(a, a).val[0]), 0);
2478EIGEN_STRONG_INLINE Packet8uc ploaddup<Packet8uc>(
const uint8_t* from) {
2479 const uint8x8_t a = vld1_u8(from);
2480 return vzip_u8(a, a).val[0];
2483EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(
const uint8_t* from) {
2484 const uint8x8_t a = vld1_u8(from);
2485 const uint8x8x2_t b = vzip_u8(a, a);
2486 return vcombine_u8(b.val[0], b.val[1]);
2489EIGEN_STRONG_INLINE Packet4s ploaddup<Packet4s>(
const int16_t* from) {
2490 return vreinterpret_s16_u32(
2491 vzip_u32(vreinterpret_u32_s16(vld1_dup_s16(from)), vreinterpret_u32_s16(vld1_dup_s16(from + 1))).val[0]);
2494EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(
const int16_t* from) {
2495 const int16x4_t a = vld1_s16(from);
2496 const int16x4x2_t b = vzip_s16(a, a);
2497 return vcombine_s16(b.val[0], b.val[1]);
2500EIGEN_STRONG_INLINE Packet4us ploaddup<Packet4us>(
const uint16_t* from) {
2501 return vreinterpret_u16_u32(
2502 vzip_u32(vreinterpret_u32_u16(vld1_dup_u16(from)), vreinterpret_u32_u16(vld1_dup_u16(from + 1))).val[0]);
2505EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(
const uint16_t* from) {
2506 const uint16x4_t a = vld1_u16(from);
2507 const uint16x4x2_t b = vzip_u16(a, a);
2508 return vcombine_u16(b.val[0], b.val[1]);
2511EIGEN_STRONG_INLINE Packet2i ploaddup<Packet2i>(
const int32_t* from) {
2512 return vld1_dup_s32(from);
2515EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int32_t* from) {
2516 return vcombine_s32(vld1_dup_s32(from), vld1_dup_s32(from + 1));
2519EIGEN_STRONG_INLINE Packet2ui ploaddup<Packet2ui>(
const uint32_t* from) {
2520 return vld1_dup_u32(from);
2523EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(
const uint32_t* from) {
2524 return vcombine_u32(vld1_dup_u32(from), vld1_dup_u32(from + 1));
2527EIGEN_STRONG_INLINE Packet2l ploaddup<Packet2l>(
const int64_t* from) {
2528 return vld1q_dup_s64(from);
2531EIGEN_STRONG_INLINE Packet2ul ploaddup<Packet2ul>(
const uint64_t* from) {
2532 return vld1q_dup_u64(from);
2536EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(
const float* from) {
2537 return vld1q_dup_f32(from);
2543EIGEN_ALWAYS_INLINE int8x8_t eigen_vld1_dup_s8(
const int8_t* ptr) {
2544#if EIGEN_COMP_CLANGAPPLE && EIGEN_ARCH_ARM64
2545 return vdup_n_s8(*ptr);
2547 return vld1_dup_s8(ptr);
2551EIGEN_ALWAYS_INLINE uint8x8_t eigen_vld1_dup_u8(
const uint8_t* ptr) {
2552#if EIGEN_COMP_CLANGAPPLE && EIGEN_ARCH_ARM64
2553 return vdup_n_u8(*ptr);
2555 return vld1_dup_u8(ptr);
2560EIGEN_STRONG_INLINE Packet4c ploadquad<Packet4c>(
const int8_t* from) {
2561 return vget_lane_s32(vreinterpret_s32_s8(eigen_vld1_dup_s8(from)), 0);
2564EIGEN_STRONG_INLINE Packet8c ploadquad<Packet8c>(
const int8_t* from) {
2565 return vreinterpret_s8_u32(
2566 vzip_u32(vreinterpret_u32_s8(eigen_vld1_dup_s8(from)), vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 1))).val[0]);
2569EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(
const int8_t* from) {
2570 const int8x8_t a = vreinterpret_s8_u32(
2571 vzip_u32(vreinterpret_u32_s8(eigen_vld1_dup_s8(from)), vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 1))).val[0]);
2572 const int8x8_t b = vreinterpret_s8_u32(
2573 vzip_u32(vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 2)), vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 3)))
2575 return vcombine_s8(a, b);
2578EIGEN_STRONG_INLINE Packet4uc ploadquad<Packet4uc>(
const uint8_t* from) {
2579 return vget_lane_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from)), 0);
2582EIGEN_STRONG_INLINE Packet8uc ploadquad<Packet8uc>(
const uint8_t* from) {
2583 return vreinterpret_u8_u32(
2584 vzip_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from)), vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 1))).val[0]);
2587EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(
const uint8_t* from) {
2588 const uint8x8_t a = vreinterpret_u8_u32(
2589 vzip_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from)), vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 1))).val[0]);
2590 const uint8x8_t b = vreinterpret_u8_u32(
2591 vzip_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 2)), vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 3)))
2593 return vcombine_u8(a, b);
2596EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(
const int16_t* from) {
2597 return vcombine_s16(vld1_dup_s16(from), vld1_dup_s16(from + 1));
2600EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(
const uint16_t* from) {
2601 return vcombine_u16(vld1_dup_u16(from), vld1_dup_u16(from + 1));
2604EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(
const int32_t* from) {
2605 return vld1q_dup_s32(from);
2608EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(
const uint32_t* from) {
2609 return vld1q_dup_u32(from);
2613EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet2f& from) {
2614 EIGEN_DEBUG_ALIGNED_STORE vst1_f32(assume_aligned<unpacket_traits<Packet2f>::alignment>(to), from);
2617EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from) {
2618 EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(assume_aligned<unpacket_traits<Packet4f>::alignment>(to), from);
2621EIGEN_STRONG_INLINE
void pstore<int8_t>(int8_t* to,
const Packet4c& from) {
2622 memcpy(to, &from,
sizeof(from));
2625EIGEN_STRONG_INLINE
void pstore<int8_t>(int8_t* to,
const Packet8c& from) {
2626 EIGEN_DEBUG_ALIGNED_STORE vst1_s8(assume_aligned<unpacket_traits<Packet8c>::alignment>(to), from);
2629EIGEN_STRONG_INLINE
void pstore<int8_t>(int8_t* to,
const Packet16c& from) {
2630 EIGEN_DEBUG_ALIGNED_STORE vst1q_s8(assume_aligned<unpacket_traits<Packet16c>::alignment>(to), from);
2633EIGEN_STRONG_INLINE
void pstore<uint8_t>(uint8_t* to,
const Packet4uc& from) {
2634 memcpy(to, &from,
sizeof(from));
2637EIGEN_STRONG_INLINE
void pstore<uint8_t>(uint8_t* to,
const Packet8uc& from) {
2638 EIGEN_DEBUG_ALIGNED_STORE vst1_u8(assume_aligned<unpacket_traits<Packet8uc>::alignment>(to), from);
2641EIGEN_STRONG_INLINE
void pstore<uint8_t>(uint8_t* to,
const Packet16uc& from) {
2642 EIGEN_DEBUG_ALIGNED_STORE vst1q_u8(assume_aligned<unpacket_traits<Packet16uc>::alignment>(to), from);
2645EIGEN_STRONG_INLINE
void pstore<int16_t>(int16_t* to,
const Packet4s& from) {
2646 EIGEN_DEBUG_ALIGNED_STORE vst1_s16(assume_aligned<unpacket_traits<Packet4s>::alignment>(to), from);
2649EIGEN_STRONG_INLINE
void pstore<int16_t>(int16_t* to,
const Packet8s& from) {
2650 EIGEN_DEBUG_ALIGNED_STORE vst1q_s16(assume_aligned<unpacket_traits<Packet8s>::alignment>(to), from);
2653EIGEN_STRONG_INLINE
void pstore<uint16_t>(uint16_t* to,
const Packet4us& from) {
2654 EIGEN_DEBUG_ALIGNED_STORE vst1_u16(assume_aligned<unpacket_traits<Packet4us>::alignment>(to), from);
2657EIGEN_STRONG_INLINE
void pstore<uint16_t>(uint16_t* to,
const Packet8us& from) {
2658 EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(assume_aligned<unpacket_traits<Packet8us>::alignment>(to), from);
2661EIGEN_STRONG_INLINE
void pstore<int32_t>(int32_t* to,
const Packet2i& from) {
2662 EIGEN_DEBUG_ALIGNED_STORE vst1_s32(assume_aligned<unpacket_traits<Packet2i>::alignment>(to), from);
2665EIGEN_STRONG_INLINE
void pstore<int32_t>(int32_t* to,
const Packet4i& from) {
2666 EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(assume_aligned<unpacket_traits<Packet4i>::alignment>(to), from);
2669EIGEN_STRONG_INLINE
void pstore<uint32_t>(uint32_t* to,
const Packet2ui& from) {
2670 EIGEN_DEBUG_ALIGNED_STORE vst1_u32(assume_aligned<unpacket_traits<Packet2ui>::alignment>(to), from);
2673EIGEN_STRONG_INLINE
void pstore<uint32_t>(uint32_t* to,
const Packet4ui& from) {
2674 EIGEN_DEBUG_ALIGNED_STORE vst1q_u32(assume_aligned<unpacket_traits<Packet4ui>::alignment>(to), from);
2677EIGEN_STRONG_INLINE
void pstore<int64_t>(int64_t* to,
const Packet2l& from) {
2678 EIGEN_DEBUG_ALIGNED_STORE vst1q_s64(assume_aligned<unpacket_traits<Packet2l>::alignment>(to), from);
2681EIGEN_STRONG_INLINE
void pstore<uint64_t>(uint64_t* to,
const Packet2ul& from) {
2682 EIGEN_DEBUG_ALIGNED_STORE vst1q_u64(assume_aligned<unpacket_traits<Packet2ul>::alignment>(to), from);
2686EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet2f& from) {
2687 EIGEN_DEBUG_UNALIGNED_STORE vst1_f32(to, from);
2690EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from) {
2691 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from);
2694EIGEN_STRONG_INLINE
void pstoreu<int8_t>(int8_t* to,
const Packet4c& from) {
2695 memcpy(to, &from,
sizeof(from));
2698EIGEN_STRONG_INLINE
void pstoreu<int8_t>(int8_t* to,
const Packet8c& from) {
2699 EIGEN_DEBUG_UNALIGNED_STORE vst1_s8(to, from);
2702EIGEN_STRONG_INLINE
void pstoreu<int8_t>(int8_t* to,
const Packet16c& from) {
2703 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s8(to, from);
2706EIGEN_STRONG_INLINE
void pstoreu<uint8_t>(uint8_t* to,
const Packet4uc& from) {
2707 memcpy(to, &from,
sizeof(from));
2710EIGEN_STRONG_INLINE
void pstoreu<uint8_t>(uint8_t* to,
const Packet8uc& from) {
2711 EIGEN_DEBUG_UNALIGNED_STORE vst1_u8(to, from);
2714EIGEN_STRONG_INLINE
void pstoreu<uint8_t>(uint8_t* to,
const Packet16uc& from) {
2715 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u8(to, from);
2718EIGEN_STRONG_INLINE
void pstoreu<int16_t>(int16_t* to,
const Packet4s& from) {
2719 EIGEN_DEBUG_UNALIGNED_STORE vst1_s16(to, from);
2722EIGEN_STRONG_INLINE
void pstoreu<int16_t>(int16_t* to,
const Packet8s& from) {
2723 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s16(to, from);
2726EIGEN_STRONG_INLINE
void pstoreu<uint16_t>(uint16_t* to,
const Packet4us& from) {
2727 EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(to, from);
2730EIGEN_STRONG_INLINE
void pstoreu<uint16_t>(uint16_t* to,
const Packet8us& from) {
2731 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(to, from);
2734EIGEN_STRONG_INLINE
void pstoreu<int32_t>(int32_t* to,
const Packet2i& from) {
2735 EIGEN_DEBUG_UNALIGNED_STORE vst1_s32(to, from);
2738EIGEN_STRONG_INLINE
void pstoreu<int32_t>(int32_t* to,
const Packet4i& from) {
2739 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from);
2742EIGEN_STRONG_INLINE
void pstoreu<uint32_t>(uint32_t* to,
const Packet2ui& from) {
2743 EIGEN_DEBUG_UNALIGNED_STORE vst1_u32(to, from);
2746EIGEN_STRONG_INLINE
void pstoreu<uint32_t>(uint32_t* to,
const Packet4ui& from) {
2747 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u32(to, from);
2750EIGEN_STRONG_INLINE
void pstoreu<int64_t>(int64_t* to,
const Packet2l& from) {
2751 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s64(to, from);
2754EIGEN_STRONG_INLINE
void pstoreu<uint64_t>(uint64_t* to,
const Packet2ul& from) {
2755 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u64(to, from);
2759EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather<float, Packet2f>(
const float* from, Index stride) {
2760 Packet2f res = vld1_dup_f32(from);
2761 res = vld1_lane_f32(from + 1 * stride, res, 1);
2765EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(
const float* from, Index stride) {
2770 Packet4f lo = vld1q_f32(from);
2771 Packet4f hi = vld1q_f32(from + 3);
2772 return vcombine_f32(vget_low_f32(vuzp1q_f32(lo, lo)), vget_high_f32(vuzp2q_f32(hi, hi)));
2774 return vcombine_f32(vld2_f32(from).val[0], vld2_f32(from + 3).val[1]);
2777 Packet4f res = vld1q_dup_f32(from);
2778 res = vld1q_lane_f32(from + 1 * stride, res, 1);
2779 res = vld1q_lane_f32(from + 2 * stride, res, 2);
2780 res = vld1q_lane_f32(from + 3 * stride, res, 3);
2784EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather<int8_t, Packet4c>(
const int8_t* from, Index stride) {
2786 for (
int i = 0; i != 4; i++)
reinterpret_cast<int8_t*
>(&res)[i] = *(from + i * stride);
2790EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather<int8_t, Packet8c>(
const int8_t* from, Index stride) {
2791 Packet8c res = vld1_dup_s8(from);
2792 res = vld1_lane_s8(from + 1 * stride, res, 1);
2793 res = vld1_lane_s8(from + 2 * stride, res, 2);
2794 res = vld1_lane_s8(from + 3 * stride, res, 3);
2795 res = vld1_lane_s8(from + 4 * stride, res, 4);
2796 res = vld1_lane_s8(from + 5 * stride, res, 5);
2797 res = vld1_lane_s8(from + 6 * stride, res, 6);
2798 res = vld1_lane_s8(from + 7 * stride, res, 7);
2802EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(
const int8_t* from, Index stride) {
2803 Packet16c res = vld1q_dup_s8(from);
2804 res = vld1q_lane_s8(from + 1 * stride, res, 1);
2805 res = vld1q_lane_s8(from + 2 * stride, res, 2);
2806 res = vld1q_lane_s8(from + 3 * stride, res, 3);
2807 res = vld1q_lane_s8(from + 4 * stride, res, 4);
2808 res = vld1q_lane_s8(from + 5 * stride, res, 5);
2809 res = vld1q_lane_s8(from + 6 * stride, res, 6);
2810 res = vld1q_lane_s8(from + 7 * stride, res, 7);
2811 res = vld1q_lane_s8(from + 8 * stride, res, 8);
2812 res = vld1q_lane_s8(from + 9 * stride, res, 9);
2813 res = vld1q_lane_s8(from + 10 * stride, res, 10);
2814 res = vld1q_lane_s8(from + 11 * stride, res, 11);
2815 res = vld1q_lane_s8(from + 12 * stride, res, 12);
2816 res = vld1q_lane_s8(from + 13 * stride, res, 13);
2817 res = vld1q_lane_s8(from + 14 * stride, res, 14);
2818 res = vld1q_lane_s8(from + 15 * stride, res, 15);
2822EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather<uint8_t, Packet4uc>(
const uint8_t* from, Index stride) {
2824 for (
int i = 0; i != 4; i++)
reinterpret_cast<uint8_t*
>(&res)[i] = *(from + i * stride);
2828EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather<uint8_t, Packet8uc>(
const uint8_t* from, Index stride) {
2829 Packet8uc res = vld1_dup_u8(from);
2830 res = vld1_lane_u8(from + 1 * stride, res, 1);
2831 res = vld1_lane_u8(from + 2 * stride, res, 2);
2832 res = vld1_lane_u8(from + 3 * stride, res, 3);
2833 res = vld1_lane_u8(from + 4 * stride, res, 4);
2834 res = vld1_lane_u8(from + 5 * stride, res, 5);
2835 res = vld1_lane_u8(from + 6 * stride, res, 6);
2836 res = vld1_lane_u8(from + 7 * stride, res, 7);
2840EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(
const uint8_t* from, Index stride) {
2841 Packet16uc res = vld1q_dup_u8(from);
2842 res = vld1q_lane_u8(from + 1 * stride, res, 1);
2843 res = vld1q_lane_u8(from + 2 * stride, res, 2);
2844 res = vld1q_lane_u8(from + 3 * stride, res, 3);
2845 res = vld1q_lane_u8(from + 4 * stride, res, 4);
2846 res = vld1q_lane_u8(from + 5 * stride, res, 5);
2847 res = vld1q_lane_u8(from + 6 * stride, res, 6);
2848 res = vld1q_lane_u8(from + 7 * stride, res, 7);
2849 res = vld1q_lane_u8(from + 8 * stride, res, 8);
2850 res = vld1q_lane_u8(from + 9 * stride, res, 9);
2851 res = vld1q_lane_u8(from + 10 * stride, res, 10);
2852 res = vld1q_lane_u8(from + 11 * stride, res, 11);
2853 res = vld1q_lane_u8(from + 12 * stride, res, 12);
2854 res = vld1q_lane_u8(from + 13 * stride, res, 13);
2855 res = vld1q_lane_u8(from + 14 * stride, res, 14);
2856 res = vld1q_lane_u8(from + 15 * stride, res, 15);
2860EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather<int16_t, Packet4s>(
const int16_t* from, Index stride) {
2861 Packet4s res = vld1_dup_s16(from);
2862 res = vld1_lane_s16(from + 1 * stride, res, 1);
2863 res = vld1_lane_s16(from + 2 * stride, res, 2);
2864 res = vld1_lane_s16(from + 3 * stride, res, 3);
2868EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(
const int16_t* from, Index stride) {
2869 Packet8s res = vld1q_dup_s16(from);
2870 res = vld1q_lane_s16(from + 1 * stride, res, 1);
2871 res = vld1q_lane_s16(from + 2 * stride, res, 2);
2872 res = vld1q_lane_s16(from + 3 * stride, res, 3);
2873 res = vld1q_lane_s16(from + 4 * stride, res, 4);
2874 res = vld1q_lane_s16(from + 5 * stride, res, 5);
2875 res = vld1q_lane_s16(from + 6 * stride, res, 6);
2876 res = vld1q_lane_s16(from + 7 * stride, res, 7);
2880EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather<uint16_t, Packet4us>(
const uint16_t* from, Index stride) {
2881 Packet4us res = vld1_dup_u16(from);
2882 res = vld1_lane_u16(from + 1 * stride, res, 1);
2883 res = vld1_lane_u16(from + 2 * stride, res, 2);
2884 res = vld1_lane_u16(from + 3 * stride, res, 3);
2888EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(
const uint16_t* from, Index stride) {
2889 Packet8us res = vld1q_dup_u16(from);
2890 res = vld1q_lane_u16(from + 1 * stride, res, 1);
2891 res = vld1q_lane_u16(from + 2 * stride, res, 2);
2892 res = vld1q_lane_u16(from + 3 * stride, res, 3);
2893 res = vld1q_lane_u16(from + 4 * stride, res, 4);
2894 res = vld1q_lane_u16(from + 5 * stride, res, 5);
2895 res = vld1q_lane_u16(from + 6 * stride, res, 6);
2896 res = vld1q_lane_u16(from + 7 * stride, res, 7);
2900EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather<int32_t, Packet2i>(
const int32_t* from, Index stride) {
2901 Packet2i res = vld1_dup_s32(from);
2902 res = vld1_lane_s32(from + 1 * stride, res, 1);
2906EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(
const int32_t* from, Index stride) {
2907 Packet4i res = vld1q_dup_s32(from);
2908 res = vld1q_lane_s32(from + 1 * stride, res, 1);
2909 res = vld1q_lane_s32(from + 2 * stride, res, 2);
2910 res = vld1q_lane_s32(from + 3 * stride, res, 3);
2914EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather<uint32_t, Packet2ui>(
const uint32_t* from, Index stride) {
2915 Packet2ui res = vld1_dup_u32(from);
2916 res = vld1_lane_u32(from + 1 * stride, res, 1);
2920EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(
const uint32_t* from, Index stride) {
2921 Packet4ui res = vld1q_dup_u32(from);
2922 res = vld1q_lane_u32(from + 1 * stride, res, 1);
2923 res = vld1q_lane_u32(from + 2 * stride, res, 2);
2924 res = vld1q_lane_u32(from + 3 * stride, res, 3);
2928EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(
const int64_t* from, Index stride) {
2929 Packet2l res = vld1q_dup_s64(from);
2930 res = vld1q_lane_s64(from + 1 * stride, res, 1);
2934EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(
const uint64_t* from, Index stride) {
2935 Packet2ul res = vld1q_dup_u64(from);
2936 res = vld1q_lane_u64(from + 1 * stride, res, 1);
2941EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<float, Packet2f>(
float* to,
const Packet2f& from, Index stride) {
2942 vst1_lane_f32(to + stride * 0, from, 0);
2943 vst1_lane_f32(to + stride * 1, from, 1);
2946EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<float, Packet4f>(
float* to,
const Packet4f& from, Index stride) {
2947 vst1q_lane_f32(to + stride * 0, from, 0);
2948 vst1q_lane_f32(to + stride * 1, from, 1);
2949 vst1q_lane_f32(to + stride * 2, from, 2);
2950 vst1q_lane_f32(to + stride * 3, from, 3);
2953EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int8_t, Packet4c>(int8_t* to,
const Packet4c& from, Index stride) {
2954 for (
int i = 0; i != 4; i++) *(to + i * stride) =
reinterpret_cast<const int8_t*
>(&from)[i];
2957EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int8_t, Packet8c>(int8_t* to,
const Packet8c& from, Index stride) {
2958 vst1_lane_s8(to + stride * 0, from, 0);
2959 vst1_lane_s8(to + stride * 1, from, 1);
2960 vst1_lane_s8(to + stride * 2, from, 2);
2961 vst1_lane_s8(to + stride * 3, from, 3);
2962 vst1_lane_s8(to + stride * 4, from, 4);
2963 vst1_lane_s8(to + stride * 5, from, 5);
2964 vst1_lane_s8(to + stride * 6, from, 6);
2965 vst1_lane_s8(to + stride * 7, from, 7);
2968EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int8_t, Packet16c>(int8_t* to,
const Packet16c& from,
2970 vst1q_lane_s8(to + stride * 0, from, 0);
2971 vst1q_lane_s8(to + stride * 1, from, 1);
2972 vst1q_lane_s8(to + stride * 2, from, 2);
2973 vst1q_lane_s8(to + stride * 3, from, 3);
2974 vst1q_lane_s8(to + stride * 4, from, 4);
2975 vst1q_lane_s8(to + stride * 5, from, 5);
2976 vst1q_lane_s8(to + stride * 6, from, 6);
2977 vst1q_lane_s8(to + stride * 7, from, 7);
2978 vst1q_lane_s8(to + stride * 8, from, 8);
2979 vst1q_lane_s8(to + stride * 9, from, 9);
2980 vst1q_lane_s8(to + stride * 10, from, 10);
2981 vst1q_lane_s8(to + stride * 11, from, 11);
2982 vst1q_lane_s8(to + stride * 12, from, 12);
2983 vst1q_lane_s8(to + stride * 13, from, 13);
2984 vst1q_lane_s8(to + stride * 14, from, 14);
2985 vst1q_lane_s8(to + stride * 15, from, 15);
2988EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint8_t, Packet4uc>(uint8_t* to,
const Packet4uc& from,
2990 for (
int i = 0; i != 4; i++) *(to + i * stride) =
reinterpret_cast<const uint8_t*
>(&from)[i];
2993EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint8_t, Packet8uc>(uint8_t* to,
const Packet8uc& from,
2995 vst1_lane_u8(to + stride * 0, from, 0);
2996 vst1_lane_u8(to + stride * 1, from, 1);
2997 vst1_lane_u8(to + stride * 2, from, 2);
2998 vst1_lane_u8(to + stride * 3, from, 3);
2999 vst1_lane_u8(to + stride * 4, from, 4);
3000 vst1_lane_u8(to + stride * 5, from, 5);
3001 vst1_lane_u8(to + stride * 6, from, 6);
3002 vst1_lane_u8(to + stride * 7, from, 7);
3005EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint8_t, Packet16uc>(uint8_t* to,
const Packet16uc& from,
3007 vst1q_lane_u8(to + stride * 0, from, 0);
3008 vst1q_lane_u8(to + stride * 1, from, 1);
3009 vst1q_lane_u8(to + stride * 2, from, 2);
3010 vst1q_lane_u8(to + stride * 3, from, 3);
3011 vst1q_lane_u8(to + stride * 4, from, 4);
3012 vst1q_lane_u8(to + stride * 5, from, 5);
3013 vst1q_lane_u8(to + stride * 6, from, 6);
3014 vst1q_lane_u8(to + stride * 7, from, 7);
3015 vst1q_lane_u8(to + stride * 8, from, 8);
3016 vst1q_lane_u8(to + stride * 9, from, 9);
3017 vst1q_lane_u8(to + stride * 10, from, 10);
3018 vst1q_lane_u8(to + stride * 11, from, 11);
3019 vst1q_lane_u8(to + stride * 12, from, 12);
3020 vst1q_lane_u8(to + stride * 13, from, 13);
3021 vst1q_lane_u8(to + stride * 14, from, 14);
3022 vst1q_lane_u8(to + stride * 15, from, 15);
3025EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int16_t, Packet4s>(int16_t* to,
const Packet4s& from,
3027 vst1_lane_s16(to + stride * 0, from, 0);
3028 vst1_lane_s16(to + stride * 1, from, 1);
3029 vst1_lane_s16(to + stride * 2, from, 2);
3030 vst1_lane_s16(to + stride * 3, from, 3);
3033EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int16_t, Packet8s>(int16_t* to,
const Packet8s& from,
3035 vst1q_lane_s16(to + stride * 0, from, 0);
3036 vst1q_lane_s16(to + stride * 1, from, 1);
3037 vst1q_lane_s16(to + stride * 2, from, 2);
3038 vst1q_lane_s16(to + stride * 3, from, 3);
3039 vst1q_lane_s16(to + stride * 4, from, 4);
3040 vst1q_lane_s16(to + stride * 5, from, 5);
3041 vst1q_lane_s16(to + stride * 6, from, 6);
3042 vst1q_lane_s16(to + stride * 7, from, 7);
3045EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint16_t, Packet4us>(uint16_t* to,
const Packet4us& from,
3047 vst1_lane_u16(to + stride * 0, from, 0);
3048 vst1_lane_u16(to + stride * 1, from, 1);
3049 vst1_lane_u16(to + stride * 2, from, 2);
3050 vst1_lane_u16(to + stride * 3, from, 3);
3053EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint16_t, Packet8us>(uint16_t* to,
const Packet8us& from,
3055 vst1q_lane_u16(to + stride * 0, from, 0);
3056 vst1q_lane_u16(to + stride * 1, from, 1);
3057 vst1q_lane_u16(to + stride * 2, from, 2);
3058 vst1q_lane_u16(to + stride * 3, from, 3);
3059 vst1q_lane_u16(to + stride * 4, from, 4);
3060 vst1q_lane_u16(to + stride * 5, from, 5);
3061 vst1q_lane_u16(to + stride * 6, from, 6);
3062 vst1q_lane_u16(to + stride * 7, from, 7);
3065EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int32_t, Packet2i>(int32_t* to,
const Packet2i& from,
3067 vst1_lane_s32(to + stride * 0, from, 0);
3068 vst1_lane_s32(to + stride * 1, from, 1);
3071EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int32_t, Packet4i>(int32_t* to,
const Packet4i& from,
3073 vst1q_lane_s32(to + stride * 0, from, 0);
3074 vst1q_lane_s32(to + stride * 1, from, 1);
3075 vst1q_lane_s32(to + stride * 2, from, 2);
3076 vst1q_lane_s32(to + stride * 3, from, 3);
3079EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint32_t, Packet2ui>(uint32_t* to,
const Packet2ui& from,
3081 vst1_lane_u32(to + stride * 0, from, 0);
3082 vst1_lane_u32(to + stride * 1, from, 1);
3085EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint32_t, Packet4ui>(uint32_t* to,
const Packet4ui& from,
3087 vst1q_lane_u32(to + stride * 0, from, 0);
3088 vst1q_lane_u32(to + stride * 1, from, 1);
3089 vst1q_lane_u32(to + stride * 2, from, 2);
3090 vst1q_lane_u32(to + stride * 3, from, 3);
3093EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int64_t, Packet2l>(int64_t* to,
const Packet2l& from,
3095 vst1q_lane_s64(to + stride * 0, from, 0);
3096 vst1q_lane_s64(to + stride * 1, from, 1);
3099EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint64_t, Packet2ul>(uint64_t* to,
const Packet2ul& from,
3101 vst1q_lane_u64(to + stride * 0, from, 0);
3102 vst1q_lane_u64(to + stride * 1, from, 1);
3106EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) {
3107 EIGEN_ARM_PREFETCH(addr);
3110EIGEN_STRONG_INLINE
void prefetch<int8_t>(
const int8_t* addr) {
3111 EIGEN_ARM_PREFETCH(addr);
3114EIGEN_STRONG_INLINE
void prefetch<uint8_t>(
const uint8_t* addr) {
3115 EIGEN_ARM_PREFETCH(addr);
3118EIGEN_STRONG_INLINE
void prefetch<int16_t>(
const int16_t* addr) {
3119 EIGEN_ARM_PREFETCH(addr);
3122EIGEN_STRONG_INLINE
void prefetch<uint16_t>(
const uint16_t* addr) {
3123 EIGEN_ARM_PREFETCH(addr);
3126EIGEN_STRONG_INLINE
void prefetch<int32_t>(
const int32_t* addr) {
3127 EIGEN_ARM_PREFETCH(addr);
3130EIGEN_STRONG_INLINE
void prefetch<uint32_t>(
const uint32_t* addr) {
3131 EIGEN_ARM_PREFETCH(addr);
3134EIGEN_STRONG_INLINE
void prefetch<int64_t>(
const int64_t* addr) {
3135 EIGEN_ARM_PREFETCH(addr);
3138EIGEN_STRONG_INLINE
void prefetch<uint64_t>(
const uint64_t* addr) {
3139 EIGEN_ARM_PREFETCH(addr);
3143EIGEN_STRONG_INLINE
float pfirst<Packet2f>(
const Packet2f& a) {
3144 return vget_lane_f32(a, 0);
3147EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
3148 return vgetq_lane_f32(a, 0);
3151EIGEN_STRONG_INLINE int8_t pfirst<Packet4c>(
const Packet4c& a) {
3152 return static_cast<int8_t
>(a & 0xff);
3155EIGEN_STRONG_INLINE int8_t pfirst<Packet8c>(
const Packet8c& a) {
3156 return vget_lane_s8(a, 0);
3159EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(
const Packet16c& a) {
3160 return vgetq_lane_s8(a, 0);
3163EIGEN_STRONG_INLINE uint8_t pfirst<Packet4uc>(
const Packet4uc& a) {
3164 return static_cast<uint8_t
>(a & 0xff);
3167EIGEN_STRONG_INLINE uint8_t pfirst<Packet8uc>(
const Packet8uc& a) {
3168 return vget_lane_u8(a, 0);
3171EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(
const Packet16uc& a) {
3172 return vgetq_lane_u8(a, 0);
3175EIGEN_STRONG_INLINE int16_t pfirst<Packet4s>(
const Packet4s& a) {
3176 return vget_lane_s16(a, 0);
3179EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(
const Packet8s& a) {
3180 return vgetq_lane_s16(a, 0);
3183EIGEN_STRONG_INLINE uint16_t pfirst<Packet4us>(
const Packet4us& a) {
3184 return vget_lane_u16(a, 0);
3187EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(
const Packet8us& a) {
3188 return vgetq_lane_u16(a, 0);
3191EIGEN_STRONG_INLINE int32_t pfirst<Packet2i>(
const Packet2i& a) {
3192 return vget_lane_s32(a, 0);
3195EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(
const Packet4i& a) {
3196 return vgetq_lane_s32(a, 0);
3199EIGEN_STRONG_INLINE uint32_t pfirst<Packet2ui>(
const Packet2ui& a) {
3200 return vget_lane_u32(a, 0);
3203EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(
const Packet4ui& a) {
3204 return vgetq_lane_u32(a, 0);
3207EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(
const Packet2l& a) {
3208 return vgetq_lane_s64(a, 0);
3211EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(
const Packet2ul& a) {
3212 return vgetq_lane_u64(a, 0);
3216EIGEN_STRONG_INLINE Packet2f preverse(
const Packet2f& a) {
3217 return vrev64_f32(a);
3220EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a) {
3221 const float32x4_t a_r64 = vrev64q_f32(a);
3222 return vcombine_f32(vget_high_f32(a_r64), vget_low_f32(a_r64));
3225EIGEN_STRONG_INLINE Packet4c preverse(
const Packet4c& a) {
3226 return vget_lane_s32(vreinterpret_s32_s8(vrev64_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
3229EIGEN_STRONG_INLINE Packet8c preverse(
const Packet8c& a) {
3230 return vrev64_s8(a);
3233EIGEN_STRONG_INLINE Packet16c preverse(
const Packet16c& a) {
3234 const int8x16_t a_r64 = vrev64q_s8(a);
3235 return vcombine_s8(vget_high_s8(a_r64), vget_low_s8(a_r64));
3238EIGEN_STRONG_INLINE Packet4uc preverse(
const Packet4uc& a) {
3239 return vget_lane_u32(vreinterpret_u32_u8(vrev64_u8(vreinterpret_u8_u32(vdup_n_u32(a)))), 0);
3242EIGEN_STRONG_INLINE Packet8uc preverse(
const Packet8uc& a) {
3243 return vrev64_u8(a);
3246EIGEN_STRONG_INLINE Packet16uc preverse(
const Packet16uc& a) {
3247 const uint8x16_t a_r64 = vrev64q_u8(a);
3248 return vcombine_u8(vget_high_u8(a_r64), vget_low_u8(a_r64));
3251EIGEN_STRONG_INLINE Packet4s preverse(
const Packet4s& a) {
3252 return vrev64_s16(a);
3255EIGEN_STRONG_INLINE Packet8s preverse(
const Packet8s& a) {
3256 const int16x8_t a_r64 = vrev64q_s16(a);
3257 return vcombine_s16(vget_high_s16(a_r64), vget_low_s16(a_r64));
3260EIGEN_STRONG_INLINE Packet4us preverse(
const Packet4us& a) {
3261 return vrev64_u16(a);
3264EIGEN_STRONG_INLINE Packet8us preverse(
const Packet8us& a) {
3265 const uint16x8_t a_r64 = vrev64q_u16(a);
3266 return vcombine_u16(vget_high_u16(a_r64), vget_low_u16(a_r64));
3269EIGEN_STRONG_INLINE Packet2i preverse(
const Packet2i& a) {
3270 return vrev64_s32(a);
3273EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a) {
3274 const int32x4_t a_r64 = vrev64q_s32(a);
3275 return vcombine_s32(vget_high_s32(a_r64), vget_low_s32(a_r64));
3278EIGEN_STRONG_INLINE Packet2ui preverse(
const Packet2ui& a) {
3279 return vrev64_u32(a);
3282EIGEN_STRONG_INLINE Packet4ui preverse(
const Packet4ui& a) {
3283 const uint32x4_t a_r64 = vrev64q_u32(a);
3284 return vcombine_u32(vget_high_u32(a_r64), vget_low_u32(a_r64));
3287EIGEN_STRONG_INLINE Packet2l preverse(
const Packet2l& a) {
3288 return vcombine_s64(vget_high_s64(a), vget_low_s64(a));
3291EIGEN_STRONG_INLINE Packet2ul preverse(
const Packet2ul& a) {
3292 return vcombine_u64(vget_high_u64(a), vget_low_u64(a));
3296EIGEN_STRONG_INLINE Packet2f pabs(
const Packet2f& a) {
3300EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
3301 return vabsq_f32(a);
3304EIGEN_STRONG_INLINE Packet4c pabs<Packet4c>(
const Packet4c& a) {
3305 return vget_lane_s32(vreinterpret_s32_s8(vabs_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
3308EIGEN_STRONG_INLINE Packet8c pabs(
const Packet8c& a) {
3312EIGEN_STRONG_INLINE Packet16c pabs(
const Packet16c& a) {
3316EIGEN_STRONG_INLINE Packet4s pabs(
const Packet4s& a) {
3320EIGEN_STRONG_INLINE Packet8s pabs(
const Packet8s& a) {
3321 return vabsq_s16(a);
3324EIGEN_STRONG_INLINE Packet2i pabs(
const Packet2i& a) {
3328EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
3329 return vabsq_s32(a);
3332EIGEN_STRONG_INLINE Packet2l pabs(
const Packet2l& a) {
3334 return vabsq_s64(a);
3339 const int64x2_t mask = vshrq_n_s64(a, 63);
3340 const uint64x2_t sum = vaddq_u64(vreinterpretq_u64_s64(a), vreinterpretq_u64_s64(mask));
3341 return veorq_s64(vreinterpretq_s64_u64(sum), mask);
3345EIGEN_STRONG_INLINE Packet2f psignbit(
const Packet2f& a) {
3346 return vreinterpret_f32_s32(vshr_n_s32(vreinterpret_s32_f32(a), 31));
3349EIGEN_STRONG_INLINE Packet4f psignbit(
const Packet4f& a) {
3350 return vreinterpretq_f32_s32(vshrq_n_s32(vreinterpretq_s32_f32(a), 31));
3354EIGEN_STRONG_INLINE Packet2f pfrexp<Packet2f>(
const Packet2f& a, Packet2f& exponent) {
3355 return pfrexp_generic(a, exponent);
3358EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(
const Packet4f& a, Packet4f& exponent) {
3359 return pfrexp_generic(a, exponent);
3363EIGEN_STRONG_INLINE Packet2f pldexp<Packet2f>(
const Packet2f& a,
const Packet2f& exponent) {
3364 return pldexp_generic(a, exponent);
3367EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(
const Packet4f& a,
const Packet4f& exponent) {
3368 return pldexp_generic(a, exponent);
3373EIGEN_STRONG_INLINE
float predux<Packet2f>(
const Packet2f& a) {
3374 return vaddv_f32(a);
3377EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a) {
3378 return vaddvq_f32(a);
3382EIGEN_STRONG_INLINE Index predux_count(
const Packet4f& a) {
3383 const uint32x4_t nonzero = vbicq_u32(vdupq_n_u32(1), vceqq_f32(a, vdupq_n_f32(0.0f)));
3384 return static_cast<Index
>(vaddvq_u32(nonzero));
3388EIGEN_STRONG_INLINE
float predux<Packet2f>(
const Packet2f& a) {
3389 return vget_lane_f32(vpadd_f32(a, a), 0);
3392EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a) {
3393 const float32x2_t sum = vadd_f32(vget_low_f32(a), vget_high_f32(a));
3394 return vget_lane_f32(vpadd_f32(sum, sum), 0);
3398EIGEN_STRONG_INLINE int8_t predux<Packet4c>(
const Packet4c& a) {
3399 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
3400 int8x8_t sum = vpadd_s8(a_dup, a_dup);
3401 sum = vpadd_s8(sum, sum);
3402 return vget_lane_s8(sum, 0);
3406EIGEN_STRONG_INLINE int8_t predux<Packet8c>(
const Packet8c& a) {
3410EIGEN_STRONG_INLINE int8_t predux<Packet16c>(
const Packet16c& a) {
3411 return vaddvq_s8(a);
3415EIGEN_STRONG_INLINE int8_t predux<Packet8c>(
const Packet8c& a) {
3416 int8x8_t sum = vpadd_s8(a, a);
3417 sum = vpadd_s8(sum, sum);
3418 sum = vpadd_s8(sum, sum);
3419 return vget_lane_s8(sum, 0);
3422EIGEN_STRONG_INLINE int8_t predux<Packet16c>(
const Packet16c& a) {
3423 int8x8_t sum = vadd_s8(vget_low_s8(a), vget_high_s8(a));
3424 sum = vpadd_s8(sum, sum);
3425 sum = vpadd_s8(sum, sum);
3426 sum = vpadd_s8(sum, sum);
3427 return vget_lane_s8(sum, 0);
3431EIGEN_STRONG_INLINE uint8_t predux<Packet4uc>(
const Packet4uc& a) {
3432 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
3433 uint8x8_t sum = vpadd_u8(a_dup, a_dup);
3434 sum = vpadd_u8(sum, sum);
3435 return vget_lane_u8(sum, 0);
3439EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(
const Packet8uc& a) {
3443EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(
const Packet16uc& a) {
3444 return vaddvq_u8(a);
3447EIGEN_STRONG_INLINE int16_t predux<Packet4s>(
const Packet4s& a) {
3448 return vaddv_s16(a);
3451EIGEN_STRONG_INLINE int16_t predux<Packet8s>(
const Packet8s& a) {
3452 return vaddvq_s16(a);
3455EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(
const Packet4us& a) {
3456 return vaddv_u16(a);
3459EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(
const Packet8us& a) {
3460 return vaddvq_u16(a);
3463EIGEN_STRONG_INLINE int32_t predux<Packet2i>(
const Packet2i& a) {
3464 return vaddv_s32(a);
3467EIGEN_STRONG_INLINE int32_t predux<Packet4i>(
const Packet4i& a) {
3468 return vaddvq_s32(a);
3471EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(
const Packet2ui& a) {
3472 return vaddv_u32(a);
3475EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(
const Packet4ui& a) {
3476 return vaddvq_u32(a);
3479EIGEN_STRONG_INLINE int64_t predux<Packet2l>(
const Packet2l& a) {
3480 return vaddvq_s64(a);
3483EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(
const Packet2ul& a) {
3484 return vaddvq_u64(a);
3488EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(
const Packet8uc& a) {
3489 uint8x8_t sum = vpadd_u8(a, a);
3490 sum = vpadd_u8(sum, sum);
3491 sum = vpadd_u8(sum, sum);
3492 return vget_lane_u8(sum, 0);
3495EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(
const Packet16uc& a) {
3496 uint8x8_t sum = vadd_u8(vget_low_u8(a), vget_high_u8(a));
3497 sum = vpadd_u8(sum, sum);
3498 sum = vpadd_u8(sum, sum);
3499 sum = vpadd_u8(sum, sum);
3500 return vget_lane_u8(sum, 0);
3503EIGEN_STRONG_INLINE int16_t predux<Packet4s>(
const Packet4s& a) {
3504 const int16x4_t sum = vpadd_s16(a, a);
3505 return vget_lane_s16(vpadd_s16(sum, sum), 0);
3508EIGEN_STRONG_INLINE int16_t predux<Packet8s>(
const Packet8s& a) {
3509 int16x4_t sum = vadd_s16(vget_low_s16(a), vget_high_s16(a));
3510 sum = vpadd_s16(sum, sum);
3511 sum = vpadd_s16(sum, sum);
3512 return vget_lane_s16(sum, 0);
3515EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(
const Packet4us& a) {
3516 const uint16x4_t sum = vpadd_u16(a, a);
3517 return vget_lane_u16(vpadd_u16(sum, sum), 0);
3520EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(
const Packet8us& a) {
3521 uint16x4_t sum = vadd_u16(vget_low_u16(a), vget_high_u16(a));
3522 sum = vpadd_u16(sum, sum);
3523 sum = vpadd_u16(sum, sum);
3524 return vget_lane_u16(sum, 0);
3527EIGEN_STRONG_INLINE int32_t predux<Packet2i>(
const Packet2i& a) {
3528 return vget_lane_s32(vpadd_s32(a, a), 0);
3531EIGEN_STRONG_INLINE int32_t predux<Packet4i>(
const Packet4i& a) {
3532 const int32x2_t sum = vadd_s32(vget_low_s32(a), vget_high_s32(a));
3533 return vget_lane_s32(vpadd_s32(sum, sum), 0);
3536EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(
const Packet2ui& a) {
3537 return vget_lane_u32(vpadd_u32(a, a), 0);
3540EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(
const Packet4ui& a) {
3541 const uint32x2_t sum = vadd_u32(vget_low_u32(a), vget_high_u32(a));
3542 return vget_lane_u32(vpadd_u32(sum, sum), 0);
3545EIGEN_STRONG_INLINE int64_t predux<Packet2l>(
const Packet2l& a) {
3546 return vgetq_lane_s64(a, 0) + vgetq_lane_s64(a, 1);
3549EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(
const Packet2ul& a) {
3550 return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1);
3555EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half(
const Packet8c& a) {
3556 return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(a, vreinterpret_s8_s32(vrev64_s32(vreinterpret_s32_s8(a))))), 0);
3559EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c predux_half(
const Packet16c& a) {
3560 return vadd_s8(vget_high_s8(a), vget_low_s8(a));
3563EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc predux_half(
const Packet8uc& a) {
3564 return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(a, vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(a))))), 0);
3567EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc predux_half(
const Packet16uc& a) {
3568 return vadd_u8(vget_high_u8(a), vget_low_u8(a));
3571EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s predux_half(
const Packet8s& a) {
3572 return vadd_s16(vget_high_s16(a), vget_low_s16(a));
3575EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us predux_half(
const Packet8us& a) {
3576 return vadd_u16(vget_high_u16(a), vget_low_u16(a));
3582EIGEN_STRONG_INLINE
float predux_mul<Packet2f>(
const Packet2f& a) {
3583 return vget_lane_f32(a, 0) * vget_lane_f32(a, 1);
3586EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a) {
3587 return predux_mul<Packet2f>(vmul_f32(vget_low_f32(a), vget_high_f32(a)));
3590EIGEN_STRONG_INLINE int8_t predux_mul<Packet4c>(
const Packet4c& a) {
3591 int8x8_t prod = vreinterpret_s8_s32(vdup_n_s32(a));
3592 prod = vmul_s8(prod, vrev16_s8(prod));
3593 return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 2);
3596EIGEN_STRONG_INLINE int8_t predux_mul<Packet8c>(
const Packet8c& a) {
3597 int8x8_t prod = vmul_s8(a, vrev16_s8(a));
3598 prod = vmul_s8(prod, vrev32_s8(prod));
3599 return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4);
3602EIGEN_STRONG_INLINE int8_t predux_mul<Packet16c>(
const Packet16c& a) {
3603 return predux_mul<Packet8c>(vmul_s8(vget_low_s8(a), vget_high_s8(a)));
3606EIGEN_STRONG_INLINE uint8_t predux_mul<Packet4uc>(
const Packet4uc& a) {
3607 uint8x8_t prod = vreinterpret_u8_u32(vdup_n_u32(a));
3608 prod = vmul_u8(prod, vrev16_u8(prod));
3609 return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 2);
3612EIGEN_STRONG_INLINE uint8_t predux_mul<Packet8uc>(
const Packet8uc& a) {
3613 uint8x8_t prod = vmul_u8(a, vrev16_u8(a));
3614 prod = vmul_u8(prod, vrev32_u8(prod));
3615 return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4);
3618EIGEN_STRONG_INLINE uint8_t predux_mul<Packet16uc>(
const Packet16uc& a) {
3619 return predux_mul<Packet8uc>(vmul_u8(vget_low_u8(a), vget_high_u8(a)));
3622EIGEN_STRONG_INLINE int16_t predux_mul<Packet4s>(
const Packet4s& a) {
3623 const int16x4_t prod = vmul_s16(a, vrev32_s16(a));
3624 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
3627EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(
const Packet8s& a) {
3631 prod = vmul_s16(vget_low_s16(a), vget_high_s16(a));
3633 prod = vmul_s16(prod, vrev32_s16(prod));
3635 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
3638EIGEN_STRONG_INLINE uint16_t predux_mul<Packet4us>(
const Packet4us& a) {
3639 const uint16x4_t prod = vmul_u16(a, vrev32_u16(a));
3640 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
3643EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(
const Packet8us& a) {
3647 prod = vmul_u16(vget_low_u16(a), vget_high_u16(a));
3649 prod = vmul_u16(prod, vrev32_u16(prod));
3651 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
3654EIGEN_STRONG_INLINE int32_t predux_mul<Packet2i>(
const Packet2i& a) {
3655 return vget_lane_s32(a, 0) * vget_lane_s32(a, 1);
3658EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(
const Packet4i& a) {
3659 return predux_mul<Packet2i>(vmul_s32(vget_low_s32(a), vget_high_s32(a)));
3662EIGEN_STRONG_INLINE uint32_t predux_mul<Packet2ui>(
const Packet2ui& a) {
3663 return vget_lane_u32(a, 0) * vget_lane_u32(a, 1);
3666EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(
const Packet4ui& a) {
3667 return predux_mul<Packet2ui>(vmul_u32(vget_low_u32(a), vget_high_u32(a)));
3670EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(
const Packet2l& a) {
3671 return vgetq_lane_s64(a, 0) * vgetq_lane_s64(a, 1);
3674EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(
const Packet2ul& a) {
3675 return vgetq_lane_u64(a, 0) * vgetq_lane_u64(a, 1);
3681EIGEN_STRONG_INLINE
float predux_min<Packet2f>(
const Packet2f& a) {
3682 return vminv_f32(a);
3685EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a) {
3686 return vminvq_f32(a);
3690EIGEN_STRONG_INLINE
float predux_min<Packet2f>(
const Packet2f& a) {
3691 return vget_lane_f32(vpmin_f32(a, a), 0);
3694EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a) {
3695 const float32x2_t min = vmin_f32(vget_low_f32(a), vget_high_f32(a));
3696 return vget_lane_f32(vpmin_f32(min, min), 0);
3700EIGEN_STRONG_INLINE int8_t predux_min<Packet4c>(
const Packet4c& a) {
3701 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
3702 int8x8_t min = vpmin_s8(a_dup, a_dup);
3703 min = vpmin_s8(min, min);
3704 return vget_lane_s8(min, 0);
3708EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(
const Packet8c& a) {
3712EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(
const Packet16c& a) {
3713 return vminvq_s8(a);
3717EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(
const Packet8c& a) {
3718 int8x8_t min = vpmin_s8(a, a);
3719 min = vpmin_s8(min, min);
3720 min = vpmin_s8(min, min);
3721 return vget_lane_s8(min, 0);
3724EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(
const Packet16c& a) {
3725 int8x8_t min = vmin_s8(vget_low_s8(a), vget_high_s8(a));
3726 min = vpmin_s8(min, min);
3727 min = vpmin_s8(min, min);
3728 min = vpmin_s8(min, min);
3729 return vget_lane_s8(min, 0);
3733EIGEN_STRONG_INLINE uint8_t predux_min<Packet4uc>(
const Packet4uc& a) {
3734 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
3735 uint8x8_t min = vpmin_u8(a_dup, a_dup);
3736 min = vpmin_u8(min, min);
3737 return vget_lane_u8(min, 0);
3741EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(
const Packet8uc& a) {
3745EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(
const Packet16uc& a) {
3746 return vminvq_u8(a);
3749EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(
const Packet4s& a) {
3750 return vminv_s16(a);
3753EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(
const Packet8s& a) {
3754 return vminvq_s16(a);
3757EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(
const Packet4us& a) {
3758 return vminv_u16(a);
3761EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(
const Packet8us& a) {
3762 return vminvq_u16(a);
3765EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(
const Packet2i& a) {
3766 return vminv_s32(a);
3769EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(
const Packet4i& a) {
3770 return vminvq_s32(a);
3773EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(
const Packet2ui& a) {
3774 return vminv_u32(a);
3777EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(
const Packet4ui& a) {
3778 return vminvq_u32(a);
3782EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(
const Packet8uc& a) {
3783 uint8x8_t min = vpmin_u8(a, a);
3784 min = vpmin_u8(min, min);
3785 min = vpmin_u8(min, min);
3786 return vget_lane_u8(min, 0);
3789EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(
const Packet16uc& a) {
3790 uint8x8_t min = vmin_u8(vget_low_u8(a), vget_high_u8(a));
3791 min = vpmin_u8(min, min);
3792 min = vpmin_u8(min, min);
3793 min = vpmin_u8(min, min);
3794 return vget_lane_u8(min, 0);
3797EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(
const Packet4s& a) {
3798 const int16x4_t min = vpmin_s16(a, a);
3799 return vget_lane_s16(vpmin_s16(min, min), 0);
3802EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(
const Packet8s& a) {
3803 int16x4_t min = vmin_s16(vget_low_s16(a), vget_high_s16(a));
3804 min = vpmin_s16(min, min);
3805 min = vpmin_s16(min, min);
3806 return vget_lane_s16(min, 0);
3809EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(
const Packet4us& a) {
3810 const uint16x4_t min = vpmin_u16(a, a);
3811 return vget_lane_u16(vpmin_u16(min, min), 0);
3814EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(
const Packet8us& a) {
3815 uint16x4_t min = vmin_u16(vget_low_u16(a), vget_high_u16(a));
3816 min = vpmin_u16(min, min);
3817 min = vpmin_u16(min, min);
3818 return vget_lane_u16(min, 0);
3821EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(
const Packet2i& a) {
3822 return vget_lane_s32(vpmin_s32(a, a), 0);
3825EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(
const Packet4i& a) {
3826 const int32x2_t min = vmin_s32(vget_low_s32(a), vget_high_s32(a));
3827 return vget_lane_s32(vpmin_s32(min, min), 0);
3830EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(
const Packet2ui& a) {
3831 return vget_lane_u32(vpmin_u32(a, a), 0);
3834EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(
const Packet4ui& a) {
3835 const uint32x2_t min = vmin_u32(vget_low_u32(a), vget_high_u32(a));
3836 return vget_lane_u32(vpmin_u32(min, min), 0);
3840EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(
const Packet2l& a) {
3841 return (std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1));
3844EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(
const Packet2ul& a) {
3845 return (std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1));
3851EIGEN_STRONG_INLINE
float predux_max<Packet2f>(
const Packet2f& a) {
3852 return vmaxv_f32(a);
3855EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a) {
3856 return vmaxvq_f32(a);
3860EIGEN_STRONG_INLINE
float predux_max<Packet2f>(
const Packet2f& a) {
3861 return vget_lane_f32(vpmax_f32(a, a), 0);
3864EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a) {
3865 const float32x2_t max = vmax_f32(vget_low_f32(a), vget_high_f32(a));
3866 return vget_lane_f32(vpmax_f32(max, max), 0);
3870EIGEN_STRONG_INLINE int8_t predux_max<Packet4c>(
const Packet4c& a) {
3871 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
3872 int8x8_t max = vpmax_s8(a_dup, a_dup);
3873 max = vpmax_s8(max, max);
3874 return vget_lane_s8(max, 0);
3878EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(
const Packet8c& a) {
3882EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(
const Packet16c& a) {
3883 return vmaxvq_s8(a);
3887EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(
const Packet8c& a) {
3888 int8x8_t max = vpmax_s8(a, a);
3889 max = vpmax_s8(max, max);
3890 max = vpmax_s8(max, max);
3891 return vget_lane_s8(max, 0);
3894EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(
const Packet16c& a) {
3895 int8x8_t max = vmax_s8(vget_low_s8(a), vget_high_s8(a));
3896 max = vpmax_s8(max, max);
3897 max = vpmax_s8(max, max);
3898 max = vpmax_s8(max, max);
3899 return vget_lane_s8(max, 0);
3903EIGEN_STRONG_INLINE uint8_t predux_max<Packet4uc>(
const Packet4uc& a) {
3904 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
3905 uint8x8_t max = vpmax_u8(a_dup, a_dup);
3906 max = vpmax_u8(max, max);
3907 return vget_lane_u8(max, 0);
3911EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(
const Packet8uc& a) {
3915EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(
const Packet16uc& a) {
3916 return vmaxvq_u8(a);
3919EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(
const Packet4s& a) {
3920 return vmaxv_s16(a);
3923EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(
const Packet8s& a) {
3924 return vmaxvq_s16(a);
3927EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(
const Packet4us& a) {
3928 return vmaxv_u16(a);
3931EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(
const Packet8us& a) {
3932 return vmaxvq_u16(a);
3935EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(
const Packet2i& a) {
3936 return vmaxv_s32(a);
3939EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(
const Packet4i& a) {
3940 return vmaxvq_s32(a);
3943EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(
const Packet2ui& a) {
3944 return vmaxv_u32(a);
3947EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(
const Packet4ui& a) {
3948 return vmaxvq_u32(a);
3952EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(
const Packet8uc& a) {
3953 uint8x8_t max = vpmax_u8(a, a);
3954 max = vpmax_u8(max, max);
3955 max = vpmax_u8(max, max);
3956 return vget_lane_u8(max, 0);
3959EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(
const Packet16uc& a) {
3960 uint8x8_t max = vmax_u8(vget_low_u8(a), vget_high_u8(a));
3961 max = vpmax_u8(max, max);
3962 max = vpmax_u8(max, max);
3963 max = vpmax_u8(max, max);
3964 return vget_lane_u8(max, 0);
3967EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(
const Packet4s& a) {
3968 const int16x4_t max = vpmax_s16(a, a);
3969 return vget_lane_s16(vpmax_s16(max, max), 0);
3972EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(
const Packet8s& a) {
3973 int16x4_t max = vmax_s16(vget_low_s16(a), vget_high_s16(a));
3974 max = vpmax_s16(max, max);
3975 max = vpmax_s16(max, max);
3976 return vget_lane_s16(max, 0);
3979EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(
const Packet4us& a) {
3980 const uint16x4_t max = vpmax_u16(a, a);
3981 return vget_lane_u16(vpmax_u16(max, max), 0);
3984EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(
const Packet8us& a) {
3985 uint16x4_t max = vmax_u16(vget_low_u16(a), vget_high_u16(a));
3986 max = vpmax_u16(max, max);
3987 max = vpmax_u16(max, max);
3988 return vget_lane_u16(max, 0);
3991EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(
const Packet2i& a) {
3992 return vget_lane_s32(vpmax_s32(a, a), 0);
3995EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(
const Packet4i& a) {
3996 const int32x2_t max = vmax_s32(vget_low_s32(a), vget_high_s32(a));
3997 return vget_lane_s32(vpmax_s32(max, max), 0);
4000EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(
const Packet2ui& a) {
4001 return vget_lane_u32(vpmax_u32(a, a), 0);
4004EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(
const Packet4ui& a) {
4005 const uint32x2_t max = vmax_u32(vget_low_u32(a), vget_high_u32(a));
4006 return vget_lane_u32(vpmax_u32(max, max), 0);
4010EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(
const Packet2l& a) {
4011 return (std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1));
4014EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(
const Packet2ul& a) {
4015 return (std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1));
4019EIGEN_STRONG_INLINE
bool predux_any(
const Packet2f& x) {
4020 return vget_lane_u64(vreinterpret_u64_f32(x), 0) != 0;
4024EIGEN_STRONG_INLINE
bool predux_any(
const Packet4f& x) {
4025 uint32x4_t u = vreinterpretq_u32_f32(x);
4027 return vget_lane_u64(vreinterpret_u64_u16(vmovn_u32(u)), 0);
4029 uint32x2_t tmp = vorr_u32(vget_low_u32(u), vget_high_u32(u));
4032 asm(
"vmov %0, %1, %P2" :
"=r"(a),
"=r"(b) :
"w"(tmp));
4041EIGEN_STRONG_INLINE
bool neon_any_bit(uint32x4_t u) {
4043 return vmaxvq_u32(u) != 0;
4045 uint32x2_t tmp = vorr_u32(vget_low_u32(u), vget_high_u32(u));
4046 return vget_lane_u32(vpmax_u32(tmp, tmp), 0) != 0;
4052EIGEN_STRONG_INLINE
bool predux_any(
const Packet4c& x) {
4053 return static_cast<int32_t
>(x) != 0;
4056EIGEN_STRONG_INLINE
bool predux_any(
const Packet4uc& x) {
4057 return static_cast<uint32_t
>(x) != 0;
4060EIGEN_STRONG_INLINE
bool predux_any(
const Packet8c& x) {
4061 return vget_lane_u64(vreinterpret_u64_s8(x), 0) != 0;
4064EIGEN_STRONG_INLINE
bool predux_any(
const Packet8uc& x) {
4065 return vget_lane_u64(vreinterpret_u64_u8(x), 0) != 0;
4068EIGEN_STRONG_INLINE
bool predux_any(
const Packet16c& x) {
4069 return detail::neon_any_bit(vreinterpretq_u32_s8(x));
4072EIGEN_STRONG_INLINE
bool predux_any(
const Packet16uc& x) {
4073 return detail::neon_any_bit(vreinterpretq_u32_u8(x));
4076EIGEN_STRONG_INLINE
bool predux_any(
const Packet4s& x) {
4077 return vget_lane_u64(vreinterpret_u64_s16(x), 0) != 0;
4080EIGEN_STRONG_INLINE
bool predux_any(
const Packet4us& x) {
4081 return vget_lane_u64(vreinterpret_u64_u16(x), 0) != 0;
4084EIGEN_STRONG_INLINE
bool predux_any(
const Packet8s& x) {
4085 return detail::neon_any_bit(vreinterpretq_u32_s16(x));
4088EIGEN_STRONG_INLINE
bool predux_any(
const Packet8us& x) {
4089 return detail::neon_any_bit(vreinterpretq_u32_u16(x));
4092EIGEN_STRONG_INLINE
bool predux_any(
const Packet2i& x) {
4093 return vget_lane_u64(vreinterpret_u64_s32(x), 0) != 0;
4096EIGEN_STRONG_INLINE
bool predux_any(
const Packet2ui& x) {
4097 return vget_lane_u64(vreinterpret_u64_u32(x), 0) != 0;
4100EIGEN_STRONG_INLINE
bool predux_any(
const Packet4i& x) {
4101 return detail::neon_any_bit(vreinterpretq_u32_s32(x));
4104EIGEN_STRONG_INLINE
bool predux_any(
const Packet4ui& x) {
4105 return detail::neon_any_bit(x);
4108EIGEN_STRONG_INLINE
bool predux_any(
const Packet2l& x) {
4109 return detail::neon_any_bit(vreinterpretq_u32_s64(x));
4112EIGEN_STRONG_INLINE
bool predux_any(
const Packet2ul& x) {
4113 return detail::neon_any_bit(vreinterpretq_u32_u64(x));
4119template <
typename Packet>
4120void zip_in_place(Packet& p1, Packet& p2);
4123EIGEN_ALWAYS_INLINE
void zip_in_place<Packet2f>(Packet2f& p1, Packet2f& p2) {
4124 const float32x2x2_t tmp = vzip_f32(p1, p2);
4130EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4f>(Packet4f& p1, Packet4f& p2) {
4131 const float32x4x2_t tmp = vzipq_f32(p1, p2);
4137EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8c>(Packet8c& p1, Packet8c& p2) {
4138 const int8x8x2_t tmp = vzip_s8(p1, p2);
4144EIGEN_ALWAYS_INLINE
void zip_in_place<Packet16c>(Packet16c& p1, Packet16c& p2) {
4145 const int8x16x2_t tmp = vzipq_s8(p1, p2);
4151EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8uc>(Packet8uc& p1, Packet8uc& p2) {
4152 const uint8x8x2_t tmp = vzip_u8(p1, p2);
4158EIGEN_ALWAYS_INLINE
void zip_in_place<Packet16uc>(Packet16uc& p1, Packet16uc& p2) {
4159 const uint8x16x2_t tmp = vzipq_u8(p1, p2);
4165EIGEN_ALWAYS_INLINE
void zip_in_place<Packet2i>(Packet2i& p1, Packet2i& p2) {
4166 const int32x2x2_t tmp = vzip_s32(p1, p2);
4172EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4i>(Packet4i& p1, Packet4i& p2) {
4173 const int32x4x2_t tmp = vzipq_s32(p1, p2);
4179EIGEN_ALWAYS_INLINE
void zip_in_place<Packet2ui>(Packet2ui& p1, Packet2ui& p2) {
4180 const uint32x2x2_t tmp = vzip_u32(p1, p2);
4186EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4ui>(Packet4ui& p1, Packet4ui& p2) {
4187 const uint32x4x2_t tmp = vzipq_u32(p1, p2);
4193EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4s>(Packet4s& p1, Packet4s& p2) {
4194 const int16x4x2_t tmp = vzip_s16(p1, p2);
4200EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8s>(Packet8s& p1, Packet8s& p2) {
4201 const int16x8x2_t tmp = vzipq_s16(p1, p2);
4207EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4us>(Packet4us& p1, Packet4us& p2) {
4208 const uint16x4x2_t tmp = vzip_u16(p1, p2);
4214EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8us>(Packet8us& p1, Packet8us& p2) {
4215 const uint16x8x2_t tmp = vzipq_u16(p1, p2);
4220template <
typename Packet>
4221EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 2>& kernel) {
4222 zip_in_place(kernel.packet[0], kernel.packet[1]);
4225template <
typename Packet>
4226EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 4>& kernel) {
4227 zip_in_place(kernel.packet[0], kernel.packet[2]);
4228 zip_in_place(kernel.packet[1], kernel.packet[3]);
4229 zip_in_place(kernel.packet[0], kernel.packet[1]);
4230 zip_in_place(kernel.packet[2], kernel.packet[3]);
4233template <
typename Packet>
4234EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 8>& kernel) {
4235 zip_in_place(kernel.packet[0], kernel.packet[4]);
4236 zip_in_place(kernel.packet[1], kernel.packet[5]);
4237 zip_in_place(kernel.packet[2], kernel.packet[6]);
4238 zip_in_place(kernel.packet[3], kernel.packet[7]);
4240 zip_in_place(kernel.packet[0], kernel.packet[2]);
4241 zip_in_place(kernel.packet[1], kernel.packet[3]);
4242 zip_in_place(kernel.packet[4], kernel.packet[6]);
4243 zip_in_place(kernel.packet[5], kernel.packet[7]);
4245 zip_in_place(kernel.packet[0], kernel.packet[1]);
4246 zip_in_place(kernel.packet[2], kernel.packet[3]);
4247 zip_in_place(kernel.packet[4], kernel.packet[5]);
4248 zip_in_place(kernel.packet[6], kernel.packet[7]);
4251template <
typename Packet>
4252EIGEN_ALWAYS_INLINE
void ptranspose_impl(PacketBlock<Packet, 16>& kernel) {
4254 for (
int i = 0; i < 4; ++i) {
4255 const int m = (1 << i);
4257 for (
int j = 0; j < m; ++j) {
4258 const int n = (1 << (3 - i));
4260 for (
int k = 0; k < n; ++k) {
4261 const int idx = 2 * j * n + k;
4262 zip_in_place(kernel.packet[idx], kernel.packet[idx + n]);
4270EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2f, 2>& kernel) {
4271 detail::ptranspose_impl(kernel);
4273EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
4274 detail::ptranspose_impl(kernel);
4277EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4c, 4>& kernel) {
4278 const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1));
4279 const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1));
4281 const int8x8x2_t zip8 = vzip_s8(a, b);
4282 const int16x4x2_t zip16 = vzip_s16(vreinterpret_s16_s8(zip8.val[0]), vreinterpret_s16_s8(zip8.val[1]));
4284 kernel.packet[0] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 0);
4285 kernel.packet[1] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 1);
4286 kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0);
4287 kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1);
4289EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8c, 8>& kernel) {
4290 detail::ptranspose_impl(kernel);
4292EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8c, 4>& kernel) {
4293 detail::ptranspose_impl(kernel);
4295EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
4296 detail::ptranspose_impl(kernel);
4298EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 8>& kernel) {
4299 detail::ptranspose_impl(kernel);
4301EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
4302 detail::ptranspose_impl(kernel);
4305EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4uc, 4>& kernel) {
4306 const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1));
4307 const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1));
4309 const uint8x8x2_t zip8 = vzip_u8(a, b);
4310 const uint16x4x2_t zip16 = vzip_u16(vreinterpret_u16_u8(zip8.val[0]), vreinterpret_u16_u8(zip8.val[1]));
4312 kernel.packet[0] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 0);
4313 kernel.packet[1] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 1);
4314 kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0);
4315 kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1);
4317EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8uc, 8>& kernel) {
4318 detail::ptranspose_impl(kernel);
4320EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8uc, 4>& kernel) {
4321 detail::ptranspose_impl(kernel);
4323EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
4324 detail::ptranspose_impl(kernel);
4326EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 8>& kernel) {
4327 detail::ptranspose_impl(kernel);
4329EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
4330 detail::ptranspose_impl(kernel);
4333EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4s, 4>& kernel) {
4334 detail::ptranspose_impl(kernel);
4336EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
4337 detail::ptranspose_impl(kernel);
4339EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
4340 detail::ptranspose_impl(kernel);
4343EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4us, 4>& kernel) {
4344 detail::ptranspose_impl(kernel);
4346EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
4347 detail::ptranspose_impl(kernel);
4349EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
4350 detail::ptranspose_impl(kernel);
4353EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2i, 2>& kernel) {
4354 detail::ptranspose_impl(kernel);
4356EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
4357 detail::ptranspose_impl(kernel);
4359EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2ui, 2>& kernel) {
4360 detail::zip_in_place(kernel.packet[0], kernel.packet[1]);
4362EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
4363 detail::ptranspose_impl(kernel);
4366EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2l, 2>& kernel) {
4368 const int64x2_t tmp1 = vzip1q_s64(kernel.packet[0], kernel.packet[1]);
4369 kernel.packet[1] = vzip2q_s64(kernel.packet[0], kernel.packet[1]);
4370 kernel.packet[0] = tmp1;
4372 const int64x1_t tmp[2][2] = {{vget_low_s64(kernel.packet[0]), vget_high_s64(kernel.packet[0])},
4373 {vget_low_s64(kernel.packet[1]), vget_high_s64(kernel.packet[1])}};
4375 kernel.packet[0] = vcombine_s64(tmp[0][0], tmp[1][0]);
4376 kernel.packet[1] = vcombine_s64(tmp[0][1], tmp[1][1]);
4379EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2ul, 2>& kernel) {
4381 const uint64x2_t tmp1 = vzip1q_u64(kernel.packet[0], kernel.packet[1]);
4382 kernel.packet[1] = vzip2q_u64(kernel.packet[0], kernel.packet[1]);
4383 kernel.packet[0] = tmp1;
4385 const uint64x1_t tmp[2][2] = {{vget_low_u64(kernel.packet[0]), vget_high_u64(kernel.packet[0])},
4386 {vget_low_u64(kernel.packet[1]), vget_high_u64(kernel.packet[1])}};
4388 kernel.packet[0] = vcombine_u64(tmp[0][0], tmp[1][0]);
4389 kernel.packet[1] = vcombine_u64(tmp[0][1], tmp[1][1]);
4394EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pselect(
const Packet2f& mask,
const Packet2f& a,
const Packet2f& b) {
4395 return vbsl_f32(vreinterpret_u32_f32(mask), a, b);
4398EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(
const Packet4f& mask,
const Packet4f& a,
const Packet4f& b) {
4399 return vbslq_f32(vreinterpretq_u32_f32(mask), a, b);
4402EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pselect(
const Packet8c& mask,
const Packet8c& a,
const Packet8c& b) {
4403 return vbsl_s8(vreinterpret_u8_s8(mask), a, b);
4406EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(
const Packet16c& mask,
const Packet16c& a,
const Packet16c& b) {
4407 return vbslq_s8(vreinterpretq_u8_s8(mask), a, b);
4410EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pselect(
const Packet8uc& mask,
const Packet8uc& a,
const Packet8uc& b) {
4411 return vbsl_u8(mask, a, b);
4414EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(
const Packet16uc& mask,
const Packet16uc& a,
4415 const Packet16uc& b) {
4416 return vbslq_u8(mask, a, b);
4419EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pselect(
const Packet4s& mask,
const Packet4s& a,
const Packet4s& b) {
4420 return vbsl_s16(vreinterpret_u16_s16(mask), a, b);
4423EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(
const Packet8s& mask,
const Packet8s& a,
const Packet8s& b) {
4424 return vbslq_s16(vreinterpretq_u16_s16(mask), a, b);
4427EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pselect(
const Packet4us& mask,
const Packet4us& a,
const Packet4us& b) {
4428 return vbsl_u16(mask, a, b);
4431EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(
const Packet8us& mask,
const Packet8us& a,
const Packet8us& b) {
4432 return vbslq_u16(mask, a, b);
4435EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pselect(
const Packet2i& mask,
const Packet2i& a,
const Packet2i& b) {
4436 return vbsl_s32(vreinterpret_u32_s32(mask), a, b);
4439EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(
const Packet4i& mask,
const Packet4i& a,
const Packet4i& b) {
4440 return vbslq_s32(vreinterpretq_u32_s32(mask), a, b);
4443EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pselect(
const Packet2ui& mask,
const Packet2ui& a,
const Packet2ui& b) {
4444 return vbsl_u32(mask, a, b);
4447EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(
const Packet4ui& mask,
const Packet4ui& a,
const Packet4ui& b) {
4448 return vbslq_u32(mask, a, b);
4451EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(
const Packet2l& mask,
const Packet2l& a,
const Packet2l& b) {
4452 return vbslq_s64(vreinterpretq_u64_s64(mask), a, b);
4455EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(
const Packet2ul& mask,
const Packet2ul& a,
const Packet2ul& b) {
4456 return vbslq_u64(mask, a, b);
4462EIGEN_STRONG_INLINE Packet2f print<Packet2f>(
const Packet2f& a) {
4463 return vrndn_f32(a);
4467EIGEN_STRONG_INLINE Packet4f print<Packet4f>(
const Packet4f& a) {
4468 return vrndnq_f32(a);
4472EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(
const Packet2f& a) {
4473 return vrndm_f32(a);
4477EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a) {
4478 return vrndmq_f32(a);
4482EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(
const Packet2f& a) {
4483 return vrndp_f32(a);
4487EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a) {
4488 return vrndpq_f32(a);
4492EIGEN_STRONG_INLINE Packet2f pround<Packet2f>(
const Packet2f& a) {
4493 return vrnda_f32(a);
4497EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(
const Packet4f& a) {
4498 return vrndaq_f32(a);
4502EIGEN_STRONG_INLINE Packet2f ptrunc<Packet2f>(
const Packet2f& a) {
4507EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(
const Packet4f& a) {
4508 return vrndq_f32(a);
4519EIGEN_STRONG_INLINE Packet4uc psqrt(
const Packet4uc& a) {
4520 uint8x8_t x = vreinterpret_u8_u32(vdup_n_u32(a));
4521 uint8x8_t res = vdup_n_u8(0);
4522 uint8x8_t add = vdup_n_u8(0x8);
4523 for (
int i = 0; i < 4; i++) {
4524 const uint8x8_t temp = vorr_u8(res, add);
4525 res = vbsl_u8(vcge_u8(x, vmul_u8(temp, temp)), temp, res);
4526 add = vshr_n_u8(add, 1);
4528 return vget_lane_u32(vreinterpret_u32_u8(res), 0);
4532EIGEN_STRONG_INLINE Packet8uc psqrt(
const Packet8uc& a) {
4533 uint8x8_t res = vdup_n_u8(0);
4534 uint8x8_t add = vdup_n_u8(0x8);
4535 for (
int i = 0; i < 4; i++) {
4536 const uint8x8_t temp = vorr_u8(res, add);
4537 res = vbsl_u8(vcge_u8(a, vmul_u8(temp, temp)), temp, res);
4538 add = vshr_n_u8(add, 1);
4544EIGEN_STRONG_INLINE Packet16uc psqrt(
const Packet16uc& a) {
4545 uint8x16_t res = vdupq_n_u8(0);
4546 uint8x16_t add = vdupq_n_u8(0x8);
4547 for (
int i = 0; i < 4; i++) {
4548 const uint8x16_t temp = vorrq_u8(res, add);
4549 res = vbslq_u8(vcgeq_u8(a, vmulq_u8(temp, temp)), temp, res);
4550 add = vshrq_n_u8(add, 1);
4556EIGEN_STRONG_INLINE Packet4us psqrt(
const Packet4us& a) {
4557 uint16x4_t res = vdup_n_u16(0);
4558 uint16x4_t add = vdup_n_u16(0x80);
4559 for (
int i = 0; i < 8; i++) {
4560 const uint16x4_t temp = vorr_u16(res, add);
4561 res = vbsl_u16(vcge_u16(a, vmul_u16(temp, temp)), temp, res);
4562 add = vshr_n_u16(add, 1);
4568EIGEN_STRONG_INLINE Packet8us psqrt(
const Packet8us& a) {
4569 uint16x8_t res = vdupq_n_u16(0);
4570 uint16x8_t add = vdupq_n_u16(0x80);
4571 for (
int i = 0; i < 8; i++) {
4572 const uint16x8_t temp = vorrq_u16(res, add);
4573 res = vbslq_u16(vcgeq_u16(a, vmulq_u16(temp, temp)), temp, res);
4574 add = vshrq_n_u16(add, 1);
4580EIGEN_STRONG_INLINE Packet2ui psqrt(
const Packet2ui& a) {
4581 uint32x2_t res = vdup_n_u32(0);
4582 uint32x2_t add = vdup_n_u32(0x8000);
4583 for (
int i = 0; i < 16; i++) {
4584 const uint32x2_t temp = vorr_u32(res, add);
4585 res = vbsl_u32(vcge_u32(a, vmul_u32(temp, temp)), temp, res);
4586 add = vshr_n_u32(add, 1);
4592EIGEN_STRONG_INLINE Packet4ui psqrt(
const Packet4ui& a) {
4593 uint32x4_t res = vdupq_n_u32(0);
4594 uint32x4_t add = vdupq_n_u32(0x8000);
4595 for (
int i = 0; i < 16; i++) {
4596 const uint32x4_t temp = vorrq_u32(res, add);
4597 res = vbslq_u32(vcgeq_u32(a, vmulq_u32(temp, temp)), temp, res);
4598 add = vshrq_n_u32(add, 1);
4603EIGEN_STRONG_INLINE Packet4f prsqrt_float_unsafe(
const Packet4f& a) {
4606 float32x4_t result = vrsqrteq_f32(a);
4607 result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result);
4608 result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result);
4612EIGEN_STRONG_INLINE Packet2f prsqrt_float_unsafe(
const Packet2f& a) {
4615 float32x2_t result = vrsqrte_f32(a);
4616 result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result);
4617 result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result);
4621template <
typename Packet>
4622Packet prsqrt_float_common(
const Packet& a) {
4623 const Packet cst_zero = pzero(a);
4624 const Packet cst_inf = pset1<Packet>(NumTraits<float>::infinity());
4625 Packet return_zero = pcmp_eq(a, cst_inf);
4626 Packet return_inf = pcmp_eq(a, cst_zero);
4627 Packet result = prsqrt_float_unsafe(a);
4628 result = pselect(return_inf, por(cst_inf, a), result);
4629 result = pandnot(result, return_zero);
4634EIGEN_STRONG_INLINE Packet4f prsqrt(
const Packet4f& a) {
4635 return prsqrt_float_common(a);
4639EIGEN_STRONG_INLINE Packet2f prsqrt(
const Packet2f& a) {
4640 return prsqrt_float_common(a);
4644EIGEN_STRONG_INLINE Packet4f preciprocal<Packet4f>(
const Packet4f& a) {
4646 const float32x4_t estimate = vrecpeq_f32(a);
4647 float32x4_t result = vmulq_f32(vrecpsq_f32(a, estimate), estimate);
4648 result = vmulq_f32(vrecpsq_f32(a, result), result);
4650 return vbslq_f32(vdupq_n_u32(0x80000000u), a, result);
4654EIGEN_STRONG_INLINE Packet2f preciprocal<Packet2f>(
const Packet2f& a) {
4656 const float32x2_t estimate = vrecpe_f32(a);
4657 float32x2_t result = vmul_f32(vrecps_f32(a, estimate), estimate);
4658 result = vmul_f32(vrecps_f32(a, result), result);
4660 return vbsl_f32(vdup_n_u32(0x80000000u), a, result);
4666EIGEN_STRONG_INLINE Packet4f psqrt(
const Packet4f& a) {
4667 return vsqrtq_f32(a);
4671EIGEN_STRONG_INLINE Packet2f psqrt(
const Packet2f& a) {
4672 return vsqrt_f32(a);
4676EIGEN_STRONG_INLINE Packet4f pdiv(
const Packet4f& a,
const Packet4f& b) {
4677 return vdivq_f32(a, b);
4681EIGEN_STRONG_INLINE Packet2f pdiv(
const Packet2f& a,
const Packet2f& b) {
4682 return vdiv_f32(a, b);
4685template <
typename Packet>
4686EIGEN_STRONG_INLINE Packet psqrt_float_common(
const Packet& a) {
4687 const Packet cst_zero = pzero(a);
4688 const Packet cst_inf = pset1<Packet>(NumTraits<float>::infinity());
4690 Packet result = pmul(a, prsqrt_float_unsafe(a));
4691 Packet a_is_zero = pcmp_eq(a, cst_zero);
4692 Packet a_is_inf = pcmp_eq(a, cst_inf);
4693 Packet return_a = por(a_is_zero, a_is_inf);
4695 result = pselect(return_a, a, result);
4700EIGEN_STRONG_INLINE Packet4f psqrt(
const Packet4f& a) {
4701 return psqrt_float_common(a);
4705EIGEN_STRONG_INLINE Packet2f psqrt(
const Packet2f& a) {
4706 return psqrt_float_common(a);
4709template <
typename Packet>
4710EIGEN_STRONG_INLINE Packet pdiv_float_common(
const Packet& a,
const Packet& b) {
4715 const Packet cst_one = pset1<Packet>(1.0f);
4716 const Packet cst_quarter = pset1<Packet>(0.25f);
4717 const Packet cst_thresh = pset1<Packet>(NumTraits<float>::highest() / 4.0f);
4719 Packet b_will_underflow = pcmp_le(cst_thresh, pabs(b));
4720 Packet f = pselect(b_will_underflow, cst_quarter, cst_one);
4721 Packet result = pmul(f, pmul(a, preciprocal(pmul(b, f))));
4726EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
4727 return pdiv_float_common(a, b);
4731EIGEN_STRONG_INLINE Packet2f pdiv<Packet2f>(
const Packet2f& a,
const Packet2f& b) {
4732 return pdiv_float_common(a, b);
4746template <
typename T>
4747uint64x2_t vreinterpretq_u64_f64(T a) {
4748 return (uint64x2_t)a;
4751template <
typename T>
4752float64x2_t vreinterpretq_f64_u64(T a) {
4753 return (float64x2_t)a;
4757#if EIGEN_COMP_MSVC_STRICT
4758using Packet2d = eigen_packet_wrapper<float64x2_t, 18>;
4759using Packet1d = eigen_packet_wrapper<float64x1_t, 19>;
4761EIGEN_ALWAYS_INLINE Packet2d make_packet2d(
double a,
double b) {
4762 double from[2] = {a, b};
4763 return vld1q_f64(from);
4767using Packet2d = float64x2_t;
4768using Packet1d = float64x1_t;
4770EIGEN_ALWAYS_INLINE Packet2d make_packet2d(
double a,
double b) {
return Packet2d{a, b}; }
4776EIGEN_STRONG_INLINE Packet2d shuffle(
const Packet2d& m,
const Packet2d& n,
int mask) {
4777 const double* a =
reinterpret_cast<const double*
>(&m);
4778 const double* b =
reinterpret_cast<const double*
>(&n);
4779 Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1)));
4783EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(
const Packet2d& a,
const Packet2d& b,
int mask) {
4784 return shuffle(a, b, mask);
4786EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(
const Packet2d& a,
const Packet2d& b) {
return shuffle(a, b, 0); }
4787EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(
const Packet2d& a,
const Packet2d& b) {
return shuffle(a, b, 3); }
4788#define vec2d_duplane(a, p) Packet2d(vdupq_laneq_f64(a, p))
4791struct packet_traits<double> : default_packet_traits {
4792 using type = Packet2d;
4793 using half = Packet2d;
4796 AlignedOnScalar = 1,
4829 HasSin = EIGEN_FAST_MATH,
4830 HasCos = EIGEN_FAST_MATH,
4831 HasTan = EIGEN_FAST_MATH,
4835 HasTanh = EIGEN_FAST_MATH,
4836 HasErf = EIGEN_FAST_MATH,
4837 HasErfc = EIGEN_FAST_MATH
4842struct unpacket_traits<Packet2d> : neon_unpacket_default<Packet2d, double> {
4843 using integer_packet = Packet2l;
4847EIGEN_STRONG_INLINE Packet2d pzero<Packet2d>(
const Packet2d& ) {
4848 return vdupq_n_f64(0.0);
4852EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
4853 return vdupq_n_f64(from);
4857EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a) {
4858 const double c[] = {0.0, 1.0};
4859 return vaddq_f64(pset1<Packet2d>(a), vld1q_f64(c));
4863EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4864 return vaddq_f64(a, b);
4868EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4869 return vsubq_f64(a, b);
4873EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d&,
const Packet2d&);
4875EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4876 const Packet2d mask = make_packet2d(numext::bit_cast<double>(0x8000000000000000ull), 0.0);
4877 return padd(a, pxor(mask, b));
4881EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
4882 return vnegq_f64(a);
4886EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4887 return vmulq_f64(a, b);
4891EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4892 return vdivq_f64(a, b);
4895#ifdef EIGEN_VECTORIZE_FMA
4898struct is_fused_madd<Packet2d> : true_type {};
4900EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
4901 return vfmaq_f64(c, a, b);
4904EIGEN_STRONG_INLINE Packet2d pnmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
4905 return vfmsq_f64(c, a, b);
4909EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
4910 return vmlaq_f64(c, a, b);
4913EIGEN_STRONG_INLINE Packet2d pnmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
4914 return vmlsq_f64(c, a, b);
4918EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4919 return vminq_f64(a, b);
4922#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
4926EIGEN_STRONG_INLINE Packet2d pmin<PropagateNumbers, Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4927 return vminnmq_f64(a, b);
4930EIGEN_STRONG_INLINE Packet2d pmax<PropagateNumbers, Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4931 return vmaxnmq_f64(a, b);
4937struct pminmax_propagates_nan<Packet2d> : bool_constant<true> {};
4940EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4941 return vmaxq_f64(a, b);
4946EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4947 return vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4951EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4952 return vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4956EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4957 return vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4961EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
4962 return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4966EIGEN_STRONG_INLINE Packet2d pnot<Packet2d>(
const Packet2d& a) {
4967 return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_f64(a)));
4971EIGEN_STRONG_INLINE Packet2d pcmp_le(
const Packet2d& a,
const Packet2d& b) {
4972 return vreinterpretq_f64_u64(vcleq_f64(a, b));
4976EIGEN_STRONG_INLINE Packet2d pcmp_lt(
const Packet2d& a,
const Packet2d& b) {
4977 return vreinterpretq_f64_u64(vcltq_f64(a, b));
4981EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(
const Packet2d& a,
const Packet2d& b) {
4982 return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_u64(vcgeq_f64(a, b))));
4986EIGEN_STRONG_INLINE Packet2d pcmp_eq(
const Packet2d& a,
const Packet2d& b) {
4987 return vreinterpretq_f64_u64(vceqq_f64(a, b));
4991EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from) {
4992 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f64(assume_aligned<unpacket_traits<Packet2d>::alignment>(from));
4996EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from) {
4997 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f64(from);
5001EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(
const double* from) {
5002 return vld1q_dup_f64(from);
5005EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from) {
5006 EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(assume_aligned<unpacket_traits<Packet2d>::alignment>(to), from);
5010EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from) {
5011 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to, from);
5015EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(
const double* from, Index stride) {
5016 Packet2d res = pset1<Packet2d>(0.0);
5017 res = vld1q_lane_f64(from + 0 * stride, res, 0);
5018 res = vld1q_lane_f64(from + 1 * stride, res, 1);
5023EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<double, Packet2d>(
double* to,
const Packet2d& from, Index stride) {
5024 vst1q_lane_f64(to + stride * 0, from, 0);
5025 vst1q_lane_f64(to + stride * 1, from, 1);
5029EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) {
5030 EIGEN_ARM_PREFETCH(addr);
5035EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
5036 return vgetq_lane_f64(a, 0);
5040EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a) {
5041 return vcombine_f64(vget_high_f64(a), vget_low_f64(a));
5045EIGEN_STRONG_INLINE Packet2d pabs(
const Packet2d& a) {
5046 return vabsq_f64(a);
5050EIGEN_STRONG_INLINE Packet2d psignbit(
const Packet2d& a) {
5051 return vreinterpretq_f64_s64(vshrq_n_s64(vreinterpretq_s64_f64(a), 63));
5055EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a) {
5056 return vaddvq_f64(a);
5060EIGEN_STRONG_INLINE Index predux_count(
const Packet2d& a) {
5062 const uint64_t zeroMaskSum = vaddvq_u64(vceqq_f64(a, vdupq_n_f64(0.0)));
5063 return static_cast<Index
>(zeroMaskSum + 2);
5067EIGEN_STRONG_INLINE
bool predux_any(
const Packet2d& a) {
5068 return vmaxvq_u32(vreinterpretq_u32_f64(a)) != 0;
5073#if EIGEN_COMP_CLANGAPPLE
5075EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a) {
5076 return (vget_low_f64(a) * vget_high_f64(a))[0];
5080EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a) {
5081 return vget_lane_f64(vmul_f64(vget_low_f64(a), vget_high_f64(a)), 0);
5087EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a) {
5088 return vminvq_f64(a);
5093EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a) {
5094 return vmaxvq_f64(a);
5097EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
5098 const float64x2_t tmp1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]);
5099 const float64x2_t tmp2 = vzip2q_f64(kernel.packet[0], kernel.packet[1]);
5101 kernel.packet[0] = tmp1;
5102 kernel.packet[1] = tmp2;
5106EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect(
const Packet2d& mask,
const Packet2d& a,
const Packet2d& b) {
5107 return vbslq_f64(vreinterpretq_u64_f64(mask), a, b);
5111EIGEN_STRONG_INLINE Packet2d print<Packet2d>(
const Packet2d& a) {
5112 return vrndnq_f64(a);
5116EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(
const Packet2d& a) {
5117 return vrndmq_f64(a);
5121EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(
const Packet2d& a) {
5122 return vrndpq_f64(a);
5126EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(
const Packet2d& a) {
5127 return vrndaq_f64(a);
5131EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(
const Packet2d& a) {
5132 return vrndq_f64(a);
5136EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(
const Packet2d& a,
const Packet2d& exponent) {
5137 return pldexp_generic(a, exponent);
5141EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(
const Packet2d& a, Packet2d& exponent) {
5142 return pfrexp_generic(a, exponent);
5146EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
5147 return vreinterpretq_f64_u64(vdupq_n_u64(from));
5151EIGEN_STRONG_INLINE Packet2d prsqrt(
const Packet2d& a) {
5153 return generic_rsqrt_newton_step<Packet2d, 3>::run(a, vrsqrteq_f64(a));
5157EIGEN_STRONG_INLINE Packet2d psqrt(
const Packet2d& _x) {
5158 return vsqrtq_f64(_x);
5163EIGEN_STRONG_INLINE Packet2d pisinf<Packet2d>(
const Packet2d& a) {
5164 return vreinterpretq_f64_u64(vcageq_f64(a, pinf<Packet2d>()));
5167EIGEN_STRONG_INLINE Packet2d pisfinite<Packet2d>(
const Packet2d& a) {
5168 return vreinterpretq_f64_u64(vcagtq_f64(pinf<Packet2d>(), a));
5173EIGEN_STRONG_INLINE Packet2d pisnan<Packet2d>(
const Packet2d& a) {
5174 return vreinterpretq_f64_u64(vcgtq_u64(vreinterpretq_u64_f64(vabsq_f64(a)), vdupq_n_u64(0x7ff0000000000000ull)));
5184#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
5185using Packet4hf = float16x4_t;
5186using Packet8hf = float16x8_t;
5192#if EIGEN_CLANG_STRICT_LESS_THAN(19, 1, 0) && !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
5193EIGEN_ALWAYS_INLINE float16x4_t vbsl_f16(uint16x4_t a, float16x4_t b, float16x4_t c) {
5194 return vreinterpret_f16_u16(vbsl_u16(a, vreinterpret_u16_f16(b), vreinterpret_u16_f16(c)));
5196EIGEN_ALWAYS_INLINE float16x8_t vbslq_f16(uint16x8_t a, float16x8_t b, float16x8_t c) {
5197 return vreinterpretq_f16_u16(vbslq_u16(a, vreinterpretq_u16_f16(b), vreinterpretq_u16_f16(c)));
5199EIGEN_ALWAYS_INLINE float16x4_t vrev64_f16(float16x4_t a) {
5200 return vreinterpret_f16_u16(vrev64_u16(vreinterpret_u16_f16(a)));
5202EIGEN_ALWAYS_INLINE float16x8_t vrev64q_f16(float16x8_t a) {
5203 return vreinterpretq_f16_u16(vrev64q_u16(vreinterpretq_u16_f16(a)));
5205EIGEN_ALWAYS_INLINE float16x8x2_t vzipq_f16(float16x8_t a, float16x8_t b) {
5206 const uint16x8x2_t r = vzipq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b));
5207 return {vreinterpretq_f16_u16(r.val[0]), vreinterpretq_f16_u16(r.val[1])};
5209EIGEN_ALWAYS_INLINE float16x8x2_t vuzpq_f16(float16x8_t a, float16x8_t b) {
5210 const uint16x8x2_t r = vuzpq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b));
5211 return {vreinterpretq_f16_u16(r.val[0]), vreinterpretq_f16_u16(r.val[1])};
5216struct packet_traits<half> : default_packet_traits {
5217 using type = Packet8hf;
5218 using half = Packet4hf;
5221 AlignedOnScalar = 1,
5245 HasTanh = packet_traits<float>::HasTanh,
5248 HasErf = EIGEN_FAST_MATH,
5255struct unpacket_traits<Packet4hf> : neon_unpacket_default<Packet4hf, half> {};
5257struct unpacket_traits<Packet8hf> : neon_unpacket_default<Packet8hf, half> {
5258 using half = Packet4hf;
5262struct pminmax_propagates_nan<Packet4hf> : bool_constant<true> {};
5264struct pminmax_propagates_nan<Packet8hf> : bool_constant<true> {};
5267EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pset1(
const half& from) {
5268 return vdupq_n_f16(half_impl::native_half(from));
5271EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pset1(
const half& from) {
5272 return vdup_n_f16(half_impl::native_half(from));
5275#define EIGEN_MAKE_HALF_BITWISE_BINOP(name, op) \
5277 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a, const Packet8hf& b) { \
5278 return vreinterpretq_f16_u16(v##op##q_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b))); \
5281 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a, const Packet4hf& b) { \
5282 return vreinterpret_f16_u16(v##op##_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b))); \
5284 static_assert(true, "Trailing semicolon required")
5286EIGEN_MAKE_HALF_BITWISE_BINOP(pand, and);
5287EIGEN_MAKE_HALF_BITWISE_BINOP(por, orr);
5288EIGEN_MAKE_HALF_BITWISE_BINOP(pxor, eor);
5289EIGEN_MAKE_HALF_BITWISE_BINOP(pandnot, bic);
5291#undef EIGEN_MAKE_HALF_BITWISE_BINOP
5294EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pload(
const half* from) {
5295 EIGEN_DEBUG_ALIGNED_LOAD
return vld1q_f16(
5296 assume_aligned<unpacket_traits<Packet8hf>::alignment>(
reinterpret_cast<const float16_t*
>(from)));
5299EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pload(
const half* from) {
5300 EIGEN_DEBUG_ALIGNED_LOAD
return vld1_f16(
5301 assume_aligned<unpacket_traits<Packet4hf>::alignment>(
reinterpret_cast<const float16_t*
>(from)));
5305EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf ploadu(
const half* from) {
5306 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1q_f16(
reinterpret_cast<const float16_t*
>(from));
5309EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf ploadu(
const half* from) {
5310 EIGEN_DEBUG_UNALIGNED_LOAD
return vld1_f16(
reinterpret_cast<const float16_t*
>(from));
5314EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf ploaddup(
const half* from) {
5316 packet = vsetq_lane_f16(half_impl::native_half(from[0]), packet, 0);
5317 packet = vsetq_lane_f16(half_impl::native_half(from[0]), packet, 1);
5318 packet = vsetq_lane_f16(half_impl::native_half(from[1]), packet, 2);
5319 packet = vsetq_lane_f16(half_impl::native_half(from[1]), packet, 3);
5320 packet = vsetq_lane_f16(half_impl::native_half(from[2]), packet, 4);
5321 packet = vsetq_lane_f16(half_impl::native_half(from[2]), packet, 5);
5322 packet = vsetq_lane_f16(half_impl::native_half(from[3]), packet, 6);
5323 packet = vsetq_lane_f16(half_impl::native_half(from[3]), packet, 7);
5327EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf ploaddup(
const half* from) {
5328 float16x4_t packet{};
5329 packet = vset_lane_f16(half_impl::native_half(from[0]), packet, 0);
5330 packet = vset_lane_f16(half_impl::native_half(from[0]), packet, 1);
5331 packet = vset_lane_f16(half_impl::native_half(from[1]), packet, 2);
5332 packet = vset_lane_f16(half_impl::native_half(from[1]), packet, 3);
5337EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf ploadquad(
const half* from) {
5340 lo = vdup_n_f16(half_impl::native_half(from[0]));
5341 hi = vdup_n_f16(half_impl::native_half(from[1]));
5342 return vcombine_f16(lo, hi);
5345EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertfirst(
const Packet8hf& a, half b) {
5346 return vsetq_lane_f16(half_impl::native_half(b), a, 0);
5348EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertfirst(
const Packet4hf& a, half b) {
5349 return vset_lane_f16(half_impl::native_half(b), a, 0);
5353EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pselect(
const Packet8hf& mask,
const Packet8hf& a,
const Packet8hf& b) {
5354 return vbslq_f16(vreinterpretq_u16_f16(mask), a, b);
5357EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pselect(
const Packet4hf& mask,
const Packet4hf& a,
const Packet4hf& b) {
5358 return vbsl_f16(vreinterpret_u16_f16(mask), a, b);
5362EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pnot(
const Packet8hf& a) {
5363 return vreinterpretq_f16_u16(vmvnq_u16(vreinterpretq_u16_f16(a)));
5366EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pnot(
const Packet4hf& a) {
5367 return vreinterpret_f16_u16(vmvn_u16(vreinterpret_u16_f16(a)));
5370EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertlast(
const Packet8hf& a, half b) {
5371 return vsetq_lane_f16(half_impl::native_half(b), a, 7);
5373EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertlast(
const Packet4hf& a, half b) {
5374 return vset_lane_f16(half_impl::native_half(b), a, 3);
5378EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstore(half* to,
const Packet8hf& from) {
5379 EIGEN_DEBUG_ALIGNED_STORE vst1q_f16(
5380 assume_aligned<unpacket_traits<Packet8hf>::alignment>(
reinterpret_cast<float16_t*
>(to)), from);
5383EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstore(half* to,
const Packet4hf& from) {
5384 EIGEN_DEBUG_ALIGNED_STORE vst1_f16(
5385 assume_aligned<unpacket_traits<Packet4hf>::alignment>(
reinterpret_cast<float16_t*
>(to)), from);
5389EIGEN_STRONG_INLINE
void pstoreu(half* to,
const Packet8hf& from) {
5390 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f16(
reinterpret_cast<float16_t*
>(to), from);
5393EIGEN_STRONG_INLINE
void pstoreu(half* to,
const Packet4hf& from) {
5394 EIGEN_DEBUG_UNALIGNED_STORE vst1_f16(
reinterpret_cast<float16_t*
>(to), from);
5398EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pgather(
const half* from, Index stride) {
5399 Packet8hf res = pset1<Packet8hf>(half(0.f));
5400 res = vsetq_lane_f16(half_impl::native_half(from[0 * stride]), res, 0);
5401 res = vsetq_lane_f16(half_impl::native_half(from[1 * stride]), res, 1);
5402 res = vsetq_lane_f16(half_impl::native_half(from[2 * stride]), res, 2);
5403 res = vsetq_lane_f16(half_impl::native_half(from[3 * stride]), res, 3);
5404 res = vsetq_lane_f16(half_impl::native_half(from[4 * stride]), res, 4);
5405 res = vsetq_lane_f16(half_impl::native_half(from[5 * stride]), res, 5);
5406 res = vsetq_lane_f16(half_impl::native_half(from[6 * stride]), res, 6);
5407 res = vsetq_lane_f16(half_impl::native_half(from[7 * stride]), res, 7);
5411EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pgather(
const half* from, Index stride) {
5412 Packet4hf res = pset1<Packet4hf>(half(0.f));
5413 res = vset_lane_f16(half_impl::native_half(from[0 * stride]), res, 0);
5414 res = vset_lane_f16(half_impl::native_half(from[1 * stride]), res, 1);
5415 res = vset_lane_f16(half_impl::native_half(from[2 * stride]), res, 2);
5416 res = vset_lane_f16(half_impl::native_half(from[3 * stride]), res, 3);
5421EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter(half* to,
const Packet8hf& from, Index stride) {
5422 to[stride * 0] = half_impl::half_from_rep(vgetq_lane_f16(from, 0));
5423 to[stride * 1] = half_impl::half_from_rep(vgetq_lane_f16(from, 1));
5424 to[stride * 2] = half_impl::half_from_rep(vgetq_lane_f16(from, 2));
5425 to[stride * 3] = half_impl::half_from_rep(vgetq_lane_f16(from, 3));
5426 to[stride * 4] = half_impl::half_from_rep(vgetq_lane_f16(from, 4));
5427 to[stride * 5] = half_impl::half_from_rep(vgetq_lane_f16(from, 5));
5428 to[stride * 6] = half_impl::half_from_rep(vgetq_lane_f16(from, 6));
5429 to[stride * 7] = half_impl::half_from_rep(vgetq_lane_f16(from, 7));
5432EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter(half* to,
const Packet4hf& from, Index stride) {
5433 to[stride * 0] = half_impl::half_from_rep(vget_lane_f16(from, 0));
5434 to[stride * 1] = half_impl::half_from_rep(vget_lane_f16(from, 1));
5435 to[stride * 2] = half_impl::half_from_rep(vget_lane_f16(from, 2));
5436 to[stride * 3] = half_impl::half_from_rep(vget_lane_f16(from, 3));
5440EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void prefetch(
const half* addr) {
5441 EIGEN_ARM_PREFETCH(addr);
5445EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half pfirst(
const Packet8hf& a) {
5446 return half(vgetq_lane_f16(a, 0));
5449EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half pfirst(
const Packet4hf& a) {
5450 return half(vget_lane_f16(a, 0));
5454EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf preverse(
const Packet8hf& a) {
5455 float16x8_t r = vrev64q_f16(a);
5456 return vcombine_f16(vget_high_f16(r), vget_low_f16(r));
5459EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf preverse(
const Packet4hf& a) {
5460 return vrev64_f16(a);
5464EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf psignbit(
const Packet8hf& a) {
5465 return vreinterpretq_f16_s16(vshrq_n_s16(vreinterpretq_s16_f16(a), 15));
5468EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf psignbit(
const Packet4hf& a) {
5469 return vreinterpret_f16_s16(vshr_n_s16(vreinterpret_s16_f16(a), 15));
5473EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool predux_any(
const Packet8hf& a) {
5474 return vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(vreinterpretq_u16_f16(a))), 0);
5477EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool predux_any(
const Packet4hf& a) {
5478 return vget_lane_u64(vreinterpret_u64_f16(a), 0);
5481EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8hf, 4>& kernel) {
5482 const float16x8x2_t zip16_1 = vzipq_f16(kernel.packet[0], kernel.packet[1]);
5483 const float16x8x2_t zip16_2 = vzipq_f16(kernel.packet[2], kernel.packet[3]);
5485 const float32x4x2_t zip32_1 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[0]), vreinterpretq_f32_f16(zip16_2.val[0]));
5486 const float32x4x2_t zip32_2 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[1]), vreinterpretq_f32_f16(zip16_2.val[1]));
5488 kernel.packet[0] = vreinterpretq_f16_f32(zip32_1.val[0]);
5489 kernel.packet[1] = vreinterpretq_f16_f32(zip32_1.val[1]);
5490 kernel.packet[2] = vreinterpretq_f16_f32(zip32_2.val[0]);
5491 kernel.packet[3] = vreinterpretq_f16_f32(zip32_2.val[1]);
5493EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4hf, 4>& kernel) {
5494 EIGEN_ALIGN16 float16x4x4_t tmp_x4;
5495 float16_t* tmp =
reinterpret_cast<float16_t*
>(&kernel);
5496 tmp_x4 = vld4_f16(tmp);
5498 kernel.packet[0] = tmp_x4.val[0];
5499 kernel.packet[1] = tmp_x4.val[1];
5500 kernel.packet[2] = tmp_x4.val[2];
5501 kernel.packet[3] = tmp_x4.val[3];
5504EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8hf, 8>& kernel) {
5505 float16x8x2_t T_1[4];
5507 T_1[0] = vuzpq_f16(kernel.packet[0], kernel.packet[1]);
5508 T_1[1] = vuzpq_f16(kernel.packet[2], kernel.packet[3]);
5509 T_1[2] = vuzpq_f16(kernel.packet[4], kernel.packet[5]);
5510 T_1[3] = vuzpq_f16(kernel.packet[6], kernel.packet[7]);
5512 float16x8x2_t T_2[4];
5513 T_2[0] = vuzpq_f16(T_1[0].val[0], T_1[1].val[0]);
5514 T_2[1] = vuzpq_f16(T_1[0].val[1], T_1[1].val[1]);
5515 T_2[2] = vuzpq_f16(T_1[2].val[0], T_1[3].val[0]);
5516 T_2[3] = vuzpq_f16(T_1[2].val[1], T_1[3].val[1]);
5518 float16x8x2_t T_3[4];
5519 T_3[0] = vuzpq_f16(T_2[0].val[0], T_2[2].val[0]);
5520 T_3[1] = vuzpq_f16(T_2[0].val[1], T_2[2].val[1]);
5521 T_3[2] = vuzpq_f16(T_2[1].val[0], T_2[3].val[0]);
5522 T_3[3] = vuzpq_f16(T_2[1].val[1], T_2[3].val[1]);
5524 kernel.packet[0] = T_3[0].val[0];
5525 kernel.packet[1] = T_3[2].val[0];
5526 kernel.packet[2] = T_3[1].val[0];
5527 kernel.packet[3] = T_3[3].val[0];
5528 kernel.packet[4] = T_3[0].val[1];
5529 kernel.packet[5] = T_3[2].val[1];
5530 kernel.packet[6] = T_3[1].val[1];
5531 kernel.packet[7] = T_3[3].val[1];
5534#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
5537EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half<Packet8hf>(
const Packet8hf& a) {
5538 return vadd_f16(vget_low_f16(a), vget_high_f16(a));
5542EIGEN_STRONG_INLINE Packet8hf plset<Packet8hf>(
const Eigen::half& a) {
5543 const float16_t f[] = {0, 1, 2, 3, 4, 5, 6, 7};
5544 Packet8hf countdown = vld1q_f16(f);
5545 return vaddq_f16(pset1<Packet8hf>(a), countdown);
5549EIGEN_STRONG_INLINE Packet4hf plset<Packet4hf>(
const Eigen::half& a) {
5550 const float16_t f[] = {0, 1, 2, 3};
5551 Packet4hf countdown = vld1_f16(f);
5552 return vadd_f16(pset1<Packet4hf>(a), countdown);
5556EIGEN_STRONG_INLINE Packet8hf padd<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5557 return vaddq_f16(a, b);
5561EIGEN_STRONG_INLINE Packet4hf padd<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5562 return vadd_f16(a, b);
5566EIGEN_STRONG_INLINE Packet8hf psub<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5567 return vsubq_f16(a, b);
5571EIGEN_STRONG_INLINE Packet4hf psub<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5572 return vsub_f16(a, b);
5576EIGEN_STRONG_INLINE Packet8hf pnegate(
const Packet8hf& a) {
5577 return vnegq_f16(a);
5581EIGEN_STRONG_INLINE Packet4hf pnegate(
const Packet4hf& a) {
5586EIGEN_STRONG_INLINE Packet8hf pmul<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5587 return vmulq_f16(a, b);
5591EIGEN_STRONG_INLINE Packet4hf pmul<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5592 return vmul_f16(a, b);
5596EIGEN_STRONG_INLINE Packet8hf pdiv<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5597 return vdivq_f16(a, b);
5601EIGEN_STRONG_INLINE Packet4hf pdiv<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5602 return vdiv_f16(a, b);
5606struct is_fused_madd<Packet8hf> : true_type {};
5608EIGEN_STRONG_INLINE Packet8hf pmadd(
const Packet8hf& a,
const Packet8hf& b,
const Packet8hf& c) {
5609 return vfmaq_f16(c, a, b);
5613struct is_fused_madd<Packet4hf> : true_type {};
5615EIGEN_STRONG_INLINE Packet4hf pmadd(
const Packet4hf& a,
const Packet4hf& b,
const Packet4hf& c) {
5616 return vfma_f16(c, a, b);
5620EIGEN_STRONG_INLINE Packet8hf pnmadd(
const Packet8hf& a,
const Packet8hf& b,
const Packet8hf& c) {
5621 return vfmsq_f16(c, a, b);
5625EIGEN_STRONG_INLINE Packet4hf pnmadd(
const Packet4hf& a,
const Packet4hf& b,
const Packet4hf& c) {
5626 return vfms_f16(c, a, b);
5630EIGEN_STRONG_INLINE Packet8hf pmin<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5631 return vminq_f16(a, b);
5635EIGEN_STRONG_INLINE Packet4hf pmin<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5636 return vmin_f16(a, b);
5639#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
5643EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNumbers, Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5644 return vminnm_f16(a, b);
5647EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNumbers, Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5648 return vminnmq_f16(a, b);
5653EIGEN_STRONG_INLINE Packet8hf pmax<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5654 return vmaxq_f16(a, b);
5658EIGEN_STRONG_INLINE Packet4hf pmax<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5659 return vmax_f16(a, b);
5662#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
5666EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNumbers, Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5667 return vmaxnm_f16(a, b);
5670EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNumbers, Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5671 return vmaxnmq_f16(a, b);
5675#define EIGEN_MAKE_ARM_FP16_CMP_8(name) \
5677 EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \
5678 return vreinterpretq_f16_u16(vc##name##q_f16(a, b)); \
5681#define EIGEN_MAKE_ARM_FP16_CMP_4(name) \
5683 EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \
5684 return vreinterpret_f16_u16(vc##name##_f16(a, b)); \
5687EIGEN_MAKE_ARM_FP16_CMP_8(eq)
5688EIGEN_MAKE_ARM_FP16_CMP_8(lt)
5689EIGEN_MAKE_ARM_FP16_CMP_8(le)
5691EIGEN_MAKE_ARM_FP16_CMP_4(eq)
5692EIGEN_MAKE_ARM_FP16_CMP_4(lt)
5693EIGEN_MAKE_ARM_FP16_CMP_4(le)
5695#undef EIGEN_MAKE_ARM_FP16_CMP_8
5696#undef EIGEN_MAKE_ARM_FP16_CMP_4
5699EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan<Packet8hf>(
const Packet8hf& a,
const Packet8hf& b) {
5700 return vreinterpretq_f16_u16(vmvnq_u16(vcgeq_f16(a, b)));
5704EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan<Packet4hf>(
const Packet4hf& a,
const Packet4hf& b) {
5705 return vreinterpret_f16_u16(vmvn_u16(vcge_f16(a, b)));
5710EIGEN_STRONG_INLINE Packet4hf pisinf<Packet4hf>(
const Packet4hf& a) {
5711 return vreinterpret_f16_u16(vcage_f16(a, pinf<Packet4hf>()));
5714EIGEN_STRONG_INLINE Packet8hf pisinf<Packet8hf>(
const Packet8hf& a) {
5715 return vreinterpretq_f16_u16(vcageq_f16(a, pinf<Packet8hf>()));
5718EIGEN_STRONG_INLINE Packet4hf pisfinite<Packet4hf>(
const Packet4hf& a) {
5719 return vreinterpret_f16_u16(vcagt_f16(pinf<Packet4hf>(), a));
5722EIGEN_STRONG_INLINE Packet8hf pisfinite<Packet8hf>(
const Packet8hf& a) {
5723 return vreinterpretq_f16_u16(vcagtq_f16(pinf<Packet8hf>(), a));
5726EIGEN_STRONG_INLINE Packet4hf pisnan<Packet4hf>(
const Packet4hf& a) {
5727 return vreinterpret_f16_u16(vmvn_u16(vcage_f16(pinf<Packet4hf>(), a)));
5730EIGEN_STRONG_INLINE Packet8hf pisnan<Packet8hf>(
const Packet8hf& a) {
5731 return vreinterpretq_f16_u16(vmvnq_u16(vcageq_f16(pinf<Packet8hf>(), a)));
5735EIGEN_STRONG_INLINE Packet8hf print<Packet8hf>(
const Packet8hf& a) {
5736 return vrndnq_f16(a);
5740EIGEN_STRONG_INLINE Packet4hf print<Packet4hf>(
const Packet4hf& a) {
5741 return vrndn_f16(a);
5745EIGEN_STRONG_INLINE Packet8hf pfloor<Packet8hf>(
const Packet8hf& a) {
5746 return vrndmq_f16(a);
5750EIGEN_STRONG_INLINE Packet4hf pfloor<Packet4hf>(
const Packet4hf& a) {
5751 return vrndm_f16(a);
5755EIGEN_STRONG_INLINE Packet8hf pceil<Packet8hf>(
const Packet8hf& a) {
5756 return vrndpq_f16(a);
5760EIGEN_STRONG_INLINE Packet4hf pceil<Packet4hf>(
const Packet4hf& a) {
5761 return vrndp_f16(a);
5765EIGEN_STRONG_INLINE Packet8hf pround<Packet8hf>(
const Packet8hf& a) {
5766 return vrndaq_f16(a);
5770EIGEN_STRONG_INLINE Packet4hf pround<Packet4hf>(
const Packet4hf& a) {
5771 return vrnda_f16(a);
5775EIGEN_STRONG_INLINE Packet8hf ptrunc<Packet8hf>(
const Packet8hf& a) {
5776 return vrndq_f16(a);
5780EIGEN_STRONG_INLINE Packet4hf ptrunc<Packet4hf>(
const Packet4hf& a) {
5785EIGEN_STRONG_INLINE Packet8hf psqrt<Packet8hf>(
const Packet8hf& a) {
5786 return vsqrtq_f16(a);
5790EIGEN_STRONG_INLINE Packet4hf psqrt<Packet4hf>(
const Packet4hf& a) {
5791 return vsqrt_f16(a);
5795EIGEN_STRONG_INLINE Packet8hf pabs<Packet8hf>(
const Packet8hf& a) {
5796 return vabsq_f16(a);
5800EIGEN_STRONG_INLINE Packet4hf pabs<Packet4hf>(
const Packet4hf& a) {
5805EIGEN_STRONG_INLINE Eigen::half predux<Packet8hf>(
const Packet8hf& a) {
5806 float16x4_t a_lo, a_hi, sum;
5808 a_lo = vget_low_f16(a);
5809 a_hi = vget_high_f16(a);
5810 sum = vpadd_f16(a_lo, a_hi);
5811 sum = vpadd_f16(sum, sum);
5812 sum = vpadd_f16(sum, sum);
5814 return half_impl::half_from_rep(vget_lane_f16(sum, 0));
5818EIGEN_STRONG_INLINE Eigen::half predux<Packet4hf>(
const Packet4hf& a) {
5821 sum = vpadd_f16(a, a);
5822 sum = vpadd_f16(sum, sum);
5823 return half_impl::half_from_rep(vget_lane_f16(sum, 0));
5827EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8hf>(
const Packet8hf& a) {
5828 float16x4_t a_lo, a_hi, prod;
5830 a_lo = vget_low_f16(a);
5831 a_hi = vget_high_f16(a);
5832 prod = vmul_f16(a_lo, a_hi);
5833 prod = vmul_f16(prod, vrev64_f16(prod));
5835 return half_impl::half_from_rep(vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1)));
5839EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet4hf>(
const Packet4hf& a) {
5841 prod = vmul_f16(a, vrev64_f16(a));
5842 return half_impl::half_from_rep(vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1)));
5846EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8hf>(
const Packet8hf& a) {
5847 return half_impl::half_from_rep(vminvq_f16(a));
5851EIGEN_STRONG_INLINE Eigen::half predux_min<Packet4hf>(
const Packet4hf& a) {
5852 return half_impl::half_from_rep(vminv_f16(a));
5856EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8hf>(
const Packet8hf& a) {
5857 return half_impl::half_from_rep(vmaxvq_f16(a));
5861EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4hf>(
const Packet4hf& a) {
5862 return half_impl::half_from_rep(vmaxv_f16(a));
5879EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half(
const Packet8hf& a) {
5880 return vcvt_f16_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(a)), vcvt_f32_f16(vget_high_f16(a))));
5884EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf plset(
const half& a) {
5885 const float f[] = {0, 1, 2, 3};
5886 float32x4_t countdown = vld1q_f32(f);
5887 float32x4_t base = vaddq_f32(vcvt_f32_f16(pset1<Packet4hf>(a)), countdown);
5888 return vcombine_f16(vcvt_f16_f32(base), vcvt_f16_f32(vaddq_f32(base, vdupq_n_f32(4))));
5891EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf plset(
const half& a) {
5892 const float f[] = {0, 1, 2, 3};
5893 Packet4f countdown = vld1q_f32(f);
5894 return vcvt_f16_f32(vaddq_f32(pset1<Packet4f>(a), countdown));
5897#define EIGEN_MAKE_HALF_UNOP(name, op) \
5899 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a) { \
5900 return vcvt_f16_f32(v##op##q_f32(vcvt_f32_f16(a))); \
5903 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a) { \
5904 return vcombine_f16(name(vget_low_f16(a)), name(vget_high_f16(a))); \
5906 static_assert(true, "Trailing semicolon required")
5907#define EIGEN_MAKE_HALF_BINOP(name, op) \
5909 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a, const Packet4hf& b) { \
5910 return vcvt_f16_f32(v##op##q_f32(vcvt_f32_f16(a), vcvt_f32_f16(b))); \
5913 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a, const Packet8hf& b) { \
5914 return vcombine_f16(name(vget_low_f16(a), vget_low_f16(b)), name(vget_high_f16(a), vget_high_f16(b))); \
5916 static_assert(true, "Trailing semicolon required")
5918EIGEN_MAKE_HALF_BINOP(padd, add);
5919EIGEN_MAKE_HALF_BINOP(psub, sub);
5922EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pnegate(
const Packet8hf& a) {
5923 return vreinterpretq_f16_u16(veorq_u16(vreinterpretq_u16_f16(a), vdupq_n_u16(0x8000)));
5926EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pnegate(
const Packet4hf& a) {
5927 return vreinterpret_f16_u16(veor_u16(vreinterpret_u16_f16(a), vdup_n_u16(0x8000)));
5930EIGEN_MAKE_HALF_BINOP(pmul, mul);
5933EIGEN_MAKE_HALF_BINOP(pdiv, div);
5936EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pdiv(
const Packet4hf& a,
const Packet4hf& b) {
5937 return vcvt_f16_f32(pdiv(vcvt_f32_f16(a), vcvt_f32_f16(b)));
5940EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pdiv(
const Packet8hf& a,
const Packet8hf& b) {
5941 return vcombine_f16(pdiv(vget_low_f16(a), vget_low_f16(b)), pdiv(vget_high_f16(a), vget_high_f16(b)));
5945#define EIGEN_MAKE_HALF_FMA(name, op) \
5947 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) { \
5948 return vcvt_f16_f32(v##op##q_f32(vcvt_f32_f16(c), vcvt_f32_f16(a), vcvt_f32_f16(b))); \
5951 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) { \
5952 return vcombine_f16(name(vget_low_f16(a), vget_low_f16(b), vget_low_f16(c)), \
5953 name(vget_high_f16(a), vget_high_f16(b), vget_high_f16(c))); \
5955 static_assert(true, "Trailing semicolon required")
5957EIGEN_MAKE_HALF_FMA(pmadd, fma);
5958EIGEN_MAKE_HALF_FMA(pnmadd, fms);
5960#undef EIGEN_MAKE_HALF_FMA
5962EIGEN_MAKE_HALF_BINOP(pmin, min);
5963EIGEN_MAKE_HALF_BINOP(pmax, max);
5965#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
5967#define EIGEN_MAKE_HALF_NUMERIC_MAXMIN(name) \
5969 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf p##name<PropagateNumbers, Packet4hf>(const Packet4hf& a, \
5970 const Packet4hf& b) { \
5971 return vcvt_f16_f32(v##name##nmq_f32(vcvt_f32_f16(a), vcvt_f32_f16(b))); \
5974 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf p##name<PropagateNumbers, Packet8hf>(const Packet8hf& a, \
5975 const Packet8hf& b) { \
5976 return vcombine_f16(p##name<PropagateNumbers, Packet4hf>(vget_low_f16(a), vget_low_f16(b)), \
5977 p##name<PropagateNumbers, Packet4hf>(vget_high_f16(a), vget_high_f16(b))); \
5979 static_assert(true, "Trailing semicolon required")
5981EIGEN_MAKE_HALF_NUMERIC_MAXMIN(min);
5982EIGEN_MAKE_HALF_NUMERIC_MAXMIN(max);
5984#undef EIGEN_MAKE_HALF_NUMERIC_MAXMIN
5988#define EIGEN_MAKE_HALF_CMP(name) \
5990 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \
5991 return vreinterpret_f16_u16(vmovn_u32(vc##name##q_f32(vcvt_f32_f16(a), vcvt_f32_f16(b)))); \
5993 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \
5994 return vcombine_f16(pcmp_##name(vget_low_f16(a), vget_low_f16(b)), \
5995 pcmp_##name(vget_high_f16(a), vget_high_f16(b))); \
5997 static_assert(true, "Trailing semicolon required")
5999EIGEN_MAKE_HALF_CMP(eq);
6000EIGEN_MAKE_HALF_CMP(lt);
6001EIGEN_MAKE_HALF_CMP(le);
6003#undef EIGEN_MAKE_HALF_CMP
6006EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan(
const Packet4hf& a,
const Packet4hf& b) {
6007 return vreinterpret_f16_u16(vmovn_u32(vmvnq_u32(vcgeq_f32(vcvt_f32_f16(a), vcvt_f32_f16(b)))));
6010EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan(
const Packet8hf& a,
const Packet8hf& b) {
6011 return vcombine_f16(pcmp_lt_or_nan(vget_low_f16(a), vget_low_f16(b)),
6012 pcmp_lt_or_nan(vget_high_f16(a), vget_high_f16(b)));
6015EIGEN_MAKE_HALF_UNOP(print, rndn);
6016EIGEN_MAKE_HALF_UNOP(pfloor, rndm);
6017EIGEN_MAKE_HALF_UNOP(pceil, rndp);
6018EIGEN_MAKE_HALF_UNOP(pround, rnda);
6019EIGEN_MAKE_HALF_UNOP(ptrunc, rnd);
6022EIGEN_MAKE_HALF_UNOP(psqrt, sqrt);
6025EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf psqrt(
const Packet4hf& a) {
6026 return vcvt_f16_f32(psqrt(vcvt_f32_f16(a)));
6029EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf psqrt(
const Packet8hf& a) {
6030 return vcombine_f16(psqrt(vget_low_f16(a)), psqrt(vget_high_f16(a)));
6035EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pabs(
const Packet8hf& a) {
6036 return vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(a), vdupq_n_u16(0x8000)));
6039EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pabs(
const Packet4hf& a) {
6040 return vreinterpret_f16_u16(vbic_u16(vreinterpret_u16_f16(a), vdup_n_u16(0x8000)));
6045#define EIGEN_HALF_FOLD_REDUX(name, op) \
6047 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet4hf& a) { \
6048 return half(name(vcvt_f32_f16(a))); \
6051 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet8hf& a) { \
6052 return half(name(p##op(vcvt_f32_f16(vget_low_f16(a)), vcvt_f32_f16(vget_high_f16(a))))); \
6054 static_assert(true, "Trailing semicolon required")
6057#define EIGEN_HALF_HORIZONTAL_REDUX(name, op) \
6059 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet8hf& a) { \
6060 return half(v##op##vq_f32(v##op##q_f32(vcvt_f32_f16(vget_low_f16(a)), vcvt_f32_f16(vget_high_f16(a))))); \
6063 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet4hf& a) { \
6064 return half(v##op##vq_f32(vcvt_f32_f16(a))); \
6066 static_assert(true, "Trailing semicolon required")
6068#define EIGEN_HALF_HORIZONTAL_REDUX(name, op) EIGEN_HALF_FOLD_REDUX(name, op)
6071EIGEN_HALF_HORIZONTAL_REDUX(predux, add);
6072EIGEN_HALF_FOLD_REDUX(predux_mul, mul);
6073EIGEN_HALF_HORIZONTAL_REDUX(predux_min, min);
6074EIGEN_HALF_HORIZONTAL_REDUX(predux_max, max);
6076#undef EIGEN_HALF_FOLD_REDUX
6077#undef EIGEN_HALF_HORIZONTAL_REDUX
6079#undef EIGEN_MAKE_HALF_UNOP
6080#undef EIGEN_MAKE_HALF_BINOP
6084#if !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
6087EIGEN_STRONG_INLINE uint16x4_t neon_half_abs_bits(
const Packet4hf& a) {
6088 return vbic_u16(vreinterpret_u16_f16(a), vdup_n_u16(0x8000));
6090EIGEN_STRONG_INLINE uint16x8_t neon_half_abs_bits(
const Packet8hf& a) {
6091 return vbicq_u16(vreinterpretq_u16_f16(a), vdupq_n_u16(0x8000));
6094EIGEN_STRONG_INLINE Packet4hf pisinf<Packet4hf>(
const Packet4hf& a) {
6095 return vreinterpret_f16_u16(vceq_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00)));
6098EIGEN_STRONG_INLINE Packet8hf pisinf<Packet8hf>(
const Packet8hf& a) {
6099 return vreinterpretq_f16_u16(vceqq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00)));
6102EIGEN_STRONG_INLINE Packet4hf pisnan<Packet4hf>(
const Packet4hf& a) {
6103 return vreinterpret_f16_u16(vcgt_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00)));
6106EIGEN_STRONG_INLINE Packet8hf pisnan<Packet8hf>(
const Packet8hf& a) {
6107 return vreinterpretq_f16_u16(vcgtq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00)));
6110EIGEN_STRONG_INLINE Packet4hf pisfinite<Packet4hf>(
const Packet4hf& a) {
6111 return vreinterpret_f16_u16(vclt_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00)));
6114EIGEN_STRONG_INLINE Packet8hf pisfinite<Packet8hf>(
const Packet8hf& a) {
6115 return vreinterpretq_f16_u16(vcltq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00)));
6125EIGEN_STRONG_INLINE Packet4ui pudiv_magic<Packet4ui>(
const Packet4ui& a, uint32_t magic, uint32_t addend,
int shift) {
6126 uint32x2_t m = vdup_n_u32(magic);
6127 uint64x2_t c = vdupq_n_u64(addend);
6128 uint64x2_t lo = vmlal_u32(c, vget_low_u32(a), m);
6129 uint64x2_t hi = vmlal_u32(c, vget_high_u32(a), m);
6131 uint32x4_t t = vuzp2q_u32(vreinterpretq_u32_u64(lo), vreinterpretq_u32_u64(hi));
6133 uint32x4_t t = vcombine_u32(vshrn_n_u64(lo, 32), vshrn_n_u64(hi, 32));
6135 return vshlq_u32(t, vdupq_n_s32(-shift));
6138EIGEN_STRONG_INLINE Packet2ui pudiv_magic<Packet2ui>(
const Packet2ui& a, uint32_t magic, uint32_t addend,
int shift) {
6139 uint32x2_t t = vshrn_n_u64(vmlal_u32(vdupq_n_u64(addend), a, vdup_n_u32(magic)), 32);
6140 return vshl_u32(t, vdup_n_s32(-shift));
6144EIGEN_STRONG_INLINE Packet4i psdiv_signed_magic<Packet4i>(
const Packet4i& a, uint32_t magic,
int shift,
6145 int ,
const Packet4i& divisor_sign) {
6146 int32x2_t m = vdup_n_s32(
static_cast<int32_t
>(magic));
6147 int64x2_t lo = vmull_s32(vget_low_s32(a), m);
6148 int64x2_t hi = vmull_s32(vget_high_s32(a), m);
6150 int32x4_t t = vuzp2q_s32(vreinterpretq_s32_s64(lo), vreinterpretq_s32_s64(hi));
6152 int32x4_t t = vcombine_s32(vshrn_n_s64(lo, 32), vshrn_n_s64(hi, 32));
6154 int32x4_t q = vsubq_s32(vshlq_s32(vaddq_s32(a, t), vdupq_n_s32(-shift)), vshrq_n_s32(a, 31));
6155 return vsubq_s32(veorq_s32(q, divisor_sign), divisor_sign);
6158EIGEN_STRONG_INLINE Packet2i psdiv_signed_magic<Packet2i>(
const Packet2i& a, uint32_t magic,
int shift,
6159 int ,
const Packet2i& divisor_sign) {
6160 int32x2_t t = vshrn_n_s64(vmull_s32(a, vdup_n_s32(
static_cast<int32_t
>(magic))), 32);
6161 int32x2_t q = vsub_s32(vshl_s32(vadd_s32(a, t), vdup_n_s32(-shift)), vshr_n_s32(a, 31));
6162 return vsub_s32(veor_s32(q, divisor_sign), divisor_sign);