Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2010 Konstantinos Margaritis <markos@freevec.org>
6// Heavily based on Gael's SSE version.
7//
8// This Source Code Form is subject to the terms of the Mozilla
9// Public License v. 2.0. If a copy of the MPL was not distributed
10// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11// SPDX-License-Identifier: MPL-2.0
12
13#ifndef EIGEN_PACKET_MATH_NEON_H
14#define EIGEN_PACKET_MATH_NEON_H
15
16// IWYU pragma: private
17#include "../../InternalHeaderCheck.h"
18
19namespace Eigen {
20
21namespace internal {
22
23#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
24#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
25#endif
26
27#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
28#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
29#endif
30
31#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
32#if EIGEN_ARCH_ARM64
33#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
34#else
35#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16
36#endif
37#endif
38
39#if EIGEN_COMP_MSVC_STRICT
40
41// In MSVC's arm_neon.h header file, all NEON vector types
42// are aliases to the same underlying type __n128.
43// We thus have to wrap them to make them different C++ types.
44// (See also bug 1428)
45using Packet2f = eigen_packet_wrapper<float32x2_t, 0>;
46using Packet4f = eigen_packet_wrapper<float32x4_t, 1>;
47using Packet4c = eigen_packet_wrapper<int32_t, 2>;
48using Packet8c = eigen_packet_wrapper<int8x8_t, 3>;
49using Packet16c = eigen_packet_wrapper<int8x16_t, 4>;
50using Packet4uc = eigen_packet_wrapper<uint32_t, 5>;
51using Packet8uc = eigen_packet_wrapper<uint8x8_t, 6>;
52using Packet16uc = eigen_packet_wrapper<uint8x16_t, 7>;
53using Packet4s = eigen_packet_wrapper<int16x4_t, 8>;
54using Packet8s = eigen_packet_wrapper<int16x8_t, 9>;
55using Packet4us = eigen_packet_wrapper<uint16x4_t, 10>;
56using Packet8us = eigen_packet_wrapper<uint16x8_t, 11>;
57using Packet2i = eigen_packet_wrapper<int32x2_t, 12>;
58using Packet4i = eigen_packet_wrapper<int32x4_t, 13>;
59using Packet2ui = eigen_packet_wrapper<uint32x2_t, 14>;
60using Packet4ui = eigen_packet_wrapper<uint32x4_t, 15>;
61using Packet2l = eigen_packet_wrapper<int64x2_t, 16>;
62using Packet2ul = eigen_packet_wrapper<uint64x2_t, 17>;
63
64EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) {
65 float from[4] = {a, b, c, d};
66 return vld1q_f32(from);
67}
68
69EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) {
70 float from[2] = {a, b};
71 return vld1_f32(from);
72}
73
74#else
75
76using Packet2f = float32x2_t;
77using Packet4f = float32x4_t;
78using Packet4c = eigen_packet_wrapper<int32_t, 2>;
79using Packet8c = int8x8_t;
80using Packet16c = int8x16_t;
81using Packet4uc = eigen_packet_wrapper<uint32_t, 5>;
82using Packet8uc = uint8x8_t;
83using Packet16uc = uint8x16_t;
84using Packet4s = int16x4_t;
85using Packet8s = int16x8_t;
86using Packet4us = uint16x4_t;
87using Packet8us = uint16x8_t;
88using Packet2i = int32x2_t;
89using Packet4i = int32x4_t;
90using Packet2ui = uint32x2_t;
91using Packet4ui = uint32x4_t;
92using Packet2l = int64x2_t;
93using Packet2ul = uint64x2_t;
94
95EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) { return Packet4f{a, b, c, d}; }
96EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) { return Packet2f{a, b}; }
97
98#endif // EIGEN_COMP_MSVC_STRICT
99
100EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask) {
101 const float* a = reinterpret_cast<const float*>(&m);
102 Packet4f res =
103 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3)));
104 return res;
105}
106
107// functionally equivalent to _mm_shuffle_ps in SSE when interleave
108// == false (i.e. shuffle<false>(m, n, mask) equals _mm_shuffle_ps(m, n, mask)),
109// interleave m and n when interleave == true. Currently used in LU/arch/InverseSize4.h
110// to enable a shared implementation for fast inversion of matrices of size 4.
111template <bool interleave>
112EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f& m, const Packet4f& n, int mask) {
113 const float* a = reinterpret_cast<const float*>(&m);
114 const float* b = reinterpret_cast<const float*>(&n);
115 Packet4f res =
116 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
117 return res;
118}
119
120template <>
121EIGEN_STRONG_INLINE Packet4f shuffle2<true>(const Packet4f& m, const Packet4f& n, int mask) {
122 const float* a = reinterpret_cast<const float*>(&m);
123 const float* b = reinterpret_cast<const float*>(&n);
124 Packet4f res =
125 make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
126 return res;
127}
128
129EIGEN_STRONG_INLINE static int eigen_neon_shuffle_mask(int p, int q, int r, int s) {
130 return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
131}
132
133EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s) {
134 return shuffle1(a, eigen_neon_shuffle_mask(p, q, r, s));
135}
136EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s) {
137 return shuffle2<false>(a, b, eigen_neon_shuffle_mask(p, q, r, s));
138}
139EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) {
140 return shuffle2<false>(a, b, eigen_neon_shuffle_mask(0, 1, 0, 1));
141}
142EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) {
143 return shuffle2<false>(b, a, eigen_neon_shuffle_mask(2, 3, 2, 3));
144}
145EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) {
146 return shuffle2<true>(a, b, eigen_neon_shuffle_mask(0, 0, 1, 1));
147}
148EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) {
149 return shuffle2<true>(a, b, eigen_neon_shuffle_mask(2, 2, 3, 3));
150}
151#define vec4f_duplane(a, p) Packet4f(vdupq_lane_f32(vget_low_f32(a), p))
152
153#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = pset1<Packet4f>(X)
154
155#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
156 const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X))
157
158#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = pset1<Packet4i>(X)
159
160#if EIGEN_ARCH_ARM64 && EIGEN_COMP_GNUC
161// __builtin_prefetch tends to do nothing on ARM64 compilers because the
162// prefetch instructions there are too detailed for __builtin_prefetch to map
163// meaningfully to them.
164#define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) :);
165#elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
166#define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR);
167#elif defined __pld
168#define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR)
169#elif EIGEN_ARCH_ARM
170#define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("pld [%[addr]]\n" ::[addr] "r"(ADDR) :);
171#else
172// by default no explicit prefetching
173#define EIGEN_ARM_PREFETCH(ADDR)
174#endif
175
176template <>
177struct packet_traits<float> : default_packet_traits {
178 using type = Packet4f;
179 using half = Packet2f;
180 enum {
181 Vectorizable = 1,
182 AlignedOnScalar = 1,
183 size = 4,
184
185 HasCmp = 1,
186 HasAdd = 1,
187 HasSub = 1,
188 HasShift = 1,
189 HasMul = 1,
190 HasNegate = 1,
191 HasAbs = 1,
192 HasArg = 0,
193 HasAbsDiff = 1,
194 HasMin = 1,
195 HasMax = 1,
196 HasConj = 1,
197 HasSetLinear = 1,
198 HasDiv = 1,
199 HasSin = EIGEN_FAST_MATH,
200 HasCos = EIGEN_FAST_MATH,
201 HasTan = EIGEN_FAST_MATH,
202 HasACos = 1,
203 HasASin = 1,
204 HasATan = 1,
205 HasATanh = 1,
206 HasSinh = 1,
207 HasCosh = 1,
208 HasASinh = 1,
209 HasACosh = 1,
210 HasLog = 1,
211 HasLog10 = 1,
212 HasExp = 1,
213 HasLog1p = 1,
214 HasExpm1 = 1,
215 HasPow = 1,
216 HasSqrt = 1,
217 HasRsqrt = 1,
218 HasCbrt = 1,
219 HasTanh = EIGEN_FAST_MATH,
220 HasErf = EIGEN_FAST_MATH,
221 HasErfc = EIGEN_FAST_MATH,
222 HasBessel = 0, // Issues with accuracy.
223 HasNdtri = 0
224 };
225};
226
227template <>
228struct packet_traits<int8_t> : default_packet_traits {
229 using type = Packet16c;
230 using half = Packet8c;
231 enum {
232 Vectorizable = 1,
233 AlignedOnScalar = 1,
234 size = 16,
235
236 HasCmp = 1,
237 HasAdd = 1,
238 HasSub = 1,
239 HasShift = 1,
240 HasMul = 1,
241 HasNegate = 1,
242 HasAbs = 1,
243 HasAbsDiff = 1,
244 HasArg = 0,
245 HasMin = 1,
246 HasMax = 1,
247 HasConj = 1,
248 HasSetLinear = 1,
249 };
250};
251
252template <>
253struct packet_traits<uint8_t> : default_packet_traits {
254 using type = Packet16uc;
255 using half = Packet8uc;
256 enum {
257 Vectorizable = 1,
258 AlignedOnScalar = 1,
259 size = 16,
260
261 HasCmp = 1,
262 HasAdd = 1,
263 HasSub = 1,
264 HasShift = 1,
265 HasMul = 1,
266 HasNegate = 0,
267 HasAbs = 1,
268 HasAbsDiff = 1,
269 HasArg = 0,
270 HasMin = 1,
271 HasMax = 1,
272 HasConj = 1,
273 HasSetLinear = 1,
274
275 HasSqrt = 1
276 };
277};
278
279template <>
280struct packet_traits<int16_t> : default_packet_traits {
281 using type = Packet8s;
282 using half = Packet4s;
283 enum {
284 Vectorizable = 1,
285 AlignedOnScalar = 1,
286 size = 8,
287
288 HasCmp = 1,
289 HasAdd = 1,
290 HasSub = 1,
291 HasShift = 1,
292 HasMul = 1,
293 HasNegate = 1,
294 HasAbs = 1,
295 HasAbsDiff = 1,
296 HasArg = 0,
297 HasMin = 1,
298 HasMax = 1,
299 HasConj = 1,
300 HasSetLinear = 1,
301 };
302};
303
304template <>
305struct packet_traits<uint16_t> : default_packet_traits {
306 using type = Packet8us;
307 using half = Packet4us;
308 enum {
309 Vectorizable = 1,
310 AlignedOnScalar = 1,
311 size = 8,
312
313 HasCmp = 1,
314 HasAdd = 1,
315 HasSub = 1,
316 HasShift = 1,
317 HasMul = 1,
318 HasNegate = 0,
319 HasAbs = 1,
320 HasAbsDiff = 1,
321 HasArg = 0,
322 HasMin = 1,
323 HasMax = 1,
324 HasConj = 1,
325 HasSetLinear = 1,
326 HasSqrt = 1
327 };
328};
329
330template <>
331struct packet_traits<int32_t> : default_packet_traits {
332 using type = Packet4i;
333 using half = Packet2i;
334 enum {
335 Vectorizable = 1,
336 AlignedOnScalar = 1,
337 size = 4,
338
339 HasCmp = 1,
340 HasAdd = 1,
341 HasSub = 1,
342 HasShift = 1,
343 HasMul = 1,
344 HasFastIntDiv = 1,
345 HasNegate = 1,
346 HasAbs = 1,
347 HasArg = 0,
348 HasAbsDiff = 1,
349 HasMin = 1,
350 HasMax = 1,
351 HasConj = 1,
352 HasSetLinear = 1,
353 };
354};
355
356template <>
357struct packet_traits<uint32_t> : default_packet_traits {
358 using type = Packet4ui;
359 using half = Packet2ui;
360 enum {
361 Vectorizable = 1,
362 AlignedOnScalar = 1,
363 size = 4,
364
365 HasCmp = 1,
366 HasAdd = 1,
367 HasSub = 1,
368 HasShift = 1,
369 HasMul = 1,
370 HasFastIntDiv = 1,
371 HasNegate = 0,
372 HasAbs = 1,
373 HasArg = 0,
374 HasAbsDiff = 1,
375 HasMin = 1,
376 HasMax = 1,
377 HasConj = 1,
378 HasSetLinear = 1,
379
380 HasSqrt = 1
381 };
382};
383
384template <>
385struct packet_traits<int64_t> : default_packet_traits {
386 using type = Packet2l;
387 using half = Packet2l;
388 enum {
389 Vectorizable = 1,
390 AlignedOnScalar = 1,
391 size = 2,
392
393 HasCmp = 1,
394 HasAdd = 1,
395 HasSub = 1,
396 HasShift = 1,
397 HasMul = 1,
398 HasNegate = 1,
399 HasAbs = 1,
400 HasArg = 0,
401 HasAbsDiff = 1,
402 HasMin = 1,
403 HasMax = 1,
404 HasConj = 1,
405 HasSetLinear = 1,
406 };
407};
408
409template <>
410struct packet_traits<uint64_t> : default_packet_traits {
411 using type = Packet2ul;
412 using half = Packet2ul;
413 enum {
414 Vectorizable = 1,
415 AlignedOnScalar = 1,
416 size = 2,
417
418 HasCmp = 1,
419 HasAdd = 1,
420 HasSub = 1,
421 HasShift = 1,
422 HasMul = 1,
423 HasNegate = 0,
424 HasAbs = 1,
425 HasArg = 0,
426 HasAbsDiff = 1,
427 HasMin = 1,
428 HasMax = 1,
429 HasConj = 1,
430 HasSetLinear = 1,
431 };
432};
433
434template <typename Packet, typename Scalar>
435struct neon_unpacket_default {
436 using type = Scalar;
437 using half = Packet;
438 static constexpr int size = sizeof(Packet) / sizeof(Scalar);
439 static constexpr int alignment = sizeof(Packet);
440 static constexpr bool vectorizable = true;
441 static constexpr bool masked_load_available = false;
442 static constexpr bool masked_store_available = false;
443};
444
445template <>
446struct unpacket_traits<Packet2f> : neon_unpacket_default<Packet2f, float> {
447 using integer_packet = Packet2i;
448};
449template <>
450struct unpacket_traits<Packet4f> : neon_unpacket_default<Packet4f, float> {
451 using half = Packet2f;
452 using integer_packet = Packet4i;
453};
454template <>
455struct unpacket_traits<Packet4c> : neon_unpacket_default<Packet4c, int8_t> {};
456template <>
457struct unpacket_traits<Packet8c> : neon_unpacket_default<Packet8c, int8_t> {
458 using half = Packet4c;
459};
460template <>
461struct unpacket_traits<Packet16c> : neon_unpacket_default<Packet16c, int8_t> {
462 using half = Packet8c;
463};
464template <>
465struct unpacket_traits<Packet4uc> : neon_unpacket_default<Packet4uc, uint8_t> {};
466template <>
467struct unpacket_traits<Packet8uc> : neon_unpacket_default<Packet8uc, uint8_t> {
468 using half = Packet4uc;
469};
470template <>
471struct unpacket_traits<Packet16uc> : neon_unpacket_default<Packet16uc, uint8_t> {
472 using half = Packet8uc;
473};
474template <>
475struct unpacket_traits<Packet4s> : neon_unpacket_default<Packet4s, int16_t> {};
476template <>
477struct unpacket_traits<Packet8s> : neon_unpacket_default<Packet8s, int16_t> {
478 using half = Packet4s;
479};
480template <>
481struct unpacket_traits<Packet4us> : neon_unpacket_default<Packet4us, uint16_t> {};
482template <>
483struct unpacket_traits<Packet8us> : neon_unpacket_default<Packet8us, uint16_t> {
484 using half = Packet4us;
485};
486template <>
487struct unpacket_traits<Packet2i> : neon_unpacket_default<Packet2i, int32_t> {};
488template <>
489struct unpacket_traits<Packet4i> : neon_unpacket_default<Packet4i, int32_t> {
490 using half = Packet2i;
491};
492template <>
493struct unpacket_traits<Packet2ui> : neon_unpacket_default<Packet2ui, uint32_t> {};
494template <>
495struct unpacket_traits<Packet4ui> : neon_unpacket_default<Packet4ui, uint32_t> {
496 using half = Packet2ui;
497};
498template <>
499struct unpacket_traits<Packet2l> : neon_unpacket_default<Packet2l, int64_t> {};
500template <>
501struct unpacket_traits<Packet2ul> : neon_unpacket_default<Packet2ul, uint64_t> {};
502
503template <>
504EIGEN_STRONG_INLINE Packet2f pzero(const Packet2f& /*a*/) {
505 return vdup_n_f32(0.0f);
506}
507
508template <>
509EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /*a*/) {
510 return vdupq_n_f32(0.0f);
511}
512
513template <>
514EIGEN_STRONG_INLINE Packet2f pset1<Packet2f>(const float& from) {
515 return vdup_n_f32(from);
516}
517template <>
518EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
519 return vdupq_n_f32(from);
520}
521template <>
522EIGEN_STRONG_INLINE Packet4c pset1<Packet4c>(const int8_t& from) {
523 return vget_lane_s32(vreinterpret_s32_s8(vdup_n_s8(from)), 0);
524}
525template <>
526EIGEN_STRONG_INLINE Packet8c pset1<Packet8c>(const int8_t& from) {
527 return vdup_n_s8(from);
528}
529template <>
530EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(const int8_t& from) {
531 return vdupq_n_s8(from);
532}
533template <>
534EIGEN_STRONG_INLINE Packet4uc pset1<Packet4uc>(const uint8_t& from) {
535 return vget_lane_u32(vreinterpret_u32_u8(vdup_n_u8(from)), 0);
536}
537template <>
538EIGEN_STRONG_INLINE Packet8uc pset1<Packet8uc>(const uint8_t& from) {
539 return vdup_n_u8(from);
540}
541template <>
542EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(const uint8_t& from) {
543 return vdupq_n_u8(from);
544}
545template <>
546EIGEN_STRONG_INLINE Packet4s pset1<Packet4s>(const int16_t& from) {
547 return vdup_n_s16(from);
548}
549template <>
550EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const int16_t& from) {
551 return vdupq_n_s16(from);
552}
553template <>
554EIGEN_STRONG_INLINE Packet4us pset1<Packet4us>(const uint16_t& from) {
555 return vdup_n_u16(from);
556}
557template <>
558EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(const uint16_t& from) {
559 return vdupq_n_u16(from);
560}
561template <>
562EIGEN_STRONG_INLINE Packet2i pset1<Packet2i>(const int32_t& from) {
563 return vdup_n_s32(from);
564}
565template <>
566EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) {
567 return vdupq_n_s32(from);
568}
569template <>
570EIGEN_STRONG_INLINE Packet2ui pset1<Packet2ui>(const uint32_t& from) {
571 return vdup_n_u32(from);
572}
573template <>
574EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(const uint32_t& from) {
575 return vdupq_n_u32(from);
576}
577template <>
578EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) {
579 return vdupq_n_s64(from);
580}
581template <>
582EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(const uint64_t& from) {
583 return vdupq_n_u64(from);
584}
585
586template <>
587EIGEN_STRONG_INLINE Packet2f pset1frombits<Packet2f>(uint32_t from) {
588 return vreinterpret_f32_u32(vdup_n_u32(from));
589}
590template <>
591EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
592 return vreinterpretq_f32_u32(vdupq_n_u32(from));
593}
594
595template <>
596EIGEN_STRONG_INLINE Packet2f plset<Packet2f>(const float& a) {
597 const float c[] = {0.0f, 1.0f};
598 return vadd_f32(pset1<Packet2f>(a), vld1_f32(c));
599}
600template <>
601EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
602 const float c[] = {0.0f, 1.0f, 2.0f, 3.0f};
603 return vaddq_f32(pset1<Packet4f>(a), vld1q_f32(c));
604}
605template <>
606EIGEN_STRONG_INLINE Packet4c plset<Packet4c>(const int8_t& a) {
607 return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_u32(vdup_n_u32(0x03020100)), vdup_n_s8(a))), 0);
608}
609template <>
610EIGEN_STRONG_INLINE Packet8c plset<Packet8c>(const int8_t& a) {
611 const int8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
612 return vadd_s8(pset1<Packet8c>(a), vld1_s8(c));
613}
614template <>
615EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(const int8_t& a) {
616 const int8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
617 return vaddq_s8(pset1<Packet16c>(a), vld1q_s8(c));
618}
619template <>
620EIGEN_STRONG_INLINE Packet4uc plset<Packet4uc>(const uint8_t& a) {
621 return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(0x03020100)), vdup_n_u8(a))), 0);
622}
623template <>
624EIGEN_STRONG_INLINE Packet8uc plset<Packet8uc>(const uint8_t& a) {
625 const uint8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
626 return vadd_u8(pset1<Packet8uc>(a), vld1_u8(c));
627}
628template <>
629EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(const uint8_t& a) {
630 const uint8_t c[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
631 return vaddq_u8(pset1<Packet16uc>(a), vld1q_u8(c));
632}
633template <>
634EIGEN_STRONG_INLINE Packet4s plset<Packet4s>(const int16_t& a) {
635 const int16_t c[] = {0, 1, 2, 3};
636 return vadd_s16(pset1<Packet4s>(a), vld1_s16(c));
637}
638template <>
639EIGEN_STRONG_INLINE Packet4us plset<Packet4us>(const uint16_t& a) {
640 const uint16_t c[] = {0, 1, 2, 3};
641 return vadd_u16(pset1<Packet4us>(a), vld1_u16(c));
642}
643template <>
644EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(const int16_t& a) {
645 const int16_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
646 return vaddq_s16(pset1<Packet8s>(a), vld1q_s16(c));
647}
648template <>
649EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(const uint16_t& a) {
650 const uint16_t c[] = {0, 1, 2, 3, 4, 5, 6, 7};
651 return vaddq_u16(pset1<Packet8us>(a), vld1q_u16(c));
652}
653template <>
654EIGEN_STRONG_INLINE Packet2i plset<Packet2i>(const int32_t& a) {
655 const int32_t c[] = {0, 1};
656 return vadd_s32(pset1<Packet2i>(a), vld1_s32(c));
657}
658template <>
659EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) {
660 const int32_t c[] = {0, 1, 2, 3};
661 return vaddq_s32(pset1<Packet4i>(a), vld1q_s32(c));
662}
663template <>
664EIGEN_STRONG_INLINE Packet2ui plset<Packet2ui>(const uint32_t& a) {
665 const uint32_t c[] = {0, 1};
666 return vadd_u32(pset1<Packet2ui>(a), vld1_u32(c));
667}
668template <>
669EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(const uint32_t& a) {
670 const uint32_t c[] = {0, 1, 2, 3};
671 return vaddq_u32(pset1<Packet4ui>(a), vld1q_u32(c));
672}
673template <>
674EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(const int64_t& a) {
675 const int64_t c[] = {0, 1};
676 return vaddq_s64(pset1<Packet2l>(a), vld1q_s64(c));
677}
678template <>
679EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(const uint64_t& a) {
680 const uint64_t c[] = {0, 1};
681 return vaddq_u64(pset1<Packet2ul>(a), vld1q_u64(c));
682}
683
684template <>
685EIGEN_STRONG_INLINE Packet2f padd<Packet2f>(const Packet2f& a, const Packet2f& b) {
686 return vadd_f32(a, b);
687}
688template <>
689EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
690 return vaddq_f32(a, b);
691}
692template <>
693EIGEN_STRONG_INLINE Packet4c padd<Packet4c>(const Packet4c& a, const Packet4c& b) {
694 return vget_lane_s32(
695 vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
696}
697template <>
698EIGEN_STRONG_INLINE Packet8c padd<Packet8c>(const Packet8c& a, const Packet8c& b) {
699 return vadd_s8(a, b);
700}
701template <>
702EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(const Packet16c& a, const Packet16c& b) {
703 return vaddq_s8(a, b);
704}
705template <>
706EIGEN_STRONG_INLINE Packet4uc padd<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
707 return vget_lane_u32(
708 vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
709}
710template <>
711EIGEN_STRONG_INLINE Packet8uc padd<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
712 return vadd_u8(a, b);
713}
714template <>
715EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
716 return vaddq_u8(a, b);
717}
718template <>
719EIGEN_STRONG_INLINE Packet4s padd<Packet4s>(const Packet4s& a, const Packet4s& b) {
720 return vadd_s16(a, b);
721}
722template <>
723EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(const Packet8s& a, const Packet8s& b) {
724 return vaddq_s16(a, b);
725}
726template <>
727EIGEN_STRONG_INLINE Packet4us padd<Packet4us>(const Packet4us& a, const Packet4us& b) {
728 return vadd_u16(a, b);
729}
730template <>
731EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(const Packet8us& a, const Packet8us& b) {
732 return vaddq_u16(a, b);
733}
734template <>
735EIGEN_STRONG_INLINE Packet2i padd<Packet2i>(const Packet2i& a, const Packet2i& b) {
736 return vadd_s32(a, b);
737}
738template <>
739EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
740 return vaddq_s32(a, b);
741}
742template <>
743EIGEN_STRONG_INLINE Packet2ui padd<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
744 return vadd_u32(a, b);
745}
746template <>
747EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
748 return vaddq_u32(a, b);
749}
750template <>
751EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) {
752 return vaddq_s64(a, b);
753}
754template <>
755EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
756 return vaddq_u64(a, b);
757}
758
759template <>
760EIGEN_STRONG_INLINE Packet2f psub<Packet2f>(const Packet2f& a, const Packet2f& b) {
761 return vsub_f32(a, b);
762}
763template <>
764EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
765 return vsubq_f32(a, b);
766}
767template <>
768EIGEN_STRONG_INLINE Packet4c psub<Packet4c>(const Packet4c& a, const Packet4c& b) {
769 return vget_lane_s32(
770 vreinterpret_s32_s8(vsub_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
771}
772template <>
773EIGEN_STRONG_INLINE Packet8c psub<Packet8c>(const Packet8c& a, const Packet8c& b) {
774 return vsub_s8(a, b);
775}
776template <>
777EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(const Packet16c& a, const Packet16c& b) {
778 return vsubq_s8(a, b);
779}
780template <>
781EIGEN_STRONG_INLINE Packet4uc psub<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
782 return vget_lane_u32(
783 vreinterpret_u32_u8(vsub_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
784}
785template <>
786EIGEN_STRONG_INLINE Packet8uc psub<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
787 return vsub_u8(a, b);
788}
789template <>
790EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
791 return vsubq_u8(a, b);
792}
793template <>
794EIGEN_STRONG_INLINE Packet4s psub<Packet4s>(const Packet4s& a, const Packet4s& b) {
795 return vsub_s16(a, b);
796}
797template <>
798EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(const Packet8s& a, const Packet8s& b) {
799 return vsubq_s16(a, b);
800}
801template <>
802EIGEN_STRONG_INLINE Packet4us psub<Packet4us>(const Packet4us& a, const Packet4us& b) {
803 return vsub_u16(a, b);
804}
805template <>
806EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(const Packet8us& a, const Packet8us& b) {
807 return vsubq_u16(a, b);
808}
809template <>
810EIGEN_STRONG_INLINE Packet2i psub<Packet2i>(const Packet2i& a, const Packet2i& b) {
811 return vsub_s32(a, b);
812}
813template <>
814EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
815 return vsubq_s32(a, b);
816}
817template <>
818EIGEN_STRONG_INLINE Packet2ui psub<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
819 return vsub_u32(a, b);
820}
821template <>
822EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
823 return vsubq_u32(a, b);
824}
825template <>
826EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) {
827 return vsubq_s64(a, b);
828}
829template <>
830EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
831 return vsubq_u64(a, b);
832}
833
834template <>
835EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(const Packet2f& a, const Packet2f& b);
836template <>
837EIGEN_STRONG_INLINE Packet2f paddsub<Packet2f>(const Packet2f& a, const Packet2f& b) {
838 Packet2f mask = make_packet2f(numext::bit_cast<float>(0x80000000u), 0.0f);
839 return padd(a, pxor(mask, b));
840}
841template <>
842EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b);
843template <>
844EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(const Packet4f& a, const Packet4f& b) {
845 Packet4f mask = make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
846 return padd(a, pxor(mask, b));
847}
848
849template <>
850EIGEN_STRONG_INLINE Packet2f pnegate(const Packet2f& a) {
851 return vneg_f32(a);
852}
853template <>
854EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
855 return vnegq_f32(a);
856}
857template <>
858EIGEN_STRONG_INLINE Packet4c pnegate(const Packet4c& a) {
859 return vget_lane_s32(vreinterpret_s32_s8(vneg_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
860}
861template <>
862EIGEN_STRONG_INLINE Packet8c pnegate(const Packet8c& a) {
863 return vneg_s8(a);
864}
865template <>
866EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) {
867 return vnegq_s8(a);
868}
869template <>
870EIGEN_STRONG_INLINE Packet4s pnegate(const Packet4s& a) {
871 return vneg_s16(a);
872}
873template <>
874EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
875 return vnegq_s16(a);
876}
877template <>
878EIGEN_STRONG_INLINE Packet2i pnegate(const Packet2i& a) {
879 return vneg_s32(a);
880}
881template <>
882EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
883 return vnegq_s32(a);
884}
885template <>
886EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) {
887#if EIGEN_ARCH_ARM64
888 return vnegq_s64(a);
889#elif !EIGEN_GNUC_STRICT_AT_LEAST(12, 0, 0)
890 return psub(pzero(a), a);
891#else
892 // NOTE: GCC>=12 refuses to emit `vsub.i64` for `0 - x`: <https://godbolt.org/z/bfaz9ao59>.
893 int64x2_t x = a;
894 int64x2_t z = vdupq_n_s64(0);
895 asm("vsub.i64 %q0, %q1, %q0" : "+w"(x) : "w"(z));
896 return x;
897#endif
898}
899
900template <>
901EIGEN_STRONG_INLINE Packet2f pmul<Packet2f>(const Packet2f& a, const Packet2f& b) {
902 return vmul_f32(a, b);
903}
904template <>
905EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
906 return vmulq_f32(a, b);
907}
908template <>
909EIGEN_STRONG_INLINE Packet4c pmul<Packet4c>(const Packet4c& a, const Packet4c& b) {
910 return vget_lane_s32(
911 vreinterpret_s32_s8(vmul_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
912}
913template <>
914EIGEN_STRONG_INLINE Packet8c pmul<Packet8c>(const Packet8c& a, const Packet8c& b) {
915 return vmul_s8(a, b);
916}
917template <>
918EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(const Packet16c& a, const Packet16c& b) {
919 return vmulq_s8(a, b);
920}
921template <>
922EIGEN_STRONG_INLINE Packet4uc pmul<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
923 return vget_lane_u32(
924 vreinterpret_u32_u8(vmul_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
925}
926template <>
927EIGEN_STRONG_INLINE Packet8uc pmul<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
928 return vmul_u8(a, b);
929}
930template <>
931EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
932 return vmulq_u8(a, b);
933}
934template <>
935EIGEN_STRONG_INLINE Packet4s pmul<Packet4s>(const Packet4s& a, const Packet4s& b) {
936 return vmul_s16(a, b);
937}
938template <>
939EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(const Packet8s& a, const Packet8s& b) {
940 return vmulq_s16(a, b);
941}
942template <>
943EIGEN_STRONG_INLINE Packet4us pmul<Packet4us>(const Packet4us& a, const Packet4us& b) {
944 return vmul_u16(a, b);
945}
946template <>
947EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(const Packet8us& a, const Packet8us& b) {
948 return vmulq_u16(a, b);
949}
950template <>
951EIGEN_STRONG_INLINE Packet2i pmul<Packet2i>(const Packet2i& a, const Packet2i& b) {
952 return vmul_s32(a, b);
953}
954template <>
955EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
956 return vmulq_s32(a, b);
957}
958template <>
959EIGEN_STRONG_INLINE Packet2ui pmul<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
960 return vmul_u32(a, b);
961}
962template <>
963EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
964 return vmulq_u32(a, b);
965}
966template <>
967EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
968 const uint32x2_t al = vmovn_u64(a);
969 const uint32x2_t bl = vmovn_u64(b);
970 const uint64x2_t hi = vpaddlq_u32(vmulq_u32(vreinterpretq_u32_u64(a), vrev64q_u32(vreinterpretq_u32_u64(b))));
971 return vmlal_u32(vshlq_n_u64(hi, 32), al, bl);
972}
973template <>
974EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(const Packet2l& a, const Packet2l& b) {
975 return vreinterpretq_s64_u64(pmul(vreinterpretq_u64_s64(a), vreinterpretq_u64_s64(b)));
976}
977
978template <>
979EIGEN_STRONG_INLINE Packet4c pdiv<Packet4c>(const Packet4c& /*a*/, const Packet4c& /*b*/) {
980 eigen_assert(false && "packet integer division are not supported by NEON");
981 return pset1<Packet4c>(0);
982}
983template <>
984EIGEN_STRONG_INLINE Packet8c pdiv<Packet8c>(const Packet8c& /*a*/, const Packet8c& /*b*/) {
985 eigen_assert(false && "packet integer division are not supported by NEON");
986 return pset1<Packet8c>(0);
987}
988template <>
989EIGEN_STRONG_INLINE Packet16c pdiv<Packet16c>(const Packet16c& /*a*/, const Packet16c& /*b*/) {
990 eigen_assert(false && "packet integer division are not supported by NEON");
991 return pset1<Packet16c>(0);
992}
993template <>
994EIGEN_STRONG_INLINE Packet4uc pdiv<Packet4uc>(const Packet4uc& /*a*/, const Packet4uc& /*b*/) {
995 eigen_assert(false && "packet integer division are not supported by NEON");
996 return pset1<Packet4uc>(0);
997}
998template <>
999EIGEN_STRONG_INLINE Packet8uc pdiv<Packet8uc>(const Packet8uc& /*a*/, const Packet8uc& /*b*/) {
1000 eigen_assert(false && "packet integer division are not supported by NEON");
1001 return pset1<Packet8uc>(0);
1002}
1003template <>
1004EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(const Packet16uc& /*a*/, const Packet16uc& /*b*/) {
1005 eigen_assert(false && "packet integer division are not supported by NEON");
1006 return pset1<Packet16uc>(0);
1007}
1008template <>
1009EIGEN_STRONG_INLINE Packet4s pdiv<Packet4s>(const Packet4s& /*a*/, const Packet4s& /*b*/) {
1010 eigen_assert(false && "packet integer division are not supported by NEON");
1011 return pset1<Packet4s>(0);
1012}
1013template <>
1014EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(const Packet8s& /*a*/, const Packet8s& /*b*/) {
1015 eigen_assert(false && "packet integer division are not supported by NEON");
1016 return pset1<Packet8s>(0);
1017}
1018template <>
1019EIGEN_STRONG_INLINE Packet4us pdiv<Packet4us>(const Packet4us& /*a*/, const Packet4us& /*b*/) {
1020 eigen_assert(false && "packet integer division are not supported by NEON");
1021 return pset1<Packet4us>(0);
1022}
1023template <>
1024EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(const Packet8us& /*a*/, const Packet8us& /*b*/) {
1025 eigen_assert(false && "packet integer division are not supported by NEON");
1026 return pset1<Packet8us>(0);
1027}
1028template <>
1029EIGEN_STRONG_INLINE Packet2i pdiv<Packet2i>(const Packet2i& /*a*/, const Packet2i& /*b*/) {
1030 eigen_assert(false && "packet integer division are not supported by NEON");
1031 return pset1<Packet2i>(0);
1032}
1033template <>
1034EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/) {
1035 eigen_assert(false && "packet integer division are not supported by NEON");
1036 return pset1<Packet4i>(0);
1037}
1038template <>
1039EIGEN_STRONG_INLINE Packet2ui pdiv<Packet2ui>(const Packet2ui& /*a*/, const Packet2ui& /*b*/) {
1040 eigen_assert(false && "packet integer division are not supported by NEON");
1041 return pset1<Packet2ui>(0);
1042}
1043template <>
1044EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(const Packet4ui& /*a*/, const Packet4ui& /*b*/) {
1045 eigen_assert(false && "packet integer division are not supported by NEON");
1046 return pset1<Packet4ui>(0);
1047}
1048template <>
1049EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(const Packet2l& /*a*/, const Packet2l& /*b*/) {
1050 eigen_assert(false && "packet integer division are not supported by NEON");
1051 return pset1<Packet2l>(0LL);
1052}
1053template <>
1054EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(const Packet2ul& /*a*/, const Packet2ul& /*b*/) {
1055 eigen_assert(false && "packet integer division are not supported by NEON");
1056 return pset1<Packet2ul>(0ULL);
1057}
1058
1059#ifdef EIGEN_VECTORIZE_FMA
1060template <>
1061struct is_fused_madd<Packet4f> : true_type {};
1062template <>
1063EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1064 return vfmaq_f32(c, a, b);
1065}
1066template <>
1067struct is_fused_madd<Packet2f> : true_type {};
1068template <>
1069EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) {
1070 return vfma_f32(c, a, b);
1071}
1072template <>
1073EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1074 return vfmsq_f32(c, a, b);
1075}
1076template <>
1077EIGEN_STRONG_INLINE Packet2f pnmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) {
1078 return vfms_f32(c, a, b);
1079}
1080#else
1081template <>
1082EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1083 return vmlaq_f32(c, a, b);
1084}
1085template <>
1086EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) {
1087 return vmla_f32(c, a, b);
1088}
1089template <>
1090EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1091 return vmlsq_f32(c, a, b);
1092}
1093template <>
1094EIGEN_STRONG_INLINE Packet2f pnmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c) {
1095 return vmls_f32(c, a, b);
1096}
1097#endif
1098// No FMA instruction for int, so use MLA unconditionally.
1099template <>
1100EIGEN_STRONG_INLINE Packet4c pmadd(const Packet4c& a, const Packet4c& b, const Packet4c& c) {
1101 return vget_lane_s32(
1102 vreinterpret_s32_s8(vmla_s8(vreinterpret_s8_s32(vdup_n_s32(c)), vreinterpret_s8_s32(vdup_n_s32(a)),
1103 vreinterpret_s8_s32(vdup_n_s32(b)))),
1104 0);
1105}
1106template <>
1107EIGEN_STRONG_INLINE Packet8c pmadd(const Packet8c& a, const Packet8c& b, const Packet8c& c) {
1108 return vmla_s8(c, a, b);
1109}
1110template <>
1111EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
1112 return vmlaq_s8(c, a, b);
1113}
1114template <>
1115EIGEN_STRONG_INLINE Packet4uc pmadd(const Packet4uc& a, const Packet4uc& b, const Packet4uc& c) {
1116 return vget_lane_u32(
1117 vreinterpret_u32_u8(vmla_u8(vreinterpret_u8_u32(vdup_n_u32(c)), vreinterpret_u8_u32(vdup_n_u32(a)),
1118 vreinterpret_u8_u32(vdup_n_u32(b)))),
1119 0);
1120}
1121template <>
1122EIGEN_STRONG_INLINE Packet8uc pmadd(const Packet8uc& a, const Packet8uc& b, const Packet8uc& c) {
1123 return vmla_u8(c, a, b);
1124}
1125template <>
1126EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c) {
1127 return vmlaq_u8(c, a, b);
1128}
1129template <>
1130EIGEN_STRONG_INLINE Packet4s pmadd(const Packet4s& a, const Packet4s& b, const Packet4s& c) {
1131 return vmla_s16(c, a, b);
1132}
1133template <>
1134EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
1135 return vmlaq_s16(c, a, b);
1136}
1137template <>
1138EIGEN_STRONG_INLINE Packet4us pmadd(const Packet4us& a, const Packet4us& b, const Packet4us& c) {
1139 return vmla_u16(c, a, b);
1140}
1141template <>
1142EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) {
1143 return vmlaq_u16(c, a, b);
1144}
1145template <>
1146EIGEN_STRONG_INLINE Packet2i pmadd(const Packet2i& a, const Packet2i& b, const Packet2i& c) {
1147 return vmla_s32(c, a, b);
1148}
1149template <>
1150EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
1151 return vmlaq_s32(c, a, b);
1152}
1153template <>
1154EIGEN_STRONG_INLINE Packet2ui pmadd(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c) {
1155 return vmla_u32(c, a, b);
1156}
1157template <>
1158EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c) {
1159 return vmlaq_u32(c, a, b);
1160}
1161
1162template <>
1163EIGEN_STRONG_INLINE Packet2f pabsdiff<Packet2f>(const Packet2f& a, const Packet2f& b) {
1164 return vabd_f32(a, b);
1165}
1166template <>
1167EIGEN_STRONG_INLINE Packet4f pabsdiff<Packet4f>(const Packet4f& a, const Packet4f& b) {
1168 return vabdq_f32(a, b);
1169}
1170template <>
1171EIGEN_STRONG_INLINE Packet4c pabsdiff<Packet4c>(const Packet4c& a, const Packet4c& b) {
1172 return vget_lane_s32(
1173 vreinterpret_s32_s8(vabd_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1174}
1175template <>
1176EIGEN_STRONG_INLINE Packet8c pabsdiff<Packet8c>(const Packet8c& a, const Packet8c& b) {
1177 return vabd_s8(a, b);
1178}
1179template <>
1180EIGEN_STRONG_INLINE Packet16c pabsdiff<Packet16c>(const Packet16c& a, const Packet16c& b) {
1181 return vabdq_s8(a, b);
1182}
1183template <>
1184EIGEN_STRONG_INLINE Packet4uc pabsdiff<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1185 return vget_lane_u32(
1186 vreinterpret_u32_u8(vabd_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1187}
1188template <>
1189EIGEN_STRONG_INLINE Packet8uc pabsdiff<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1190 return vabd_u8(a, b);
1191}
1192template <>
1193EIGEN_STRONG_INLINE Packet16uc pabsdiff<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1194 return vabdq_u8(a, b);
1195}
1196template <>
1197EIGEN_STRONG_INLINE Packet4s pabsdiff<Packet4s>(const Packet4s& a, const Packet4s& b) {
1198 return vabd_s16(a, b);
1199}
1200template <>
1201EIGEN_STRONG_INLINE Packet8s pabsdiff<Packet8s>(const Packet8s& a, const Packet8s& b) {
1202 return vabdq_s16(a, b);
1203}
1204template <>
1205EIGEN_STRONG_INLINE Packet4us pabsdiff<Packet4us>(const Packet4us& a, const Packet4us& b) {
1206 return vabd_u16(a, b);
1207}
1208template <>
1209EIGEN_STRONG_INLINE Packet8us pabsdiff<Packet8us>(const Packet8us& a, const Packet8us& b) {
1210 return vabdq_u16(a, b);
1211}
1212template <>
1213EIGEN_STRONG_INLINE Packet2i pabsdiff<Packet2i>(const Packet2i& a, const Packet2i& b) {
1214 return vabd_s32(a, b);
1215}
1216template <>
1217EIGEN_STRONG_INLINE Packet4i pabsdiff<Packet4i>(const Packet4i& a, const Packet4i& b) {
1218 return vabdq_s32(a, b);
1219}
1220template <>
1221EIGEN_STRONG_INLINE Packet2ui pabsdiff<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1222 return vabd_u32(a, b);
1223}
1224template <>
1225EIGEN_STRONG_INLINE Packet4ui pabsdiff<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1226 return vabdq_u32(a, b);
1227}
1228
1229template <>
1230EIGEN_STRONG_INLINE Packet2f pmin<Packet2f>(const Packet2f& a, const Packet2f& b) {
1231 return vmin_f32(a, b);
1232}
1233template <>
1234EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
1235 return vminq_f32(a, b);
1236}
1237
1238#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1239// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8
1240// systems).
1241template <>
1242EIGEN_STRONG_INLINE Packet4f pmin<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) {
1243 return vminnmq_f32(a, b);
1244}
1245template <>
1246EIGEN_STRONG_INLINE Packet2f pmin<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) {
1247 return vminnm_f32(a, b);
1248}
1249#endif
1250
1251template <>
1252struct pminmax_propagates_nan<Packet2f> : bool_constant<true> {};
1253template <>
1254struct pminmax_propagates_nan<Packet4f> : bool_constant<true> {};
1255
1256template <>
1257EIGEN_STRONG_INLINE Packet4c pmin<Packet4c>(const Packet4c& a, const Packet4c& b) {
1258 return vget_lane_s32(
1259 vreinterpret_s32_s8(vmin_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1260}
1261template <>
1262EIGEN_STRONG_INLINE Packet8c pmin<Packet8c>(const Packet8c& a, const Packet8c& b) {
1263 return vmin_s8(a, b);
1264}
1265template <>
1266EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(const Packet16c& a, const Packet16c& b) {
1267 return vminq_s8(a, b);
1268}
1269template <>
1270EIGEN_STRONG_INLINE Packet4uc pmin<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1271 return vget_lane_u32(
1272 vreinterpret_u32_u8(vmin_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1273}
1274template <>
1275EIGEN_STRONG_INLINE Packet8uc pmin<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1276 return vmin_u8(a, b);
1277}
1278template <>
1279EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1280 return vminq_u8(a, b);
1281}
1282template <>
1283EIGEN_STRONG_INLINE Packet4s pmin<Packet4s>(const Packet4s& a, const Packet4s& b) {
1284 return vmin_s16(a, b);
1285}
1286template <>
1287EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(const Packet8s& a, const Packet8s& b) {
1288 return vminq_s16(a, b);
1289}
1290template <>
1291EIGEN_STRONG_INLINE Packet4us pmin<Packet4us>(const Packet4us& a, const Packet4us& b) {
1292 return vmin_u16(a, b);
1293}
1294template <>
1295EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(const Packet8us& a, const Packet8us& b) {
1296 return vminq_u16(a, b);
1297}
1298template <>
1299EIGEN_STRONG_INLINE Packet2i pmin<Packet2i>(const Packet2i& a, const Packet2i& b) {
1300 return vmin_s32(a, b);
1301}
1302template <>
1303EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
1304 return vminq_s32(a, b);
1305}
1306template <>
1307EIGEN_STRONG_INLINE Packet2ui pmin<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1308 return vmin_u32(a, b);
1309}
1310template <>
1311EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1312 return vminq_u32(a, b);
1313}
1314template <>
1315EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1316 return vsubq_u64(a, vqsubq_u64(a, b));
1317}
1318
1319template <>
1320EIGEN_STRONG_INLINE Packet2f pmax<Packet2f>(const Packet2f& a, const Packet2f& b) {
1321 return vmax_f32(a, b);
1322}
1323template <>
1324EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
1325 return vmaxq_f32(a, b);
1326}
1327
1328#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1329// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8
1330// systems).
1331template <>
1332EIGEN_STRONG_INLINE Packet4f pmax<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) {
1333 return vmaxnmq_f32(a, b);
1334}
1335template <>
1336EIGEN_STRONG_INLINE Packet2f pmax<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) {
1337 return vmaxnm_f32(a, b);
1338}
1339#endif
1340
1341template <>
1342EIGEN_STRONG_INLINE Packet4c pmax<Packet4c>(const Packet4c& a, const Packet4c& b) {
1343 return vget_lane_s32(
1344 vreinterpret_s32_s8(vmax_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1345}
1346template <>
1347EIGEN_STRONG_INLINE Packet8c pmax<Packet8c>(const Packet8c& a, const Packet8c& b) {
1348 return vmax_s8(a, b);
1349}
1350template <>
1351EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(const Packet16c& a, const Packet16c& b) {
1352 return vmaxq_s8(a, b);
1353}
1354template <>
1355EIGEN_STRONG_INLINE Packet4uc pmax<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1356 return vget_lane_u32(
1357 vreinterpret_u32_u8(vmax_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1358}
1359template <>
1360EIGEN_STRONG_INLINE Packet8uc pmax<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1361 return vmax_u8(a, b);
1362}
1363template <>
1364EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1365 return vmaxq_u8(a, b);
1366}
1367template <>
1368EIGEN_STRONG_INLINE Packet4s pmax<Packet4s>(const Packet4s& a, const Packet4s& b) {
1369 return vmax_s16(a, b);
1370}
1371template <>
1372EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(const Packet8s& a, const Packet8s& b) {
1373 return vmaxq_s16(a, b);
1374}
1375template <>
1376EIGEN_STRONG_INLINE Packet4us pmax<Packet4us>(const Packet4us& a, const Packet4us& b) {
1377 return vmax_u16(a, b);
1378}
1379template <>
1380EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(const Packet8us& a, const Packet8us& b) {
1381 return vmaxq_u16(a, b);
1382}
1383template <>
1384EIGEN_STRONG_INLINE Packet2i pmax<Packet2i>(const Packet2i& a, const Packet2i& b) {
1385 return vmax_s32(a, b);
1386}
1387template <>
1388EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
1389 return vmaxq_s32(a, b);
1390}
1391template <>
1392EIGEN_STRONG_INLINE Packet2ui pmax<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1393 return vmax_u32(a, b);
1394}
1395template <>
1396EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1397 return vmaxq_u32(a, b);
1398}
1399template <>
1400EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1401 return vaddq_u64(b, vqsubq_u64(a, b));
1402}
1403
1404// GCC defines the ARM32 vcge/vcgt float intrinsics as generic vector comparisons and, unless
1405// -funsafe-math-optimizations is given, expands them one lane at a time through VFP, because Advanced SIMD
1406// flushes subnormal inputs. Emit the Advanced SIMD comparison directly, as Clang does.
1407#if EIGEN_COMP_GNUC_STRICT && !EIGEN_ARCH_ARM64
1408EIGEN_STRONG_INLINE uint32x2_t neon_vcge_f32(float32x2_t a, float32x2_t b) {
1409 uint32x2_t r;
1410 __asm__("vcge.f32 %P0, %P1, %P2" : "=w"(r) : "w"(a), "w"(b));
1411 return r;
1412}
1413EIGEN_STRONG_INLINE uint32x4_t neon_vcgeq_f32(float32x4_t a, float32x4_t b) {
1414 uint32x4_t r;
1415 __asm__("vcge.f32 %q0, %q1, %q2" : "=w"(r) : "w"(a), "w"(b));
1416 return r;
1417}
1418EIGEN_STRONG_INLINE uint32x2_t neon_vcgt_f32(float32x2_t a, float32x2_t b) {
1419 uint32x2_t r;
1420 __asm__("vcgt.f32 %P0, %P1, %P2" : "=w"(r) : "w"(a), "w"(b));
1421 return r;
1422}
1423EIGEN_STRONG_INLINE uint32x4_t neon_vcgtq_f32(float32x4_t a, float32x4_t b) {
1424 uint32x4_t r;
1425 __asm__("vcgt.f32 %q0, %q1, %q2" : "=w"(r) : "w"(a), "w"(b));
1426 return r;
1427}
1428#else
1429EIGEN_STRONG_INLINE uint32x2_t neon_vcge_f32(float32x2_t a, float32x2_t b) { return vcge_f32(a, b); }
1430EIGEN_STRONG_INLINE uint32x4_t neon_vcgeq_f32(float32x4_t a, float32x4_t b) { return vcgeq_f32(a, b); }
1431EIGEN_STRONG_INLINE uint32x2_t neon_vcgt_f32(float32x2_t a, float32x2_t b) { return vcgt_f32(a, b); }
1432EIGEN_STRONG_INLINE uint32x4_t neon_vcgtq_f32(float32x4_t a, float32x4_t b) { return vcgtq_f32(a, b); }
1433#endif
1434
1435template <>
1436EIGEN_STRONG_INLINE Packet2f pcmp_le<Packet2f>(const Packet2f& a, const Packet2f& b) {
1437 return vreinterpret_f32_u32(neon_vcge_f32(b, a));
1438}
1439template <>
1440EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b) {
1441 return vreinterpretq_f32_u32(neon_vcgeq_f32(b, a));
1442}
1443template <>
1444EIGEN_STRONG_INLINE Packet4c pcmp_le<Packet4c>(const Packet4c& a, const Packet4c& b) {
1445 return vget_lane_s32(
1446 vreinterpret_s32_u8(vcle_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1447}
1448template <>
1449EIGEN_STRONG_INLINE Packet8c pcmp_le<Packet8c>(const Packet8c& a, const Packet8c& b) {
1450 return vreinterpret_s8_u8(vcle_s8(a, b));
1451}
1452template <>
1453EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(const Packet16c& a, const Packet16c& b) {
1454 return vreinterpretq_s8_u8(vcleq_s8(a, b));
1455}
1456template <>
1457EIGEN_STRONG_INLINE Packet4uc pcmp_le<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1458 return vget_lane_u32(
1459 vreinterpret_u32_u8(vcle_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1460}
1461template <>
1462EIGEN_STRONG_INLINE Packet8uc pcmp_le<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1463 return vcle_u8(a, b);
1464}
1465template <>
1466EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1467 return vcleq_u8(a, b);
1468}
1469template <>
1470EIGEN_STRONG_INLINE Packet4s pcmp_le<Packet4s>(const Packet4s& a, const Packet4s& b) {
1471 return vreinterpret_s16_u16(vcle_s16(a, b));
1472}
1473template <>
1474EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(const Packet8s& a, const Packet8s& b) {
1475 return vreinterpretq_s16_u16(vcleq_s16(a, b));
1476}
1477template <>
1478EIGEN_STRONG_INLINE Packet4us pcmp_le<Packet4us>(const Packet4us& a, const Packet4us& b) {
1479 return vcle_u16(a, b);
1480}
1481template <>
1482EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(const Packet8us& a, const Packet8us& b) {
1483 return vcleq_u16(a, b);
1484}
1485template <>
1486EIGEN_STRONG_INLINE Packet2i pcmp_le<Packet2i>(const Packet2i& a, const Packet2i& b) {
1487 return vreinterpret_s32_u32(vcle_s32(a, b));
1488}
1489template <>
1490EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(const Packet4i& a, const Packet4i& b) {
1491 return vreinterpretq_s32_u32(vcleq_s32(a, b));
1492}
1493template <>
1494EIGEN_STRONG_INLINE Packet2ui pcmp_le<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1495 return vcle_u32(a, b);
1496}
1497template <>
1498EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1499 return vcleq_u32(a, b);
1500}
1501
1502template <>
1503EIGEN_STRONG_INLINE Packet2f pcmp_lt<Packet2f>(const Packet2f& a, const Packet2f& b) {
1504 return vreinterpret_f32_u32(neon_vcgt_f32(b, a));
1505}
1506template <>
1507EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b) {
1508 return vreinterpretq_f32_u32(neon_vcgtq_f32(b, a));
1509}
1510template <>
1511EIGEN_STRONG_INLINE Packet4c pcmp_lt<Packet4c>(const Packet4c& a, const Packet4c& b) {
1512 return vget_lane_s32(
1513 vreinterpret_s32_u8(vclt_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1514}
1515template <>
1516EIGEN_STRONG_INLINE Packet8c pcmp_lt<Packet8c>(const Packet8c& a, const Packet8c& b) {
1517 return vreinterpret_s8_u8(vclt_s8(a, b));
1518}
1519template <>
1520EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(const Packet16c& a, const Packet16c& b) {
1521 return vreinterpretq_s8_u8(vcltq_s8(a, b));
1522}
1523template <>
1524EIGEN_STRONG_INLINE Packet4uc pcmp_lt<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1525 return vget_lane_u32(
1526 vreinterpret_u32_u8(vclt_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1527}
1528template <>
1529EIGEN_STRONG_INLINE Packet8uc pcmp_lt<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1530 return vclt_u8(a, b);
1531}
1532template <>
1533EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1534 return vcltq_u8(a, b);
1535}
1536template <>
1537EIGEN_STRONG_INLINE Packet4s pcmp_lt<Packet4s>(const Packet4s& a, const Packet4s& b) {
1538 return vreinterpret_s16_u16(vclt_s16(a, b));
1539}
1540template <>
1541EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(const Packet8s& a, const Packet8s& b) {
1542 return vreinterpretq_s16_u16(vcltq_s16(a, b));
1543}
1544template <>
1545EIGEN_STRONG_INLINE Packet4us pcmp_lt<Packet4us>(const Packet4us& a, const Packet4us& b) {
1546 return vclt_u16(a, b);
1547}
1548template <>
1549EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(const Packet8us& a, const Packet8us& b) {
1550 return vcltq_u16(a, b);
1551}
1552template <>
1553EIGEN_STRONG_INLINE Packet2i pcmp_lt<Packet2i>(const Packet2i& a, const Packet2i& b) {
1554 return vreinterpret_s32_u32(vclt_s32(a, b));
1555}
1556template <>
1557EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(const Packet4i& a, const Packet4i& b) {
1558 return vreinterpretq_s32_u32(vcltq_s32(a, b));
1559}
1560template <>
1561EIGEN_STRONG_INLINE Packet2ui pcmp_lt<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1562 return vclt_u32(a, b);
1563}
1564template <>
1565EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1566 return vcltq_u32(a, b);
1567}
1568
1569template <>
1570EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(const Packet2l& a, const Packet2l& b) {
1571#if EIGEN_ARCH_ARM64
1572 return vreinterpretq_s64_u64(vcltq_s64(a, b));
1573#else
1574 return vshrq_n_s64(vqsubq_s64(a, b), 63);
1575#endif
1576}
1577template <>
1578EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1579#if EIGEN_ARCH_ARM64
1580 return vcltq_u64(a, b);
1581#else
1582 const uint64x2_t flag = vshrq_n_u64(vreinterpretq_u64_u8(vdupq_n_u8(0xFF)), 1);
1583 return vreinterpretq_u64_s64(vshrq_n_s64(vreinterpretq_s64_u64(vqaddq_u64(vqsubq_u64(b, a), flag)), 63));
1584#endif
1585}
1586template <>
1587EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(const Packet2l& a, const Packet2l& b) {
1588#if EIGEN_ARCH_ARM64
1589 return vreinterpretq_s64_u64(vcleq_s64(a, b));
1590#else
1591 return vreinterpretq_s64_u8(vmvnq_u8(vreinterpretq_u8_s64(pcmp_lt(b, a))));
1592#endif
1593}
1594template <>
1595EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1596#if EIGEN_ARCH_ARM64
1597 return vcleq_u64(a, b);
1598#else
1599 return vreinterpretq_u64_u8(vmvnq_u8(vreinterpretq_u8_u64(pcmp_lt(b, a))));
1600#endif
1601}
1602
1603template <>
1604EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(const Packet2l& a, const Packet2l& b) {
1605 return vbslq_s64(vreinterpretq_u64_s64(pcmp_lt(a, b)), a, b);
1606}
1607template <>
1608EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(const Packet2l& a, const Packet2l& b) {
1609 return vbslq_s64(vreinterpretq_u64_s64(pcmp_lt(b, a)), a, b);
1610}
1611
1612template <>
1613EIGEN_STRONG_INLINE Packet2f pcmp_eq<Packet2f>(const Packet2f& a, const Packet2f& b) {
1614 return vreinterpret_f32_u32(vceq_f32(a, b));
1615}
1616template <>
1617EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(const Packet4f& a, const Packet4f& b) {
1618 return vreinterpretq_f32_u32(vceqq_f32(a, b));
1619}
1620template <>
1621EIGEN_STRONG_INLINE Packet4c pcmp_eq<Packet4c>(const Packet4c& a, const Packet4c& b) {
1622 return vget_lane_s32(
1623 vreinterpret_s32_u8(vceq_s8(vreinterpret_s8_s32(vdup_n_s32(a)), vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1624}
1625template <>
1626EIGEN_STRONG_INLINE Packet8c pcmp_eq<Packet8c>(const Packet8c& a, const Packet8c& b) {
1627 return vreinterpret_s8_u8(vceq_s8(a, b));
1628}
1629template <>
1630EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(const Packet16c& a, const Packet16c& b) {
1631 return vreinterpretq_s8_u8(vceqq_s8(a, b));
1632}
1633template <>
1634EIGEN_STRONG_INLINE Packet4uc pcmp_eq<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1635 return vget_lane_u32(
1636 vreinterpret_u32_u8(vceq_u8(vreinterpret_u8_u32(vdup_n_u32(a)), vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1637}
1638template <>
1639EIGEN_STRONG_INLINE Packet8uc pcmp_eq<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1640 return vceq_u8(a, b);
1641}
1642template <>
1643EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1644 return vceqq_u8(a, b);
1645}
1646template <>
1647EIGEN_STRONG_INLINE Packet4s pcmp_eq<Packet4s>(const Packet4s& a, const Packet4s& b) {
1648 return vreinterpret_s16_u16(vceq_s16(a, b));
1649}
1650template <>
1651EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(const Packet8s& a, const Packet8s& b) {
1652 return vreinterpretq_s16_u16(vceqq_s16(a, b));
1653}
1654template <>
1655EIGEN_STRONG_INLINE Packet4us pcmp_eq<Packet4us>(const Packet4us& a, const Packet4us& b) {
1656 return vceq_u16(a, b);
1657}
1658template <>
1659EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(const Packet8us& a, const Packet8us& b) {
1660 return vceqq_u16(a, b);
1661}
1662template <>
1663EIGEN_STRONG_INLINE Packet2i pcmp_eq<Packet2i>(const Packet2i& a, const Packet2i& b) {
1664 return vreinterpret_s32_u32(vceq_s32(a, b));
1665}
1666template <>
1667EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(const Packet4i& a, const Packet4i& b) {
1668 return vreinterpretq_s32_u32(vceqq_s32(a, b));
1669}
1670template <>
1671EIGEN_STRONG_INLINE Packet2ui pcmp_eq<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1672 return vceq_u32(a, b);
1673}
1674template <>
1675EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1676 return vceqq_u32(a, b);
1677}
1678template <>
1679EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1680#if EIGEN_ARCH_ARM64
1681 return vceqq_u64(a, b);
1682#else
1683 const uint32x4_t eq = vceqq_u32(vreinterpretq_u32_u64(a), vreinterpretq_u32_u64(b));
1684 return vreinterpretq_u64_u32(vandq_u32(eq, vrev64q_u32(eq)));
1685#endif
1686}
1687template <>
1688EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(const Packet2l& a, const Packet2l& b) {
1689#if EIGEN_ARCH_ARM64
1690 return vreinterpretq_s64_u64(vceqq_s64(a, b));
1691#else
1692 return vreinterpretq_s64_u64(pcmp_eq(vreinterpretq_u64_s64(a), vreinterpretq_u64_s64(b)));
1693#endif
1694}
1695
1696template <>
1697EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan<Packet2f>(const Packet2f& a, const Packet2f& b) {
1698 return vreinterpret_f32_u32(vmvn_u32(neon_vcge_f32(a, b)));
1699}
1700template <>
1701EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b) {
1702 return vreinterpretq_f32_u32(vmvnq_u32(neon_vcgeq_f32(a, b)));
1703}
1704
1705// Classify with absolute comparisons against infinity: |a| >= inf holds only for +-inf, inf > |a| only for finite
1706// values, and |a| <= inf fails only for NaN. Each is one instruction (FACGE/FACGT, VACGE/VACGT), is unaffected by
1707// flushing subnormal inputs, and avoids the self-comparison a != a, which Clang rewrites into two comparisons.
1708template <>
1709EIGEN_STRONG_INLINE Packet2f pisinf<Packet2f>(const Packet2f& a) {
1710 return vreinterpret_f32_u32(vcage_f32(a, pinf<Packet2f>()));
1711}
1712template <>
1713EIGEN_STRONG_INLINE Packet4f pisinf<Packet4f>(const Packet4f& a) {
1714 return vreinterpretq_f32_u32(vcageq_f32(a, pinf<Packet4f>()));
1715}
1716template <>
1717EIGEN_STRONG_INLINE Packet2f pisfinite<Packet2f>(const Packet2f& a) {
1718 return vreinterpret_f32_u32(vcagt_f32(pinf<Packet2f>(), a));
1719}
1720template <>
1721EIGEN_STRONG_INLINE Packet4f pisfinite<Packet4f>(const Packet4f& a) {
1722 return vreinterpretq_f32_u32(vcagtq_f32(pinf<Packet4f>(), a));
1723}
1724template <>
1725EIGEN_STRONG_INLINE Packet2f pisnan<Packet2f>(const Packet2f& a) {
1726 return vreinterpret_f32_u32(vmvn_u32(vcage_f32(pinf<Packet2f>(), a)));
1727}
1728template <>
1729EIGEN_STRONG_INLINE Packet4f pisnan<Packet4f>(const Packet4f& a) {
1730 return vreinterpretq_f32_u32(vmvnq_u32(vcageq_f32(pinf<Packet4f>(), a)));
1731}
1732
1733// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
1734template <>
1735EIGEN_STRONG_INLINE Packet2f pand<Packet2f>(const Packet2f& a, const Packet2f& b) {
1736 return vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1737}
1738template <>
1739EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
1740 return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1741}
1742template <>
1743EIGEN_STRONG_INLINE Packet4c pand<Packet4c>(const Packet4c& a, const Packet4c& b) {
1744 return a & b;
1745}
1746template <>
1747EIGEN_STRONG_INLINE Packet8c pand<Packet8c>(const Packet8c& a, const Packet8c& b) {
1748 return vand_s8(a, b);
1749}
1750template <>
1751EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(const Packet16c& a, const Packet16c& b) {
1752 return vandq_s8(a, b);
1753}
1754template <>
1755EIGEN_STRONG_INLINE Packet4uc pand<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1756 return a & b;
1757}
1758template <>
1759EIGEN_STRONG_INLINE Packet8uc pand<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1760 return vand_u8(a, b);
1761}
1762template <>
1763EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1764 return vandq_u8(a, b);
1765}
1766template <>
1767EIGEN_STRONG_INLINE Packet4s pand<Packet4s>(const Packet4s& a, const Packet4s& b) {
1768 return vand_s16(a, b);
1769}
1770template <>
1771EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(const Packet8s& a, const Packet8s& b) {
1772 return vandq_s16(a, b);
1773}
1774template <>
1775EIGEN_STRONG_INLINE Packet4us pand<Packet4us>(const Packet4us& a, const Packet4us& b) {
1776 return vand_u16(a, b);
1777}
1778template <>
1779EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(const Packet8us& a, const Packet8us& b) {
1780 return vandq_u16(a, b);
1781}
1782template <>
1783EIGEN_STRONG_INLINE Packet2i pand<Packet2i>(const Packet2i& a, const Packet2i& b) {
1784 return vand_s32(a, b);
1785}
1786template <>
1787EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
1788 return vandq_s32(a, b);
1789}
1790template <>
1791EIGEN_STRONG_INLINE Packet2ui pand<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1792 return vand_u32(a, b);
1793}
1794template <>
1795EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1796 return vandq_u32(a, b);
1797}
1798template <>
1799EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) {
1800 return vandq_s64(a, b);
1801}
1802template <>
1803EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1804 return vandq_u64(a, b);
1805}
1806
1807template <>
1808EIGEN_STRONG_INLINE Packet2f por<Packet2f>(const Packet2f& a, const Packet2f& b) {
1809 return vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1810}
1811template <>
1812EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
1813 return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1814}
1815template <>
1816EIGEN_STRONG_INLINE Packet4c por<Packet4c>(const Packet4c& a, const Packet4c& b) {
1817 return a | b;
1818}
1819template <>
1820EIGEN_STRONG_INLINE Packet8c por<Packet8c>(const Packet8c& a, const Packet8c& b) {
1821 return vorr_s8(a, b);
1822}
1823template <>
1824EIGEN_STRONG_INLINE Packet16c por<Packet16c>(const Packet16c& a, const Packet16c& b) {
1825 return vorrq_s8(a, b);
1826}
1827template <>
1828EIGEN_STRONG_INLINE Packet4uc por<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1829 return a | b;
1830}
1831template <>
1832EIGEN_STRONG_INLINE Packet8uc por<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1833 return vorr_u8(a, b);
1834}
1835template <>
1836EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1837 return vorrq_u8(a, b);
1838}
1839template <>
1840EIGEN_STRONG_INLINE Packet4s por<Packet4s>(const Packet4s& a, const Packet4s& b) {
1841 return vorr_s16(a, b);
1842}
1843template <>
1844EIGEN_STRONG_INLINE Packet8s por<Packet8s>(const Packet8s& a, const Packet8s& b) {
1845 return vorrq_s16(a, b);
1846}
1847template <>
1848EIGEN_STRONG_INLINE Packet4us por<Packet4us>(const Packet4us& a, const Packet4us& b) {
1849 return vorr_u16(a, b);
1850}
1851template <>
1852EIGEN_STRONG_INLINE Packet8us por<Packet8us>(const Packet8us& a, const Packet8us& b) {
1853 return vorrq_u16(a, b);
1854}
1855template <>
1856EIGEN_STRONG_INLINE Packet2i por<Packet2i>(const Packet2i& a, const Packet2i& b) {
1857 return vorr_s32(a, b);
1858}
1859template <>
1860EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
1861 return vorrq_s32(a, b);
1862}
1863template <>
1864EIGEN_STRONG_INLINE Packet2ui por<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1865 return vorr_u32(a, b);
1866}
1867template <>
1868EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1869 return vorrq_u32(a, b);
1870}
1871template <>
1872EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b) {
1873 return vorrq_s64(a, b);
1874}
1875template <>
1876EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1877 return vorrq_u64(a, b);
1878}
1879
1880template <>
1881EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(const Packet2f& a, const Packet2f& b) {
1882 return vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1883}
1884template <>
1885EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
1886 return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1887}
1888template <>
1889EIGEN_STRONG_INLINE Packet4c pxor<Packet4c>(const Packet4c& a, const Packet4c& b) {
1890 return a ^ b;
1891}
1892template <>
1893EIGEN_STRONG_INLINE Packet8c pxor<Packet8c>(const Packet8c& a, const Packet8c& b) {
1894 return veor_s8(a, b);
1895}
1896template <>
1897EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(const Packet16c& a, const Packet16c& b) {
1898 return veorq_s8(a, b);
1899}
1900template <>
1901EIGEN_STRONG_INLINE Packet4uc pxor<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1902 return a ^ b;
1903}
1904template <>
1905EIGEN_STRONG_INLINE Packet8uc pxor<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1906 return veor_u8(a, b);
1907}
1908template <>
1909EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1910 return veorq_u8(a, b);
1911}
1912template <>
1913EIGEN_STRONG_INLINE Packet4s pxor<Packet4s>(const Packet4s& a, const Packet4s& b) {
1914 return veor_s16(a, b);
1915}
1916template <>
1917EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(const Packet8s& a, const Packet8s& b) {
1918 return veorq_s16(a, b);
1919}
1920template <>
1921EIGEN_STRONG_INLINE Packet4us pxor<Packet4us>(const Packet4us& a, const Packet4us& b) {
1922 return veor_u16(a, b);
1923}
1924template <>
1925EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(const Packet8us& a, const Packet8us& b) {
1926 return veorq_u16(a, b);
1927}
1928template <>
1929EIGEN_STRONG_INLINE Packet2i pxor<Packet2i>(const Packet2i& a, const Packet2i& b) {
1930 return veor_s32(a, b);
1931}
1932template <>
1933EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
1934 return veorq_s32(a, b);
1935}
1936template <>
1937EIGEN_STRONG_INLINE Packet2ui pxor<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
1938 return veor_u32(a, b);
1939}
1940template <>
1941EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1942 return veorq_u32(a, b);
1943}
1944template <>
1945EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b) {
1946 return veorq_s64(a, b);
1947}
1948template <>
1949EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1950 return veorq_u64(a, b);
1951}
1952
1953template <>
1954EIGEN_STRONG_INLINE Packet2f pandnot<Packet2f>(const Packet2f& a, const Packet2f& b) {
1955 return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a), vreinterpret_u32_f32(b)));
1956}
1957template <>
1958EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
1959 return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a), vreinterpretq_u32_f32(b)));
1960}
1961template <>
1962EIGEN_STRONG_INLINE Packet4c pandnot<Packet4c>(const Packet4c& a, const Packet4c& b) {
1963 return a & ~b;
1964}
1965template <>
1966EIGEN_STRONG_INLINE Packet8c pandnot<Packet8c>(const Packet8c& a, const Packet8c& b) {
1967 return vbic_s8(a, b);
1968}
1969template <>
1970EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(const Packet16c& a, const Packet16c& b) {
1971 return vbicq_s8(a, b);
1972}
1973template <>
1974EIGEN_STRONG_INLINE Packet4uc pandnot<Packet4uc>(const Packet4uc& a, const Packet4uc& b) {
1975 return a & ~b;
1976}
1977template <>
1978EIGEN_STRONG_INLINE Packet8uc pandnot<Packet8uc>(const Packet8uc& a, const Packet8uc& b) {
1979 return vbic_u8(a, b);
1980}
1981template <>
1982EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1983 return vbicq_u8(a, b);
1984}
1985template <>
1986EIGEN_STRONG_INLINE Packet4s pandnot<Packet4s>(const Packet4s& a, const Packet4s& b) {
1987 return vbic_s16(a, b);
1988}
1989template <>
1990EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(const Packet8s& a, const Packet8s& b) {
1991 return vbicq_s16(a, b);
1992}
1993template <>
1994EIGEN_STRONG_INLINE Packet4us pandnot<Packet4us>(const Packet4us& a, const Packet4us& b) {
1995 return vbic_u16(a, b);
1996}
1997template <>
1998EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(const Packet8us& a, const Packet8us& b) {
1999 return vbicq_u16(a, b);
2000}
2001template <>
2002EIGEN_STRONG_INLINE Packet2i pandnot<Packet2i>(const Packet2i& a, const Packet2i& b) {
2003 return vbic_s32(a, b);
2004}
2005template <>
2006EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
2007 return vbicq_s32(a, b);
2008}
2009template <>
2010EIGEN_STRONG_INLINE Packet2ui pandnot<Packet2ui>(const Packet2ui& a, const Packet2ui& b) {
2011 return vbic_u32(a, b);
2012}
2013template <>
2014EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
2015 return vbicq_u32(a, b);
2016}
2017template <>
2018EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b) {
2019 return vbicq_s64(a, b);
2020}
2021template <>
2022EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
2023 return vbicq_u64(a, b);
2024}
2025
2026template <>
2027EIGEN_STRONG_INLINE Packet2f pnot<Packet2f>(const Packet2f& a) {
2028 return vreinterpret_f32_u32(vmvn_u32(vreinterpret_u32_f32(a)));
2029}
2030template <>
2031EIGEN_STRONG_INLINE Packet4f pnot<Packet4f>(const Packet4f& a) {
2032 return vreinterpretq_f32_u32(vmvnq_u32(vreinterpretq_u32_f32(a)));
2033}
2034template <>
2035EIGEN_STRONG_INLINE Packet4c pnot<Packet4c>(const Packet4c& a) {
2036 return ~a;
2037}
2038template <>
2039EIGEN_STRONG_INLINE Packet8c pnot<Packet8c>(const Packet8c& a) {
2040 return vmvn_s8(a);
2041}
2042template <>
2043EIGEN_STRONG_INLINE Packet16c pnot<Packet16c>(const Packet16c& a) {
2044 return vmvnq_s8(a);
2045}
2046template <>
2047EIGEN_STRONG_INLINE Packet4uc pnot<Packet4uc>(const Packet4uc& a) {
2048 return ~a;
2049}
2050template <>
2051EIGEN_STRONG_INLINE Packet8uc pnot<Packet8uc>(const Packet8uc& a) {
2052 return vmvn_u8(a);
2053}
2054template <>
2055EIGEN_STRONG_INLINE Packet16uc pnot<Packet16uc>(const Packet16uc& a) {
2056 return vmvnq_u8(a);
2057}
2058template <>
2059EIGEN_STRONG_INLINE Packet4s pnot<Packet4s>(const Packet4s& a) {
2060 return vmvn_s16(a);
2061}
2062template <>
2063EIGEN_STRONG_INLINE Packet8s pnot<Packet8s>(const Packet8s& a) {
2064 return vmvnq_s16(a);
2065}
2066template <>
2067EIGEN_STRONG_INLINE Packet4us pnot<Packet4us>(const Packet4us& a) {
2068 return vmvn_u16(a);
2069}
2070template <>
2071EIGEN_STRONG_INLINE Packet8us pnot<Packet8us>(const Packet8us& a) {
2072 return vmvnq_u16(a);
2073}
2074template <>
2075EIGEN_STRONG_INLINE Packet2i pnot<Packet2i>(const Packet2i& a) {
2076 return vmvn_s32(a);
2077}
2078template <>
2079EIGEN_STRONG_INLINE Packet4i pnot<Packet4i>(const Packet4i& a) {
2080 return vmvnq_s32(a);
2081}
2082template <>
2083EIGEN_STRONG_INLINE Packet2ui pnot<Packet2ui>(const Packet2ui& a) {
2084 return vmvn_u32(a);
2085}
2086template <>
2087EIGEN_STRONG_INLINE Packet4ui pnot<Packet4ui>(const Packet4ui& a) {
2088 return vmvnq_u32(a);
2089}
2090template <>
2091EIGEN_STRONG_INLINE Packet2l pnot<Packet2l>(const Packet2l& a) {
2092 return vreinterpretq_s64_s32(vmvnq_s32(vreinterpretq_s32_s64(a)));
2093}
2094template <>
2095EIGEN_STRONG_INLINE Packet2ul pnot<Packet2ul>(const Packet2ul& a) {
2096 return vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(a)));
2097}
2098
2099template <int N>
2100EIGEN_STRONG_INLINE Packet4c parithmetic_shift_right(const Packet4c& a) {
2101 return vget_lane_s32(vreinterpret_s32_s8(vshr_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
2102}
2103template <int N>
2104EIGEN_STRONG_INLINE Packet8c parithmetic_shift_right(const Packet8c& a) {
2105 return vshr_n_s8(a, N);
2106}
2107template <int N>
2108EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(const Packet16c& a) {
2109 return vshrq_n_s8(a, N);
2110}
2111template <int N>
2112EIGEN_STRONG_INLINE Packet4uc parithmetic_shift_right(const Packet4uc& a) {
2113 return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
2114}
2115template <int N>
2116EIGEN_STRONG_INLINE Packet8uc parithmetic_shift_right(const Packet8uc& a) {
2117 return vshr_n_u8(a, N);
2118}
2119template <int N>
2120EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(const Packet16uc& a) {
2121 return vshrq_n_u8(a, N);
2122}
2123template <int N>
2124EIGEN_STRONG_INLINE Packet4s parithmetic_shift_right(const Packet4s& a) {
2125 return vshr_n_s16(a, N);
2126}
2127template <int N>
2128EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(const Packet8s& a) {
2129 return vshrq_n_s16(a, N);
2130}
2131template <int N>
2132EIGEN_STRONG_INLINE Packet4us parithmetic_shift_right(const Packet4us& a) {
2133 return vshr_n_u16(a, N);
2134}
2135template <int N>
2136EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(const Packet8us& a) {
2137 return vshrq_n_u16(a, N);
2138}
2139template <int N>
2140EIGEN_STRONG_INLINE Packet2i parithmetic_shift_right(const Packet2i& a) {
2141 return vshr_n_s32(a, N);
2142}
2143template <int N>
2144EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) {
2145 return vshrq_n_s32(a, N);
2146}
2147template <int N>
2148EIGEN_STRONG_INLINE Packet2ui parithmetic_shift_right(const Packet2ui& a) {
2149 return vshr_n_u32(a, N);
2150}
2151template <int N>
2152EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(const Packet4ui& a) {
2153 return vshrq_n_u32(a, N);
2154}
2155template <int N>
2156EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(const Packet2l& a) {
2157 return vshrq_n_s64(a, N);
2158}
2159template <int N>
2160EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(const Packet2ul& a) {
2161 return vshrq_n_u64(a, N);
2162}
2163
2164template <int N>
2165EIGEN_STRONG_INLINE Packet4c plogical_shift_right(const Packet4c& a) {
2166 return vget_lane_s32(vreinterpret_s32_u8(vshr_n_u8(vreinterpret_u8_s32(vdup_n_s32(a)), N)), 0);
2167}
2168template <int N>
2169EIGEN_STRONG_INLINE Packet8c plogical_shift_right(const Packet8c& a) {
2170 return vreinterpret_s8_u8(vshr_n_u8(vreinterpret_u8_s8(a), N));
2171}
2172template <int N>
2173EIGEN_STRONG_INLINE Packet16c plogical_shift_right(const Packet16c& a) {
2174 return vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(a), N));
2175}
2176template <int N>
2177EIGEN_STRONG_INLINE Packet4uc plogical_shift_right(const Packet4uc& a) {
2178 return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
2179}
2180template <int N>
2181EIGEN_STRONG_INLINE Packet8uc plogical_shift_right(const Packet8uc& a) {
2182 return vshr_n_u8(a, N);
2183}
2184template <int N>
2185EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(const Packet16uc& a) {
2186 return vshrq_n_u8(a, N);
2187}
2188template <int N>
2189EIGEN_STRONG_INLINE Packet4s plogical_shift_right(const Packet4s& a) {
2190 return vreinterpret_s16_u16(vshr_n_u16(vreinterpret_u16_s16(a), N));
2191}
2192template <int N>
2193EIGEN_STRONG_INLINE Packet8s plogical_shift_right(const Packet8s& a) {
2194 return vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_s16(a), N));
2195}
2196template <int N>
2197EIGEN_STRONG_INLINE Packet4us plogical_shift_right(const Packet4us& a) {
2198 return vshr_n_u16(a, N);
2199}
2200template <int N>
2201EIGEN_STRONG_INLINE Packet8us plogical_shift_right(const Packet8us& a) {
2202 return vshrq_n_u16(a, N);
2203}
2204template <int N>
2205EIGEN_STRONG_INLINE Packet2i plogical_shift_right(const Packet2i& a) {
2206 return vreinterpret_s32_u32(vshr_n_u32(vreinterpret_u32_s32(a), N));
2207}
2208template <int N>
2209EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) {
2210 return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a), N));
2211}
2212template <int N>
2213EIGEN_STRONG_INLINE Packet2ui plogical_shift_right(const Packet2ui& a) {
2214 return vshr_n_u32(a, N);
2215}
2216template <int N>
2217EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) {
2218 return vshrq_n_u32(a, N);
2219}
2220template <int N>
2221EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
2222 return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a), N));
2223}
2224template <int N>
2225EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(const Packet2ul& a) {
2226 return vshrq_n_u64(a, N);
2227}
2228
2229template <int N>
2230EIGEN_STRONG_INLINE Packet4c plogical_shift_left(const Packet4c& a) {
2231 return vget_lane_s32(vreinterpret_s32_s8(vshl_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0);
2232}
2233template <int N>
2234EIGEN_STRONG_INLINE Packet8c plogical_shift_left(const Packet8c& a) {
2235 return vshl_n_s8(a, N);
2236}
2237template <int N>
2238EIGEN_STRONG_INLINE Packet16c plogical_shift_left(const Packet16c& a) {
2239 return vshlq_n_s8(a, N);
2240}
2241template <int N>
2242EIGEN_STRONG_INLINE Packet4uc plogical_shift_left(const Packet4uc& a) {
2243 return vget_lane_u32(vreinterpret_u32_u8(vshl_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0);
2244}
2245template <int N>
2246EIGEN_STRONG_INLINE Packet8uc plogical_shift_left(const Packet8uc& a) {
2247 return vshl_n_u8(a, N);
2248}
2249template <int N>
2250EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(const Packet16uc& a) {
2251 return vshlq_n_u8(a, N);
2252}
2253template <int N>
2254EIGEN_STRONG_INLINE Packet4s plogical_shift_left(const Packet4s& a) {
2255 return vshl_n_s16(a, N);
2256}
2257template <int N>
2258EIGEN_STRONG_INLINE Packet8s plogical_shift_left(const Packet8s& a) {
2259 return vshlq_n_s16(a, N);
2260}
2261template <int N>
2262EIGEN_STRONG_INLINE Packet4us plogical_shift_left(const Packet4us& a) {
2263 return vshl_n_u16(a, N);
2264}
2265template <int N>
2266EIGEN_STRONG_INLINE Packet8us plogical_shift_left(const Packet8us& a) {
2267 return vshlq_n_u16(a, N);
2268}
2269template <int N>
2270EIGEN_STRONG_INLINE Packet2i plogical_shift_left(const Packet2i& a) {
2271 return vshl_n_s32(a, N);
2272}
2273template <int N>
2274EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) {
2275 return vshlq_n_s32(a, N);
2276}
2277template <int N>
2278EIGEN_STRONG_INLINE Packet2ui plogical_shift_left(const Packet2ui& a) {
2279 return vshl_n_u32(a, N);
2280}
2281template <int N>
2282EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) {
2283 return vshlq_n_u32(a, N);
2284}
2285template <int N>
2286EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
2287 return vshlq_n_s64(a, N);
2288}
2289template <int N>
2290EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(const Packet2ul& a) {
2291 return vshlq_n_u64(a, N);
2292}
2293
2294template <>
2295EIGEN_STRONG_INLINE Packet2f pload<Packet2f>(const float* from) {
2296 EIGEN_DEBUG_ALIGNED_LOAD return vld1_f32(assume_aligned<unpacket_traits<Packet2f>::alignment>(from));
2297}
2298template <>
2299EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
2300 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(assume_aligned<unpacket_traits<Packet4f>::alignment>(from));
2301}
2302template <>
2303EIGEN_STRONG_INLINE Packet4c pload<Packet4c>(const int8_t* from) {
2304 Packet4c res;
2305 memcpy(&res, from, sizeof(Packet4c));
2306 return res;
2307}
2308template <>
2309EIGEN_STRONG_INLINE Packet8c pload<Packet8c>(const int8_t* from) {
2310 EIGEN_DEBUG_ALIGNED_LOAD return vld1_s8(assume_aligned<unpacket_traits<Packet8c>::alignment>(from));
2311}
2312template <>
2313EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(const int8_t* from) {
2314 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s8(assume_aligned<unpacket_traits<Packet16c>::alignment>(from));
2315}
2316template <>
2317EIGEN_STRONG_INLINE Packet4uc pload<Packet4uc>(const uint8_t* from) {
2318 Packet4uc res;
2319 memcpy(&res, from, sizeof(Packet4uc));
2320 return res;
2321}
2322template <>
2323EIGEN_STRONG_INLINE Packet8uc pload<Packet8uc>(const uint8_t* from) {
2324 EIGEN_DEBUG_ALIGNED_LOAD return vld1_u8(assume_aligned<unpacket_traits<Packet8uc>::alignment>(from));
2325}
2326template <>
2327EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(const uint8_t* from) {
2328 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u8(assume_aligned<unpacket_traits<Packet16uc>::alignment>(from));
2329}
2330template <>
2331EIGEN_STRONG_INLINE Packet4s pload<Packet4s>(const int16_t* from) {
2332 EIGEN_DEBUG_ALIGNED_LOAD return vld1_s16(assume_aligned<unpacket_traits<Packet4s>::alignment>(from));
2333}
2334template <>
2335EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(const int16_t* from) {
2336 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s16(assume_aligned<unpacket_traits<Packet8s>::alignment>(from));
2337}
2338template <>
2339EIGEN_STRONG_INLINE Packet4us pload<Packet4us>(const uint16_t* from) {
2340 EIGEN_DEBUG_ALIGNED_LOAD return vld1_u16(assume_aligned<unpacket_traits<Packet4us>::alignment>(from));
2341}
2342template <>
2343EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(const uint16_t* from) {
2344 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u16(assume_aligned<unpacket_traits<Packet8us>::alignment>(from));
2345}
2346template <>
2347EIGEN_STRONG_INLINE Packet2i pload<Packet2i>(const int32_t* from) {
2348 EIGEN_DEBUG_ALIGNED_LOAD return vld1_s32(assume_aligned<unpacket_traits<Packet2i>::alignment>(from));
2349}
2350template <>
2351EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) {
2352 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(assume_aligned<unpacket_traits<Packet4i>::alignment>(from));
2353}
2354template <>
2355EIGEN_STRONG_INLINE Packet2ui pload<Packet2ui>(const uint32_t* from) {
2356 EIGEN_DEBUG_ALIGNED_LOAD return vld1_u32(assume_aligned<unpacket_traits<Packet2ui>::alignment>(from));
2357}
2358template <>
2359EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(const uint32_t* from) {
2360 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u32(assume_aligned<unpacket_traits<Packet4ui>::alignment>(from));
2361}
2362template <>
2363EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(const int64_t* from) {
2364 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s64(assume_aligned<unpacket_traits<Packet2l>::alignment>(from));
2365}
2366template <>
2367EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(const uint64_t* from) {
2368 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u64(assume_aligned<unpacket_traits<Packet2ul>::alignment>(from));
2369}
2370
2371template <>
2372EIGEN_STRONG_INLINE Packet2f ploadu<Packet2f>(const float* from) {
2373 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f32(from);
2374}
2375template <>
2376EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
2377 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from);
2378}
2379template <>
2380EIGEN_STRONG_INLINE Packet4c ploadu<Packet4c>(const int8_t* from) {
2381 Packet4c res;
2382 memcpy(&res, from, sizeof(Packet4c));
2383 return res;
2384}
2385template <>
2386EIGEN_STRONG_INLINE Packet8c ploadu<Packet8c>(const int8_t* from) {
2387 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s8(from);
2388}
2389template <>
2390EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(const int8_t* from) {
2391 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s8(from);
2392}
2393template <>
2394EIGEN_STRONG_INLINE Packet4uc ploadu<Packet4uc>(const uint8_t* from) {
2395 Packet4uc res;
2396 memcpy(&res, from, sizeof(Packet4uc));
2397 return res;
2398}
2399template <>
2400EIGEN_STRONG_INLINE Packet8uc ploadu<Packet8uc>(const uint8_t* from) {
2401 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u8(from);
2402}
2403template <>
2404EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(const uint8_t* from) {
2405 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u8(from);
2406}
2407template <>
2408EIGEN_STRONG_INLINE Packet4s ploadu<Packet4s>(const int16_t* from) {
2409 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s16(from);
2410}
2411template <>
2412EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(const int16_t* from) {
2413 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s16(from);
2414}
2415template <>
2416EIGEN_STRONG_INLINE Packet4us ploadu<Packet4us>(const uint16_t* from) {
2417 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u16(from);
2418}
2419template <>
2420EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(const uint16_t* from) {
2421 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u16(from);
2422}
2423template <>
2424EIGEN_STRONG_INLINE Packet2i ploadu<Packet2i>(const int32_t* from) {
2425 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s32(from);
2426}
2427template <>
2428EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) {
2429 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from);
2430}
2431template <>
2432EIGEN_STRONG_INLINE Packet2ui ploadu<Packet2ui>(const uint32_t* from) {
2433 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u32(from);
2434}
2435template <>
2436EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(const uint32_t* from) {
2437 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u32(from);
2438}
2439template <>
2440EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(const int64_t* from) {
2441 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s64(from);
2442}
2443template <>
2444EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(const uint64_t* from) {
2445 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u64(from);
2446}
2447
2448template <>
2449EIGEN_STRONG_INLINE Packet2f ploaddup<Packet2f>(const float* from) {
2450 return vld1_dup_f32(from);
2451}
2452template <>
2453EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
2454 return vcombine_f32(vld1_dup_f32(from), vld1_dup_f32(from + 1));
2455}
2456template <>
2457EIGEN_STRONG_INLINE Packet4c ploaddup<Packet4c>(const int8_t* from) {
2458 const int8x8_t a = vreinterpret_s8_s32(vdup_n_s32(pload<Packet4c>(from)));
2459 return vget_lane_s32(vreinterpret_s32_s8(vzip_s8(a, a).val[0]), 0);
2460}
2461template <>
2462EIGEN_STRONG_INLINE Packet8c ploaddup<Packet8c>(const int8_t* from) {
2463 const int8x8_t a = vld1_s8(from);
2464 return vzip_s8(a, a).val[0];
2465}
2466template <>
2467EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(const int8_t* from) {
2468 const int8x8_t a = vld1_s8(from);
2469 const int8x8x2_t b = vzip_s8(a, a);
2470 return vcombine_s8(b.val[0], b.val[1]);
2471}
2472template <>
2473EIGEN_STRONG_INLINE Packet4uc ploaddup<Packet4uc>(const uint8_t* from) {
2474 const uint8x8_t a = vreinterpret_u8_u32(vdup_n_u32(pload<Packet4uc>(from)));
2475 return vget_lane_u32(vreinterpret_u32_u8(vzip_u8(a, a).val[0]), 0);
2476}
2477template <>
2478EIGEN_STRONG_INLINE Packet8uc ploaddup<Packet8uc>(const uint8_t* from) {
2479 const uint8x8_t a = vld1_u8(from);
2480 return vzip_u8(a, a).val[0];
2481}
2482template <>
2483EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(const uint8_t* from) {
2484 const uint8x8_t a = vld1_u8(from);
2485 const uint8x8x2_t b = vzip_u8(a, a);
2486 return vcombine_u8(b.val[0], b.val[1]);
2487}
2488template <>
2489EIGEN_STRONG_INLINE Packet4s ploaddup<Packet4s>(const int16_t* from) {
2490 return vreinterpret_s16_u32(
2491 vzip_u32(vreinterpret_u32_s16(vld1_dup_s16(from)), vreinterpret_u32_s16(vld1_dup_s16(from + 1))).val[0]);
2492}
2493template <>
2494EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(const int16_t* from) {
2495 const int16x4_t a = vld1_s16(from);
2496 const int16x4x2_t b = vzip_s16(a, a);
2497 return vcombine_s16(b.val[0], b.val[1]);
2498}
2499template <>
2500EIGEN_STRONG_INLINE Packet4us ploaddup<Packet4us>(const uint16_t* from) {
2501 return vreinterpret_u16_u32(
2502 vzip_u32(vreinterpret_u32_u16(vld1_dup_u16(from)), vreinterpret_u32_u16(vld1_dup_u16(from + 1))).val[0]);
2503}
2504template <>
2505EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(const uint16_t* from) {
2506 const uint16x4_t a = vld1_u16(from);
2507 const uint16x4x2_t b = vzip_u16(a, a);
2508 return vcombine_u16(b.val[0], b.val[1]);
2509}
2510template <>
2511EIGEN_STRONG_INLINE Packet2i ploaddup<Packet2i>(const int32_t* from) {
2512 return vld1_dup_s32(from);
2513}
2514template <>
2515EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from) {
2516 return vcombine_s32(vld1_dup_s32(from), vld1_dup_s32(from + 1));
2517}
2518template <>
2519EIGEN_STRONG_INLINE Packet2ui ploaddup<Packet2ui>(const uint32_t* from) {
2520 return vld1_dup_u32(from);
2521}
2522template <>
2523EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(const uint32_t* from) {
2524 return vcombine_u32(vld1_dup_u32(from), vld1_dup_u32(from + 1));
2525}
2526template <>
2527EIGEN_STRONG_INLINE Packet2l ploaddup<Packet2l>(const int64_t* from) {
2528 return vld1q_dup_s64(from);
2529}
2530template <>
2531EIGEN_STRONG_INLINE Packet2ul ploaddup<Packet2ul>(const uint64_t* from) {
2532 return vld1q_dup_u64(from);
2533}
2534
2535template <>
2536EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(const float* from) {
2537 return vld1q_dup_f32(from);
2538}
2539
2540// WORKAROUND: Apple Clang 17.0.0 (and Homebrew Clang 21.1.8) at -O0 optimization
2541// generate incorrect code for vld1_dup_[su]8, ignoring the pointer offset.
2542// We use vdup_n_s8(*from) to force a safe scalar load before broadcast.
2543EIGEN_ALWAYS_INLINE int8x8_t eigen_vld1_dup_s8(const int8_t* ptr) {
2544#if EIGEN_COMP_CLANGAPPLE && EIGEN_ARCH_ARM64
2545 return vdup_n_s8(*ptr);
2546#else
2547 return vld1_dup_s8(ptr);
2548#endif
2549}
2550
2551EIGEN_ALWAYS_INLINE uint8x8_t eigen_vld1_dup_u8(const uint8_t* ptr) {
2552#if EIGEN_COMP_CLANGAPPLE && EIGEN_ARCH_ARM64
2553 return vdup_n_u8(*ptr);
2554#else
2555 return vld1_dup_u8(ptr);
2556#endif
2557}
2558
2559template <>
2560EIGEN_STRONG_INLINE Packet4c ploadquad<Packet4c>(const int8_t* from) {
2561 return vget_lane_s32(vreinterpret_s32_s8(eigen_vld1_dup_s8(from)), 0);
2562}
2563template <>
2564EIGEN_STRONG_INLINE Packet8c ploadquad<Packet8c>(const int8_t* from) {
2565 return vreinterpret_s8_u32(
2566 vzip_u32(vreinterpret_u32_s8(eigen_vld1_dup_s8(from)), vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 1))).val[0]);
2567}
2568template <>
2569EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(const int8_t* from) {
2570 const int8x8_t a = vreinterpret_s8_u32(
2571 vzip_u32(vreinterpret_u32_s8(eigen_vld1_dup_s8(from)), vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 1))).val[0]);
2572 const int8x8_t b = vreinterpret_s8_u32(
2573 vzip_u32(vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 2)), vreinterpret_u32_s8(eigen_vld1_dup_s8(from + 3)))
2574 .val[0]);
2575 return vcombine_s8(a, b);
2576}
2577template <>
2578EIGEN_STRONG_INLINE Packet4uc ploadquad<Packet4uc>(const uint8_t* from) {
2579 return vget_lane_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from)), 0);
2580}
2581template <>
2582EIGEN_STRONG_INLINE Packet8uc ploadquad<Packet8uc>(const uint8_t* from) {
2583 return vreinterpret_u8_u32(
2584 vzip_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from)), vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 1))).val[0]);
2585}
2586template <>
2587EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(const uint8_t* from) {
2588 const uint8x8_t a = vreinterpret_u8_u32(
2589 vzip_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from)), vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 1))).val[0]);
2590 const uint8x8_t b = vreinterpret_u8_u32(
2591 vzip_u32(vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 2)), vreinterpret_u32_u8(eigen_vld1_dup_u8(from + 3)))
2592 .val[0]);
2593 return vcombine_u8(a, b);
2594}
2595template <>
2596EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(const int16_t* from) {
2597 return vcombine_s16(vld1_dup_s16(from), vld1_dup_s16(from + 1));
2598}
2599template <>
2600EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(const uint16_t* from) {
2601 return vcombine_u16(vld1_dup_u16(from), vld1_dup_u16(from + 1));
2602}
2603template <>
2604EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(const int32_t* from) {
2605 return vld1q_dup_s32(from);
2606}
2607template <>
2608EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(const uint32_t* from) {
2609 return vld1q_dup_u32(from);
2610}
2611
2612template <>
2613EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet2f& from) {
2614 EIGEN_DEBUG_ALIGNED_STORE vst1_f32(assume_aligned<unpacket_traits<Packet2f>::alignment>(to), from);
2615}
2616template <>
2617EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
2618 EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(assume_aligned<unpacket_traits<Packet4f>::alignment>(to), from);
2619}
2620template <>
2621EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet4c& from) {
2622 memcpy(to, &from, sizeof(from));
2623}
2624template <>
2625EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet8c& from) {
2626 EIGEN_DEBUG_ALIGNED_STORE vst1_s8(assume_aligned<unpacket_traits<Packet8c>::alignment>(to), from);
2627}
2628template <>
2629EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet16c& from) {
2630 EIGEN_DEBUG_ALIGNED_STORE vst1q_s8(assume_aligned<unpacket_traits<Packet16c>::alignment>(to), from);
2631}
2632template <>
2633EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet4uc& from) {
2634 memcpy(to, &from, sizeof(from));
2635}
2636template <>
2637EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet8uc& from) {
2638 EIGEN_DEBUG_ALIGNED_STORE vst1_u8(assume_aligned<unpacket_traits<Packet8uc>::alignment>(to), from);
2639}
2640template <>
2641EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet16uc& from) {
2642 EIGEN_DEBUG_ALIGNED_STORE vst1q_u8(assume_aligned<unpacket_traits<Packet16uc>::alignment>(to), from);
2643}
2644template <>
2645EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet4s& from) {
2646 EIGEN_DEBUG_ALIGNED_STORE vst1_s16(assume_aligned<unpacket_traits<Packet4s>::alignment>(to), from);
2647}
2648template <>
2649EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet8s& from) {
2650 EIGEN_DEBUG_ALIGNED_STORE vst1q_s16(assume_aligned<unpacket_traits<Packet8s>::alignment>(to), from);
2651}
2652template <>
2653EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet4us& from) {
2654 EIGEN_DEBUG_ALIGNED_STORE vst1_u16(assume_aligned<unpacket_traits<Packet4us>::alignment>(to), from);
2655}
2656template <>
2657EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet8us& from) {
2658 EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(assume_aligned<unpacket_traits<Packet8us>::alignment>(to), from);
2659}
2660template <>
2661EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet2i& from) {
2662 EIGEN_DEBUG_ALIGNED_STORE vst1_s32(assume_aligned<unpacket_traits<Packet2i>::alignment>(to), from);
2663}
2664template <>
2665EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) {
2666 EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(assume_aligned<unpacket_traits<Packet4i>::alignment>(to), from);
2667}
2668template <>
2669EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet2ui& from) {
2670 EIGEN_DEBUG_ALIGNED_STORE vst1_u32(assume_aligned<unpacket_traits<Packet2ui>::alignment>(to), from);
2671}
2672template <>
2673EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet4ui& from) {
2674 EIGEN_DEBUG_ALIGNED_STORE vst1q_u32(assume_aligned<unpacket_traits<Packet4ui>::alignment>(to), from);
2675}
2676template <>
2677EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet2l& from) {
2678 EIGEN_DEBUG_ALIGNED_STORE vst1q_s64(assume_aligned<unpacket_traits<Packet2l>::alignment>(to), from);
2679}
2680template <>
2681EIGEN_STRONG_INLINE void pstore<uint64_t>(uint64_t* to, const Packet2ul& from) {
2682 EIGEN_DEBUG_ALIGNED_STORE vst1q_u64(assume_aligned<unpacket_traits<Packet2ul>::alignment>(to), from);
2683}
2684
2685template <>
2686EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet2f& from) {
2687 EIGEN_DEBUG_UNALIGNED_STORE vst1_f32(to, from);
2688}
2689template <>
2690EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
2691 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from);
2692}
2693template <>
2694EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet4c& from) {
2695 memcpy(to, &from, sizeof(from));
2696}
2697template <>
2698EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet8c& from) {
2699 EIGEN_DEBUG_UNALIGNED_STORE vst1_s8(to, from);
2700}
2701template <>
2702EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet16c& from) {
2703 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s8(to, from);
2704}
2705template <>
2706EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet4uc& from) {
2707 memcpy(to, &from, sizeof(from));
2708}
2709template <>
2710EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet8uc& from) {
2711 EIGEN_DEBUG_UNALIGNED_STORE vst1_u8(to, from);
2712}
2713template <>
2714EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet16uc& from) {
2715 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u8(to, from);
2716}
2717template <>
2718EIGEN_STRONG_INLINE void pstoreu<int16_t>(int16_t* to, const Packet4s& from) {
2719 EIGEN_DEBUG_UNALIGNED_STORE vst1_s16(to, from);
2720}
2721template <>
2722EIGEN_STRONG_INLINE void pstoreu<int16_t>(int16_t* to, const Packet8s& from) {
2723 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s16(to, from);
2724}
2725template <>
2726EIGEN_STRONG_INLINE void pstoreu<uint16_t>(uint16_t* to, const Packet4us& from) {
2727 EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(to, from);
2728}
2729template <>
2730EIGEN_STRONG_INLINE void pstoreu<uint16_t>(uint16_t* to, const Packet8us& from) {
2731 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(to, from);
2732}
2733template <>
2734EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet2i& from) {
2735 EIGEN_DEBUG_UNALIGNED_STORE vst1_s32(to, from);
2736}
2737template <>
2738EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from) {
2739 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from);
2740}
2741template <>
2742EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet2ui& from) {
2743 EIGEN_DEBUG_UNALIGNED_STORE vst1_u32(to, from);
2744}
2745template <>
2746EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet4ui& from) {
2747 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u32(to, from);
2748}
2749template <>
2750EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet2l& from) {
2751 EIGEN_DEBUG_UNALIGNED_STORE vst1q_s64(to, from);
2752}
2753template <>
2754EIGEN_STRONG_INLINE void pstoreu<uint64_t>(uint64_t* to, const Packet2ul& from) {
2755 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u64(to, from);
2756}
2757
2758template <>
2759EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather<float, Packet2f>(const float* from, Index stride) {
2760 Packet2f res = vld1_dup_f32(from);
2761 res = vld1_lane_f32(from + 1 * stride, res, 1);
2762 return res;
2763}
2764template <>
2765EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
2766 if (stride == 2) {
2767 // Overlap the loads so the second ends at the last gathered coefficient, from[6].
2768#if EIGEN_ARCH_ARM64
2769 // lo = [0 1 2 3], hi = [3 4 5 6]. GCC turns the equivalent vuzp2q(vrev64q(lo), hi) into a slower TBL.
2770 Packet4f lo = vld1q_f32(from);
2771 Packet4f hi = vld1q_f32(from + 3);
2772 return vcombine_f32(vget_low_f32(vuzp1q_f32(lo, lo)), vget_high_f32(vuzp2q_f32(hi, hi)));
2773#else
2774 return vcombine_f32(vld2_f32(from).val[0], vld2_f32(from + 3).val[1]);
2775#endif
2776 }
2777 Packet4f res = vld1q_dup_f32(from);
2778 res = vld1q_lane_f32(from + 1 * stride, res, 1);
2779 res = vld1q_lane_f32(from + 2 * stride, res, 2);
2780 res = vld1q_lane_f32(from + 3 * stride, res, 3);
2781 return res;
2782}
2783template <>
2784EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather<int8_t, Packet4c>(const int8_t* from, Index stride) {
2785 Packet4c res;
2786 for (int i = 0; i != 4; i++) reinterpret_cast<int8_t*>(&res)[i] = *(from + i * stride);
2787 return res;
2788}
2789template <>
2790EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather<int8_t, Packet8c>(const int8_t* from, Index stride) {
2791 Packet8c res = vld1_dup_s8(from);
2792 res = vld1_lane_s8(from + 1 * stride, res, 1);
2793 res = vld1_lane_s8(from + 2 * stride, res, 2);
2794 res = vld1_lane_s8(from + 3 * stride, res, 3);
2795 res = vld1_lane_s8(from + 4 * stride, res, 4);
2796 res = vld1_lane_s8(from + 5 * stride, res, 5);
2797 res = vld1_lane_s8(from + 6 * stride, res, 6);
2798 res = vld1_lane_s8(from + 7 * stride, res, 7);
2799 return res;
2800}
2801template <>
2802EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(const int8_t* from, Index stride) {
2803 Packet16c res = vld1q_dup_s8(from);
2804 res = vld1q_lane_s8(from + 1 * stride, res, 1);
2805 res = vld1q_lane_s8(from + 2 * stride, res, 2);
2806 res = vld1q_lane_s8(from + 3 * stride, res, 3);
2807 res = vld1q_lane_s8(from + 4 * stride, res, 4);
2808 res = vld1q_lane_s8(from + 5 * stride, res, 5);
2809 res = vld1q_lane_s8(from + 6 * stride, res, 6);
2810 res = vld1q_lane_s8(from + 7 * stride, res, 7);
2811 res = vld1q_lane_s8(from + 8 * stride, res, 8);
2812 res = vld1q_lane_s8(from + 9 * stride, res, 9);
2813 res = vld1q_lane_s8(from + 10 * stride, res, 10);
2814 res = vld1q_lane_s8(from + 11 * stride, res, 11);
2815 res = vld1q_lane_s8(from + 12 * stride, res, 12);
2816 res = vld1q_lane_s8(from + 13 * stride, res, 13);
2817 res = vld1q_lane_s8(from + 14 * stride, res, 14);
2818 res = vld1q_lane_s8(from + 15 * stride, res, 15);
2819 return res;
2820}
2821template <>
2822EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather<uint8_t, Packet4uc>(const uint8_t* from, Index stride) {
2823 Packet4uc res;
2824 for (int i = 0; i != 4; i++) reinterpret_cast<uint8_t*>(&res)[i] = *(from + i * stride);
2825 return res;
2826}
2827template <>
2828EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather<uint8_t, Packet8uc>(const uint8_t* from, Index stride) {
2829 Packet8uc res = vld1_dup_u8(from);
2830 res = vld1_lane_u8(from + 1 * stride, res, 1);
2831 res = vld1_lane_u8(from + 2 * stride, res, 2);
2832 res = vld1_lane_u8(from + 3 * stride, res, 3);
2833 res = vld1_lane_u8(from + 4 * stride, res, 4);
2834 res = vld1_lane_u8(from + 5 * stride, res, 5);
2835 res = vld1_lane_u8(from + 6 * stride, res, 6);
2836 res = vld1_lane_u8(from + 7 * stride, res, 7);
2837 return res;
2838}
2839template <>
2840EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(const uint8_t* from, Index stride) {
2841 Packet16uc res = vld1q_dup_u8(from);
2842 res = vld1q_lane_u8(from + 1 * stride, res, 1);
2843 res = vld1q_lane_u8(from + 2 * stride, res, 2);
2844 res = vld1q_lane_u8(from + 3 * stride, res, 3);
2845 res = vld1q_lane_u8(from + 4 * stride, res, 4);
2846 res = vld1q_lane_u8(from + 5 * stride, res, 5);
2847 res = vld1q_lane_u8(from + 6 * stride, res, 6);
2848 res = vld1q_lane_u8(from + 7 * stride, res, 7);
2849 res = vld1q_lane_u8(from + 8 * stride, res, 8);
2850 res = vld1q_lane_u8(from + 9 * stride, res, 9);
2851 res = vld1q_lane_u8(from + 10 * stride, res, 10);
2852 res = vld1q_lane_u8(from + 11 * stride, res, 11);
2853 res = vld1q_lane_u8(from + 12 * stride, res, 12);
2854 res = vld1q_lane_u8(from + 13 * stride, res, 13);
2855 res = vld1q_lane_u8(from + 14 * stride, res, 14);
2856 res = vld1q_lane_u8(from + 15 * stride, res, 15);
2857 return res;
2858}
2859template <>
2860EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather<int16_t, Packet4s>(const int16_t* from, Index stride) {
2861 Packet4s res = vld1_dup_s16(from);
2862 res = vld1_lane_s16(from + 1 * stride, res, 1);
2863 res = vld1_lane_s16(from + 2 * stride, res, 2);
2864 res = vld1_lane_s16(from + 3 * stride, res, 3);
2865 return res;
2866}
2867template <>
2868EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(const int16_t* from, Index stride) {
2869 Packet8s res = vld1q_dup_s16(from);
2870 res = vld1q_lane_s16(from + 1 * stride, res, 1);
2871 res = vld1q_lane_s16(from + 2 * stride, res, 2);
2872 res = vld1q_lane_s16(from + 3 * stride, res, 3);
2873 res = vld1q_lane_s16(from + 4 * stride, res, 4);
2874 res = vld1q_lane_s16(from + 5 * stride, res, 5);
2875 res = vld1q_lane_s16(from + 6 * stride, res, 6);
2876 res = vld1q_lane_s16(from + 7 * stride, res, 7);
2877 return res;
2878}
2879template <>
2880EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather<uint16_t, Packet4us>(const uint16_t* from, Index stride) {
2881 Packet4us res = vld1_dup_u16(from);
2882 res = vld1_lane_u16(from + 1 * stride, res, 1);
2883 res = vld1_lane_u16(from + 2 * stride, res, 2);
2884 res = vld1_lane_u16(from + 3 * stride, res, 3);
2885 return res;
2886}
2887template <>
2888EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(const uint16_t* from, Index stride) {
2889 Packet8us res = vld1q_dup_u16(from);
2890 res = vld1q_lane_u16(from + 1 * stride, res, 1);
2891 res = vld1q_lane_u16(from + 2 * stride, res, 2);
2892 res = vld1q_lane_u16(from + 3 * stride, res, 3);
2893 res = vld1q_lane_u16(from + 4 * stride, res, 4);
2894 res = vld1q_lane_u16(from + 5 * stride, res, 5);
2895 res = vld1q_lane_u16(from + 6 * stride, res, 6);
2896 res = vld1q_lane_u16(from + 7 * stride, res, 7);
2897 return res;
2898}
2899template <>
2900EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather<int32_t, Packet2i>(const int32_t* from, Index stride) {
2901 Packet2i res = vld1_dup_s32(from);
2902 res = vld1_lane_s32(from + 1 * stride, res, 1);
2903 return res;
2904}
2905template <>
2906EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride) {
2907 Packet4i res = vld1q_dup_s32(from);
2908 res = vld1q_lane_s32(from + 1 * stride, res, 1);
2909 res = vld1q_lane_s32(from + 2 * stride, res, 2);
2910 res = vld1q_lane_s32(from + 3 * stride, res, 3);
2911 return res;
2912}
2913template <>
2914EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather<uint32_t, Packet2ui>(const uint32_t* from, Index stride) {
2915 Packet2ui res = vld1_dup_u32(from);
2916 res = vld1_lane_u32(from + 1 * stride, res, 1);
2917 return res;
2918}
2919template <>
2920EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(const uint32_t* from, Index stride) {
2921 Packet4ui res = vld1q_dup_u32(from);
2922 res = vld1q_lane_u32(from + 1 * stride, res, 1);
2923 res = vld1q_lane_u32(from + 2 * stride, res, 2);
2924 res = vld1q_lane_u32(from + 3 * stride, res, 3);
2925 return res;
2926}
2927template <>
2928EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(const int64_t* from, Index stride) {
2929 Packet2l res = vld1q_dup_s64(from);
2930 res = vld1q_lane_s64(from + 1 * stride, res, 1);
2931 return res;
2932}
2933template <>
2934EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(const uint64_t* from, Index stride) {
2935 Packet2ul res = vld1q_dup_u64(from);
2936 res = vld1q_lane_u64(from + 1 * stride, res, 1);
2937 return res;
2938}
2939
2940template <>
2941EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet2f>(float* to, const Packet2f& from, Index stride) {
2942 vst1_lane_f32(to + stride * 0, from, 0);
2943 vst1_lane_f32(to + stride * 1, from, 1);
2944}
2945template <>
2946EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
2947 vst1q_lane_f32(to + stride * 0, from, 0);
2948 vst1q_lane_f32(to + stride * 1, from, 1);
2949 vst1q_lane_f32(to + stride * 2, from, 2);
2950 vst1q_lane_f32(to + stride * 3, from, 3);
2951}
2952template <>
2953EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet4c>(int8_t* to, const Packet4c& from, Index stride) {
2954 for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast<const int8_t*>(&from)[i];
2955}
2956template <>
2957EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet8c>(int8_t* to, const Packet8c& from, Index stride) {
2958 vst1_lane_s8(to + stride * 0, from, 0);
2959 vst1_lane_s8(to + stride * 1, from, 1);
2960 vst1_lane_s8(to + stride * 2, from, 2);
2961 vst1_lane_s8(to + stride * 3, from, 3);
2962 vst1_lane_s8(to + stride * 4, from, 4);
2963 vst1_lane_s8(to + stride * 5, from, 5);
2964 vst1_lane_s8(to + stride * 6, from, 6);
2965 vst1_lane_s8(to + stride * 7, from, 7);
2966}
2967template <>
2968EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet16c>(int8_t* to, const Packet16c& from,
2969 Index stride) {
2970 vst1q_lane_s8(to + stride * 0, from, 0);
2971 vst1q_lane_s8(to + stride * 1, from, 1);
2972 vst1q_lane_s8(to + stride * 2, from, 2);
2973 vst1q_lane_s8(to + stride * 3, from, 3);
2974 vst1q_lane_s8(to + stride * 4, from, 4);
2975 vst1q_lane_s8(to + stride * 5, from, 5);
2976 vst1q_lane_s8(to + stride * 6, from, 6);
2977 vst1q_lane_s8(to + stride * 7, from, 7);
2978 vst1q_lane_s8(to + stride * 8, from, 8);
2979 vst1q_lane_s8(to + stride * 9, from, 9);
2980 vst1q_lane_s8(to + stride * 10, from, 10);
2981 vst1q_lane_s8(to + stride * 11, from, 11);
2982 vst1q_lane_s8(to + stride * 12, from, 12);
2983 vst1q_lane_s8(to + stride * 13, from, 13);
2984 vst1q_lane_s8(to + stride * 14, from, 14);
2985 vst1q_lane_s8(to + stride * 15, from, 15);
2986}
2987template <>
2988EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet4uc>(uint8_t* to, const Packet4uc& from,
2989 Index stride) {
2990 for (int i = 0; i != 4; i++) *(to + i * stride) = reinterpret_cast<const uint8_t*>(&from)[i];
2991}
2992template <>
2993EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet8uc>(uint8_t* to, const Packet8uc& from,
2994 Index stride) {
2995 vst1_lane_u8(to + stride * 0, from, 0);
2996 vst1_lane_u8(to + stride * 1, from, 1);
2997 vst1_lane_u8(to + stride * 2, from, 2);
2998 vst1_lane_u8(to + stride * 3, from, 3);
2999 vst1_lane_u8(to + stride * 4, from, 4);
3000 vst1_lane_u8(to + stride * 5, from, 5);
3001 vst1_lane_u8(to + stride * 6, from, 6);
3002 vst1_lane_u8(to + stride * 7, from, 7);
3003}
3004template <>
3005EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet16uc>(uint8_t* to, const Packet16uc& from,
3006 Index stride) {
3007 vst1q_lane_u8(to + stride * 0, from, 0);
3008 vst1q_lane_u8(to + stride * 1, from, 1);
3009 vst1q_lane_u8(to + stride * 2, from, 2);
3010 vst1q_lane_u8(to + stride * 3, from, 3);
3011 vst1q_lane_u8(to + stride * 4, from, 4);
3012 vst1q_lane_u8(to + stride * 5, from, 5);
3013 vst1q_lane_u8(to + stride * 6, from, 6);
3014 vst1q_lane_u8(to + stride * 7, from, 7);
3015 vst1q_lane_u8(to + stride * 8, from, 8);
3016 vst1q_lane_u8(to + stride * 9, from, 9);
3017 vst1q_lane_u8(to + stride * 10, from, 10);
3018 vst1q_lane_u8(to + stride * 11, from, 11);
3019 vst1q_lane_u8(to + stride * 12, from, 12);
3020 vst1q_lane_u8(to + stride * 13, from, 13);
3021 vst1q_lane_u8(to + stride * 14, from, 14);
3022 vst1q_lane_u8(to + stride * 15, from, 15);
3023}
3024template <>
3025EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet4s>(int16_t* to, const Packet4s& from,
3026 Index stride) {
3027 vst1_lane_s16(to + stride * 0, from, 0);
3028 vst1_lane_s16(to + stride * 1, from, 1);
3029 vst1_lane_s16(to + stride * 2, from, 2);
3030 vst1_lane_s16(to + stride * 3, from, 3);
3031}
3032template <>
3033EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet8s>(int16_t* to, const Packet8s& from,
3034 Index stride) {
3035 vst1q_lane_s16(to + stride * 0, from, 0);
3036 vst1q_lane_s16(to + stride * 1, from, 1);
3037 vst1q_lane_s16(to + stride * 2, from, 2);
3038 vst1q_lane_s16(to + stride * 3, from, 3);
3039 vst1q_lane_s16(to + stride * 4, from, 4);
3040 vst1q_lane_s16(to + stride * 5, from, 5);
3041 vst1q_lane_s16(to + stride * 6, from, 6);
3042 vst1q_lane_s16(to + stride * 7, from, 7);
3043}
3044template <>
3045EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet4us>(uint16_t* to, const Packet4us& from,
3046 Index stride) {
3047 vst1_lane_u16(to + stride * 0, from, 0);
3048 vst1_lane_u16(to + stride * 1, from, 1);
3049 vst1_lane_u16(to + stride * 2, from, 2);
3050 vst1_lane_u16(to + stride * 3, from, 3);
3051}
3052template <>
3053EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet8us>(uint16_t* to, const Packet8us& from,
3054 Index stride) {
3055 vst1q_lane_u16(to + stride * 0, from, 0);
3056 vst1q_lane_u16(to + stride * 1, from, 1);
3057 vst1q_lane_u16(to + stride * 2, from, 2);
3058 vst1q_lane_u16(to + stride * 3, from, 3);
3059 vst1q_lane_u16(to + stride * 4, from, 4);
3060 vst1q_lane_u16(to + stride * 5, from, 5);
3061 vst1q_lane_u16(to + stride * 6, from, 6);
3062 vst1q_lane_u16(to + stride * 7, from, 7);
3063}
3064template <>
3065EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet2i>(int32_t* to, const Packet2i& from,
3066 Index stride) {
3067 vst1_lane_s32(to + stride * 0, from, 0);
3068 vst1_lane_s32(to + stride * 1, from, 1);
3069}
3070template <>
3071EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from,
3072 Index stride) {
3073 vst1q_lane_s32(to + stride * 0, from, 0);
3074 vst1q_lane_s32(to + stride * 1, from, 1);
3075 vst1q_lane_s32(to + stride * 2, from, 2);
3076 vst1q_lane_s32(to + stride * 3, from, 3);
3077}
3078template <>
3079EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet2ui>(uint32_t* to, const Packet2ui& from,
3080 Index stride) {
3081 vst1_lane_u32(to + stride * 0, from, 0);
3082 vst1_lane_u32(to + stride * 1, from, 1);
3083}
3084template <>
3085EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from,
3086 Index stride) {
3087 vst1q_lane_u32(to + stride * 0, from, 0);
3088 vst1q_lane_u32(to + stride * 1, from, 1);
3089 vst1q_lane_u32(to + stride * 2, from, 2);
3090 vst1q_lane_u32(to + stride * 3, from, 3);
3091}
3092template <>
3093EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int64_t, Packet2l>(int64_t* to, const Packet2l& from,
3094 Index stride) {
3095 vst1q_lane_s64(to + stride * 0, from, 0);
3096 vst1q_lane_s64(to + stride * 1, from, 1);
3097}
3098template <>
3099EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint64_t, Packet2ul>(uint64_t* to, const Packet2ul& from,
3100 Index stride) {
3101 vst1q_lane_u64(to + stride * 0, from, 0);
3102 vst1q_lane_u64(to + stride * 1, from, 1);
3103}
3104
3105template <>
3106EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
3107 EIGEN_ARM_PREFETCH(addr);
3108}
3109template <>
3110EIGEN_STRONG_INLINE void prefetch<int8_t>(const int8_t* addr) {
3111 EIGEN_ARM_PREFETCH(addr);
3112}
3113template <>
3114EIGEN_STRONG_INLINE void prefetch<uint8_t>(const uint8_t* addr) {
3115 EIGEN_ARM_PREFETCH(addr);
3116}
3117template <>
3118EIGEN_STRONG_INLINE void prefetch<int16_t>(const int16_t* addr) {
3119 EIGEN_ARM_PREFETCH(addr);
3120}
3121template <>
3122EIGEN_STRONG_INLINE void prefetch<uint16_t>(const uint16_t* addr) {
3123 EIGEN_ARM_PREFETCH(addr);
3124}
3125template <>
3126EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) {
3127 EIGEN_ARM_PREFETCH(addr);
3128}
3129template <>
3130EIGEN_STRONG_INLINE void prefetch<uint32_t>(const uint32_t* addr) {
3131 EIGEN_ARM_PREFETCH(addr);
3132}
3133template <>
3134EIGEN_STRONG_INLINE void prefetch<int64_t>(const int64_t* addr) {
3135 EIGEN_ARM_PREFETCH(addr);
3136}
3137template <>
3138EIGEN_STRONG_INLINE void prefetch<uint64_t>(const uint64_t* addr) {
3139 EIGEN_ARM_PREFETCH(addr);
3140}
3141
3142template <>
3143EIGEN_STRONG_INLINE float pfirst<Packet2f>(const Packet2f& a) {
3144 return vget_lane_f32(a, 0);
3145}
3146template <>
3147EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
3148 return vgetq_lane_f32(a, 0);
3149}
3150template <>
3151EIGEN_STRONG_INLINE int8_t pfirst<Packet4c>(const Packet4c& a) {
3152 return static_cast<int8_t>(a & 0xff);
3153}
3154template <>
3155EIGEN_STRONG_INLINE int8_t pfirst<Packet8c>(const Packet8c& a) {
3156 return vget_lane_s8(a, 0);
3157}
3158template <>
3159EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(const Packet16c& a) {
3160 return vgetq_lane_s8(a, 0);
3161}
3162template <>
3163EIGEN_STRONG_INLINE uint8_t pfirst<Packet4uc>(const Packet4uc& a) {
3164 return static_cast<uint8_t>(a & 0xff);
3165}
3166template <>
3167EIGEN_STRONG_INLINE uint8_t pfirst<Packet8uc>(const Packet8uc& a) {
3168 return vget_lane_u8(a, 0);
3169}
3170template <>
3171EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(const Packet16uc& a) {
3172 return vgetq_lane_u8(a, 0);
3173}
3174template <>
3175EIGEN_STRONG_INLINE int16_t pfirst<Packet4s>(const Packet4s& a) {
3176 return vget_lane_s16(a, 0);
3177}
3178template <>
3179EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(const Packet8s& a) {
3180 return vgetq_lane_s16(a, 0);
3181}
3182template <>
3183EIGEN_STRONG_INLINE uint16_t pfirst<Packet4us>(const Packet4us& a) {
3184 return vget_lane_u16(a, 0);
3185}
3186template <>
3187EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(const Packet8us& a) {
3188 return vgetq_lane_u16(a, 0);
3189}
3190template <>
3191EIGEN_STRONG_INLINE int32_t pfirst<Packet2i>(const Packet2i& a) {
3192 return vget_lane_s32(a, 0);
3193}
3194template <>
3195EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) {
3196 return vgetq_lane_s32(a, 0);
3197}
3198template <>
3199EIGEN_STRONG_INLINE uint32_t pfirst<Packet2ui>(const Packet2ui& a) {
3200 return vget_lane_u32(a, 0);
3201}
3202template <>
3203EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(const Packet4ui& a) {
3204 return vgetq_lane_u32(a, 0);
3205}
3206template <>
3207EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(const Packet2l& a) {
3208 return vgetq_lane_s64(a, 0);
3209}
3210template <>
3211EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(const Packet2ul& a) {
3212 return vgetq_lane_u64(a, 0);
3213}
3214
3215template <>
3216EIGEN_STRONG_INLINE Packet2f preverse(const Packet2f& a) {
3217 return vrev64_f32(a);
3218}
3219template <>
3220EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
3221 const float32x4_t a_r64 = vrev64q_f32(a);
3222 return vcombine_f32(vget_high_f32(a_r64), vget_low_f32(a_r64));
3223}
3224template <>
3225EIGEN_STRONG_INLINE Packet4c preverse(const Packet4c& a) {
3226 return vget_lane_s32(vreinterpret_s32_s8(vrev64_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
3227}
3228template <>
3229EIGEN_STRONG_INLINE Packet8c preverse(const Packet8c& a) {
3230 return vrev64_s8(a);
3231}
3232template <>
3233EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) {
3234 const int8x16_t a_r64 = vrev64q_s8(a);
3235 return vcombine_s8(vget_high_s8(a_r64), vget_low_s8(a_r64));
3236}
3237template <>
3238EIGEN_STRONG_INLINE Packet4uc preverse(const Packet4uc& a) {
3239 return vget_lane_u32(vreinterpret_u32_u8(vrev64_u8(vreinterpret_u8_u32(vdup_n_u32(a)))), 0);
3240}
3241template <>
3242EIGEN_STRONG_INLINE Packet8uc preverse(const Packet8uc& a) {
3243 return vrev64_u8(a);
3244}
3245template <>
3246EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) {
3247 const uint8x16_t a_r64 = vrev64q_u8(a);
3248 return vcombine_u8(vget_high_u8(a_r64), vget_low_u8(a_r64));
3249}
3250template <>
3251EIGEN_STRONG_INLINE Packet4s preverse(const Packet4s& a) {
3252 return vrev64_s16(a);
3253}
3254template <>
3255EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a) {
3256 const int16x8_t a_r64 = vrev64q_s16(a);
3257 return vcombine_s16(vget_high_s16(a_r64), vget_low_s16(a_r64));
3258}
3259template <>
3260EIGEN_STRONG_INLINE Packet4us preverse(const Packet4us& a) {
3261 return vrev64_u16(a);
3262}
3263template <>
3264EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) {
3265 const uint16x8_t a_r64 = vrev64q_u16(a);
3266 return vcombine_u16(vget_high_u16(a_r64), vget_low_u16(a_r64));
3267}
3268template <>
3269EIGEN_STRONG_INLINE Packet2i preverse(const Packet2i& a) {
3270 return vrev64_s32(a);
3271}
3272template <>
3273EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
3274 const int32x4_t a_r64 = vrev64q_s32(a);
3275 return vcombine_s32(vget_high_s32(a_r64), vget_low_s32(a_r64));
3276}
3277template <>
3278EIGEN_STRONG_INLINE Packet2ui preverse(const Packet2ui& a) {
3279 return vrev64_u32(a);
3280}
3281template <>
3282EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a) {
3283 const uint32x4_t a_r64 = vrev64q_u32(a);
3284 return vcombine_u32(vget_high_u32(a_r64), vget_low_u32(a_r64));
3285}
3286template <>
3287EIGEN_STRONG_INLINE Packet2l preverse(const Packet2l& a) {
3288 return vcombine_s64(vget_high_s64(a), vget_low_s64(a));
3289}
3290template <>
3291EIGEN_STRONG_INLINE Packet2ul preverse(const Packet2ul& a) {
3292 return vcombine_u64(vget_high_u64(a), vget_low_u64(a));
3293}
3294
3295template <>
3296EIGEN_STRONG_INLINE Packet2f pabs(const Packet2f& a) {
3297 return vabs_f32(a);
3298}
3299template <>
3300EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) {
3301 return vabsq_f32(a);
3302}
3303template <>
3304EIGEN_STRONG_INLINE Packet4c pabs<Packet4c>(const Packet4c& a) {
3305 return vget_lane_s32(vreinterpret_s32_s8(vabs_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0);
3306}
3307template <>
3308EIGEN_STRONG_INLINE Packet8c pabs(const Packet8c& a) {
3309 return vabs_s8(a);
3310}
3311template <>
3312EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) {
3313 return vabsq_s8(a);
3314}
3315template <>
3316EIGEN_STRONG_INLINE Packet4s pabs(const Packet4s& a) {
3317 return vabs_s16(a);
3318}
3319template <>
3320EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) {
3321 return vabsq_s16(a);
3322}
3323template <>
3324EIGEN_STRONG_INLINE Packet2i pabs(const Packet2i& a) {
3325 return vabs_s32(a);
3326}
3327template <>
3328EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) {
3329 return vabsq_s32(a);
3330}
3331template <>
3332EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) {
3333#if EIGEN_ARCH_ARM64
3334 return vabsq_s64(a);
3335#else
3336 // NOTE: From <https://graphics.stanford.edu/~seander/bithacks.html#IntegerAbs>. The addition is
3337 // done in the unsigned domain: at `a == INT64_MIN`, `mask == -1`, so the signed `a + mask` is UB
3338 // (it wraps to INT64_MIN on every real target, which is exactly what's needed here).
3339 const int64x2_t mask = vshrq_n_s64(a, 63);
3340 const uint64x2_t sum = vaddq_u64(vreinterpretq_u64_s64(a), vreinterpretq_u64_s64(mask));
3341 return veorq_s64(vreinterpretq_s64_u64(sum), mask);
3342#endif
3343}
3344template <>
3345EIGEN_STRONG_INLINE Packet2f psignbit(const Packet2f& a) {
3346 return vreinterpret_f32_s32(vshr_n_s32(vreinterpret_s32_f32(a), 31));
3347}
3348template <>
3349EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) {
3350 return vreinterpretq_f32_s32(vshrq_n_s32(vreinterpretq_s32_f32(a), 31));
3351}
3352
3353template <>
3354EIGEN_STRONG_INLINE Packet2f pfrexp<Packet2f>(const Packet2f& a, Packet2f& exponent) {
3355 return pfrexp_generic(a, exponent);
3356}
3357template <>
3358EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) {
3359 return pfrexp_generic(a, exponent);
3360}
3361
3362template <>
3363EIGEN_STRONG_INLINE Packet2f pldexp<Packet2f>(const Packet2f& a, const Packet2f& exponent) {
3364 return pldexp_generic(a, exponent);
3365}
3366template <>
3367EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
3368 return pldexp_generic(a, exponent);
3369}
3370
3371#if EIGEN_ARCH_ARM64
3372template <>
3373EIGEN_STRONG_INLINE float predux<Packet2f>(const Packet2f& a) {
3374 return vaddv_f32(a);
3375}
3376template <>
3377EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
3378 return vaddvq_f32(a);
3379}
3380
3381template <>
3382EIGEN_STRONG_INLINE Index predux_count(const Packet4f& a) {
3383 const uint32x4_t nonzero = vbicq_u32(vdupq_n_u32(1), vceqq_f32(a, vdupq_n_f32(0.0f)));
3384 return static_cast<Index>(vaddvq_u32(nonzero));
3385}
3386#else
3387template <>
3388EIGEN_STRONG_INLINE float predux<Packet2f>(const Packet2f& a) {
3389 return vget_lane_f32(vpadd_f32(a, a), 0);
3390}
3391template <>
3392EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
3393 const float32x2_t sum = vadd_f32(vget_low_f32(a), vget_high_f32(a));
3394 return vget_lane_f32(vpadd_f32(sum, sum), 0);
3395}
3396#endif
3397template <>
3398EIGEN_STRONG_INLINE int8_t predux<Packet4c>(const Packet4c& a) {
3399 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
3400 int8x8_t sum = vpadd_s8(a_dup, a_dup);
3401 sum = vpadd_s8(sum, sum);
3402 return vget_lane_s8(sum, 0);
3403}
3404#if EIGEN_ARCH_ARM64
3405template <>
3406EIGEN_STRONG_INLINE int8_t predux<Packet8c>(const Packet8c& a) {
3407 return vaddv_s8(a);
3408}
3409template <>
3410EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a) {
3411 return vaddvq_s8(a);
3412}
3413#else
3414template <>
3415EIGEN_STRONG_INLINE int8_t predux<Packet8c>(const Packet8c& a) {
3416 int8x8_t sum = vpadd_s8(a, a);
3417 sum = vpadd_s8(sum, sum);
3418 sum = vpadd_s8(sum, sum);
3419 return vget_lane_s8(sum, 0);
3420}
3421template <>
3422EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a) {
3423 int8x8_t sum = vadd_s8(vget_low_s8(a), vget_high_s8(a));
3424 sum = vpadd_s8(sum, sum);
3425 sum = vpadd_s8(sum, sum);
3426 sum = vpadd_s8(sum, sum);
3427 return vget_lane_s8(sum, 0);
3428}
3429#endif
3430template <>
3431EIGEN_STRONG_INLINE uint8_t predux<Packet4uc>(const Packet4uc& a) {
3432 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
3433 uint8x8_t sum = vpadd_u8(a_dup, a_dup);
3434 sum = vpadd_u8(sum, sum);
3435 return vget_lane_u8(sum, 0);
3436}
3437#if EIGEN_ARCH_ARM64
3438template <>
3439EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(const Packet8uc& a) {
3440 return vaddv_u8(a);
3441}
3442template <>
3443EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(const Packet16uc& a) {
3444 return vaddvq_u8(a);
3445}
3446template <>
3447EIGEN_STRONG_INLINE int16_t predux<Packet4s>(const Packet4s& a) {
3448 return vaddv_s16(a);
3449}
3450template <>
3451EIGEN_STRONG_INLINE int16_t predux<Packet8s>(const Packet8s& a) {
3452 return vaddvq_s16(a);
3453}
3454template <>
3455EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(const Packet4us& a) {
3456 return vaddv_u16(a);
3457}
3458template <>
3459EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(const Packet8us& a) {
3460 return vaddvq_u16(a);
3461}
3462template <>
3463EIGEN_STRONG_INLINE int32_t predux<Packet2i>(const Packet2i& a) {
3464 return vaddv_s32(a);
3465}
3466template <>
3467EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) {
3468 return vaddvq_s32(a);
3469}
3470template <>
3471EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(const Packet2ui& a) {
3472 return vaddv_u32(a);
3473}
3474template <>
3475EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a) {
3476 return vaddvq_u32(a);
3477}
3478template <>
3479EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a) {
3480 return vaddvq_s64(a);
3481}
3482template <>
3483EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(const Packet2ul& a) {
3484 return vaddvq_u64(a);
3485}
3486#else
3487template <>
3488EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(const Packet8uc& a) {
3489 uint8x8_t sum = vpadd_u8(a, a);
3490 sum = vpadd_u8(sum, sum);
3491 sum = vpadd_u8(sum, sum);
3492 return vget_lane_u8(sum, 0);
3493}
3494template <>
3495EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(const Packet16uc& a) {
3496 uint8x8_t sum = vadd_u8(vget_low_u8(a), vget_high_u8(a));
3497 sum = vpadd_u8(sum, sum);
3498 sum = vpadd_u8(sum, sum);
3499 sum = vpadd_u8(sum, sum);
3500 return vget_lane_u8(sum, 0);
3501}
3502template <>
3503EIGEN_STRONG_INLINE int16_t predux<Packet4s>(const Packet4s& a) {
3504 const int16x4_t sum = vpadd_s16(a, a);
3505 return vget_lane_s16(vpadd_s16(sum, sum), 0);
3506}
3507template <>
3508EIGEN_STRONG_INLINE int16_t predux<Packet8s>(const Packet8s& a) {
3509 int16x4_t sum = vadd_s16(vget_low_s16(a), vget_high_s16(a));
3510 sum = vpadd_s16(sum, sum);
3511 sum = vpadd_s16(sum, sum);
3512 return vget_lane_s16(sum, 0);
3513}
3514template <>
3515EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(const Packet4us& a) {
3516 const uint16x4_t sum = vpadd_u16(a, a);
3517 return vget_lane_u16(vpadd_u16(sum, sum), 0);
3518}
3519template <>
3520EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(const Packet8us& a) {
3521 uint16x4_t sum = vadd_u16(vget_low_u16(a), vget_high_u16(a));
3522 sum = vpadd_u16(sum, sum);
3523 sum = vpadd_u16(sum, sum);
3524 return vget_lane_u16(sum, 0);
3525}
3526template <>
3527EIGEN_STRONG_INLINE int32_t predux<Packet2i>(const Packet2i& a) {
3528 return vget_lane_s32(vpadd_s32(a, a), 0);
3529}
3530template <>
3531EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) {
3532 const int32x2_t sum = vadd_s32(vget_low_s32(a), vget_high_s32(a));
3533 return vget_lane_s32(vpadd_s32(sum, sum), 0);
3534}
3535template <>
3536EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(const Packet2ui& a) {
3537 return vget_lane_u32(vpadd_u32(a, a), 0);
3538}
3539template <>
3540EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a) {
3541 const uint32x2_t sum = vadd_u32(vget_low_u32(a), vget_high_u32(a));
3542 return vget_lane_u32(vpadd_u32(sum, sum), 0);
3543}
3544template <>
3545EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a) {
3546 return vgetq_lane_s64(a, 0) + vgetq_lane_s64(a, 1);
3547}
3548template <>
3549EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(const Packet2ul& a) {
3550 return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1);
3551}
3552#endif
3553
3554template <>
3555EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half(const Packet8c& a) {
3556 return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(a, vreinterpret_s8_s32(vrev64_s32(vreinterpret_s32_s8(a))))), 0);
3557}
3558template <>
3559EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c predux_half(const Packet16c& a) {
3560 return vadd_s8(vget_high_s8(a), vget_low_s8(a));
3561}
3562template <>
3563EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc predux_half(const Packet8uc& a) {
3564 return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(a, vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(a))))), 0);
3565}
3566template <>
3567EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc predux_half(const Packet16uc& a) {
3568 return vadd_u8(vget_high_u8(a), vget_low_u8(a));
3569}
3570template <>
3571EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s predux_half(const Packet8s& a) {
3572 return vadd_s16(vget_high_s16(a), vget_low_s16(a));
3573}
3574template <>
3575EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us predux_half(const Packet8us& a) {
3576 return vadd_u16(vget_high_u16(a), vget_low_u16(a));
3577}
3578
3579// Other reduction functions:
3580// mul
3581template <>
3582EIGEN_STRONG_INLINE float predux_mul<Packet2f>(const Packet2f& a) {
3583 return vget_lane_f32(a, 0) * vget_lane_f32(a, 1);
3584}
3585template <>
3586EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
3587 return predux_mul<Packet2f>(vmul_f32(vget_low_f32(a), vget_high_f32(a)));
3588}
3589template <>
3590EIGEN_STRONG_INLINE int8_t predux_mul<Packet4c>(const Packet4c& a) {
3591 int8x8_t prod = vreinterpret_s8_s32(vdup_n_s32(a));
3592 prod = vmul_s8(prod, vrev16_s8(prod));
3593 return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 2);
3594}
3595template <>
3596EIGEN_STRONG_INLINE int8_t predux_mul<Packet8c>(const Packet8c& a) {
3597 int8x8_t prod = vmul_s8(a, vrev16_s8(a));
3598 prod = vmul_s8(prod, vrev32_s8(prod));
3599 return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4);
3600}
3601template <>
3602EIGEN_STRONG_INLINE int8_t predux_mul<Packet16c>(const Packet16c& a) {
3603 return predux_mul<Packet8c>(vmul_s8(vget_low_s8(a), vget_high_s8(a)));
3604}
3605template <>
3606EIGEN_STRONG_INLINE uint8_t predux_mul<Packet4uc>(const Packet4uc& a) {
3607 uint8x8_t prod = vreinterpret_u8_u32(vdup_n_u32(a));
3608 prod = vmul_u8(prod, vrev16_u8(prod));
3609 return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 2);
3610}
3611template <>
3612EIGEN_STRONG_INLINE uint8_t predux_mul<Packet8uc>(const Packet8uc& a) {
3613 uint8x8_t prod = vmul_u8(a, vrev16_u8(a));
3614 prod = vmul_u8(prod, vrev32_u8(prod));
3615 return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4);
3616}
3617template <>
3618EIGEN_STRONG_INLINE uint8_t predux_mul<Packet16uc>(const Packet16uc& a) {
3619 return predux_mul<Packet8uc>(vmul_u8(vget_low_u8(a), vget_high_u8(a)));
3620}
3621template <>
3622EIGEN_STRONG_INLINE int16_t predux_mul<Packet4s>(const Packet4s& a) {
3623 const int16x4_t prod = vmul_s16(a, vrev32_s16(a));
3624 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
3625}
3626template <>
3627EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(const Packet8s& a) {
3628 int16x4_t prod;
3629
3630 // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8|
3631 prod = vmul_s16(vget_low_s16(a), vget_high_s16(a));
3632 // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8|
3633 prod = vmul_s16(prod, vrev32_s16(prod));
3634 // Multiply |a1*a5*a2*a6*a3*a7*a4*a8|
3635 return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
3636}
3637template <>
3638EIGEN_STRONG_INLINE uint16_t predux_mul<Packet4us>(const Packet4us& a) {
3639 const uint16x4_t prod = vmul_u16(a, vrev32_u16(a));
3640 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
3641}
3642template <>
3643EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(const Packet8us& a) {
3644 uint16x4_t prod;
3645
3646 // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8|
3647 prod = vmul_u16(vget_low_u16(a), vget_high_u16(a));
3648 // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8|
3649 prod = vmul_u16(prod, vrev32_u16(prod));
3650 // Multiply |a1*a5*a2*a6*a3*a7*a4*a8|
3651 return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
3652}
3653template <>
3654EIGEN_STRONG_INLINE int32_t predux_mul<Packet2i>(const Packet2i& a) {
3655 return vget_lane_s32(a, 0) * vget_lane_s32(a, 1);
3656}
3657template <>
3658EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a) {
3659 return predux_mul<Packet2i>(vmul_s32(vget_low_s32(a), vget_high_s32(a)));
3660}
3661template <>
3662EIGEN_STRONG_INLINE uint32_t predux_mul<Packet2ui>(const Packet2ui& a) {
3663 return vget_lane_u32(a, 0) * vget_lane_u32(a, 1);
3664}
3665template <>
3666EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(const Packet4ui& a) {
3667 return predux_mul<Packet2ui>(vmul_u32(vget_low_u32(a), vget_high_u32(a)));
3668}
3669template <>
3670EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(const Packet2l& a) {
3671 return vgetq_lane_s64(a, 0) * vgetq_lane_s64(a, 1);
3672}
3673template <>
3674EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(const Packet2ul& a) {
3675 return vgetq_lane_u64(a, 0) * vgetq_lane_u64(a, 1);
3676}
3677
3678// min
3679#if EIGEN_ARCH_ARM64
3680template <>
3681EIGEN_STRONG_INLINE float predux_min<Packet2f>(const Packet2f& a) {
3682 return vminv_f32(a);
3683}
3684template <>
3685EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
3686 return vminvq_f32(a);
3687}
3688#else
3689template <>
3690EIGEN_STRONG_INLINE float predux_min<Packet2f>(const Packet2f& a) {
3691 return vget_lane_f32(vpmin_f32(a, a), 0);
3692}
3693template <>
3694EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
3695 const float32x2_t min = vmin_f32(vget_low_f32(a), vget_high_f32(a));
3696 return vget_lane_f32(vpmin_f32(min, min), 0);
3697}
3698#endif
3699template <>
3700EIGEN_STRONG_INLINE int8_t predux_min<Packet4c>(const Packet4c& a) {
3701 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
3702 int8x8_t min = vpmin_s8(a_dup, a_dup);
3703 min = vpmin_s8(min, min);
3704 return vget_lane_s8(min, 0);
3705}
3706#if EIGEN_ARCH_ARM64
3707template <>
3708EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(const Packet8c& a) {
3709 return vminv_s8(a);
3710}
3711template <>
3712EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(const Packet16c& a) {
3713 return vminvq_s8(a);
3714}
3715#else
3716template <>
3717EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(const Packet8c& a) {
3718 int8x8_t min = vpmin_s8(a, a);
3719 min = vpmin_s8(min, min);
3720 min = vpmin_s8(min, min);
3721 return vget_lane_s8(min, 0);
3722}
3723template <>
3724EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(const Packet16c& a) {
3725 int8x8_t min = vmin_s8(vget_low_s8(a), vget_high_s8(a));
3726 min = vpmin_s8(min, min);
3727 min = vpmin_s8(min, min);
3728 min = vpmin_s8(min, min);
3729 return vget_lane_s8(min, 0);
3730}
3731#endif
3732template <>
3733EIGEN_STRONG_INLINE uint8_t predux_min<Packet4uc>(const Packet4uc& a) {
3734 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
3735 uint8x8_t min = vpmin_u8(a_dup, a_dup);
3736 min = vpmin_u8(min, min);
3737 return vget_lane_u8(min, 0);
3738}
3739#if EIGEN_ARCH_ARM64
3740template <>
3741EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(const Packet8uc& a) {
3742 return vminv_u8(a);
3743}
3744template <>
3745EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(const Packet16uc& a) {
3746 return vminvq_u8(a);
3747}
3748template <>
3749EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(const Packet4s& a) {
3750 return vminv_s16(a);
3751}
3752template <>
3753EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(const Packet8s& a) {
3754 return vminvq_s16(a);
3755}
3756template <>
3757EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(const Packet4us& a) {
3758 return vminv_u16(a);
3759}
3760template <>
3761EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(const Packet8us& a) {
3762 return vminvq_u16(a);
3763}
3764template <>
3765EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(const Packet2i& a) {
3766 return vminv_s32(a);
3767}
3768template <>
3769EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) {
3770 return vminvq_s32(a);
3771}
3772template <>
3773EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(const Packet2ui& a) {
3774 return vminv_u32(a);
3775}
3776template <>
3777EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a) {
3778 return vminvq_u32(a);
3779}
3780#else
3781template <>
3782EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(const Packet8uc& a) {
3783 uint8x8_t min = vpmin_u8(a, a);
3784 min = vpmin_u8(min, min);
3785 min = vpmin_u8(min, min);
3786 return vget_lane_u8(min, 0);
3787}
3788template <>
3789EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(const Packet16uc& a) {
3790 uint8x8_t min = vmin_u8(vget_low_u8(a), vget_high_u8(a));
3791 min = vpmin_u8(min, min);
3792 min = vpmin_u8(min, min);
3793 min = vpmin_u8(min, min);
3794 return vget_lane_u8(min, 0);
3795}
3796template <>
3797EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(const Packet4s& a) {
3798 const int16x4_t min = vpmin_s16(a, a);
3799 return vget_lane_s16(vpmin_s16(min, min), 0);
3800}
3801template <>
3802EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(const Packet8s& a) {
3803 int16x4_t min = vmin_s16(vget_low_s16(a), vget_high_s16(a));
3804 min = vpmin_s16(min, min);
3805 min = vpmin_s16(min, min);
3806 return vget_lane_s16(min, 0);
3807}
3808template <>
3809EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(const Packet4us& a) {
3810 const uint16x4_t min = vpmin_u16(a, a);
3811 return vget_lane_u16(vpmin_u16(min, min), 0);
3812}
3813template <>
3814EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(const Packet8us& a) {
3815 uint16x4_t min = vmin_u16(vget_low_u16(a), vget_high_u16(a));
3816 min = vpmin_u16(min, min);
3817 min = vpmin_u16(min, min);
3818 return vget_lane_u16(min, 0);
3819}
3820template <>
3821EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(const Packet2i& a) {
3822 return vget_lane_s32(vpmin_s32(a, a), 0);
3823}
3824template <>
3825EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) {
3826 const int32x2_t min = vmin_s32(vget_low_s32(a), vget_high_s32(a));
3827 return vget_lane_s32(vpmin_s32(min, min), 0);
3828}
3829template <>
3830EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(const Packet2ui& a) {
3831 return vget_lane_u32(vpmin_u32(a, a), 0);
3832}
3833template <>
3834EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a) {
3835 const uint32x2_t min = vmin_u32(vget_low_u32(a), vget_high_u32(a));
3836 return vget_lane_u32(vpmin_u32(min, min), 0);
3837}
3838#endif
3839template <>
3840EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(const Packet2l& a) {
3841 return (std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1));
3842}
3843template <>
3844EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(const Packet2ul& a) {
3845 return (std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1));
3846}
3847
3848// max
3849#if EIGEN_ARCH_ARM64
3850template <>
3851EIGEN_STRONG_INLINE float predux_max<Packet2f>(const Packet2f& a) {
3852 return vmaxv_f32(a);
3853}
3854template <>
3855EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
3856 return vmaxvq_f32(a);
3857}
3858#else
3859template <>
3860EIGEN_STRONG_INLINE float predux_max<Packet2f>(const Packet2f& a) {
3861 return vget_lane_f32(vpmax_f32(a, a), 0);
3862}
3863template <>
3864EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
3865 const float32x2_t max = vmax_f32(vget_low_f32(a), vget_high_f32(a));
3866 return vget_lane_f32(vpmax_f32(max, max), 0);
3867}
3868#endif
3869template <>
3870EIGEN_STRONG_INLINE int8_t predux_max<Packet4c>(const Packet4c& a) {
3871 const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
3872 int8x8_t max = vpmax_s8(a_dup, a_dup);
3873 max = vpmax_s8(max, max);
3874 return vget_lane_s8(max, 0);
3875}
3876#if EIGEN_ARCH_ARM64
3877template <>
3878EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(const Packet8c& a) {
3879 return vmaxv_s8(a);
3880}
3881template <>
3882EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(const Packet16c& a) {
3883 return vmaxvq_s8(a);
3884}
3885#else
3886template <>
3887EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(const Packet8c& a) {
3888 int8x8_t max = vpmax_s8(a, a);
3889 max = vpmax_s8(max, max);
3890 max = vpmax_s8(max, max);
3891 return vget_lane_s8(max, 0);
3892}
3893template <>
3894EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(const Packet16c& a) {
3895 int8x8_t max = vmax_s8(vget_low_s8(a), vget_high_s8(a));
3896 max = vpmax_s8(max, max);
3897 max = vpmax_s8(max, max);
3898 max = vpmax_s8(max, max);
3899 return vget_lane_s8(max, 0);
3900}
3901#endif
3902template <>
3903EIGEN_STRONG_INLINE uint8_t predux_max<Packet4uc>(const Packet4uc& a) {
3904 const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
3905 uint8x8_t max = vpmax_u8(a_dup, a_dup);
3906 max = vpmax_u8(max, max);
3907 return vget_lane_u8(max, 0);
3908}
3909#if EIGEN_ARCH_ARM64
3910template <>
3911EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(const Packet8uc& a) {
3912 return vmaxv_u8(a);
3913}
3914template <>
3915EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(const Packet16uc& a) {
3916 return vmaxvq_u8(a);
3917}
3918template <>
3919EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(const Packet4s& a) {
3920 return vmaxv_s16(a);
3921}
3922template <>
3923EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(const Packet8s& a) {
3924 return vmaxvq_s16(a);
3925}
3926template <>
3927EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(const Packet4us& a) {
3928 return vmaxv_u16(a);
3929}
3930template <>
3931EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(const Packet8us& a) {
3932 return vmaxvq_u16(a);
3933}
3934template <>
3935EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(const Packet2i& a) {
3936 return vmaxv_s32(a);
3937}
3938template <>
3939EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) {
3940 return vmaxvq_s32(a);
3941}
3942template <>
3943EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(const Packet2ui& a) {
3944 return vmaxv_u32(a);
3945}
3946template <>
3947EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a) {
3948 return vmaxvq_u32(a);
3949}
3950#else
3951template <>
3952EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(const Packet8uc& a) {
3953 uint8x8_t max = vpmax_u8(a, a);
3954 max = vpmax_u8(max, max);
3955 max = vpmax_u8(max, max);
3956 return vget_lane_u8(max, 0);
3957}
3958template <>
3959EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(const Packet16uc& a) {
3960 uint8x8_t max = vmax_u8(vget_low_u8(a), vget_high_u8(a));
3961 max = vpmax_u8(max, max);
3962 max = vpmax_u8(max, max);
3963 max = vpmax_u8(max, max);
3964 return vget_lane_u8(max, 0);
3965}
3966template <>
3967EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(const Packet4s& a) {
3968 const int16x4_t max = vpmax_s16(a, a);
3969 return vget_lane_s16(vpmax_s16(max, max), 0);
3970}
3971template <>
3972EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(const Packet8s& a) {
3973 int16x4_t max = vmax_s16(vget_low_s16(a), vget_high_s16(a));
3974 max = vpmax_s16(max, max);
3975 max = vpmax_s16(max, max);
3976 return vget_lane_s16(max, 0);
3977}
3978template <>
3979EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(const Packet4us& a) {
3980 const uint16x4_t max = vpmax_u16(a, a);
3981 return vget_lane_u16(vpmax_u16(max, max), 0);
3982}
3983template <>
3984EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(const Packet8us& a) {
3985 uint16x4_t max = vmax_u16(vget_low_u16(a), vget_high_u16(a));
3986 max = vpmax_u16(max, max);
3987 max = vpmax_u16(max, max);
3988 return vget_lane_u16(max, 0);
3989}
3990template <>
3991EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(const Packet2i& a) {
3992 return vget_lane_s32(vpmax_s32(a, a), 0);
3993}
3994template <>
3995EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) {
3996 const int32x2_t max = vmax_s32(vget_low_s32(a), vget_high_s32(a));
3997 return vget_lane_s32(vpmax_s32(max, max), 0);
3998}
3999template <>
4000EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(const Packet2ui& a) {
4001 return vget_lane_u32(vpmax_u32(a, a), 0);
4002}
4003template <>
4004EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a) {
4005 const uint32x2_t max = vmax_u32(vget_low_u32(a), vget_high_u32(a));
4006 return vget_lane_u32(vpmax_u32(max, max), 0);
4007}
4008#endif
4009template <>
4010EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(const Packet2l& a) {
4011 return (std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1));
4012}
4013template <>
4014EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(const Packet2ul& a) {
4015 return (std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1));
4016}
4017
4018template <>
4019EIGEN_STRONG_INLINE bool predux_any(const Packet2f& x) {
4020 return vget_lane_u64(vreinterpret_u64_f32(x), 0) != 0;
4021}
4022
4023template <>
4024EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) {
4025 uint32x4_t u = vreinterpretq_u32_f32(x);
4026#if EIGEN_ARCH_ARM64
4027 return vget_lane_u64(vreinterpret_u64_u16(vmovn_u32(u)), 0);
4028#else
4029 uint32x2_t tmp = vorr_u32(vget_low_u32(u), vget_high_u32(u));
4030 uint32_t a, b;
4031 // GCC and Clang refuse to emit this instruction.
4032 asm("vmov %0, %1, %P2" : "=r"(a), "=r"(b) : "w"(tmp));
4033 return a | b;
4034#endif
4035}
4036
4037namespace detail {
4038// Whether any bit of a 128-bit register is set. Integer masks reduce through these bits rather than through the
4039// generic horizontal sum. On ARMv7, the 32-bit lane move after vpmax keeps predux_all's dependency chain as short as
4040// the sum's; a two-register vmov lengthens it.
4041EIGEN_STRONG_INLINE bool neon_any_bit(uint32x4_t u) {
4042#if EIGEN_ARCH_ARM64
4043 return vmaxvq_u32(u) != 0;
4044#else
4045 uint32x2_t tmp = vorr_u32(vget_low_u32(u), vget_high_u32(u));
4046 return vget_lane_u32(vpmax_u32(tmp, tmp), 0) != 0;
4047#endif
4048}
4049} // namespace detail
4050
4051template <>
4052EIGEN_STRONG_INLINE bool predux_any(const Packet4c& x) {
4053 return static_cast<int32_t>(x) != 0;
4054}
4055template <>
4056EIGEN_STRONG_INLINE bool predux_any(const Packet4uc& x) {
4057 return static_cast<uint32_t>(x) != 0;
4058}
4059template <>
4060EIGEN_STRONG_INLINE bool predux_any(const Packet8c& x) {
4061 return vget_lane_u64(vreinterpret_u64_s8(x), 0) != 0;
4062}
4063template <>
4064EIGEN_STRONG_INLINE bool predux_any(const Packet8uc& x) {
4065 return vget_lane_u64(vreinterpret_u64_u8(x), 0) != 0;
4066}
4067template <>
4068EIGEN_STRONG_INLINE bool predux_any(const Packet16c& x) {
4069 return detail::neon_any_bit(vreinterpretq_u32_s8(x));
4070}
4071template <>
4072EIGEN_STRONG_INLINE bool predux_any(const Packet16uc& x) {
4073 return detail::neon_any_bit(vreinterpretq_u32_u8(x));
4074}
4075template <>
4076EIGEN_STRONG_INLINE bool predux_any(const Packet4s& x) {
4077 return vget_lane_u64(vreinterpret_u64_s16(x), 0) != 0;
4078}
4079template <>
4080EIGEN_STRONG_INLINE bool predux_any(const Packet4us& x) {
4081 return vget_lane_u64(vreinterpret_u64_u16(x), 0) != 0;
4082}
4083template <>
4084EIGEN_STRONG_INLINE bool predux_any(const Packet8s& x) {
4085 return detail::neon_any_bit(vreinterpretq_u32_s16(x));
4086}
4087template <>
4088EIGEN_STRONG_INLINE bool predux_any(const Packet8us& x) {
4089 return detail::neon_any_bit(vreinterpretq_u32_u16(x));
4090}
4091template <>
4092EIGEN_STRONG_INLINE bool predux_any(const Packet2i& x) {
4093 return vget_lane_u64(vreinterpret_u64_s32(x), 0) != 0;
4094}
4095template <>
4096EIGEN_STRONG_INLINE bool predux_any(const Packet2ui& x) {
4097 return vget_lane_u64(vreinterpret_u64_u32(x), 0) != 0;
4098}
4099template <>
4100EIGEN_STRONG_INLINE bool predux_any(const Packet4i& x) {
4101 return detail::neon_any_bit(vreinterpretq_u32_s32(x));
4102}
4103template <>
4104EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& x) {
4105 return detail::neon_any_bit(x);
4106}
4107template <>
4108EIGEN_STRONG_INLINE bool predux_any(const Packet2l& x) {
4109 return detail::neon_any_bit(vreinterpretq_u32_s64(x));
4110}
4111template <>
4112EIGEN_STRONG_INLINE bool predux_any(const Packet2ul& x) {
4113 return detail::neon_any_bit(vreinterpretq_u32_u64(x));
4114}
4115
4116// Helpers for ptranspose.
4117namespace detail {
4118
4119template <typename Packet>
4120void zip_in_place(Packet& p1, Packet& p2);
4121
4122template <>
4123EIGEN_ALWAYS_INLINE void zip_in_place<Packet2f>(Packet2f& p1, Packet2f& p2) {
4124 const float32x2x2_t tmp = vzip_f32(p1, p2);
4125 p1 = tmp.val[0];
4126 p2 = tmp.val[1];
4127}
4128
4129template <>
4130EIGEN_ALWAYS_INLINE void zip_in_place<Packet4f>(Packet4f& p1, Packet4f& p2) {
4131 const float32x4x2_t tmp = vzipq_f32(p1, p2);
4132 p1 = tmp.val[0];
4133 p2 = tmp.val[1];
4134}
4135
4136template <>
4137EIGEN_ALWAYS_INLINE void zip_in_place<Packet8c>(Packet8c& p1, Packet8c& p2) {
4138 const int8x8x2_t tmp = vzip_s8(p1, p2);
4139 p1 = tmp.val[0];
4140 p2 = tmp.val[1];
4141}
4142
4143template <>
4144EIGEN_ALWAYS_INLINE void zip_in_place<Packet16c>(Packet16c& p1, Packet16c& p2) {
4145 const int8x16x2_t tmp = vzipq_s8(p1, p2);
4146 p1 = tmp.val[0];
4147 p2 = tmp.val[1];
4148}
4149
4150template <>
4151EIGEN_ALWAYS_INLINE void zip_in_place<Packet8uc>(Packet8uc& p1, Packet8uc& p2) {
4152 const uint8x8x2_t tmp = vzip_u8(p1, p2);
4153 p1 = tmp.val[0];
4154 p2 = tmp.val[1];
4155}
4156
4157template <>
4158EIGEN_ALWAYS_INLINE void zip_in_place<Packet16uc>(Packet16uc& p1, Packet16uc& p2) {
4159 const uint8x16x2_t tmp = vzipq_u8(p1, p2);
4160 p1 = tmp.val[0];
4161 p2 = tmp.val[1];
4162}
4163
4164template <>
4165EIGEN_ALWAYS_INLINE void zip_in_place<Packet2i>(Packet2i& p1, Packet2i& p2) {
4166 const int32x2x2_t tmp = vzip_s32(p1, p2);
4167 p1 = tmp.val[0];
4168 p2 = tmp.val[1];
4169}
4170
4171template <>
4172EIGEN_ALWAYS_INLINE void zip_in_place<Packet4i>(Packet4i& p1, Packet4i& p2) {
4173 const int32x4x2_t tmp = vzipq_s32(p1, p2);
4174 p1 = tmp.val[0];
4175 p2 = tmp.val[1];
4176}
4177
4178template <>
4179EIGEN_ALWAYS_INLINE void zip_in_place<Packet2ui>(Packet2ui& p1, Packet2ui& p2) {
4180 const uint32x2x2_t tmp = vzip_u32(p1, p2);
4181 p1 = tmp.val[0];
4182 p2 = tmp.val[1];
4183}
4184
4185template <>
4186EIGEN_ALWAYS_INLINE void zip_in_place<Packet4ui>(Packet4ui& p1, Packet4ui& p2) {
4187 const uint32x4x2_t tmp = vzipq_u32(p1, p2);
4188 p1 = tmp.val[0];
4189 p2 = tmp.val[1];
4190}
4191
4192template <>
4193EIGEN_ALWAYS_INLINE void zip_in_place<Packet4s>(Packet4s& p1, Packet4s& p2) {
4194 const int16x4x2_t tmp = vzip_s16(p1, p2);
4195 p1 = tmp.val[0];
4196 p2 = tmp.val[1];
4197}
4198
4199template <>
4200EIGEN_ALWAYS_INLINE void zip_in_place<Packet8s>(Packet8s& p1, Packet8s& p2) {
4201 const int16x8x2_t tmp = vzipq_s16(p1, p2);
4202 p1 = tmp.val[0];
4203 p2 = tmp.val[1];
4204}
4205
4206template <>
4207EIGEN_ALWAYS_INLINE void zip_in_place<Packet4us>(Packet4us& p1, Packet4us& p2) {
4208 const uint16x4x2_t tmp = vzip_u16(p1, p2);
4209 p1 = tmp.val[0];
4210 p2 = tmp.val[1];
4211}
4212
4213template <>
4214EIGEN_ALWAYS_INLINE void zip_in_place<Packet8us>(Packet8us& p1, Packet8us& p2) {
4215 const uint16x8x2_t tmp = vzipq_u16(p1, p2);
4216 p1 = tmp.val[0];
4217 p2 = tmp.val[1];
4218}
4219
4220template <typename Packet>
4221EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 2>& kernel) {
4222 zip_in_place(kernel.packet[0], kernel.packet[1]);
4223}
4224
4225template <typename Packet>
4226EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 4>& kernel) {
4227 zip_in_place(kernel.packet[0], kernel.packet[2]);
4228 zip_in_place(kernel.packet[1], kernel.packet[3]);
4229 zip_in_place(kernel.packet[0], kernel.packet[1]);
4230 zip_in_place(kernel.packet[2], kernel.packet[3]);
4231}
4232
4233template <typename Packet>
4234EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 8>& kernel) {
4235 zip_in_place(kernel.packet[0], kernel.packet[4]);
4236 zip_in_place(kernel.packet[1], kernel.packet[5]);
4237 zip_in_place(kernel.packet[2], kernel.packet[6]);
4238 zip_in_place(kernel.packet[3], kernel.packet[7]);
4239
4240 zip_in_place(kernel.packet[0], kernel.packet[2]);
4241 zip_in_place(kernel.packet[1], kernel.packet[3]);
4242 zip_in_place(kernel.packet[4], kernel.packet[6]);
4243 zip_in_place(kernel.packet[5], kernel.packet[7]);
4244
4245 zip_in_place(kernel.packet[0], kernel.packet[1]);
4246 zip_in_place(kernel.packet[2], kernel.packet[3]);
4247 zip_in_place(kernel.packet[4], kernel.packet[5]);
4248 zip_in_place(kernel.packet[6], kernel.packet[7]);
4249}
4250
4251template <typename Packet>
4252EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 16>& kernel) {
4253 EIGEN_UNROLL_LOOP
4254 for (int i = 0; i < 4; ++i) {
4255 const int m = (1 << i);
4256 EIGEN_UNROLL_LOOP
4257 for (int j = 0; j < m; ++j) {
4258 const int n = (1 << (3 - i));
4259 EIGEN_UNROLL_LOOP
4260 for (int k = 0; k < n; ++k) {
4261 const int idx = 2 * j * n + k;
4262 zip_in_place(kernel.packet[idx], kernel.packet[idx + n]);
4263 }
4264 }
4265 }
4266}
4267
4268} // namespace detail
4269
4270EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2f, 2>& kernel) {
4271 detail::ptranspose_impl(kernel);
4272}
4273EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
4274 detail::ptranspose_impl(kernel);
4275}
4276
4277EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4c, 4>& kernel) {
4278 const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1));
4279 const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1));
4280
4281 const int8x8x2_t zip8 = vzip_s8(a, b);
4282 const int16x4x2_t zip16 = vzip_s16(vreinterpret_s16_s8(zip8.val[0]), vreinterpret_s16_s8(zip8.val[1]));
4283
4284 kernel.packet[0] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 0);
4285 kernel.packet[1] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 1);
4286 kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0);
4287 kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1);
4288}
4289EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8c, 8>& kernel) {
4290 detail::ptranspose_impl(kernel);
4291}
4292EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8c, 4>& kernel) {
4293 detail::ptranspose_impl(kernel);
4294}
4295EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
4296 detail::ptranspose_impl(kernel);
4297}
4298EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 8>& kernel) {
4299 detail::ptranspose_impl(kernel);
4300}
4301EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
4302 detail::ptranspose_impl(kernel);
4303}
4304
4305EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4uc, 4>& kernel) {
4306 const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1));
4307 const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1));
4308
4309 const uint8x8x2_t zip8 = vzip_u8(a, b);
4310 const uint16x4x2_t zip16 = vzip_u16(vreinterpret_u16_u8(zip8.val[0]), vreinterpret_u16_u8(zip8.val[1]));
4311
4312 kernel.packet[0] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 0);
4313 kernel.packet[1] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 1);
4314 kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0);
4315 kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1);
4316}
4317EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8uc, 8>& kernel) {
4318 detail::ptranspose_impl(kernel);
4319}
4320EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8uc, 4>& kernel) {
4321 detail::ptranspose_impl(kernel);
4322}
4323EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
4324 detail::ptranspose_impl(kernel);
4325}
4326EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 8>& kernel) {
4327 detail::ptranspose_impl(kernel);
4328}
4329EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
4330 detail::ptranspose_impl(kernel);
4331}
4332
4333EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4s, 4>& kernel) {
4334 detail::ptranspose_impl(kernel);
4335}
4336EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
4337 detail::ptranspose_impl(kernel);
4338}
4339EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
4340 detail::ptranspose_impl(kernel);
4341}
4342
4343EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4us, 4>& kernel) {
4344 detail::ptranspose_impl(kernel);
4345}
4346EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
4347 detail::ptranspose_impl(kernel);
4348}
4349EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
4350 detail::ptranspose_impl(kernel);
4351}
4352
4353EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2i, 2>& kernel) {
4354 detail::ptranspose_impl(kernel);
4355}
4356EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
4357 detail::ptranspose_impl(kernel);
4358}
4359EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2ui, 2>& kernel) {
4360 detail::zip_in_place(kernel.packet[0], kernel.packet[1]);
4361}
4362EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
4363 detail::ptranspose_impl(kernel);
4364}
4365
4366EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2l, 2>& kernel) {
4367#if EIGEN_ARCH_ARM64
4368 const int64x2_t tmp1 = vzip1q_s64(kernel.packet[0], kernel.packet[1]);
4369 kernel.packet[1] = vzip2q_s64(kernel.packet[0], kernel.packet[1]);
4370 kernel.packet[0] = tmp1;
4371#else
4372 const int64x1_t tmp[2][2] = {{vget_low_s64(kernel.packet[0]), vget_high_s64(kernel.packet[0])},
4373 {vget_low_s64(kernel.packet[1]), vget_high_s64(kernel.packet[1])}};
4374
4375 kernel.packet[0] = vcombine_s64(tmp[0][0], tmp[1][0]);
4376 kernel.packet[1] = vcombine_s64(tmp[0][1], tmp[1][1]);
4377#endif
4378}
4379EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2ul, 2>& kernel) {
4380#if EIGEN_ARCH_ARM64
4381 const uint64x2_t tmp1 = vzip1q_u64(kernel.packet[0], kernel.packet[1]);
4382 kernel.packet[1] = vzip2q_u64(kernel.packet[0], kernel.packet[1]);
4383 kernel.packet[0] = tmp1;
4384#else
4385 const uint64x1_t tmp[2][2] = {{vget_low_u64(kernel.packet[0]), vget_high_u64(kernel.packet[0])},
4386 {vget_low_u64(kernel.packet[1]), vget_high_u64(kernel.packet[1])}};
4387
4388 kernel.packet[0] = vcombine_u64(tmp[0][0], tmp[1][0]);
4389 kernel.packet[1] = vcombine_u64(tmp[0][1], tmp[1][1]);
4390#endif
4391}
4392
4393template <>
4394EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pselect(const Packet2f& mask, const Packet2f& a, const Packet2f& b) {
4395 return vbsl_f32(vreinterpret_u32_f32(mask), a, b);
4396}
4397template <>
4398EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) {
4399 return vbslq_f32(vreinterpretq_u32_f32(mask), a, b);
4400}
4401template <>
4402EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pselect(const Packet8c& mask, const Packet8c& a, const Packet8c& b) {
4403 return vbsl_s8(vreinterpret_u8_s8(mask), a, b);
4404}
4405template <>
4406EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b) {
4407 return vbslq_s8(vreinterpretq_u8_s8(mask), a, b);
4408}
4409template <>
4410EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pselect(const Packet8uc& mask, const Packet8uc& a, const Packet8uc& b) {
4411 return vbsl_u8(mask, a, b);
4412}
4413template <>
4414EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a,
4415 const Packet16uc& b) {
4416 return vbslq_u8(mask, a, b);
4417}
4418template <>
4419EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pselect(const Packet4s& mask, const Packet4s& a, const Packet4s& b) {
4420 return vbsl_s16(vreinterpret_u16_s16(mask), a, b);
4421}
4422template <>
4423EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) {
4424 return vbslq_s16(vreinterpretq_u16_s16(mask), a, b);
4425}
4426template <>
4427EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pselect(const Packet4us& mask, const Packet4us& a, const Packet4us& b) {
4428 return vbsl_u16(mask, a, b);
4429}
4430template <>
4431EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) {
4432 return vbslq_u16(mask, a, b);
4433}
4434template <>
4435EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pselect(const Packet2i& mask, const Packet2i& a, const Packet2i& b) {
4436 return vbsl_s32(vreinterpret_u32_s32(mask), a, b);
4437}
4438template <>
4439EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) {
4440 return vbslq_s32(vreinterpretq_u32_s32(mask), a, b);
4441}
4442template <>
4443EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pselect(const Packet2ui& mask, const Packet2ui& a, const Packet2ui& b) {
4444 return vbsl_u32(mask, a, b);
4445}
4446template <>
4447EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) {
4448 return vbslq_u32(mask, a, b);
4449}
4450template <>
4451EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) {
4452 return vbslq_s64(vreinterpretq_u64_s64(mask), a, b);
4453}
4454template <>
4455EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b) {
4456 return vbslq_u64(mask, a, b);
4457}
4458
4459// Use armv8 rounding intrinsics if available.
4460#if EIGEN_ARCH_ARMV8
4461template <>
4462EIGEN_STRONG_INLINE Packet2f print<Packet2f>(const Packet2f& a) {
4463 return vrndn_f32(a);
4464}
4465
4466template <>
4467EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
4468 return vrndnq_f32(a);
4469}
4470
4471template <>
4472EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(const Packet2f& a) {
4473 return vrndm_f32(a);
4474}
4475
4476template <>
4477EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
4478 return vrndmq_f32(a);
4479}
4480
4481template <>
4482EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(const Packet2f& a) {
4483 return vrndp_f32(a);
4484}
4485
4486template <>
4487EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
4488 return vrndpq_f32(a);
4489}
4490
4491template <>
4492EIGEN_STRONG_INLINE Packet2f pround<Packet2f>(const Packet2f& a) {
4493 return vrnda_f32(a);
4494}
4495
4496template <>
4497EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) {
4498 return vrndaq_f32(a);
4499}
4500
4501template <>
4502EIGEN_STRONG_INLINE Packet2f ptrunc<Packet2f>(const Packet2f& a) {
4503 return vrnd_f32(a);
4504}
4505
4506template <>
4507EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
4508 return vrndq_f32(a);
4509}
4510#endif
4511
4518template <>
4519EIGEN_STRONG_INLINE Packet4uc psqrt(const Packet4uc& a) {
4520 uint8x8_t x = vreinterpret_u8_u32(vdup_n_u32(a));
4521 uint8x8_t res = vdup_n_u8(0);
4522 uint8x8_t add = vdup_n_u8(0x8);
4523 for (int i = 0; i < 4; i++) {
4524 const uint8x8_t temp = vorr_u8(res, add);
4525 res = vbsl_u8(vcge_u8(x, vmul_u8(temp, temp)), temp, res);
4526 add = vshr_n_u8(add, 1);
4527 }
4528 return vget_lane_u32(vreinterpret_u32_u8(res), 0);
4529}
4531template <>
4532EIGEN_STRONG_INLINE Packet8uc psqrt(const Packet8uc& a) {
4533 uint8x8_t res = vdup_n_u8(0);
4534 uint8x8_t add = vdup_n_u8(0x8);
4535 for (int i = 0; i < 4; i++) {
4536 const uint8x8_t temp = vorr_u8(res, add);
4537 res = vbsl_u8(vcge_u8(a, vmul_u8(temp, temp)), temp, res);
4538 add = vshr_n_u8(add, 1);
4539 }
4540 return res;
4541}
4543template <>
4544EIGEN_STRONG_INLINE Packet16uc psqrt(const Packet16uc& a) {
4545 uint8x16_t res = vdupq_n_u8(0);
4546 uint8x16_t add = vdupq_n_u8(0x8);
4547 for (int i = 0; i < 4; i++) {
4548 const uint8x16_t temp = vorrq_u8(res, add);
4549 res = vbslq_u8(vcgeq_u8(a, vmulq_u8(temp, temp)), temp, res);
4550 add = vshrq_n_u8(add, 1);
4551 }
4552 return res;
4553}
4555template <>
4556EIGEN_STRONG_INLINE Packet4us psqrt(const Packet4us& a) {
4557 uint16x4_t res = vdup_n_u16(0);
4558 uint16x4_t add = vdup_n_u16(0x80);
4559 for (int i = 0; i < 8; i++) {
4560 const uint16x4_t temp = vorr_u16(res, add);
4561 res = vbsl_u16(vcge_u16(a, vmul_u16(temp, temp)), temp, res);
4562 add = vshr_n_u16(add, 1);
4563 }
4564 return res;
4565}
4567template <>
4568EIGEN_STRONG_INLINE Packet8us psqrt(const Packet8us& a) {
4569 uint16x8_t res = vdupq_n_u16(0);
4570 uint16x8_t add = vdupq_n_u16(0x80);
4571 for (int i = 0; i < 8; i++) {
4572 const uint16x8_t temp = vorrq_u16(res, add);
4573 res = vbslq_u16(vcgeq_u16(a, vmulq_u16(temp, temp)), temp, res);
4574 add = vshrq_n_u16(add, 1);
4575 }
4576 return res;
4577}
4579template <>
4580EIGEN_STRONG_INLINE Packet2ui psqrt(const Packet2ui& a) {
4581 uint32x2_t res = vdup_n_u32(0);
4582 uint32x2_t add = vdup_n_u32(0x8000);
4583 for (int i = 0; i < 16; i++) {
4584 const uint32x2_t temp = vorr_u32(res, add);
4585 res = vbsl_u32(vcge_u32(a, vmul_u32(temp, temp)), temp, res);
4586 add = vshr_n_u32(add, 1);
4587 }
4588 return res;
4589}
4591template <>
4592EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) {
4593 uint32x4_t res = vdupq_n_u32(0);
4594 uint32x4_t add = vdupq_n_u32(0x8000);
4595 for (int i = 0; i < 16; i++) {
4596 const uint32x4_t temp = vorrq_u32(res, add);
4597 res = vbslq_u32(vcgeq_u32(a, vmulq_u32(temp, temp)), temp, res);
4598 add = vshrq_n_u32(add, 1);
4599 }
4600 return res;
4601}
4602
4603EIGEN_STRONG_INLINE Packet4f prsqrt_float_unsafe(const Packet4f& a) {
4604 // Compute approximate reciprocal sqrt.
4605 // Does not correctly handle +/- 0 or +inf
4606 float32x4_t result = vrsqrteq_f32(a);
4607 result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result);
4608 result = vmulq_f32(vrsqrtsq_f32(vmulq_f32(a, result), result), result);
4609 return result;
4610}
4611
4612EIGEN_STRONG_INLINE Packet2f prsqrt_float_unsafe(const Packet2f& a) {
4613 // Compute approximate reciprocal sqrt.
4614 // Does not correctly handle +/- 0 or +inf
4615 float32x2_t result = vrsqrte_f32(a);
4616 result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result);
4617 result = vmul_f32(vrsqrts_f32(vmul_f32(a, result), result), result);
4618 return result;
4619}
4620
4621template <typename Packet>
4622Packet prsqrt_float_common(const Packet& a) {
4623 const Packet cst_zero = pzero(a);
4624 const Packet cst_inf = pset1<Packet>(NumTraits<float>::infinity());
4625 Packet return_zero = pcmp_eq(a, cst_inf);
4626 Packet return_inf = pcmp_eq(a, cst_zero);
4627 Packet result = prsqrt_float_unsafe(a);
4628 result = pselect(return_inf, por(cst_inf, a), result);
4629 result = pandnot(result, return_zero);
4630 return result;
4631}
4632
4633template <>
4634EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) {
4635 return prsqrt_float_common(a);
4636}
4637
4638template <>
4639EIGEN_STRONG_INLINE Packet2f prsqrt(const Packet2f& a) {
4640 return prsqrt_float_common(a);
4641}
4642
4643template <>
4644EIGEN_STRONG_INLINE Packet4f preciprocal<Packet4f>(const Packet4f& a) {
4645 // Compute approximate reciprocal.
4646 const float32x4_t estimate = vrecpeq_f32(a);
4647 float32x4_t result = vmulq_f32(vrecpsq_f32(a, estimate), estimate);
4648 result = vmulq_f32(vrecpsq_f32(a, result), result);
4649 // 1 / a has the sign of a. The estimate is +-inf for |a| < 2^-128, which the steps turn into -+inf.
4650 return vbslq_f32(vdupq_n_u32(0x80000000u), a, result);
4651}
4652
4653template <>
4654EIGEN_STRONG_INLINE Packet2f preciprocal<Packet2f>(const Packet2f& a) {
4655 // Compute approximate reciprocal.
4656 const float32x2_t estimate = vrecpe_f32(a);
4657 float32x2_t result = vmul_f32(vrecps_f32(a, estimate), estimate);
4658 result = vmul_f32(vrecps_f32(a, result), result);
4659 // 1 / a has the sign of a. The estimate is +-inf for |a| < 2^-128, which the steps turn into -+inf.
4660 return vbsl_f32(vdup_n_u32(0x80000000u), a, result);
4661}
4662
4663// Unfortunately vsqrt_f32 is only available for A64.
4664#if EIGEN_ARCH_ARM64
4665template <>
4666EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) {
4667 return vsqrtq_f32(a);
4668}
4669
4670template <>
4671EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& a) {
4672 return vsqrt_f32(a);
4673}
4674
4675template <>
4676EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) {
4677 return vdivq_f32(a, b);
4678}
4679
4680template <>
4681EIGEN_STRONG_INLINE Packet2f pdiv(const Packet2f& a, const Packet2f& b) {
4682 return vdiv_f32(a, b);
4683}
4684#else
4685template <typename Packet>
4686EIGEN_STRONG_INLINE Packet psqrt_float_common(const Packet& a) {
4687 const Packet cst_zero = pzero(a);
4688 const Packet cst_inf = pset1<Packet>(NumTraits<float>::infinity());
4689
4690 Packet result = pmul(a, prsqrt_float_unsafe(a));
4691 Packet a_is_zero = pcmp_eq(a, cst_zero);
4692 Packet a_is_inf = pcmp_eq(a, cst_inf);
4693 Packet return_a = por(a_is_zero, a_is_inf);
4694
4695 result = pselect(return_a, a, result);
4696 return result;
4697}
4698
4699template <>
4700EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) {
4701 return psqrt_float_common(a);
4702}
4703
4704template <>
4705EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& a) {
4706 return psqrt_float_common(a);
4707}
4708
4709template <typename Packet>
4710EIGEN_STRONG_INLINE Packet pdiv_float_common(const Packet& a, const Packet& b) {
4711 // if b is large, NEON intrinsics will flush preciprocal(b) to zero
4712 // avoid underflow with the following manipulation:
4713 // a / b = f * (a * reciprocal(f * b))
4714
4715 const Packet cst_one = pset1<Packet>(1.0f);
4716 const Packet cst_quarter = pset1<Packet>(0.25f);
4717 const Packet cst_thresh = pset1<Packet>(NumTraits<float>::highest() / 4.0f);
4718
4719 Packet b_will_underflow = pcmp_le(cst_thresh, pabs(b));
4720 Packet f = pselect(b_will_underflow, cst_quarter, cst_one);
4721 Packet result = pmul(f, pmul(a, preciprocal(pmul(b, f))));
4722 return result;
4723}
4724
4725template <>
4726EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
4727 return pdiv_float_common(a, b);
4728}
4729
4730template <>
4731EIGEN_STRONG_INLINE Packet2f pdiv<Packet2f>(const Packet2f& a, const Packet2f& b) {
4732 return pdiv_float_common(a, b);
4733}
4734#endif
4735
4736//---------- double ----------
4737
4738#if EIGEN_ARCH_ARM64
4739
4740#if EIGEN_COMP_GNUC
4741// Bug 907: workaround missing declarations of the following two functions in the ADK
4742// Defining these functions as templates ensures that if these intrinsics are
4743// already defined in arm_neon.h, then our workaround doesn't cause a conflict
4744// and has lower priority in overload resolution.
4745// This doesn't work with MSVC though, since the function names are macros.
4746template <typename T>
4747uint64x2_t vreinterpretq_u64_f64(T a) {
4748 return (uint64x2_t)a;
4749}
4750
4751template <typename T>
4752float64x2_t vreinterpretq_f64_u64(T a) {
4753 return (float64x2_t)a;
4754}
4755#endif
4756
4757#if EIGEN_COMP_MSVC_STRICT
4758using Packet2d = eigen_packet_wrapper<float64x2_t, 18>;
4759using Packet1d = eigen_packet_wrapper<float64x1_t, 19>;
4760
4761EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) {
4762 double from[2] = {a, b};
4763 return vld1q_f64(from);
4764}
4765
4766#else
4767using Packet2d = float64x2_t;
4768using Packet1d = float64x1_t;
4769
4770EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) { return Packet2d{a, b}; }
4771#endif
4772
4773// functionally equivalent to _mm_shuffle_pd in SSE (i.e. shuffle(m, n, mask) equals _mm_shuffle_pd(m,n,mask))
4774// Currently used in LU/arch/InverseSize4.h to enable a shared implementation
4775// for fast inversion of matrices of size 4.
4776EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask) {
4777 const double* a = reinterpret_cast<const double*>(&m);
4778 const double* b = reinterpret_cast<const double*>(&n);
4779 Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1)));
4780 return res;
4781}
4782
4783EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(const Packet2d& a, const Packet2d& b, int mask) {
4784 return shuffle(a, b, mask);
4785}
4786EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 0); }
4787EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 3); }
4788#define vec2d_duplane(a, p) Packet2d(vdupq_laneq_f64(a, p))
4789
4790template <>
4791struct packet_traits<double> : default_packet_traits {
4792 using type = Packet2d;
4793 using half = Packet2d;
4794 enum {
4795 Vectorizable = 1,
4796 AlignedOnScalar = 1,
4797 size = 2,
4798
4799 HasCmp = 1,
4800 HasAdd = 1,
4801 HasSub = 1,
4802 HasShift = 1,
4803 HasMul = 1,
4804 HasNegate = 1,
4805 HasAbs = 1,
4806 HasArg = 0,
4807 HasAbsDiff = 1,
4808 HasMin = 1,
4809 HasMax = 1,
4810 HasConj = 1,
4811 HasSetLinear = 1,
4812
4813 HasDiv = 1,
4814
4815#if EIGEN_ARCH_ARM64
4816 HasExp = 1,
4817 HasLog = 1,
4818 HasLog10 = 1,
4819 HasLog1p = 1,
4820 HasExpm1 = 1,
4821 HasPow = 1,
4822 HasATan = 1,
4823 HasATanh = 1,
4824 HasSinh = 1,
4825 HasCosh = 1,
4826 HasASinh = 1,
4827 HasACosh = 1,
4828#endif
4829 HasSin = EIGEN_FAST_MATH,
4830 HasCos = EIGEN_FAST_MATH,
4831 HasTan = EIGEN_FAST_MATH,
4832 HasSqrt = 1,
4833 HasRsqrt = 1,
4834 HasCbrt = 1,
4835 HasTanh = EIGEN_FAST_MATH,
4836 HasErf = EIGEN_FAST_MATH,
4837 HasErfc = EIGEN_FAST_MATH
4838 };
4839};
4840
4841template <>
4842struct unpacket_traits<Packet2d> : neon_unpacket_default<Packet2d, double> {
4843 using integer_packet = Packet2l;
4844};
4845
4846template <>
4847EIGEN_STRONG_INLINE Packet2d pzero<Packet2d>(const Packet2d& /*a*/) {
4848 return vdupq_n_f64(0.0);
4849}
4850
4851template <>
4852EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
4853 return vdupq_n_f64(from);
4854}
4855
4856template <>
4857EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
4858 const double c[] = {0.0, 1.0};
4859 return vaddq_f64(pset1<Packet2d>(a), vld1q_f64(c));
4860}
4861
4862template <>
4863EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
4864 return vaddq_f64(a, b);
4865}
4866
4867template <>
4868EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
4869 return vsubq_f64(a, b);
4870}
4871
4872template <>
4873EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d&, const Packet2d&);
4874template <>
4875EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(const Packet2d& a, const Packet2d& b) {
4876 const Packet2d mask = make_packet2d(numext::bit_cast<double>(0x8000000000000000ull), 0.0);
4877 return padd(a, pxor(mask, b));
4878}
4879
4880template <>
4881EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
4882 return vnegq_f64(a);
4883}
4884
4885template <>
4886EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
4887 return vmulq_f64(a, b);
4888}
4889
4890template <>
4891EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
4892 return vdivq_f64(a, b);
4893}
4894
4895#ifdef EIGEN_VECTORIZE_FMA
4896// See bug 936. See above comment about FMA for float.
4897template <>
4898struct is_fused_madd<Packet2d> : true_type {};
4899template <>
4900EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
4901 return vfmaq_f64(c, a, b);
4902}
4903template <>
4904EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
4905 return vfmsq_f64(c, a, b);
4906}
4907#else
4908template <>
4909EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
4910 return vmlaq_f64(c, a, b);
4911}
4912template <>
4913EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
4914 return vmlsq_f64(c, a, b);
4915}
4916#endif
4917template <>
4918EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
4919 return vminq_f64(a, b);
4920}
4921
4922#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
4923// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8
4924// systems).
4925template <>
4926EIGEN_STRONG_INLINE Packet2d pmin<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) {
4927 return vminnmq_f64(a, b);
4928}
4929template <>
4930EIGEN_STRONG_INLINE Packet2d pmax<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) {
4931 return vmaxnmq_f64(a, b);
4932}
4933
4934#endif
4935
4936template <>
4937struct pminmax_propagates_nan<Packet2d> : bool_constant<true> {};
4938
4939template <>
4940EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
4941 return vmaxq_f64(a, b);
4942}
4943
4944// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
4945template <>
4946EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
4947 return vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4948}
4949
4950template <>
4951EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
4952 return vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4953}
4954
4955template <>
4956EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
4957 return vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4958}
4959
4960template <>
4961EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
4962 return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a), vreinterpretq_u64_f64(b)));
4963}
4964
4965template <>
4966EIGEN_STRONG_INLINE Packet2d pnot<Packet2d>(const Packet2d& a) {
4967 return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_f64(a)));
4968}
4969
4970template <>
4971EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b) {
4972 return vreinterpretq_f64_u64(vcleq_f64(a, b));
4973}
4974
4975template <>
4976EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b) {
4977 return vreinterpretq_f64_u64(vcltq_f64(a, b));
4978}
4979
4980template <>
4981EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) {
4982 return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_u64(vcgeq_f64(a, b))));
4983}
4984
4985template <>
4986EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) {
4987 return vreinterpretq_f64_u64(vceqq_f64(a, b));
4988}
4989
4990template <>
4991EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
4992 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f64(assume_aligned<unpacket_traits<Packet2d>::alignment>(from));
4993}
4994
4995template <>
4996EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
4997 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f64(from);
4998}
4999
5000template <>
5001EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) {
5002 return vld1q_dup_f64(from);
5003}
5004template <>
5005EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
5006 EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(assume_aligned<unpacket_traits<Packet2d>::alignment>(to), from);
5007}
5008
5009template <>
5010EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
5011 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to, from);
5012}
5013
5014template <>
5015EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
5016 Packet2d res = pset1<Packet2d>(0.0);
5017 res = vld1q_lane_f64(from + 0 * stride, res, 0);
5018 res = vld1q_lane_f64(from + 1 * stride, res, 1);
5019 return res;
5020}
5021
5022template <>
5023EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
5024 vst1q_lane_f64(to + stride * 0, from, 0);
5025 vst1q_lane_f64(to + stride * 1, from, 1);
5026}
5027
5028template <>
5029EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
5030 EIGEN_ARM_PREFETCH(addr);
5031}
5032
5033// FIXME only store the 2 first elements ?
5034template <>
5035EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
5036 return vgetq_lane_f64(a, 0);
5037}
5038
5039template <>
5040EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
5041 return vcombine_f64(vget_high_f64(a), vget_low_f64(a));
5042}
5043
5044template <>
5045EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) {
5046 return vabsq_f64(a);
5047}
5048
5049template <>
5050EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) {
5051 return vreinterpretq_f64_s64(vshrq_n_s64(vreinterpretq_s64_f64(a), 63));
5052}
5053
5054template <>
5055EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
5056 return vaddvq_f64(a);
5057}
5058
5059template <>
5060EIGEN_STRONG_INLINE Index predux_count(const Packet2d& a) {
5061 // Each zero lane contributes UINT64_MAX, so unsigned 2 + sum(mask) is the number of nonzero lanes.
5062 const uint64_t zeroMaskSum = vaddvq_u64(vceqq_f64(a, vdupq_n_f64(0.0)));
5063 return static_cast<Index>(zeroMaskSum + 2);
5064}
5065
5066template <>
5067EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) {
5068 return vmaxvq_u32(vreinterpretq_u32_f64(a)) != 0;
5069}
5070
5071// Other reduction functions:
5072// mul
5073#if EIGEN_COMP_CLANGAPPLE
5074template <>
5075EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
5076 return (vget_low_f64(a) * vget_high_f64(a))[0];
5077}
5078#else
5079template <>
5080EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
5081 return vget_lane_f64(vmul_f64(vget_low_f64(a), vget_high_f64(a)), 0);
5082}
5083#endif
5084
5085// min
5086template <>
5087EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
5088 return vminvq_f64(a);
5089}
5090
5091// max
5092template <>
5093EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
5094 return vmaxvq_f64(a);
5095}
5096
5097EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
5098 const float64x2_t tmp1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]);
5099 const float64x2_t tmp2 = vzip2q_f64(kernel.packet[0], kernel.packet[1]);
5100
5101 kernel.packet[0] = tmp1;
5102 kernel.packet[1] = tmp2;
5103}
5104
5105template <>
5106EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) {
5107 return vbslq_f64(vreinterpretq_u64_f64(mask), a, b);
5108}
5109
5110template <>
5111EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
5112 return vrndnq_f64(a);
5113}
5114
5115template <>
5116EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
5117 return vrndmq_f64(a);
5118}
5119
5120template <>
5121EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
5122 return vrndpq_f64(a);
5123}
5124
5125template <>
5126EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) {
5127 return vrndaq_f64(a);
5128}
5129
5130template <>
5131EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
5132 return vrndq_f64(a);
5133}
5134
5135template <>
5136EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
5137 return pldexp_generic(a, exponent);
5138}
5139
5140template <>
5141EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent) {
5142 return pfrexp_generic(a, exponent);
5143}
5144
5145template <>
5146EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
5147 return vreinterpretq_f64_u64(vdupq_n_u64(from));
5148}
5149
5150template <>
5151EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) {
5152 // Do Newton iterations for 1/sqrt(x).
5153 return generic_rsqrt_newton_step<Packet2d, /*Steps=*/3>::run(a, vrsqrteq_f64(a));
5154}
5155
5156template <>
5157EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x) {
5158 return vsqrtq_f64(_x);
5159}
5160
5161// Absolute comparisons as for Packet4f.
5162template <>
5163EIGEN_STRONG_INLINE Packet2d pisinf<Packet2d>(const Packet2d& a) {
5164 return vreinterpretq_f64_u64(vcageq_f64(a, pinf<Packet2d>()));
5165}
5166template <>
5167EIGEN_STRONG_INLINE Packet2d pisfinite<Packet2d>(const Packet2d& a) {
5168 return vreinterpretq_f64_u64(vcagtq_f64(pinf<Packet2d>(), a));
5169}
5170// |bits| > inf bits as an unsigned comparison. !vcage(inf, a) is one instruction shorter inside a select, but Clang's
5171// inliner charges more for the intrinsic, and pexp_complex<Packet1cd> sits within a few units of its threshold.
5172template <>
5173EIGEN_STRONG_INLINE Packet2d pisnan<Packet2d>(const Packet2d& a) {
5174 return vreinterpretq_f64_u64(vcgtq_u64(vreinterpretq_u64_f64(vabsq_f64(a)), vdupq_n_u64(0x7ff0000000000000ull)));
5175}
5176
5177#endif // EIGEN_ARCH_ARM64
5178
5179// Do we have fp16 and support Neon intrinsics?
5180// FIXME: This disables vectorization on ARMv7 even though FP16 vector operations should
5181// be available with `__ARM_FEATURE_FP16_VECTOR_ARITHMETIC`. However, since the internal
5182// feature test macro `EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC` requires `EIGEN_ARCH_ARM64`
5183// in addition to the ISA feature test, the guard here also only checks for ARM64.
5184#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
5185using Packet4hf = float16x4_t;
5186using Packet8hf = float16x8_t;
5187
5188// Clang <19.1.0 does not provide `f16` intrinsics for pure data movement instructions.
5189// Polyfill them with `u16`.
5190//
5191// See: <https://github.com/llvm/llvm-project/pull/87467>.
5192#if EIGEN_CLANG_STRICT_LESS_THAN(19, 1, 0) && !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
5193EIGEN_ALWAYS_INLINE float16x4_t vbsl_f16(uint16x4_t a, float16x4_t b, float16x4_t c) {
5194 return vreinterpret_f16_u16(vbsl_u16(a, vreinterpret_u16_f16(b), vreinterpret_u16_f16(c)));
5195}
5196EIGEN_ALWAYS_INLINE float16x8_t vbslq_f16(uint16x8_t a, float16x8_t b, float16x8_t c) {
5197 return vreinterpretq_f16_u16(vbslq_u16(a, vreinterpretq_u16_f16(b), vreinterpretq_u16_f16(c)));
5198}
5199EIGEN_ALWAYS_INLINE float16x4_t vrev64_f16(float16x4_t a) {
5200 return vreinterpret_f16_u16(vrev64_u16(vreinterpret_u16_f16(a)));
5201}
5202EIGEN_ALWAYS_INLINE float16x8_t vrev64q_f16(float16x8_t a) {
5203 return vreinterpretq_f16_u16(vrev64q_u16(vreinterpretq_u16_f16(a)));
5204}
5205EIGEN_ALWAYS_INLINE float16x8x2_t vzipq_f16(float16x8_t a, float16x8_t b) {
5206 const uint16x8x2_t r = vzipq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b));
5207 return {vreinterpretq_f16_u16(r.val[0]), vreinterpretq_f16_u16(r.val[1])};
5208}
5209EIGEN_ALWAYS_INLINE float16x8x2_t vuzpq_f16(float16x8_t a, float16x8_t b) {
5210 const uint16x8x2_t r = vuzpq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b));
5211 return {vreinterpretq_f16_u16(r.val[0]), vreinterpretq_f16_u16(r.val[1])};
5212}
5213#endif // EIGEN_CLANG_STRICT_LESS_THAN(19, 1, 0) && !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
5214
5215template <>
5216struct packet_traits<half> : default_packet_traits {
5217 using type = Packet8hf;
5218 using half = Packet4hf;
5219 enum {
5220 Vectorizable = 1,
5221 AlignedOnScalar = 1,
5222 size = 8,
5223
5224 HasCmp = 1,
5225 HasCast = 1,
5226 HasAdd = 1,
5227 HasSub = 1,
5228 HasShift = 1,
5229 HasMul = 1,
5230 HasNegate = 1,
5231 HasAbs = 1,
5232 HasArg = 0,
5233 HasAbsDiff = 0,
5234 HasMin = 1,
5235 HasMax = 1,
5236 HasConj = 1,
5237 HasSetLinear = 1,
5238 HasInsert = 1,
5239 HasReduxp = 1,
5240 HasDiv = 1,
5241 HasSin = 0,
5242 HasCos = 0,
5243 HasLog = 0,
5244 HasExp = 0,
5245 HasTanh = packet_traits<float>::HasTanh, // tanh<half> calls tanh<float>
5246 HasSqrt = 1,
5247 HasRsqrt = 1,
5248 HasErf = EIGEN_FAST_MATH,
5249 HasBessel = 0, // Issues with accuracy.
5250 HasNdtri = 0
5251 };
5252};
5253
5254template <>
5255struct unpacket_traits<Packet4hf> : neon_unpacket_default<Packet4hf, half> {};
5256template <>
5257struct unpacket_traits<Packet8hf> : neon_unpacket_default<Packet8hf, half> {
5258 using half = Packet4hf;
5259};
5260
5261template <>
5262struct pminmax_propagates_nan<Packet4hf> : bool_constant<true> {};
5263template <>
5264struct pminmax_propagates_nan<Packet8hf> : bool_constant<true> {};
5265
5266template <>
5267EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pset1(const half& from) {
5268 return vdupq_n_f16(half_impl::native_half(from));
5269}
5270template <>
5271EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pset1(const half& from) {
5272 return vdup_n_f16(half_impl::native_half(from));
5273}
5274
5275#define EIGEN_MAKE_HALF_BITWISE_BINOP(name, op) \
5276 template <> \
5277 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a, const Packet8hf& b) { \
5278 return vreinterpretq_f16_u16(v##op##q_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b))); \
5279 } \
5280 template <> \
5281 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a, const Packet4hf& b) { \
5282 return vreinterpret_f16_u16(v##op##_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b))); \
5283 } \
5284 static_assert(true, "Trailing semicolon required")
5285
5286EIGEN_MAKE_HALF_BITWISE_BINOP(pand, and);
5287EIGEN_MAKE_HALF_BITWISE_BINOP(por, orr);
5288EIGEN_MAKE_HALF_BITWISE_BINOP(pxor, eor);
5289EIGEN_MAKE_HALF_BITWISE_BINOP(pandnot, bic);
5290
5291#undef EIGEN_MAKE_HALF_BITWISE_BINOP
5292
5293template <>
5294EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pload(const half* from) {
5295 EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f16(
5296 assume_aligned<unpacket_traits<Packet8hf>::alignment>(reinterpret_cast<const float16_t*>(from)));
5297}
5298template <>
5299EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pload(const half* from) {
5300 EIGEN_DEBUG_ALIGNED_LOAD return vld1_f16(
5301 assume_aligned<unpacket_traits<Packet4hf>::alignment>(reinterpret_cast<const float16_t*>(from)));
5302}
5303
5304template <>
5305EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf ploadu(const half* from) {
5306 EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f16(reinterpret_cast<const float16_t*>(from));
5307}
5308template <>
5309EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf ploadu(const half* from) {
5310 EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f16(reinterpret_cast<const float16_t*>(from));
5311}
5312
5313template <>
5314EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf ploaddup(const half* from) {
5315 Packet8hf packet{};
5316 packet = vsetq_lane_f16(half_impl::native_half(from[0]), packet, 0);
5317 packet = vsetq_lane_f16(half_impl::native_half(from[0]), packet, 1);
5318 packet = vsetq_lane_f16(half_impl::native_half(from[1]), packet, 2);
5319 packet = vsetq_lane_f16(half_impl::native_half(from[1]), packet, 3);
5320 packet = vsetq_lane_f16(half_impl::native_half(from[2]), packet, 4);
5321 packet = vsetq_lane_f16(half_impl::native_half(from[2]), packet, 5);
5322 packet = vsetq_lane_f16(half_impl::native_half(from[3]), packet, 6);
5323 packet = vsetq_lane_f16(half_impl::native_half(from[3]), packet, 7);
5324 return packet;
5325}
5326template <>
5327EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf ploaddup(const half* from) {
5328 float16x4_t packet{};
5329 packet = vset_lane_f16(half_impl::native_half(from[0]), packet, 0);
5330 packet = vset_lane_f16(half_impl::native_half(from[0]), packet, 1);
5331 packet = vset_lane_f16(half_impl::native_half(from[1]), packet, 2);
5332 packet = vset_lane_f16(half_impl::native_half(from[1]), packet, 3);
5333 return packet;
5334}
5335
5336template <>
5337EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf ploadquad(const half* from) {
5338 Packet4hf lo, hi;
5339 // Not vld1_dup_f16: GCC implements it as a scalar __fp16 load, which breaks strict aliasing on uint16_t storage.
5340 lo = vdup_n_f16(half_impl::native_half(from[0]));
5341 hi = vdup_n_f16(half_impl::native_half(from[1]));
5342 return vcombine_f16(lo, hi);
5343}
5344
5345EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertfirst(const Packet8hf& a, half b) {
5346 return vsetq_lane_f16(half_impl::native_half(b), a, 0);
5347}
5348EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertfirst(const Packet4hf& a, half b) {
5349 return vset_lane_f16(half_impl::native_half(b), a, 0);
5350}
5351
5352template <>
5353EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pselect(const Packet8hf& mask, const Packet8hf& a, const Packet8hf& b) {
5354 return vbslq_f16(vreinterpretq_u16_f16(mask), a, b);
5355}
5356template <>
5357EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pselect(const Packet4hf& mask, const Packet4hf& a, const Packet4hf& b) {
5358 return vbsl_f16(vreinterpret_u16_f16(mask), a, b);
5359}
5360
5361template <>
5362EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pnot(const Packet8hf& a) {
5363 return vreinterpretq_f16_u16(vmvnq_u16(vreinterpretq_u16_f16(a)));
5364}
5365template <>
5366EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pnot(const Packet4hf& a) {
5367 return vreinterpret_f16_u16(vmvn_u16(vreinterpret_u16_f16(a)));
5368}
5369
5370EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertlast(const Packet8hf& a, half b) {
5371 return vsetq_lane_f16(half_impl::native_half(b), a, 7);
5372}
5373EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertlast(const Packet4hf& a, half b) {
5374 return vset_lane_f16(half_impl::native_half(b), a, 3);
5375}
5376
5377template <>
5378EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore(half* to, const Packet8hf& from) {
5379 EIGEN_DEBUG_ALIGNED_STORE vst1q_f16(
5380 assume_aligned<unpacket_traits<Packet8hf>::alignment>(reinterpret_cast<float16_t*>(to)), from);
5381}
5382template <>
5383EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore(half* to, const Packet4hf& from) {
5384 EIGEN_DEBUG_ALIGNED_STORE vst1_f16(
5385 assume_aligned<unpacket_traits<Packet4hf>::alignment>(reinterpret_cast<float16_t*>(to)), from);
5386}
5387
5388template <>
5389EIGEN_STRONG_INLINE void pstoreu(half* to, const Packet8hf& from) {
5390 EIGEN_DEBUG_UNALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from);
5391}
5392template <>
5393EIGEN_STRONG_INLINE void pstoreu(half* to, const Packet4hf& from) {
5394 EIGEN_DEBUG_UNALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from);
5395}
5396
5397template <>
5398EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pgather(const half* from, Index stride) {
5399 Packet8hf res = pset1<Packet8hf>(half(0.f));
5400 res = vsetq_lane_f16(half_impl::native_half(from[0 * stride]), res, 0);
5401 res = vsetq_lane_f16(half_impl::native_half(from[1 * stride]), res, 1);
5402 res = vsetq_lane_f16(half_impl::native_half(from[2 * stride]), res, 2);
5403 res = vsetq_lane_f16(half_impl::native_half(from[3 * stride]), res, 3);
5404 res = vsetq_lane_f16(half_impl::native_half(from[4 * stride]), res, 4);
5405 res = vsetq_lane_f16(half_impl::native_half(from[5 * stride]), res, 5);
5406 res = vsetq_lane_f16(half_impl::native_half(from[6 * stride]), res, 6);
5407 res = vsetq_lane_f16(half_impl::native_half(from[7 * stride]), res, 7);
5408 return res;
5409}
5410template <>
5411EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pgather(const half* from, Index stride) {
5412 Packet4hf res = pset1<Packet4hf>(half(0.f));
5413 res = vset_lane_f16(half_impl::native_half(from[0 * stride]), res, 0);
5414 res = vset_lane_f16(half_impl::native_half(from[1 * stride]), res, 1);
5415 res = vset_lane_f16(half_impl::native_half(from[2 * stride]), res, 2);
5416 res = vset_lane_f16(half_impl::native_half(from[3 * stride]), res, 3);
5417 return res;
5418}
5419
5420template <>
5421EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(half* to, const Packet8hf& from, Index stride) {
5422 to[stride * 0] = half_impl::half_from_rep(vgetq_lane_f16(from, 0));
5423 to[stride * 1] = half_impl::half_from_rep(vgetq_lane_f16(from, 1));
5424 to[stride * 2] = half_impl::half_from_rep(vgetq_lane_f16(from, 2));
5425 to[stride * 3] = half_impl::half_from_rep(vgetq_lane_f16(from, 3));
5426 to[stride * 4] = half_impl::half_from_rep(vgetq_lane_f16(from, 4));
5427 to[stride * 5] = half_impl::half_from_rep(vgetq_lane_f16(from, 5));
5428 to[stride * 6] = half_impl::half_from_rep(vgetq_lane_f16(from, 6));
5429 to[stride * 7] = half_impl::half_from_rep(vgetq_lane_f16(from, 7));
5430}
5431template <>
5432EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(half* to, const Packet4hf& from, Index stride) {
5433 to[stride * 0] = half_impl::half_from_rep(vget_lane_f16(from, 0));
5434 to[stride * 1] = half_impl::half_from_rep(vget_lane_f16(from, 1));
5435 to[stride * 2] = half_impl::half_from_rep(vget_lane_f16(from, 2));
5436 to[stride * 3] = half_impl::half_from_rep(vget_lane_f16(from, 3));
5437}
5438
5439template <>
5440EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void prefetch(const half* addr) {
5441 EIGEN_ARM_PREFETCH(addr);
5442}
5443
5444template <>
5445EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half pfirst(const Packet8hf& a) {
5446 return half(vgetq_lane_f16(a, 0));
5447}
5448template <>
5449EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half pfirst(const Packet4hf& a) {
5450 return half(vget_lane_f16(a, 0));
5451}
5452
5453template <>
5454EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf preverse(const Packet8hf& a) {
5455 float16x8_t r = vrev64q_f16(a);
5456 return vcombine_f16(vget_high_f16(r), vget_low_f16(r));
5457}
5458template <>
5459EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf preverse(const Packet4hf& a) {
5460 return vrev64_f16(a);
5461}
5462
5463template <>
5464EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf psignbit(const Packet8hf& a) {
5465 return vreinterpretq_f16_s16(vshrq_n_s16(vreinterpretq_s16_f16(a), 15));
5466}
5467template <>
5468EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf psignbit(const Packet4hf& a) {
5469 return vreinterpret_f16_s16(vshr_n_s16(vreinterpret_s16_f16(a), 15));
5470}
5471
5472template <>
5473EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const Packet8hf& a) {
5474 return vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(vreinterpretq_u16_f16(a))), 0);
5475}
5476template <>
5477EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const Packet4hf& a) {
5478 return vget_lane_u64(vreinterpret_u64_f16(a), 0);
5479}
5480
5481EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8hf, 4>& kernel) {
5482 const float16x8x2_t zip16_1 = vzipq_f16(kernel.packet[0], kernel.packet[1]);
5483 const float16x8x2_t zip16_2 = vzipq_f16(kernel.packet[2], kernel.packet[3]);
5484
5485 const float32x4x2_t zip32_1 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[0]), vreinterpretq_f32_f16(zip16_2.val[0]));
5486 const float32x4x2_t zip32_2 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[1]), vreinterpretq_f32_f16(zip16_2.val[1]));
5487
5488 kernel.packet[0] = vreinterpretq_f16_f32(zip32_1.val[0]);
5489 kernel.packet[1] = vreinterpretq_f16_f32(zip32_1.val[1]);
5490 kernel.packet[2] = vreinterpretq_f16_f32(zip32_2.val[0]);
5491 kernel.packet[3] = vreinterpretq_f16_f32(zip32_2.val[1]);
5492}
5493EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4hf, 4>& kernel) {
5494 EIGEN_ALIGN16 float16x4x4_t tmp_x4;
5495 float16_t* tmp = reinterpret_cast<float16_t*>(&kernel);
5496 tmp_x4 = vld4_f16(tmp);
5497
5498 kernel.packet[0] = tmp_x4.val[0];
5499 kernel.packet[1] = tmp_x4.val[1];
5500 kernel.packet[2] = tmp_x4.val[2];
5501 kernel.packet[3] = tmp_x4.val[3];
5502}
5503
5504EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8hf, 8>& kernel) {
5505 float16x8x2_t T_1[4];
5506
5507 T_1[0] = vuzpq_f16(kernel.packet[0], kernel.packet[1]);
5508 T_1[1] = vuzpq_f16(kernel.packet[2], kernel.packet[3]);
5509 T_1[2] = vuzpq_f16(kernel.packet[4], kernel.packet[5]);
5510 T_1[3] = vuzpq_f16(kernel.packet[6], kernel.packet[7]);
5511
5512 float16x8x2_t T_2[4];
5513 T_2[0] = vuzpq_f16(T_1[0].val[0], T_1[1].val[0]);
5514 T_2[1] = vuzpq_f16(T_1[0].val[1], T_1[1].val[1]);
5515 T_2[2] = vuzpq_f16(T_1[2].val[0], T_1[3].val[0]);
5516 T_2[3] = vuzpq_f16(T_1[2].val[1], T_1[3].val[1]);
5517
5518 float16x8x2_t T_3[4];
5519 T_3[0] = vuzpq_f16(T_2[0].val[0], T_2[2].val[0]);
5520 T_3[1] = vuzpq_f16(T_2[0].val[1], T_2[2].val[1]);
5521 T_3[2] = vuzpq_f16(T_2[1].val[0], T_2[3].val[0]);
5522 T_3[3] = vuzpq_f16(T_2[1].val[1], T_2[3].val[1]);
5523
5524 kernel.packet[0] = T_3[0].val[0];
5525 kernel.packet[1] = T_3[2].val[0];
5526 kernel.packet[2] = T_3[1].val[0];
5527 kernel.packet[3] = T_3[3].val[0];
5528 kernel.packet[4] = T_3[0].val[1];
5529 kernel.packet[5] = T_3[2].val[1];
5530 kernel.packet[6] = T_3[1].val[1];
5531 kernel.packet[7] = T_3[3].val[1];
5532}
5533
5534#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
5535
5536template <>
5537EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half<Packet8hf>(const Packet8hf& a) {
5538 return vadd_f16(vget_low_f16(a), vget_high_f16(a));
5539}
5540
5541template <>
5542EIGEN_STRONG_INLINE Packet8hf plset<Packet8hf>(const Eigen::half& a) {
5543 const float16_t f[] = {0, 1, 2, 3, 4, 5, 6, 7};
5544 Packet8hf countdown = vld1q_f16(f);
5545 return vaddq_f16(pset1<Packet8hf>(a), countdown);
5546}
5547
5548template <>
5549EIGEN_STRONG_INLINE Packet4hf plset<Packet4hf>(const Eigen::half& a) {
5550 const float16_t f[] = {0, 1, 2, 3};
5551 Packet4hf countdown = vld1_f16(f);
5552 return vadd_f16(pset1<Packet4hf>(a), countdown);
5553}
5554
5555template <>
5556EIGEN_STRONG_INLINE Packet8hf padd<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5557 return vaddq_f16(a, b);
5558}
5559
5560template <>
5561EIGEN_STRONG_INLINE Packet4hf padd<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5562 return vadd_f16(a, b);
5563}
5564
5565template <>
5566EIGEN_STRONG_INLINE Packet8hf psub<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5567 return vsubq_f16(a, b);
5568}
5569
5570template <>
5571EIGEN_STRONG_INLINE Packet4hf psub<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5572 return vsub_f16(a, b);
5573}
5574
5575template <>
5576EIGEN_STRONG_INLINE Packet8hf pnegate(const Packet8hf& a) {
5577 return vnegq_f16(a);
5578}
5579
5580template <>
5581EIGEN_STRONG_INLINE Packet4hf pnegate(const Packet4hf& a) {
5582 return vneg_f16(a);
5583}
5584
5585template <>
5586EIGEN_STRONG_INLINE Packet8hf pmul<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5587 return vmulq_f16(a, b);
5588}
5589
5590template <>
5591EIGEN_STRONG_INLINE Packet4hf pmul<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5592 return vmul_f16(a, b);
5593}
5594
5595template <>
5596EIGEN_STRONG_INLINE Packet8hf pdiv<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5597 return vdivq_f16(a, b);
5598}
5599
5600template <>
5601EIGEN_STRONG_INLINE Packet4hf pdiv<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5602 return vdiv_f16(a, b);
5603}
5604
5605template <>
5606struct is_fused_madd<Packet8hf> : true_type {};
5607template <>
5608EIGEN_STRONG_INLINE Packet8hf pmadd(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) {
5609 return vfmaq_f16(c, a, b);
5610}
5611
5612template <>
5613struct is_fused_madd<Packet4hf> : true_type {};
5614template <>
5615EIGEN_STRONG_INLINE Packet4hf pmadd(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) {
5616 return vfma_f16(c, a, b);
5617}
5618
5619template <>
5620EIGEN_STRONG_INLINE Packet8hf pnmadd(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) {
5621 return vfmsq_f16(c, a, b);
5622}
5623
5624template <>
5625EIGEN_STRONG_INLINE Packet4hf pnmadd(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) {
5626 return vfms_f16(c, a, b);
5627}
5628
5629template <>
5630EIGEN_STRONG_INLINE Packet8hf pmin<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5631 return vminq_f16(a, b);
5632}
5633
5634template <>
5635EIGEN_STRONG_INLINE Packet4hf pmin<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5636 return vmin_f16(a, b);
5637}
5638
5639#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
5640// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8
5641// systems).
5642template <>
5643EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNumbers, Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5644 return vminnm_f16(a, b);
5645}
5646template <>
5647EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNumbers, Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5648 return vminnmq_f16(a, b);
5649}
5650#endif
5651
5652template <>
5653EIGEN_STRONG_INLINE Packet8hf pmax<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5654 return vmaxq_f16(a, b);
5655}
5656
5657template <>
5658EIGEN_STRONG_INLINE Packet4hf pmax<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5659 return vmax_f16(a, b);
5660}
5661
5662#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
5663// numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8
5664// systems).
5665template <>
5666EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNumbers, Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5667 return vmaxnm_f16(a, b);
5668}
5669template <>
5670EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNumbers, Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5671 return vmaxnmq_f16(a, b);
5672}
5673#endif
5674
5675#define EIGEN_MAKE_ARM_FP16_CMP_8(name) \
5676 template <> \
5677 EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \
5678 return vreinterpretq_f16_u16(vc##name##q_f16(a, b)); \
5679 }
5680
5681#define EIGEN_MAKE_ARM_FP16_CMP_4(name) \
5682 template <> \
5683 EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \
5684 return vreinterpret_f16_u16(vc##name##_f16(a, b)); \
5685 }
5686
5687EIGEN_MAKE_ARM_FP16_CMP_8(eq)
5688EIGEN_MAKE_ARM_FP16_CMP_8(lt)
5689EIGEN_MAKE_ARM_FP16_CMP_8(le)
5690
5691EIGEN_MAKE_ARM_FP16_CMP_4(eq)
5692EIGEN_MAKE_ARM_FP16_CMP_4(lt)
5693EIGEN_MAKE_ARM_FP16_CMP_4(le)
5694
5695#undef EIGEN_MAKE_ARM_FP16_CMP_8
5696#undef EIGEN_MAKE_ARM_FP16_CMP_4
5697
5698template <>
5699EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
5700 return vreinterpretq_f16_u16(vmvnq_u16(vcgeq_f16(a, b)));
5701}
5702
5703template <>
5704EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
5705 return vreinterpret_f16_u16(vmvn_u16(vcge_f16(a, b)));
5706}
5707
5708// Absolute comparisons as for Packet4f.
5709template <>
5710EIGEN_STRONG_INLINE Packet4hf pisinf<Packet4hf>(const Packet4hf& a) {
5711 return vreinterpret_f16_u16(vcage_f16(a, pinf<Packet4hf>()));
5712}
5713template <>
5714EIGEN_STRONG_INLINE Packet8hf pisinf<Packet8hf>(const Packet8hf& a) {
5715 return vreinterpretq_f16_u16(vcageq_f16(a, pinf<Packet8hf>()));
5716}
5717template <>
5718EIGEN_STRONG_INLINE Packet4hf pisfinite<Packet4hf>(const Packet4hf& a) {
5719 return vreinterpret_f16_u16(vcagt_f16(pinf<Packet4hf>(), a));
5720}
5721template <>
5722EIGEN_STRONG_INLINE Packet8hf pisfinite<Packet8hf>(const Packet8hf& a) {
5723 return vreinterpretq_f16_u16(vcagtq_f16(pinf<Packet8hf>(), a));
5724}
5725template <>
5726EIGEN_STRONG_INLINE Packet4hf pisnan<Packet4hf>(const Packet4hf& a) {
5727 return vreinterpret_f16_u16(vmvn_u16(vcage_f16(pinf<Packet4hf>(), a)));
5728}
5729template <>
5730EIGEN_STRONG_INLINE Packet8hf pisnan<Packet8hf>(const Packet8hf& a) {
5731 return vreinterpretq_f16_u16(vmvnq_u16(vcageq_f16(pinf<Packet8hf>(), a)));
5732}
5733
5734template <>
5735EIGEN_STRONG_INLINE Packet8hf print<Packet8hf>(const Packet8hf& a) {
5736 return vrndnq_f16(a);
5737}
5738
5739template <>
5740EIGEN_STRONG_INLINE Packet4hf print<Packet4hf>(const Packet4hf& a) {
5741 return vrndn_f16(a);
5742}
5743
5744template <>
5745EIGEN_STRONG_INLINE Packet8hf pfloor<Packet8hf>(const Packet8hf& a) {
5746 return vrndmq_f16(a);
5747}
5748
5749template <>
5750EIGEN_STRONG_INLINE Packet4hf pfloor<Packet4hf>(const Packet4hf& a) {
5751 return vrndm_f16(a);
5752}
5753
5754template <>
5755EIGEN_STRONG_INLINE Packet8hf pceil<Packet8hf>(const Packet8hf& a) {
5756 return vrndpq_f16(a);
5757}
5758
5759template <>
5760EIGEN_STRONG_INLINE Packet4hf pceil<Packet4hf>(const Packet4hf& a) {
5761 return vrndp_f16(a);
5762}
5763
5764template <>
5765EIGEN_STRONG_INLINE Packet8hf pround<Packet8hf>(const Packet8hf& a) {
5766 return vrndaq_f16(a);
5767}
5768
5769template <>
5770EIGEN_STRONG_INLINE Packet4hf pround<Packet4hf>(const Packet4hf& a) {
5771 return vrnda_f16(a);
5772}
5773
5774template <>
5775EIGEN_STRONG_INLINE Packet8hf ptrunc<Packet8hf>(const Packet8hf& a) {
5776 return vrndq_f16(a);
5777}
5778
5779template <>
5780EIGEN_STRONG_INLINE Packet4hf ptrunc<Packet4hf>(const Packet4hf& a) {
5781 return vrnd_f16(a);
5782}
5783
5784template <>
5785EIGEN_STRONG_INLINE Packet8hf psqrt<Packet8hf>(const Packet8hf& a) {
5786 return vsqrtq_f16(a);
5787}
5788
5789template <>
5790EIGEN_STRONG_INLINE Packet4hf psqrt<Packet4hf>(const Packet4hf& a) {
5791 return vsqrt_f16(a);
5792}
5793
5794template <>
5795EIGEN_STRONG_INLINE Packet8hf pabs<Packet8hf>(const Packet8hf& a) {
5796 return vabsq_f16(a);
5797}
5798
5799template <>
5800EIGEN_STRONG_INLINE Packet4hf pabs<Packet4hf>(const Packet4hf& a) {
5801 return vabs_f16(a);
5802}
5803
5804template <>
5805EIGEN_STRONG_INLINE Eigen::half predux<Packet8hf>(const Packet8hf& a) {
5806 float16x4_t a_lo, a_hi, sum;
5807
5808 a_lo = vget_low_f16(a);
5809 a_hi = vget_high_f16(a);
5810 sum = vpadd_f16(a_lo, a_hi);
5811 sum = vpadd_f16(sum, sum);
5812 sum = vpadd_f16(sum, sum);
5813
5814 return half_impl::half_from_rep(vget_lane_f16(sum, 0));
5815}
5816
5817template <>
5818EIGEN_STRONG_INLINE Eigen::half predux<Packet4hf>(const Packet4hf& a) {
5819 float16x4_t sum;
5820
5821 sum = vpadd_f16(a, a);
5822 sum = vpadd_f16(sum, sum);
5823 return half_impl::half_from_rep(vget_lane_f16(sum, 0));
5824}
5825
5826template <>
5827EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8hf>(const Packet8hf& a) {
5828 float16x4_t a_lo, a_hi, prod;
5829
5830 a_lo = vget_low_f16(a);
5831 a_hi = vget_high_f16(a);
5832 prod = vmul_f16(a_lo, a_hi);
5833 prod = vmul_f16(prod, vrev64_f16(prod));
5834
5835 return half_impl::half_from_rep(vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1)));
5836}
5837
5838template <>
5839EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet4hf>(const Packet4hf& a) {
5840 float16x4_t prod;
5841 prod = vmul_f16(a, vrev64_f16(a));
5842 return half_impl::half_from_rep(vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1)));
5843}
5844
5845template <>
5846EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8hf>(const Packet8hf& a) {
5847 return half_impl::half_from_rep(vminvq_f16(a));
5848}
5849
5850template <>
5851EIGEN_STRONG_INLINE Eigen::half predux_min<Packet4hf>(const Packet4hf& a) {
5852 return half_impl::half_from_rep(vminv_f16(a));
5853}
5854
5855template <>
5856EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8hf>(const Packet8hf& a) {
5857 return half_impl::half_from_rep(vmaxvq_f16(a));
5858}
5859
5860template <>
5861EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4hf>(const Packet4hf& a) {
5862 return half_impl::half_from_rep(vmaxv_f16(a));
5863}
5864
5865#else
5866
5867// Even if we do not have native vector arithmetic on fp16 types, we can still
5868// assume the presence of conversion instructions to/from fp32 on AArch64
5869// ([ref]). As a result, packet arithmetic on AArch64 without `+fp16` is
5870// implemented through conversion to fp32. Beware that this comes at the
5871// expense of additional instructions for casting and potential differences in
5872// floating-point error accumulation. The performance is still much better than
5873// manual fp16, however.
5874//
5875// [ref]:
5876// <https://developer.arm.com/documentation/101028/0012/13--Advanced-SIMD--Neon--intrinsics#availability-of-16-bit-floating-point-vector-interchange-types>
5877
5878template <>
5879EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half(const Packet8hf& a) {
5880 return vcvt_f16_f32(vaddq_f32(vcvt_f32_f16(vget_low_f16(a)), vcvt_f32_f16(vget_high_f16(a))));
5881}
5882
5883template <>
5884EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf plset(const half& a) {
5885 const float f[] = {0, 1, 2, 3};
5886 float32x4_t countdown = vld1q_f32(f);
5887 float32x4_t base = vaddq_f32(vcvt_f32_f16(pset1<Packet4hf>(a)), countdown);
5888 return vcombine_f16(vcvt_f16_f32(base), vcvt_f16_f32(vaddq_f32(base, vdupq_n_f32(4))));
5889}
5890template <>
5891EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf plset(const half& a) {
5892 const float f[] = {0, 1, 2, 3};
5893 Packet4f countdown = vld1q_f32(f);
5894 return vcvt_f16_f32(vaddq_f32(pset1<Packet4f>(a), countdown));
5895}
5896
5897#define EIGEN_MAKE_HALF_UNOP(name, op) \
5898 template <> \
5899 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a) { \
5900 return vcvt_f16_f32(v##op##q_f32(vcvt_f32_f16(a))); \
5901 } \
5902 template <> \
5903 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a) { \
5904 return vcombine_f16(name(vget_low_f16(a)), name(vget_high_f16(a))); \
5905 } \
5906 static_assert(true, "Trailing semicolon required")
5907#define EIGEN_MAKE_HALF_BINOP(name, op) \
5908 template <> \
5909 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a, const Packet4hf& b) { \
5910 return vcvt_f16_f32(v##op##q_f32(vcvt_f32_f16(a), vcvt_f32_f16(b))); \
5911 } \
5912 template <> \
5913 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a, const Packet8hf& b) { \
5914 return vcombine_f16(name(vget_low_f16(a), vget_low_f16(b)), name(vget_high_f16(a), vget_high_f16(b))); \
5915 } \
5916 static_assert(true, "Trailing semicolon required")
5917
5918EIGEN_MAKE_HALF_BINOP(padd, add);
5919EIGEN_MAKE_HALF_BINOP(psub, sub);
5920
5921template <>
5922EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pnegate(const Packet8hf& a) {
5923 return vreinterpretq_f16_u16(veorq_u16(vreinterpretq_u16_f16(a), vdupq_n_u16(0x8000)));
5924}
5925template <>
5926EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pnegate(const Packet4hf& a) {
5927 return vreinterpret_f16_u16(veor_u16(vreinterpret_u16_f16(a), vdup_n_u16(0x8000)));
5928}
5929
5930EIGEN_MAKE_HALF_BINOP(pmul, mul);
5931
5932#if EIGEN_ARCH_ARM64
5933EIGEN_MAKE_HALF_BINOP(pdiv, div);
5934#else
5935template <>
5936EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pdiv(const Packet4hf& a, const Packet4hf& b) {
5937 return vcvt_f16_f32(pdiv(vcvt_f32_f16(a), vcvt_f32_f16(b)));
5938}
5939template <>
5940EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pdiv(const Packet8hf& a, const Packet8hf& b) {
5941 return vcombine_f16(pdiv(vget_low_f16(a), vget_low_f16(b)), pdiv(vget_high_f16(a), vget_high_f16(b)));
5942}
5943#endif
5944
5945#define EIGEN_MAKE_HALF_FMA(name, op) \
5946 template <> \
5947 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf name(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) { \
5948 return vcvt_f16_f32(v##op##q_f32(vcvt_f32_f16(c), vcvt_f32_f16(a), vcvt_f32_f16(b))); \
5949 } \
5950 template <> \
5951 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf name(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) { \
5952 return vcombine_f16(name(vget_low_f16(a), vget_low_f16(b), vget_low_f16(c)), \
5953 name(vget_high_f16(a), vget_high_f16(b), vget_high_f16(c))); \
5954 } \
5955 static_assert(true, "Trailing semicolon required")
5956
5957EIGEN_MAKE_HALF_FMA(pmadd, fma);
5958EIGEN_MAKE_HALF_FMA(pnmadd, fms);
5959
5960#undef EIGEN_MAKE_HALF_FMA
5961
5962EIGEN_MAKE_HALF_BINOP(pmin, min);
5963EIGEN_MAKE_HALF_BINOP(pmax, max);
5964
5965#ifdef __ARM_FEATURE_NUMERIC_MAXMIN
5966
5967#define EIGEN_MAKE_HALF_NUMERIC_MAXMIN(name) \
5968 template <> \
5969 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf p##name<PropagateNumbers, Packet4hf>(const Packet4hf& a, \
5970 const Packet4hf& b) { \
5971 return vcvt_f16_f32(v##name##nmq_f32(vcvt_f32_f16(a), vcvt_f32_f16(b))); \
5972 } \
5973 template <> \
5974 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf p##name<PropagateNumbers, Packet8hf>(const Packet8hf& a, \
5975 const Packet8hf& b) { \
5976 return vcombine_f16(p##name<PropagateNumbers, Packet4hf>(vget_low_f16(a), vget_low_f16(b)), \
5977 p##name<PropagateNumbers, Packet4hf>(vget_high_f16(a), vget_high_f16(b))); \
5978 } \
5979 static_assert(true, "Trailing semicolon required")
5980
5981EIGEN_MAKE_HALF_NUMERIC_MAXMIN(min);
5982EIGEN_MAKE_HALF_NUMERIC_MAXMIN(max);
5983
5984#undef EIGEN_MAKE_HALF_NUMERIC_MAXMIN
5985
5986#endif
5987
5988#define EIGEN_MAKE_HALF_CMP(name) \
5989 template <> \
5990 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \
5991 return vreinterpret_f16_u16(vmovn_u32(vc##name##q_f32(vcvt_f32_f16(a), vcvt_f32_f16(b)))); \
5992 } \
5993 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \
5994 return vcombine_f16(pcmp_##name(vget_low_f16(a), vget_low_f16(b)), \
5995 pcmp_##name(vget_high_f16(a), vget_high_f16(b))); \
5996 } \
5997 static_assert(true, "Trailing semicolon required")
5998
5999EIGEN_MAKE_HALF_CMP(eq);
6000EIGEN_MAKE_HALF_CMP(lt);
6001EIGEN_MAKE_HALF_CMP(le);
6002
6003#undef EIGEN_MAKE_HALF_CMP
6004
6005template <>
6006EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan(const Packet4hf& a, const Packet4hf& b) {
6007 return vreinterpret_f16_u16(vmovn_u32(vmvnq_u32(vcgeq_f32(vcvt_f32_f16(a), vcvt_f32_f16(b)))));
6008}
6009template <>
6010EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan(const Packet8hf& a, const Packet8hf& b) {
6011 return vcombine_f16(pcmp_lt_or_nan(vget_low_f16(a), vget_low_f16(b)),
6012 pcmp_lt_or_nan(vget_high_f16(a), vget_high_f16(b)));
6013}
6014
6015EIGEN_MAKE_HALF_UNOP(print, rndn);
6016EIGEN_MAKE_HALF_UNOP(pfloor, rndm);
6017EIGEN_MAKE_HALF_UNOP(pceil, rndp);
6018EIGEN_MAKE_HALF_UNOP(pround, rnda);
6019EIGEN_MAKE_HALF_UNOP(ptrunc, rnd);
6020
6021#if EIGEN_ARCH_ARM64
6022EIGEN_MAKE_HALF_UNOP(psqrt, sqrt);
6023#else
6024template <>
6025EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf psqrt(const Packet4hf& a) {
6026 return vcvt_f16_f32(psqrt(vcvt_f32_f16(a)));
6027}
6028template <>
6029EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf psqrt(const Packet8hf& a) {
6030 return vcombine_f16(psqrt(vget_low_f16(a)), psqrt(vget_high_f16(a)));
6031}
6032#endif
6033
6034template <>
6035EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pabs(const Packet8hf& a) {
6036 return vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(a), vdupq_n_u16(0x8000)));
6037}
6038template <>
6039EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pabs(const Packet4hf& a) {
6040 return vreinterpret_f16_u16(vbic_u16(vreinterpret_u16_f16(a), vdup_n_u16(0x8000)));
6041}
6042
6043// NOTE: On AArch64, we can use horizontal vector reductions for `add`, `min`, and `max`.
6044// However, the fallback through `predux<Packet${N}f>` is still necessary for `mul`.
6045#define EIGEN_HALF_FOLD_REDUX(name, op) \
6046 template <> \
6047 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet4hf& a) { \
6048 return half(name(vcvt_f32_f16(a))); \
6049 } \
6050 template <> \
6051 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet8hf& a) { \
6052 return half(name(p##op(vcvt_f32_f16(vget_low_f16(a)), vcvt_f32_f16(vget_high_f16(a))))); \
6053 } \
6054 static_assert(true, "Trailing semicolon required")
6055
6056#if EIGEN_ARCH_ARM64
6057#define EIGEN_HALF_HORIZONTAL_REDUX(name, op) \
6058 template <> \
6059 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet8hf& a) { \
6060 return half(v##op##vq_f32(v##op##q_f32(vcvt_f32_f16(vget_low_f16(a)), vcvt_f32_f16(vget_high_f16(a))))); \
6061 } \
6062 template <> \
6063 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half name(const Packet4hf& a) { \
6064 return half(v##op##vq_f32(vcvt_f32_f16(a))); \
6065 } \
6066 static_assert(true, "Trailing semicolon required")
6067#else
6068#define EIGEN_HALF_HORIZONTAL_REDUX(name, op) EIGEN_HALF_FOLD_REDUX(name, op)
6069#endif
6070
6071EIGEN_HALF_HORIZONTAL_REDUX(predux, add);
6072EIGEN_HALF_FOLD_REDUX(predux_mul, mul);
6073EIGEN_HALF_HORIZONTAL_REDUX(predux_min, min);
6074EIGEN_HALF_HORIZONTAL_REDUX(predux_max, max);
6075
6076#undef EIGEN_HALF_FOLD_REDUX
6077#undef EIGEN_HALF_HORIZONTAL_REDUX
6078
6079#undef EIGEN_MAKE_HALF_UNOP
6080#undef EIGEN_MAKE_HALF_BINOP
6081
6082#endif // end EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
6083
6084#if !EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
6085// Without FP16 arithmetic, half comparisons convert lane by lane. Classify on the raw bits instead, as for bfloat16:
6086// |a| ==, >, < 0x7c00.
6087EIGEN_STRONG_INLINE uint16x4_t neon_half_abs_bits(const Packet4hf& a) {
6088 return vbic_u16(vreinterpret_u16_f16(a), vdup_n_u16(0x8000));
6089}
6090EIGEN_STRONG_INLINE uint16x8_t neon_half_abs_bits(const Packet8hf& a) {
6091 return vbicq_u16(vreinterpretq_u16_f16(a), vdupq_n_u16(0x8000));
6092}
6093template <>
6094EIGEN_STRONG_INLINE Packet4hf pisinf<Packet4hf>(const Packet4hf& a) {
6095 return vreinterpret_f16_u16(vceq_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00)));
6096}
6097template <>
6098EIGEN_STRONG_INLINE Packet8hf pisinf<Packet8hf>(const Packet8hf& a) {
6099 return vreinterpretq_f16_u16(vceqq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00)));
6100}
6101template <>
6102EIGEN_STRONG_INLINE Packet4hf pisnan<Packet4hf>(const Packet4hf& a) {
6103 return vreinterpret_f16_u16(vcgt_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00)));
6104}
6105template <>
6106EIGEN_STRONG_INLINE Packet8hf pisnan<Packet8hf>(const Packet8hf& a) {
6107 return vreinterpretq_f16_u16(vcgtq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00)));
6108}
6109template <>
6110EIGEN_STRONG_INLINE Packet4hf pisfinite<Packet4hf>(const Packet4hf& a) {
6111 return vreinterpret_f16_u16(vclt_u16(neon_half_abs_bits(a), vdup_n_u16(0x7c00)));
6112}
6113template <>
6114EIGEN_STRONG_INLINE Packet8hf pisfinite<Packet8hf>(const Packet8hf& a) {
6115 return vreinterpretq_f16_u16(vcltq_u16(neon_half_abs_bits(a), vdupq_n_u16(0x7c00)));
6116}
6117#endif
6118
6119#endif // end EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
6120
6121/*---------------- division by an invariant integer ----------------*/
6122
6123// pudiv_magic and psdiv_signed_magic (IntegerDivision.h) for 32-bit lanes. vshl by a negative count shifts right.
6124template <>
6125EIGEN_STRONG_INLINE Packet4ui pudiv_magic<Packet4ui>(const Packet4ui& a, uint32_t magic, uint32_t addend, int shift) {
6126 uint32x2_t m = vdup_n_u32(magic);
6127 uint64x2_t c = vdupq_n_u64(addend);
6128 uint64x2_t lo = vmlal_u32(c, vget_low_u32(a), m);
6129 uint64x2_t hi = vmlal_u32(c, vget_high_u32(a), m);
6130#if EIGEN_ARCH_ARM64
6131 uint32x4_t t = vuzp2q_u32(vreinterpretq_u32_u64(lo), vreinterpretq_u32_u64(hi));
6132#else
6133 uint32x4_t t = vcombine_u32(vshrn_n_u64(lo, 32), vshrn_n_u64(hi, 32));
6134#endif
6135 return vshlq_u32(t, vdupq_n_s32(-shift));
6136}
6137template <>
6138EIGEN_STRONG_INLINE Packet2ui pudiv_magic<Packet2ui>(const Packet2ui& a, uint32_t magic, uint32_t addend, int shift) {
6139 uint32x2_t t = vshrn_n_u64(vmlal_u32(vdupq_n_u64(addend), a, vdup_n_u32(magic)), 32);
6140 return vshl_u32(t, vdup_n_s32(-shift));
6141}
6142
6143template <>
6144EIGEN_STRONG_INLINE Packet4i psdiv_signed_magic<Packet4i>(const Packet4i& a, uint32_t magic, int shift,
6145 int /*total_shift*/, const Packet4i& divisor_sign) {
6146 int32x2_t m = vdup_n_s32(static_cast<int32_t>(magic));
6147 int64x2_t lo = vmull_s32(vget_low_s32(a), m);
6148 int64x2_t hi = vmull_s32(vget_high_s32(a), m);
6149#if EIGEN_ARCH_ARM64
6150 int32x4_t t = vuzp2q_s32(vreinterpretq_s32_s64(lo), vreinterpretq_s32_s64(hi));
6151#else
6152 int32x4_t t = vcombine_s32(vshrn_n_s64(lo, 32), vshrn_n_s64(hi, 32));
6153#endif
6154 int32x4_t q = vsubq_s32(vshlq_s32(vaddq_s32(a, t), vdupq_n_s32(-shift)), vshrq_n_s32(a, 31));
6155 return vsubq_s32(veorq_s32(q, divisor_sign), divisor_sign);
6156}
6157template <>
6158EIGEN_STRONG_INLINE Packet2i psdiv_signed_magic<Packet2i>(const Packet2i& a, uint32_t magic, int shift,
6159 int /*total_shift*/, const Packet2i& divisor_sign) {
6160 int32x2_t t = vshrn_n_s64(vmull_s32(a, vdup_n_s32(static_cast<int32_t>(magic))), 32);
6161 int32x2_t q = vsub_s32(vshl_s32(vadd_s32(a, t), vdup_n_s32(-shift)), vshr_n_s32(a, 31));
6162 return vsub_s32(veor_s32(q, divisor_sign), divisor_sign);
6163}
6164
6165} // end namespace internal
6166
6167} // end namespace Eigen
6168
6169#endif // EIGEN_PACKET_MATH_NEON_H