11#ifndef EIGEN_PACKET_MATH_ZVECTOR_H
12#define EIGEN_PACKET_MATH_ZVECTOR_H
15#include "../../InternalHeaderCheck.h"
21#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 16
25#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
26#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
29#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
30#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
33using Packet4i = __vector int;
34using Packet4ui = __vector
unsigned int;
35using Packet4bi = __vector __bool int;
36using Packet8i = __vector
short int;
37using Packet16uc = __vector
unsigned char;
38using Packet2d = __vector double;
39using Packet2ul = __vector
unsigned long long;
40using Packet2l = __vector
long long;
43using Packet4f = __vector float;
47 numext::uint32_t ui[4];
49 numext::uint64_t ul[2];
63#define EIGEN_DECLARE_CONST_FAST_Packet4i(NAME, X) Packet4i p4i_##NAME = reinterpret_cast<Packet4i>(vec_splat_s32(X))
65#define EIGEN_DECLARE_CONST_FAST_Packet2d(NAME, X) Packet2d p2d_##NAME = reinterpret_cast<Packet2d>(vec_splat_s64(X))
67#define EIGEN_DECLARE_CONST_FAST_Packet2l(NAME, X) Packet2l p2l_##NAME = reinterpret_cast<Packet2l>(vec_splat_s64(X))
69#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) Packet4i p4i_##NAME = pset1<Packet4i>(X)
71#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) Packet2d p2d_##NAME = pset1<Packet2d>(X)
73#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) Packet2l p2l_##NAME = pset1<Packet2l>(X)
76static EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0);
77static EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1);
79static EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0);
80static EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0);
81static EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1);
83static Packet2d p2d_ONE = {1.0, 1.0};
84static Packet2d p2d_ZERO_ = {numext::bit_cast<double>(0x8000000000000000ull),
85 numext::bit_cast<double>(0x8000000000000000ull)};
87#define EIGEN_DECLARE_CONST_FAST_Packet4f(NAME, X) Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X))
89#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) Packet4f p4f_##NAME = pset1<Packet4f>(X)
91#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
92 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
94static EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0);
95static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1, -1);
96static Packet4f p4f_MZERO = {0x80000000, 0x80000000, 0x80000000, 0x80000000};
98static Packet4i p4i_COUNTDOWN = {0, 1, 2, 3};
99static Packet4f p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0};
100static Packet2d p2d_COUNTDOWN =
reinterpret_cast<Packet2d
>(
101 vec_sld(
reinterpret_cast<Packet16uc
>(p2d_ZERO),
reinterpret_cast<Packet16uc
>(p2d_ONE), 8));
103static Packet16uc p16uc_PSET64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7};
104static Packet16uc p16uc_DUPLICATE32_HI = {0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7};
107#define EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0
109#define EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & EIGEN_MASK_ALIGNMENT)
114static Packet16uc p16uc_FORWARD = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
115static Packet16uc p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3};
116static Packet16uc p16uc_REVERSE64 = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7};
118static Packet16uc p16uc_PSET32_WODD =
119 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
121static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
123static Packet16uc p16uc_PSET64_LO = (Packet16uc)vec_mergel(
124 (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);
125static Packet16uc p16uc_TRANSPOSE64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
126static Packet16uc p16uc_TRANSPOSE64_LO = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
128static Packet16uc p16uc_COMPLEX32_REV =
129 vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8);
131static Packet16uc p16uc_COMPLEX32_REV2 =
132 vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8);
134#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
135#define EIGEN_ZVECTOR_PREFETCH(ADDR) __builtin_prefetch(ADDR);
137#define EIGEN_ZVECTOR_PREFETCH(ADDR) asm(" pfd [%[addr]]\n" ::[addr] "r"(ADDR) : "cc");
141struct packet_traits<int> : default_packet_traits {
142 using type = Packet4i;
143 using half = Packet4i;
157struct packet_traits<float> : default_packet_traits {
158 using type = Packet4f;
159 using half = Packet4f;
186struct packet_traits<double> : default_packet_traits {
187 using type = Packet2d;
188 using half = Packet2d;
212struct unpacket_traits<Packet4i> {
218 masked_load_available =
false,
219 masked_store_available =
false
221 using half = Packet4i;
224struct unpacket_traits<Packet4f> {
230 masked_load_available =
false,
231 masked_store_available =
false
233 using half = Packet4f;
234 using integer_packet = Packet4i;
237struct unpacket_traits<Packet2d> {
243 masked_load_available =
false,
244 masked_store_available =
false
246 using half = Packet2d;
247 using integer_packet = Packet2l;
251struct unpacket_traits<Packet2l> {
252 using type = numext::int64_t;
253 using half = Packet2l;
254 static constexpr int size = 2;
255 static constexpr int alignment =
Aligned16;
256 static constexpr bool vectorizable =
false;
257 static constexpr bool masked_load_available =
false;
258 static constexpr bool masked_store_available =
false;
262EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4f, 4>& kernel);
264inline std::ostream& operator<<(std::ostream& s,
const Packet4i& v) {
267 s << vt.i[0] <<
", " << vt.i[1] <<
", " << vt.i[2] <<
", " << vt.i[3];
271inline std::ostream& operator<<(std::ostream& s,
const Packet4ui& v) {
274 s << vt.ui[0] <<
", " << vt.ui[1] <<
", " << vt.ui[2] <<
", " << vt.ui[3];
278inline std::ostream& operator<<(std::ostream& s,
const Packet2l& v) {
281 s << vt.l[0] <<
", " << vt.l[1];
285inline std::ostream& operator<<(std::ostream& s,
const Packet2ul& v) {
288 s << vt.ul[0] <<
", " << vt.ul[1];
292inline std::ostream& operator<<(std::ostream& s,
const Packet2d& v) {
295 s << vt.d[0] <<
", " << vt.d[1];
299inline std::ostream& operator<<(std::ostream& s,
const Packet4f& v) {
302 s << vt.f[0] <<
", " << vt.f[1] <<
", " << vt.f[2] <<
", " << vt.f[3];
307EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int* from) {
308 EIGEN_DEBUG_ALIGNED_LOAD
309 return vec_xl(0, from);
313EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from) {
314 EIGEN_DEBUG_ALIGNED_LOAD
315 return vec_xl(0, from);
319EIGEN_STRONG_INLINE
void pstore<int>(
int* to,
const Packet4i& from) {
320 EIGEN_DEBUG_ALIGNED_STORE
321 vec_xst(from, 0, to);
325EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from) {
326 EIGEN_DEBUG_ALIGNED_STORE
327 vec_xst(from, 0, to);
331EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(
const Packet4f& a, Packet4f& exponent) {
332 return pfrexp_generic(a, exponent);
336EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(
const Packet2d& a, Packet2d& exponent) {
337 return pfrexp_generic(a, exponent);
341EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int& from) {
342 return vec_splats(from);
345EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
346 return vec_splats(from);
349EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(
const numext::int64_t& from) {
350 return vec_splats(
static_cast<long long>(from));
354EIGEN_STRONG_INLINE
void pbroadcast4<Packet4i>(
const int* a, Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3) {
355 a3 = pload<Packet4i>(a);
356 a0 = vec_splat(a3, 0);
357 a1 = vec_splat(a3, 1);
358 a2 = vec_splat(a3, 2);
359 a3 = vec_splat(a3, 3);
363EIGEN_STRONG_INLINE
void pbroadcast4<Packet2d>(
const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2,
365 a1 = pload<Packet2d>(a);
366 a0 = vec_splat(a1, 0);
367 a1 = vec_splat(a1, 1);
368 a3 = pload<Packet2d>(a + 2);
369 a2 = vec_splat(a3, 0);
370 a3 = vec_splat(a3, 1);
374EIGEN_DEVICE_FUNC
inline Packet4i pgather<int, Packet4i>(
const int* from, Index stride) {
375 EIGEN_ALIGN16
int ai[4];
376 ai[0] = from[0 * stride];
377 ai[1] = from[1 * stride];
378 ai[2] = from[2 * stride];
379 ai[3] = from[3 * stride];
380 return pload<Packet4i>(ai);
384EIGEN_DEVICE_FUNC
inline Packet2d pgather<double, Packet2d>(
const double* from, Index stride) {
385 EIGEN_ALIGN16
double af[2];
386 af[0] = from[0 * stride];
387 af[1] = from[1 * stride];
388 return pload<Packet2d>(af);
392EIGEN_DEVICE_FUNC
inline void pscatter<int, Packet4i>(
int* to,
const Packet4i& from, Index stride) {
393 EIGEN_ALIGN16
int ai[4];
394 pstore<int>((
int*)ai, from);
395 to[0 * stride] = ai[0];
396 to[1 * stride] = ai[1];
397 to[2 * stride] = ai[2];
398 to[3 * stride] = ai[3];
402EIGEN_DEVICE_FUNC
inline void pscatter<double, Packet2d>(
double* to,
const Packet2d& from, Index stride) {
403 EIGEN_ALIGN16
double af[2];
404 pstore<double>(af, from);
405 to[0 * stride] = af[0];
406 to[1 * stride] = af[1];
410EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
414EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
419EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
423EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
428EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
432EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
437EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
441EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
446EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
450EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
455EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
456 return padd<Packet4i>(pmul<Packet4i>(a, b), c);
459struct is_fused_madd<Packet2d> : true_type {};
461EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
462 return vec_madd(a, b, c);
466EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int& a) {
467 return padd<Packet4i>(pset1<Packet4i>(a), p4i_COUNTDOWN);
470EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a) {
471 return padd<Packet2d>(pset1<Packet2d>(a), p2d_COUNTDOWN);
475EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
476 return vec_min(a, b);
481EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
482 return __builtin_s390_vfmindb(a, b, 3);
486EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
487 return vec_max(a, b);
490EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
491 return __builtin_s390_vfmaxdb(a, b, 3);
495EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
496 return vec_and(a, b);
499EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
500 return vec_and(a, b);
504EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
508EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
513EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
514 return vec_xor(a, b);
517EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
518 return vec_xor(a, b);
522EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
523 return pand<Packet4i>(a, vec_nor(b, b));
526EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
527 return vec_and(a, vec_nor(b, b));
531EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
535EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
539EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
540 return vec_and(a, b);
543EIGEN_STRONG_INLINE Packet2l por<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
547EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
548 return vec_xor(a, b);
551EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
552 return vec_and(a, vec_nor(b, b));
555EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
556 return reinterpret_cast<Packet2l
>(vec_cmpeq(a, b));
559EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
560 return reinterpret_cast<Packet2l
>(vec_cmplt(a, b));
563EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
564 return reinterpret_cast<Packet2l
>(vec_cmple(a, b));
567EIGEN_STRONG_INLINE Packet2l pselect<Packet2l>(
const Packet2l& mask,
const Packet2l& a,
const Packet2l& b) {
568 return vec_sel(b, a,
reinterpret_cast<Packet2ul
>(mask));
572struct cast_impl<Packet2l, Packet2d> {
573 EIGEN_DEVICE_FUNC
static inline Packet2d run(
const Packet2l& a) {
return vec_double(a); }
577struct cast_impl<Packet2d, Packet2l> {
578 EIGEN_DEVICE_FUNC
static inline Packet2l run(
const Packet2d& a) {
579 return vec_signed(vec_sel(p2d_ZERO, a, vec_cmpeq(a, a)));
584EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(
const Packet2d& a) {
586 return __builtin_s390_vfidb(a, 0, 1);
589EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(
const Packet2d& a) {
593EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(
const Packet2d& a) {
597EIGEN_STRONG_INLINE Packet2d print<Packet2d>(
const Packet2d& a) {
599 return __builtin_s390_vfidb(a, 4, 4);
602EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(
const Packet2d& a) {
604 return __builtin_s390_vfidb(a, 4, 5);
607EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
608 return pnot(pcmp_le(b, a));
612EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from) {
613 return pload<Packet4i>(from);
616EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from) {
617 return pload<Packet2d>(from);
621EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int* from) {
622 unsigned long long bits;
623 memcpy(&bits, from,
sizeof(bits));
624 const Packet4i p =
reinterpret_cast<Packet4i
>(vec_splats(bits));
625 return vec_mergeh(p, p);
629EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(
const double* from) {
630 return pset1<Packet2d>(from[0]);
634EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from) {
635 pstore<int>(to, from);
638EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from) {
639 pstore<double>(to, from);
643EIGEN_STRONG_INLINE
void prefetch<int>(
const int* addr) {
644 EIGEN_ZVECTOR_PREFETCH(addr);
647EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) {
648 EIGEN_ZVECTOR_PREFETCH(addr);
652EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(
const Packet2l& a) {
653 return Packet2l{parithmetic_shift_right<N>(a[0]), parithmetic_shift_right<N>(a[1])};
656EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(
const Packet4i& a) {
657 return Packet4i{parithmetic_shift_right<N>(a[0]), parithmetic_shift_right<N>(a[1]), parithmetic_shift_right<N>(a[2]),
658 parithmetic_shift_right<N>(a[3])};
662EIGEN_STRONG_INLINE Packet2l plogical_shift_right(
const Packet2l& a) {
663 return Packet2l{plogical_shift_right<N>(a[0]), plogical_shift_right<N>(a[1])};
666EIGEN_STRONG_INLINE Packet4i plogical_shift_right(
const Packet4i& a) {
667 return Packet4i{plogical_shift_right<N>(a[0]), plogical_shift_right<N>(a[1]), plogical_shift_right<N>(a[2]),
668 plogical_shift_right<N>(a[3])};
672EIGEN_STRONG_INLINE Packet2l plogical_shift_left(
const Packet2l& a) {
673 return Packet2l{plogical_shift_left<N>(a[0]), plogical_shift_left<N>(a[1])};
676EIGEN_STRONG_INLINE Packet4i plogical_shift_left(
const Packet4i& a) {
677 return Packet4i{plogical_shift_left<N>(a[0]), plogical_shift_left<N>(a[1]), plogical_shift_left<N>(a[2]),
678 plogical_shift_left<N>(a[3])};
682EIGEN_STRONG_INLINE
int pfirst<Packet4i>(
const Packet4i& a) {
683 EIGEN_ALIGN16
int x[4];
688EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
689 EIGEN_ALIGN16
double x[2];
695EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a) {
696 return reinterpret_cast<Packet4i
>(
697 vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE32));
701EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a) {
702 return reinterpret_cast<Packet2d
>(
703 vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE64));
707EIGEN_STRONG_INLINE Packet4i pabs<Packet4i>(
const Packet4i& a) {
711EIGEN_STRONG_INLINE Packet2d pabs<Packet2d>(
const Packet2d& a) {
716EIGEN_STRONG_INLINE
int predux<Packet4i>(
const Packet4i& a) {
718 b = vec_sld(a, a, 8);
719 sum = padd<Packet4i>(a, b);
720 b = vec_sld(sum, sum, 4);
721 sum = padd<Packet4i>(sum, b);
726EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a) {
728 b =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4i
>(a),
reinterpret_cast<Packet4i
>(a), 8));
729 sum = padd<Packet2d>(a, b);
734EIGEN_STRONG_INLINE
bool predux_any(
const Packet2d& a) {
735 const Packet2ul zero = {0, 0};
736 return vec_any_ne(
reinterpret_cast<Packet2ul
>(a), zero);
742EIGEN_STRONG_INLINE
int predux_mul<Packet4i>(
const Packet4i& a) {
743 EIGEN_ALIGN16
int aux[4];
745 return aux[0] * aux[1] * aux[2] * aux[3];
749EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a) {
751 pmul(a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4i
>(a),
reinterpret_cast<Packet4i
>(a), 8))));
756EIGEN_STRONG_INLINE
int predux_min<Packet4i>(
const Packet4i& a) {
758 b = pmin<Packet4i>(a, vec_sld(a, a, 8));
759 res = pmin<Packet4i>(b, vec_sld(b, b, 4));
764EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a) {
765 return pfirst(pmin<Packet2d>(
766 a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4i
>(a),
reinterpret_cast<Packet4i
>(a), 8))));
771EIGEN_STRONG_INLINE
int predux_max<Packet4i>(
const Packet4i& a) {
773 b = pmax<Packet4i>(a, vec_sld(a, a, 8));
774 res = pmax<Packet4i>(b, vec_sld(b, b, 4));
780EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a) {
781 return pfirst(pmax<Packet2d>(
782 a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4i
>(a),
reinterpret_cast<Packet4i
>(a), 8))));
785EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
786 Packet4i t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
787 Packet4i t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
788 Packet4i t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
789 Packet4i t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
790 kernel.packet[0] = vec_mergeh(t0, t2);
791 kernel.packet[1] = vec_mergel(t0, t2);
792 kernel.packet[2] = vec_mergeh(t1, t3);
793 kernel.packet[3] = vec_mergel(t1, t3);
796EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
797 Packet2d t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
798 Packet2d t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
799 kernel.packet[0] = t0;
800 kernel.packet[1] = t1;
804EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from) {
805 EIGEN_DEBUG_ALIGNED_LOAD
806 return vec_xl(0, from);
810EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from) {
811 EIGEN_DEBUG_ALIGNED_STORE
812 vec_xst(from, 0, to);
816EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
817 return vec_splats(from);
821EIGEN_STRONG_INLINE
void pbroadcast4<Packet4f>(
const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
822 a3 = pload<Packet4f>(a);
823 a0 = vec_splat(a3, 0);
824 a1 = vec_splat(a3, 1);
825 a2 = vec_splat(a3, 2);
826 a3 = vec_splat(a3, 3);
830EIGEN_DEVICE_FUNC
inline Packet4f pgather<float, Packet4f>(
const float* from, Index stride) {
831 EIGEN_ALIGN16
float af[4];
832 af[0] = from[0 * stride];
833 af[1] = from[1 * stride];
834 af[2] = from[2 * stride];
835 af[3] = from[3 * stride];
836 return pload<Packet4f>(af);
840EIGEN_DEVICE_FUNC
inline void pscatter<float, Packet4f>(
float* to,
const Packet4f& from, Index stride) {
841 EIGEN_ALIGN16
float af[4];
842 pstore<float>((
float*)af, from);
843 to[0 * stride] = af[0];
844 to[1 * stride] = af[1];
845 to[2 * stride] = af[2];
846 to[3 * stride] = af[3];
850EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
854EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
858EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
862EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
866EIGEN_STRONG_INLINE Packet4f pnegate<Packet4f>(
const Packet4f& a) {
870struct is_fused_madd<Packet4f> : true_type {};
872EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
873 return vec_madd(a, b, c);
877EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
878 return __builtin_s390_vfminsb(a, b, 3);
881EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
882 return __builtin_s390_vfmaxsb(a, b, 3);
885EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
886 return vec_and(a, b);
889EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
893EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
894 return vec_xor(a, b);
897EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
898 return vec_and(a, vec_nor(b, b));
901EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(
const Packet4f& a) {
903 return __builtin_s390_vfisb(a, 0, 1);
906EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a) {
910EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a) {
914EIGEN_STRONG_INLINE Packet4f print<Packet4f>(
const Packet4f& a) {
916 return __builtin_s390_vfisb(a, 4, 4);
919EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(
const Packet4f& a) {
921 return __builtin_s390_vfisb(a, 4, 5);
924EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
925 return pnot(pcmp_le(b, a));
928EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>(
const Packet4f& a) {
932EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
933 EIGEN_ALIGN16
float x[4];
939EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from) {
940 unsigned long long bits;
941 memcpy(&bits, from,
sizeof(bits));
942 const Packet4f p =
reinterpret_cast<Packet4f
>(vec_splats(bits));
943 return vec_mergeh(p, p);
947EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a) {
948 return reinterpret_cast<Packet4f
>(
949 vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE32));
953EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a) {
955 b = vec_sld(a, a, 8);
956 sum = padd<Packet4f>(a, b);
957 b = vec_sld(sum, sum, 4);
958 sum = padd<Packet4f>(sum, b);
963EIGEN_STRONG_INLINE
bool predux_any(
const Packet4f& a) {
964 const Packet4ui zero = {0, 0, 0, 0};
965 return vec_any_ne(
reinterpret_cast<Packet4ui
>(a), zero);
971EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a) {
973 prod = pmul(a, vec_sld(a, a, 8));
974 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
979EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a) {
981 b = pmin<Packet4f>(a, vec_sld(a, a, 8));
982 res = pmin<Packet4f>(b, vec_sld(b, b, 4));
988EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a) {
990 b = pmax<Packet4f>(a, vec_sld(a, a, 8));
991 res = pmax<Packet4f>(b, vec_sld(b, b, 4));
995EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
996 Packet4f t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
997 Packet4f t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
998 Packet4f t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
999 Packet4f t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
1000 kernel.packet[0] = vec_mergeh(t0, t2);
1001 kernel.packet[1] = vec_mergel(t0, t2);
1002 kernel.packet[2] = vec_mergeh(t1, t3);
1003 kernel.packet[3] = vec_mergel(t1, t3);
1007EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(
const Packet4f& a,
const Packet4f& exponent) {
1008 return pldexp_generic(a, exponent);
1012EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(
const Packet2d& a,
const Packet2d& exponent) {
1014 const Packet2d max_exponent = pset1<Packet2d>(2099.0);
1015 const Packet2d last_max = pset1<Packet2d>(1022.0);
1016 const Packet2l e = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
1017 const Packet2l b = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(last_max)), last_max));
1018 const Packet2l bias = {1023, 1023};
1019 const Packet2l even = {-2, -2};
1020 const Packet2l t = psub(e, b) & even;
1021 const Packet2d c1 =
reinterpret_cast<Packet2d
>(plogical_shift_left<51>(t + bias + bias));
1022 const Packet2d c2 =
reinterpret_cast<Packet2d
>(plogical_shift_left<52>(psub(e, t) + bias));
1023 return pldexp_apply_factors(a, c1, c2);
1027EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) {
1028 EIGEN_ZVECTOR_PREFETCH(addr);
1031EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from) {
1032 return pload<Packet4f>(from);
1035EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from) {
1036 pstore<float>(to, from);
1039EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a) {
1040 return padd<Packet4f>(pset1<Packet4f>(a), p4f_COUNTDOWN);
1043#if !defined(vec_float) || !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 13)
1044#pragma GCC warning "float->int and int->float conversion is simulated. compile for z15 for improved performance"
1046struct cast_impl<Packet4i, Packet4f> {
1047 EIGEN_DEVICE_FUNC
static inline Packet4f run(
const Packet4i& a) {
1048 return Packet4f{float(a[0]), float(a[1]), float(a[2]), float(a[3])};
1053struct cast_impl<Packet4f, Packet4i> {
1055 EIGEN_DEVICE_FUNC
static inline Packet4i run(
const Packet4f& a) {
1056 return Packet4i{to_int(a[0]), to_int(a[1]), to_int(a[2]), to_int(a[3])};
1058 EIGEN_DEVICE_FUNC
static inline int to_int(
float x) {
return x == x ? int(x) : 0; }
1062struct cast_impl<Packet4i, Packet4f> {
1063 EIGEN_DEVICE_FUNC
static inline Packet4f run(
const Packet4i& a) {
return vec_float(a); }
1067struct cast_impl<Packet4f, Packet4i> {
1068 EIGEN_DEVICE_FUNC
static inline Packet4i run(
const Packet4f& a) {
1069 return vec_signed(vec_sel(p4f_ZERO, a, vec_cmpeq(a, a)));
1075EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
1076 return pset1<Packet4f>(Eigen::numext::bit_cast<float>(from));
1079EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
1080 return pset1<Packet2d>(Eigen::numext::bit_cast<double>(from));
@ Aligned16
Definition Constants.h:238