11#ifndef EIGEN_PACKET_MATH_ALTIVEC_H
12#define EIGEN_PACKET_MATH_ALTIVEC_H
15#include "../../InternalHeaderCheck.h"
21#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4
25#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
26#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
30#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
31#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
34using Packet4f = __vector float;
35using Packet4i = __vector int;
36using Packet4ui = __vector
unsigned int;
37using Packet4bi = __vector __bool int;
38using Packet8s = __vector
short int;
39using Packet8us = __vector
unsigned short int;
40using Packet8bi = __vector __bool short;
41using Packet16c = __vector
signed char;
42using Packet16uc = __vector
unsigned char;
43using Packet8bf = eigen_packet_wrapper<__vector unsigned short int, 0>;
47#define EIGEN_DECLARE_CONST_FAST_Packet4f(NAME, X) Packet4f p4f_##NAME = {X, X, X, X}
49#define EIGEN_DECLARE_CONST_FAST_Packet4i(NAME, X) Packet4i p4i_##NAME = vec_splat_s32(X)
51#define EIGEN_DECLARE_CONST_FAST_Packet4ui(NAME, X) Packet4ui p4ui_##NAME = {X, X, X, X}
53#define EIGEN_DECLARE_CONST_FAST_Packet8us(NAME, X) Packet8us p8us_##NAME = {X, X, X, X, X, X, X, X}
55#define EIGEN_DECLARE_CONST_FAST_Packet16uc(NAME, X) \
56 Packet16uc p16uc_##NAME = {X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X}
58#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) Packet4f p4f_##NAME = pset1<Packet4f>(X)
60#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) Packet4i p4i_##NAME = pset1<Packet4i>(X)
62#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) Packet2d p2d_##NAME = pset1<Packet2d>(X)
64#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) Packet2l p2l_##NAME = pset1<Packet2l>(X)
66#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
67 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
70#define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride))
71#define __UNPACK_TYPE__(PACKETNAME) typename unpacket_traits<PACKETNAME>::type
74static EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0);
75static EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0);
76static EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1);
77static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16, -16);
78static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1, -1);
79static EIGEN_DECLARE_CONST_FAST_Packet4ui(SIGN, 0x80000000u);
80static EIGEN_DECLARE_CONST_FAST_Packet4ui(PREV0DOT5, 0x3EFFFFFFu);
81static EIGEN_DECLARE_CONST_FAST_Packet8us(ONE, 1);
82static Packet4f p4f_MZERO =
83 (Packet4f)vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1);
85static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0);
88static Packet4f p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0};
89static Packet4i p4i_COUNTDOWN = {0, 1, 2, 3};
90static Packet8s p8s_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7};
91static Packet8us p8us_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7};
93static Packet16c p16c_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
94static Packet16uc p16uc_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
96static Packet16uc p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3};
97static Packet16uc p16uc_REVERSE16 = {14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1};
98static Packet16uc p16uc_REVERSE8 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
101static Packet16uc p16uc_DUPLICATE32_HI = {0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7};
103static const Packet16uc p16uc_DUPLICATE16_EVEN = {0, 1, 0, 1, 4, 5, 4, 5, 8, 9, 8, 9, 12, 13, 12, 13};
104static const Packet16uc p16uc_DUPLICATE16_ODD = {2, 3, 2, 3, 6, 7, 6, 7, 10, 11, 10, 11, 14, 15, 14, 15};
106static Packet16uc p16uc_QUADRUPLICATE16_HI = {0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3};
107static Packet16uc p16uc_QUADRUPLICATE16 = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3};
109static Packet16uc p16uc_MERGEE16 = {0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29};
110static Packet16uc p16uc_MERGEO16 = {2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31};
112static Packet16uc p16uc_MERGEH16 = {0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29};
114static Packet16uc p16uc_MERGEL16 = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31};
120static Packet16uc p16uc_FORWARD = vec_lvsl(0, (
float*)0);
121static Packet16uc p16uc_PSET32_WODD =
122 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
124static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
126static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc)vec_abs(p4i_MINUS16), 3),
129static Packet16uc p16uc_FORWARD = p16uc_REVERSE32;
130static Packet16uc p16uc_PSET32_WODD =
131 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
133static Packet16uc p16uc_PSET32_WEVEN =
134 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
136static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc)vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO,
140static Packet16uc p16uc_PSET64_HI = (Packet16uc)vec_mergeh(
141 (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);
142static Packet16uc p16uc_PSET64_LO = (Packet16uc)vec_mergel(
143 (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);
144static Packet16uc p16uc_TRANSPOSE64_HI =
145 p16uc_PSET64_HI + p16uc_HALF64_0_16;
146static Packet16uc p16uc_TRANSPOSE64_LO =
147 p16uc_PSET64_LO + p16uc_HALF64_0_16;
149static Packet16uc p16uc_COMPLEX32_REV =
150 vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8);
152#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
153#define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR);
155#define EIGEN_PPC_PREFETCH(ADDR) asm(" dcbt [%[addr]]\n" ::[addr] "r"(ADDR) : "cc");
159#define LOAD_STORE_UNROLL_16 _Pragma("unroll 16")
161#define LOAD_STORE_UNROLL_16 _Pragma("GCC unroll(16)")
165struct packet_traits<float> : default_packet_traits {
166 using type = Packet4f;
167 using half = Packet4f;
180 HasSin = EIGEN_FAST_MATH,
181 HasCos = EIGEN_FAST_MATH,
182 HasTan = EIGEN_FAST_MATH,
191#ifdef EIGEN_VECTORIZE_VSX
201 HasTanh = EIGEN_FAST_MATH,
202 HasErf = EIGEN_FAST_MATH,
203 HasErfc = EIGEN_FAST_MATH,
214struct packet_traits<bfloat16> : default_packet_traits {
215 using type = Packet8bf;
216 using half = Packet8bf;
229 HasSin = EIGEN_FAST_MATH,
230 HasCos = EIGEN_FAST_MATH,
233#ifdef EIGEN_VECTORIZE_VSX
251struct packet_traits<int> : default_packet_traits {
252 using type = Packet4i;
253 using half = Packet4i;
263#if defined(_ARCH_PWR10) && (EIGEN_COMP_LLVM || EIGEN_GNUC_STRICT_AT_LEAST(11, 0, 0))
273struct packet_traits<short int> : default_packet_traits {
274 using type = Packet8s;
275 using half = Packet8s;
290struct packet_traits<unsigned short int> : default_packet_traits {
291 using type = Packet8us;
292 using half = Packet8us;
307struct packet_traits<signed char> : default_packet_traits {
308 using type = Packet16c;
309 using half = Packet16c;
324struct packet_traits<unsigned char> : default_packet_traits {
325 using type = Packet16uc;
326 using half = Packet16uc;
341struct unpacket_traits<Packet4f> {
343 using half = Packet4f;
344 using integer_packet = Packet4i;
349 masked_load_available =
false,
350 masked_store_available =
false
354struct unpacket_traits<Packet4i> {
356 using half = Packet4i;
361 masked_load_available =
false,
362 masked_store_available =
false
366struct unpacket_traits<Packet8s> {
367 using type =
short int;
368 using half = Packet8s;
373 masked_load_available =
false,
374 masked_store_available =
false
378struct unpacket_traits<Packet8us> {
379 using type =
unsigned short int;
380 using half = Packet8us;
385 masked_load_available =
false,
386 masked_store_available =
false
391struct unpacket_traits<Packet16c> {
392 using type =
signed char;
393 using half = Packet16c;
398 masked_load_available =
false,
399 masked_store_available =
false
403struct unpacket_traits<Packet16uc> {
404 using type =
unsigned char;
405 using half = Packet16uc;
410 masked_load_available =
false,
411 masked_store_available =
false
416struct unpacket_traits<Packet8bf> {
417 using type = bfloat16;
418 using half = Packet8bf;
423 masked_load_available =
false,
424 masked_store_available =
false
428template <
typename Packet>
429EIGEN_STRONG_INLINE Packet pload_common(
const __UNPACK_TYPE__(Packet) * from) {
432 EIGEN_UNUSED_VARIABLE(from);
433 EIGEN_DEBUG_ALIGNED_LOAD
434#ifdef EIGEN_VECTORIZE_VSX
435 return vec_xl(0,
const_cast<__UNPACK_TYPE__(Packet)*
>(from));
437 return vec_ld(0, from);
443EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from) {
444 return pload_common<Packet4f>(from);
448EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int* from) {
449 return pload_common<Packet4i>(from);
453EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(
const short int* from) {
454 return pload_common<Packet8s>(from);
458EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(
const unsigned short int* from) {
459 return pload_common<Packet8us>(from);
463EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(
const signed char* from) {
464 return pload_common<Packet16c>(from);
468EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(
const unsigned char* from) {
469 return pload_common<Packet16uc>(from);
473EIGEN_STRONG_INLINE Packet8bf pload<Packet8bf>(
const bfloat16* from) {
474 return pload_common<Packet8us>(
reinterpret_cast<const unsigned short int*
>(from));
477template <
typename Packet>
478EIGEN_ALWAYS_INLINE Packet pload_ignore(
const __UNPACK_TYPE__(Packet) * from) {
481 EIGEN_UNUSED_VARIABLE(from);
482 EIGEN_DEBUG_ALIGNED_LOAD
485#pragma GCC diagnostic push
486#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
488#ifdef EIGEN_VECTORIZE_VSX
489 return vec_xl(0,
const_cast<__UNPACK_TYPE__(Packet)*
>(from));
491 return vec_ld(0, from);
494#pragma GCC diagnostic pop
499EIGEN_ALWAYS_INLINE Packet8bf pload_ignore<Packet8bf>(
const bfloat16* from) {
500 return pload_ignore<Packet8us>(
reinterpret_cast<const unsigned short int*
>(from));
503template <
typename Packet>
504EIGEN_ALWAYS_INLINE Packet pload_partial_common(
const __UNPACK_TYPE__(Packet) * from,
const Index n,
505 const Index offset) {
508 const Index packet_size = unpacket_traits<Packet>::size;
509 eigen_internal_assert(n + offset <= packet_size &&
"number of elements plus offset will read past end of packet");
510 const Index size =
sizeof(__UNPACK_TYPE__(Packet));
512 EIGEN_UNUSED_VARIABLE(packet_size);
513 EIGEN_DEBUG_ALIGNED_LOAD
514 EIGEN_UNUSED_VARIABLE(from);
515 Packet load = vec_xl_len(
const_cast<__UNPACK_TYPE__(Packet)*
>(from), n * size);
517 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
519 load = Packet(vec_sro(Packet16uc(load), shift));
521 load = Packet(vec_slo(Packet16uc(load), shift));
527 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size];
528 unsigned char* load2 =
reinterpret_cast<unsigned char*
>(load + offset);
529 unsigned char* from2 =
reinterpret_cast<unsigned char*
>(
const_cast<__UNPACK_TYPE__(Packet)*
>(from));
532 pstoreu(load2, ploadu<Packet16uc>(from2));
534 memcpy((
void*)load2, (
void*)from2, n2);
536 return pload_ignore<Packet>(load);
538 return Packet(pset1<Packet16uc>(0));
544EIGEN_ALWAYS_INLINE Packet4f pload_partial<Packet4f>(
const float* from,
const Index n,
const Index offset) {
545 return pload_partial_common<Packet4f>(from, n, offset);
549EIGEN_ALWAYS_INLINE Packet4i pload_partial<Packet4i>(
const int* from,
const Index n,
const Index offset) {
550 return pload_partial_common<Packet4i>(from, n, offset);
554EIGEN_ALWAYS_INLINE Packet8s pload_partial<Packet8s>(
const short int* from,
const Index n,
const Index offset) {
555 return pload_partial_common<Packet8s>(from, n, offset);
559EIGEN_ALWAYS_INLINE Packet8us pload_partial<Packet8us>(
const unsigned short int* from,
const Index n,
560 const Index offset) {
561 return pload_partial_common<Packet8us>(from, n, offset);
565EIGEN_ALWAYS_INLINE Packet8bf pload_partial<Packet8bf>(
const bfloat16* from,
const Index n,
const Index offset) {
566 return pload_partial_common<Packet8us>(
reinterpret_cast<const unsigned short int*
>(from), n, offset);
570EIGEN_ALWAYS_INLINE Packet16c pload_partial<Packet16c>(
const signed char* from,
const Index n,
const Index offset) {
571 return pload_partial_common<Packet16c>(from, n, offset);
575EIGEN_ALWAYS_INLINE Packet16uc pload_partial<Packet16uc>(
const unsigned char* from,
const Index n,
const Index offset) {
576 return pload_partial_common<Packet16uc>(from, n, offset);
579template <
typename Packet>
580EIGEN_STRONG_INLINE
void pstore_common(__UNPACK_TYPE__(Packet) * to,
const Packet& from) {
583 EIGEN_UNUSED_VARIABLE(to);
584 EIGEN_DEBUG_ALIGNED_STORE
585#ifdef EIGEN_VECTORIZE_VSX
586 vec_xst(from, 0, to);
593EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from) {
594 pstore_common<Packet4f>(to, from);
598EIGEN_STRONG_INLINE
void pstore<int>(
int* to,
const Packet4i& from) {
599 pstore_common<Packet4i>(to, from);
603EIGEN_STRONG_INLINE
void pstore<short int>(
short int* to,
const Packet8s& from) {
604 pstore_common<Packet8s>(to, from);
608EIGEN_STRONG_INLINE
void pstore<unsigned short int>(
unsigned short int* to,
const Packet8us& from) {
609 pstore_common<Packet8us>(to, from);
613EIGEN_STRONG_INLINE
void pstore<bfloat16>(bfloat16* to,
const Packet8bf& from) {
614 pstore_common<Packet8us>(
reinterpret_cast<unsigned short int*
>(to), from.m_val);
618EIGEN_STRONG_INLINE
void pstore<signed char>(
signed char* to,
const Packet16c& from) {
619 pstore_common<Packet16c>(to, from);
623EIGEN_STRONG_INLINE
void pstore<unsigned char>(
unsigned char* to,
const Packet16uc& from) {
624 pstore_common<Packet16uc>(to, from);
627template <
typename Packet>
628EIGEN_ALWAYS_INLINE
void pstore_partial_common(__UNPACK_TYPE__(Packet) * to,
const Packet& from,
const Index n,
629 const Index offset) {
632 const Index packet_size = unpacket_traits<Packet>::size;
633 eigen_internal_assert(n + offset <= packet_size &&
"number of elements plus offset will write past end of packet");
634 const Index size =
sizeof(__UNPACK_TYPE__(Packet));
636 EIGEN_UNUSED_VARIABLE(packet_size);
637 EIGEN_UNUSED_VARIABLE(to);
638 EIGEN_DEBUG_ALIGNED_STORE
641 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
643 store = Packet(vec_slo(Packet16uc(store), shift));
645 store = Packet(vec_sro(Packet16uc(store), shift));
648 vec_xst_len(store, to, n * size);
651 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size];
653 unsigned char* store2 =
reinterpret_cast<unsigned char*
>(store + offset);
654 unsigned char* to2 =
reinterpret_cast<unsigned char*
>(to);
657 pstore(to2, ploadu<Packet16uc>(store2));
659 memcpy((
void*)to2, (
void*)store2, n2);
666EIGEN_ALWAYS_INLINE
void pstore_partial<float>(
float* to,
const Packet4f& from,
const Index n,
const Index offset) {
667 pstore_partial_common<Packet4f>(to, from, n, offset);
671EIGEN_ALWAYS_INLINE
void pstore_partial<int>(
int* to,
const Packet4i& from,
const Index n,
const Index offset) {
672 pstore_partial_common<Packet4i>(to, from, n, offset);
676EIGEN_ALWAYS_INLINE
void pstore_partial<short int>(
short int* to,
const Packet8s& from,
const Index n,
677 const Index offset) {
678 pstore_partial_common<Packet8s>(to, from, n, offset);
682EIGEN_ALWAYS_INLINE
void pstore_partial<unsigned short int>(
unsigned short int* to,
const Packet8us& from,
683 const Index n,
const Index offset) {
684 pstore_partial_common<Packet8us>(to, from, n, offset);
688EIGEN_ALWAYS_INLINE
void pstore_partial<bfloat16>(bfloat16* to,
const Packet8bf& from,
const Index n,
689 const Index offset) {
690 pstore_partial_common<Packet8us>(
reinterpret_cast<unsigned short int*
>(to), from.m_val, n, offset);
694EIGEN_ALWAYS_INLINE
void pstore_partial<signed char>(
signed char* to,
const Packet16c& from,
const Index n,
695 const Index offset) {
696 pstore_partial_common<Packet16c>(to, from, n, offset);
700EIGEN_ALWAYS_INLINE
void pstore_partial<unsigned char>(
unsigned char* to,
const Packet16uc& from,
const Index n,
701 const Index offset) {
702 pstore_partial_common<Packet16uc>(to, from, n, offset);
705template <
typename Packet>
706EIGEN_STRONG_INLINE Packet pset1_size4(
const __UNPACK_TYPE__(Packet) & from) {
707 Packet v = {from, from, from, from};
711template <
typename Packet>
712EIGEN_STRONG_INLINE Packet pset1_size8(
const __UNPACK_TYPE__(Packet) & from) {
713 Packet v = {from, from, from, from, from, from, from, from};
717template <
typename Packet>
718EIGEN_STRONG_INLINE Packet pset1_size16(
const __UNPACK_TYPE__(Packet) & from) {
719 Packet v = {from, from, from, from, from, from, from, from, from, from, from, from, from, from, from, from};
724EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
725 return pset1_size4<Packet4f>(from);
729EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int& from) {
730 return pset1_size4<Packet4i>(from);
734EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(
const short int& from) {
735 return pset1_size8<Packet8s>(from);
739EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(
const unsigned short int& from) {
740 return pset1_size8<Packet8us>(from);
744EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(
const signed char& from) {
745 return pset1_size16<Packet16c>(from);
749EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(
const unsigned char& from) {
750 return pset1_size16<Packet16uc>(from);
754EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(
unsigned int from) {
755 return reinterpret_cast<Packet4f
>(pset1<Packet4i>(from));
759EIGEN_STRONG_INLINE Packet8bf pset1<Packet8bf>(
const bfloat16& from) {
760 return pset1_size8<Packet8us>(
reinterpret_cast<const unsigned short int&
>(from));
763template <
typename Packet>
764EIGEN_STRONG_INLINE
void pbroadcast4_common(
const __UNPACK_TYPE__(Packet) * a, Packet& a0, Packet& a1, Packet& a2,
766 a3 = pload<Packet>(a);
767 a0 = vec_splat(a3, 0);
768 a1 = vec_splat(a3, 1);
769 a2 = vec_splat(a3, 2);
770 a3 = vec_splat(a3, 3);
774EIGEN_STRONG_INLINE
void pbroadcast4<Packet4f>(
const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
775 pbroadcast4_common<Packet4f>(a, a0, a1, a2, a3);
778EIGEN_STRONG_INLINE
void pbroadcast4<Packet4i>(
const int* a, Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3) {
779 pbroadcast4_common<Packet4i>(a, a0, a1, a2, a3);
782template <
typename Packet>
783EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pgather_common(
const __UNPACK_TYPE__(Packet) * from, Index stride,
784 const Index n = unpacket_traits<Packet>::size) {
785 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits<Packet>::size];
786 eigen_internal_assert(n <= unpacket_traits<Packet>::size &&
"number of elements will gather past end of packet");
788 if (n == unpacket_traits<Packet>::size) {
789 return ploadu<Packet>(from);
791 return ploadu_partial<Packet>(from, n);
795 for (Index i = 0; i < n; i++) {
796 a[i] = from[i * stride];
799 return pload_ignore<Packet>(a);
804EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f pgather<float, Packet4f>(
const float* from, Index stride) {
805 return pgather_common<Packet4f>(from, stride);
809EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i pgather<int, Packet4i>(
const int* from, Index stride) {
810 return pgather_common<Packet4i>(from, stride);
814EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s pgather<short int, Packet8s>(
const short int* from, Index stride) {
815 return pgather_common<Packet8s>(from, stride);
819EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us pgather<unsigned short int, Packet8us>(
const unsigned short int* from,
821 return pgather_common<Packet8us>(from, stride);
825EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf pgather<bfloat16, Packet8bf>(
const bfloat16* from, Index stride) {
826 return pgather_common<Packet8bf>(from, stride);
830EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c pgather<signed char, Packet16c>(
const signed char* from, Index stride) {
831 return pgather_common<Packet16c>(from, stride);
835EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc pgather<unsigned char, Packet16uc>(
const unsigned char* from,
837 return pgather_common<Packet16uc>(from, stride);
841EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f pgather_partial<float, Packet4f>(
const float* from, Index stride,
843 return pgather_common<Packet4f>(from, stride, n);
847EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i pgather_partial<int, Packet4i>(
const int* from, Index stride,
849 return pgather_common<Packet4i>(from, stride, n);
853EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s pgather_partial<short int, Packet8s>(
const short int* from, Index stride,
855 return pgather_common<Packet8s>(from, stride, n);
859EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us
860pgather_partial<unsigned short int, Packet8us>(
const unsigned short int* from, Index stride,
const Index n) {
861 return pgather_common<Packet8us>(from, stride, n);
865EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf pgather_partial<bfloat16, Packet8bf>(
const bfloat16* from, Index stride,
867 return pgather_common<Packet8bf>(from, stride, n);
871EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c pgather_partial<signed char, Packet16c>(
const signed char* from,
872 Index stride,
const Index n) {
873 return pgather_common<Packet16c>(from, stride, n);
877EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc pgather_partial<unsigned char, Packet16uc>(
const unsigned char* from,
880 return pgather_common<Packet16uc>(from, stride, n);
883template <
typename Packet>
884EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_common(__UNPACK_TYPE__(Packet) * to,
const Packet& from,
886 const Index n = unpacket_traits<Packet>::size) {
887 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits<Packet>::size];
888 eigen_internal_assert(n <= unpacket_traits<Packet>::size &&
"number of elements will scatter past end of packet");
890 if (n == unpacket_traits<Packet>::size) {
891 return pstoreu(to, from);
893 return pstoreu_partial(to, from, n);
896 pstore<__UNPACK_TYPE__(Packet)>(a, from);
898 for (Index i = 0; i < n; i++) {
899 to[i * stride] = a[i];
905EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<float, Packet4f>(
float* to,
const Packet4f& from, Index stride) {
906 pscatter_common<Packet4f>(to, from, stride);
910EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<int, Packet4i>(
int* to,
const Packet4i& from, Index stride) {
911 pscatter_common<Packet4i>(to, from, stride);
915EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<short int, Packet8s>(
short int* to,
const Packet8s& from,
917 pscatter_common<Packet8s>(to, from, stride);
921EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<unsigned short int, Packet8us>(
unsigned short int* to,
922 const Packet8us& from,
924 pscatter_common<Packet8us>(to, from, stride);
928EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<bfloat16, Packet8bf>(bfloat16* to,
const Packet8bf& from,
930 pscatter_common<Packet8bf>(to, from, stride);
934EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<signed char, Packet16c>(
signed char* to,
const Packet16c& from,
936 pscatter_common<Packet16c>(to, from, stride);
940EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<unsigned char, Packet16uc>(
unsigned char* to,
941 const Packet16uc& from, Index stride) {
942 pscatter_common<Packet16uc>(to, from, stride);
946EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<float, Packet4f>(
float* to,
const Packet4f& from,
947 Index stride,
const Index n) {
948 pscatter_common<Packet4f>(to, from, stride, n);
952EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<int, Packet4i>(
int* to,
const Packet4i& from, Index stride,
954 pscatter_common<Packet4i>(to, from, stride, n);
958EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<short int, Packet8s>(
short int* to,
const Packet8s& from,
959 Index stride,
const Index n) {
960 pscatter_common<Packet8s>(to, from, stride, n);
964EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<unsigned short int, Packet8us>(
unsigned short int* to,
965 const Packet8us& from,
968 pscatter_common<Packet8us>(to, from, stride, n);
972EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<bfloat16, Packet8bf>(bfloat16* to,
const Packet8bf& from,
973 Index stride,
const Index n) {
974 pscatter_common<Packet8bf>(to, from, stride, n);
978EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<signed char, Packet16c>(
signed char* to,
979 const Packet16c& from, Index stride,
981 pscatter_common<Packet16c>(to, from, stride, n);
985EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<unsigned char, Packet16uc>(
unsigned char* to,
986 const Packet16uc& from,
987 Index stride,
const Index n) {
988 pscatter_common<Packet16uc>(to, from, stride, n);
992EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a) {
993 return pset1<Packet4f>(a) + p4f_COUNTDOWN;
996EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int& a) {
997 return pset1<Packet4i>(a) + p4i_COUNTDOWN;
1000EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(
const short int& a) {
1001 return pset1<Packet8s>(a) + p8s_COUNTDOWN;
1004EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(
const unsigned short int& a) {
1005 return pset1<Packet8us>(a) + p8us_COUNTDOWN;
1008EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(
const signed char& a) {
1009 return pset1<Packet16c>(a) + p16c_COUNTDOWN;
1012EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(
const unsigned char& a) {
1013 return pset1<Packet16uc>(a) + p16uc_COUNTDOWN;
1017EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1021EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1025EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1029EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1033EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1037EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1041EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1046EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1050EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1054EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1058EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1062EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1066EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1071EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
1072#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1075 return vec_xor(a, p4f_MZERO);
1079EIGEN_STRONG_INLINE Packet16c pnegate(
const Packet16c& a) {
1080#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1083 return reinterpret_cast<Packet16c
>(p4i_ZERO) - a;
1087EIGEN_STRONG_INLINE Packet8s pnegate(
const Packet8s& a) {
1088#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1091 return reinterpret_cast<Packet8s
>(p4i_ZERO) - a;
1095EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
1096#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1099 return p4i_ZERO - a;
1104EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1105 return vec_madd(a, b, p4f_MZERO);
1108EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1112EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1113 return vec_mul(a, b);
1116EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1117 return vec_mul(a, b);
1120EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1121 return vec_mul(a, b);
1124EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1125 return vec_mul(a, b);
1129EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1131 Packet4f t, y_0, y_1;
1137 t = vec_nmsub(y_0, b, p4f_ONE);
1138 y_1 = vec_madd(y_0, t, y_0);
1140 y_1 = vec_sel(y_0, y_1, vec_cmpeq(t, t));
1142 return vec_madd(a, y_1, p4f_MZERO);
1144 return vec_div(a, b);
1149EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1150#if defined(_ARCH_PWR10) && (EIGEN_COMP_LLVM || EIGEN_GNUC_STRICT_AT_LEAST(11, 0, 0))
1151 return vec_div(a, b);
1153 EIGEN_UNUSED_VARIABLE(a);
1154 EIGEN_UNUSED_VARIABLE(b);
1155 eigen_assert(
false &&
"packet integer division are not supported by AltiVec");
1156 return pset1<Packet4i>(0);
1161struct is_fused_madd<Packet4f> : true_type {};
1163EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1164 return vec_madd(a, b, c);
1168EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
1172EIGEN_STRONG_INLINE Packet8s pmadd(
const Packet8s& a,
const Packet8s& b,
const Packet8s& c) {
1173 return vec_madd(a, b, c);
1176EIGEN_STRONG_INLINE Packet8us pmadd(
const Packet8us& a,
const Packet8us& b,
const Packet8us& c) {
1177 return vec_madd(a, b, c);
1180#ifdef EIGEN_VECTORIZE_VSX
1182EIGEN_STRONG_INLINE Packet4f pmsub(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1183 return vec_msub(a, b, c);
1186EIGEN_STRONG_INLINE Packet4f pnmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1187 return vec_nmsub(a, b, c);
1190EIGEN_STRONG_INLINE Packet4f pnmsub(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
1191 return vec_nmadd(a, b, c);
1196EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1197#ifdef EIGEN_VECTORIZE_VSX
1200 __asm__(
"xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa"(ret) :
"wa"(a),
"wa"(b));
1203 return vec_min(a, b);
1207EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1208 return vec_min(a, b);
1211EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1212 return vec_min(a, b);
1215EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1216 return vec_min(a, b);
1219EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1220 return vec_min(a, b);
1223EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1224 return vec_min(a, b);
1228EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1229#ifdef EIGEN_VECTORIZE_VSX
1232 __asm__(
"xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa"(ret) :
"wa"(a),
"wa"(b));
1235 return vec_max(a, b);
1239EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1240 return vec_max(a, b);
1243EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1244 return vec_max(a, b);
1247EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1248 return vec_max(a, b);
1251EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1252 return vec_max(a, b);
1255EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1256 return vec_max(a, b);
1260EIGEN_STRONG_INLINE Packet4f pcmp_le(
const Packet4f& a,
const Packet4f& b) {
1261 return reinterpret_cast<Packet4f
>(vec_cmple(a, b));
1265#if defined(EIGEN_VECTORIZE_VSX) || EIGEN_COMP_CLANG
1267EIGEN_STRONG_INLINE Packet4f pcmp_lt(
const Packet4f& a,
const Packet4f& b) {
1268 return reinterpret_cast<Packet4f
>(vec_cmplt(a, b));
1272EIGEN_STRONG_INLINE Packet4f pcmp_eq(
const Packet4f& a,
const Packet4f& b) {
1273 return reinterpret_cast<Packet4f
>(vec_cmpeq(a, b));
1276EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(
const Packet4f& a,
const Packet4f& b) {
1277 Packet4f c =
reinterpret_cast<Packet4f
>(vec_cmpge(a, b));
1278 return vec_nor(c, c);
1281#ifdef EIGEN_VECTORIZE_VSX
1283EIGEN_STRONG_INLINE Packet4i pcmp_le(
const Packet4i& a,
const Packet4i& b) {
1284 return reinterpret_cast<Packet4i
>(vec_cmple(a, b));
1288EIGEN_STRONG_INLINE Packet4i pcmp_lt(
const Packet4i& a,
const Packet4i& b) {
1289 return reinterpret_cast<Packet4i
>(vec_cmplt(a, b));
1292EIGEN_STRONG_INLINE Packet4i pcmp_eq(
const Packet4i& a,
const Packet4i& b) {
1293 return reinterpret_cast<Packet4i
>(vec_cmpeq(a, b));
1295#ifdef EIGEN_VECTORIZE_VSX
1297EIGEN_STRONG_INLINE Packet8s pcmp_le(
const Packet8s& a,
const Packet8s& b) {
1298 return reinterpret_cast<Packet8s
>(vec_cmple(a, b));
1302EIGEN_STRONG_INLINE Packet8s pcmp_lt(
const Packet8s& a,
const Packet8s& b) {
1303 return reinterpret_cast<Packet8s
>(vec_cmplt(a, b));
1306EIGEN_STRONG_INLINE Packet8s pcmp_eq(
const Packet8s& a,
const Packet8s& b) {
1307 return reinterpret_cast<Packet8s
>(vec_cmpeq(a, b));
1309#ifdef EIGEN_VECTORIZE_VSX
1311EIGEN_STRONG_INLINE Packet8us pcmp_le(
const Packet8us& a,
const Packet8us& b) {
1312 return reinterpret_cast<Packet8us
>(vec_cmple(a, b));
1316EIGEN_STRONG_INLINE Packet8us pcmp_lt(
const Packet8us& a,
const Packet8us& b) {
1317 return reinterpret_cast<Packet8us
>(vec_cmplt(a, b));
1320EIGEN_STRONG_INLINE Packet8us pcmp_eq(
const Packet8us& a,
const Packet8us& b) {
1321 return reinterpret_cast<Packet8us
>(vec_cmpeq(a, b));
1323#ifdef EIGEN_VECTORIZE_VSX
1325EIGEN_STRONG_INLINE Packet16c pcmp_le(
const Packet16c& a,
const Packet16c& b) {
1326 return reinterpret_cast<Packet16c
>(vec_cmple(a, b));
1330EIGEN_STRONG_INLINE Packet16c pcmp_lt(
const Packet16c& a,
const Packet16c& b) {
1331 return reinterpret_cast<Packet16c
>(vec_cmplt(a, b));
1334EIGEN_STRONG_INLINE Packet16c pcmp_eq(
const Packet16c& a,
const Packet16c& b) {
1335 return reinterpret_cast<Packet16c
>(vec_cmpeq(a, b));
1337#ifdef EIGEN_VECTORIZE_VSX
1339EIGEN_STRONG_INLINE Packet16uc pcmp_le(
const Packet16uc& a,
const Packet16uc& b) {
1340 return reinterpret_cast<Packet16uc
>(vec_cmple(a, b));
1344EIGEN_STRONG_INLINE Packet16uc pcmp_lt(
const Packet16uc& a,
const Packet16uc& b) {
1345 return reinterpret_cast<Packet16uc
>(vec_cmplt(a, b));
1348EIGEN_STRONG_INLINE Packet16uc pcmp_eq(
const Packet16uc& a,
const Packet16uc& b) {
1349 return reinterpret_cast<Packet16uc
>(vec_cmpeq(a, b));
1353EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1354 return vec_and(a, b);
1357EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1358 return vec_and(a, b);
1361EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1362 return vec_and(a, b);
1365EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1366 return vec_and(a, b);
1369EIGEN_STRONG_INLINE Packet8bf pand<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
1370 return pand<Packet8us>(a, b);
1374EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1375 return vec_or(a, b);
1378EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1379 return vec_or(a, b);
1382EIGEN_STRONG_INLINE Packet8s por<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1383 return vec_or(a, b);
1386EIGEN_STRONG_INLINE Packet8us por<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1387 return vec_or(a, b);
1390EIGEN_STRONG_INLINE Packet8bf por<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
1391 return por<Packet8us>(a, b);
1395EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1396 return vec_xor(a, b);
1399EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1400 return vec_xor(a, b);
1403EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1404 return vec_xor(a, b);
1407EIGEN_STRONG_INLINE Packet8bf pxor<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
1408 return pxor<Packet8us>(a, b);
1412EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1413 return vec_andc(a, b);
1416EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1417 return vec_andc(a, b);
1421EIGEN_STRONG_INLINE Packet4f pselect(
const Packet4f& mask,
const Packet4f& a,
const Packet4f& b) {
1422 return vec_sel(b, a,
reinterpret_cast<Packet4ui
>(mask));
1426EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(
const Packet4f& a) {
1427 Packet4f t = vec_add(
1428 reinterpret_cast<Packet4f
>(vec_or(vec_and(
reinterpret_cast<Packet4ui
>(a), p4ui_SIGN), p4ui_PREV0DOT5)), a);
1431#ifdef EIGEN_VECTORIZE_VSX
1432 __asm__(
"xvrspiz %x0, %x1\n\t" :
"=&wa"(res) :
"wa"(t));
1434 __asm__(
"vrfiz %0, %1\n\t" :
"=v"(res) :
"v"(t));
1440EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a) {
1444EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a) {
1445 return vec_floor(a);
1448EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(
const Packet4f& a) {
1449 return vec_trunc(a);
1451#ifdef EIGEN_VECTORIZE_VSX
1453EIGEN_STRONG_INLINE Packet4f print<Packet4f>(
const Packet4f& a) {
1456 __asm__(
"xvrspic %x0, %x1\n\t" :
"=&wa"(res) :
"wa"(a));
1462template <
typename Packet>
1463EIGEN_STRONG_INLINE Packet ploadu_common(
const __UNPACK_TYPE__(Packet) * from) {
1464 EIGEN_DEBUG_UNALIGNED_LOAD
1465#if defined(EIGEN_VECTORIZE_VSX)
1466 return vec_xl(0,
const_cast<__UNPACK_TYPE__(Packet)*
>(from));
1468 Packet16uc MSQ = vec_ld(0, (
unsigned char*)from);
1469 Packet16uc LSQ = vec_ld(15, (
unsigned char*)from);
1470 Packet16uc mask = vec_lvsl(0, from);
1472 return (Packet)vec_perm(MSQ, LSQ, mask);
1477EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from) {
1478 return ploadu_common<Packet4f>(from);
1481EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from) {
1482 return ploadu_common<Packet4i>(from);
1485EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(
const short int* from) {
1486 return ploadu_common<Packet8s>(from);
1489EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(
const unsigned short int* from) {
1490 return ploadu_common<Packet8us>(from);
1493EIGEN_STRONG_INLINE Packet8bf ploadu<Packet8bf>(
const bfloat16* from) {
1494 return ploadu_common<Packet8us>(
reinterpret_cast<const unsigned short int*
>(from));
1497EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(
const signed char* from) {
1498 return ploadu_common<Packet16c>(from);
1501EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(
const unsigned char* from) {
1502 return ploadu_common<Packet16uc>(from);
1505template <
typename Packet>
1506EIGEN_ALWAYS_INLINE Packet ploadu_partial_common(
const __UNPACK_TYPE__(Packet) * from,
const Index n,
1507 const Index offset) {
1508 const Index packet_size = unpacket_traits<Packet>::size;
1509 eigen_internal_assert(n + offset <= packet_size &&
"number of elements plus offset will read past end of packet");
1510 const Index size =
sizeof(__UNPACK_TYPE__(Packet));
1512 EIGEN_UNUSED_VARIABLE(packet_size);
1513 EIGEN_DEBUG_ALIGNED_LOAD
1514 EIGEN_DEBUG_UNALIGNED_LOAD
1515 Packet load = vec_xl_len(
const_cast<__UNPACK_TYPE__(Packet)*
>(from), n * size);
1517 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
1519 load = Packet(vec_sro(Packet16uc(load), shift));
1521 load = Packet(vec_slo(Packet16uc(load), shift));
1527 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size];
1528 unsigned char* load2 =
reinterpret_cast<unsigned char*
>(load + offset);
1529 unsigned char* from2 =
reinterpret_cast<unsigned char*
>(
const_cast<__UNPACK_TYPE__(Packet)*
>(from));
1530 Index n2 = n * size;
1532 pstoreu(load2, ploadu<Packet16uc>(from2));
1534 memcpy((
void*)load2, (
void*)from2, n2);
1536 return pload_ignore<Packet>(load);
1538 return Packet(pset1<Packet16uc>(0));
1544EIGEN_ALWAYS_INLINE Packet4f ploadu_partial<Packet4f>(
const float* from,
const Index n,
const Index offset) {
1545 return ploadu_partial_common<Packet4f>(from, n, offset);
1548EIGEN_ALWAYS_INLINE Packet4i ploadu_partial<Packet4i>(
const int* from,
const Index n,
const Index offset) {
1549 return ploadu_partial_common<Packet4i>(from, n, offset);
1552EIGEN_ALWAYS_INLINE Packet8s ploadu_partial<Packet8s>(
const short int* from,
const Index n,
const Index offset) {
1553 return ploadu_partial_common<Packet8s>(from, n, offset);
1556EIGEN_ALWAYS_INLINE Packet8us ploadu_partial<Packet8us>(
const unsigned short int* from,
const Index n,
1557 const Index offset) {
1558 return ploadu_partial_common<Packet8us>(from, n, offset);
1561EIGEN_ALWAYS_INLINE Packet8bf ploadu_partial<Packet8bf>(
const bfloat16* from,
const Index n,
const Index offset) {
1562 return ploadu_partial_common<Packet8us>(
reinterpret_cast<const unsigned short int*
>(from), n, offset);
1565EIGEN_ALWAYS_INLINE Packet16c ploadu_partial<Packet16c>(
const signed char* from,
const Index n,
const Index offset) {
1566 return ploadu_partial_common<Packet16c>(from, n, offset);
1569EIGEN_ALWAYS_INLINE Packet16uc ploadu_partial<Packet16uc>(
const unsigned char* from,
const Index n,
1570 const Index offset) {
1571 return ploadu_partial_common<Packet16uc>(from, n, offset);
1574template <
typename Packet>
1575EIGEN_STRONG_INLINE Packet ploaddup_common(
const __UNPACK_TYPE__(Packet) * from) {
1577#ifdef EIGEN_VECTORIZE_VSX
1578 unsigned long long bits;
1579 memcpy(&bits, from,
sizeof(bits));
1580 const Packet p =
reinterpret_cast<Packet
>(vec_splats(bits));
1582 Packet p =
reinterpret_cast<Packet
>(vec_splat_s32(0));
1583 memcpy(&p, from,
sizeof(Packet) / 2);
1585 return vec_mergeh(p, p);
1588EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from) {
1589 return ploaddup_common<Packet4f>(from);
1592EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int* from) {
1593 return ploaddup_common<Packet4i>(from);
1597EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(
const short int* from) {
1598 return ploaddup_common<Packet8s>(from);
1602EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(
const unsigned short int* from) {
1603 return ploaddup_common<Packet8us>(from);
1607EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(
const short int* from) {
1609 if ((std::ptrdiff_t(from) % 16) == 0)
1610 p = pload<Packet8s>(from);
1612 p = ploadu<Packet8s>(from);
1613 return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI);
1617EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(
const unsigned short int* from) {
1619 if ((std::ptrdiff_t(from) % 16) == 0)
1620 p = pload<Packet8us>(from);
1622 p = ploadu<Packet8us>(from);
1623 return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI);
1627EIGEN_STRONG_INLINE Packet8bf ploadquad<Packet8bf>(
const bfloat16* from) {
1628 return ploadquad<Packet8us>(
reinterpret_cast<const unsigned short int*
>(from));
1632EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(
const signed char* from) {
1633 return ploaddup_common<Packet16c>(from);
1637EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(
const unsigned char* from) {
1638 return ploaddup_common<Packet16uc>(from);
1642EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(
const signed char* from) {
1644 if ((std::ptrdiff_t(from) % 16) == 0)
1645 p = pload<Packet16c>(from);
1647 p = ploadu<Packet16c>(from);
1648 return vec_perm(p, p, p16uc_QUADRUPLICATE16);
1652EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(
const unsigned char* from) {
1654 if ((std::ptrdiff_t(from) % 16) == 0)
1655 p = pload<Packet16uc>(from);
1657 p = ploadu<Packet16uc>(from);
1658 return vec_perm(p, p, p16uc_QUADRUPLICATE16);
1661template <
typename Packet>
1662EIGEN_STRONG_INLINE
void pstoreu_common(__UNPACK_TYPE__(Packet) * to,
const Packet& from) {
1663 EIGEN_DEBUG_UNALIGNED_STORE
1664#if defined(EIGEN_VECTORIZE_VSX)
1665 vec_xst(from, 0, to);
1669 Packet16uc MSQ, LSQ, edges;
1670 Packet16uc edgeAlign, align;
1672 MSQ = vec_ld(0, (
unsigned char*)to);
1673 LSQ = vec_ld(15, (
unsigned char*)to);
1674 edgeAlign = vec_lvsl(0, to);
1675 edges = vec_perm(LSQ, MSQ, edgeAlign);
1676 align = vec_lvsr(0, to);
1677 MSQ = vec_perm(edges, (Packet16uc)from, align);
1678 LSQ = vec_perm((Packet16uc)from, edges, align);
1679 vec_st(LSQ, 15, (
unsigned char*)to);
1680 vec_st(MSQ, 0, (
unsigned char*)to);
1684EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from) {
1685 pstoreu_common<Packet4f>(to, from);
1688EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from) {
1689 pstoreu_common<Packet4i>(to, from);
1692EIGEN_STRONG_INLINE
void pstoreu<short int>(
short int* to,
const Packet8s& from) {
1693 pstoreu_common<Packet8s>(to, from);
1696EIGEN_STRONG_INLINE
void pstoreu<unsigned short int>(
unsigned short int* to,
const Packet8us& from) {
1697 pstoreu_common<Packet8us>(to, from);
1700EIGEN_STRONG_INLINE
void pstoreu<bfloat16>(bfloat16* to,
const Packet8bf& from) {
1701 pstoreu_common<Packet8us>(
reinterpret_cast<unsigned short int*
>(to), from.m_val);
1704EIGEN_STRONG_INLINE
void pstoreu<signed char>(
signed char* to,
const Packet16c& from) {
1705 pstoreu_common<Packet16c>(to, from);
1708EIGEN_STRONG_INLINE
void pstoreu<unsigned char>(
unsigned char* to,
const Packet16uc& from) {
1709 pstoreu_common<Packet16uc>(to, from);
1712template <
typename Packet>
1713EIGEN_ALWAYS_INLINE
void pstoreu_partial_common(__UNPACK_TYPE__(Packet) * to,
const Packet& from,
const Index n,
1714 const Index offset) {
1715 const Index packet_size = unpacket_traits<Packet>::size;
1716 eigen_internal_assert(n + offset <= packet_size &&
"number of elements plus offset will write past end of packet");
1717 const Index size =
sizeof(__UNPACK_TYPE__(Packet));
1719 EIGEN_UNUSED_VARIABLE(packet_size);
1720 EIGEN_DEBUG_UNALIGNED_STORE
1721 Packet store = from;
1723 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
1725 store = Packet(vec_slo(Packet16uc(store), shift));
1727 store = Packet(vec_sro(Packet16uc(store), shift));
1730 vec_xst_len(store, to, n * size);
1733 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size];
1734 pstore(store, from);
1735 unsigned char* store2 =
reinterpret_cast<unsigned char*
>(store + offset);
1736 unsigned char* to2 =
reinterpret_cast<unsigned char*
>(to);
1737 Index n2 = n * size;
1739 pstoreu(to2, ploadu<Packet16uc>(store2));
1741 memcpy((
void*)to2, (
void*)store2, n2);
1748EIGEN_ALWAYS_INLINE
void pstoreu_partial<float>(
float* to,
const Packet4f& from,
const Index n,
const Index offset) {
1749 pstoreu_partial_common<Packet4f>(to, from, n, offset);
1752EIGEN_ALWAYS_INLINE
void pstoreu_partial<int>(
int* to,
const Packet4i& from,
const Index n,
const Index offset) {
1753 pstoreu_partial_common<Packet4i>(to, from, n, offset);
1756EIGEN_ALWAYS_INLINE
void pstoreu_partial<short int>(
short int* to,
const Packet8s& from,
const Index n,
1757 const Index offset) {
1758 pstoreu_partial_common<Packet8s>(to, from, n, offset);
1761EIGEN_ALWAYS_INLINE
void pstoreu_partial<unsigned short int>(
unsigned short int* to,
const Packet8us& from,
1762 const Index n,
const Index offset) {
1763 pstoreu_partial_common<Packet8us>(to, from, n, offset);
1766EIGEN_ALWAYS_INLINE
void pstoreu_partial<bfloat16>(bfloat16* to,
const Packet8bf& from,
const Index n,
1767 const Index offset) {
1768 pstoreu_partial_common<Packet8us>(
reinterpret_cast<unsigned short int*
>(to), from, n, offset);
1771EIGEN_ALWAYS_INLINE
void pstoreu_partial<signed char>(
signed char* to,
const Packet16c& from,
const Index n,
1772 const Index offset) {
1773 pstoreu_partial_common<Packet16c>(to, from, n, offset);
1776EIGEN_ALWAYS_INLINE
void pstoreu_partial<unsigned char>(
unsigned char* to,
const Packet16uc& from,
const Index n,
1777 const Index offset) {
1778 pstoreu_partial_common<Packet16uc>(to, from, n, offset);
1782EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) {
1783 EIGEN_PPC_PREFETCH(addr);
1786EIGEN_STRONG_INLINE
void prefetch<int>(
const int* addr) {
1787 EIGEN_PPC_PREFETCH(addr);
1791EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
1792 EIGEN_ALIGN16
float x;
1797EIGEN_STRONG_INLINE
int pfirst<Packet4i>(
const Packet4i& a) {
1798 EIGEN_ALIGN16
int x;
1803template <
typename Packet>
1804EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) pfirst_common(
const Packet& a) {
1805 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) x;
1811EIGEN_STRONG_INLINE
short int pfirst<Packet8s>(
const Packet8s& a) {
1812 return pfirst_common<Packet8s>(a);
1816EIGEN_STRONG_INLINE
unsigned short int pfirst<Packet8us>(
const Packet8us& a) {
1817 return pfirst_common<Packet8us>(a);
1821EIGEN_STRONG_INLINE
signed char pfirst<Packet16c>(
const Packet16c& a) {
1822 return pfirst_common<Packet16c>(a);
1826EIGEN_STRONG_INLINE
unsigned char pfirst<Packet16uc>(
const Packet16uc& a) {
1827 return pfirst_common<Packet16uc>(a);
1831EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a) {
1832 return reinterpret_cast<Packet4f
>(
1833 vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE32));
1836EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a) {
1837 return reinterpret_cast<Packet4i
>(
1838 vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE32));
1841EIGEN_STRONG_INLINE Packet8s preverse(
const Packet8s& a) {
1842 return reinterpret_cast<Packet8s
>(
1843 vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE16));
1846EIGEN_STRONG_INLINE Packet8us preverse(
const Packet8us& a) {
1847 return reinterpret_cast<Packet8us
>(
1848 vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE16));
1851EIGEN_STRONG_INLINE Packet16c preverse(
const Packet16c& a) {
1852 return vec_perm(a, a, p16uc_REVERSE8);
1855EIGEN_STRONG_INLINE Packet16uc preverse(
const Packet16uc& a) {
1856 return vec_perm(a, a, p16uc_REVERSE8);
1859EIGEN_STRONG_INLINE Packet8bf preverse(
const Packet8bf& a) {
1860 return preverse<Packet8us>(a);
1864EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
1868EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
1872EIGEN_STRONG_INLINE Packet8s pabs(
const Packet8s& a) {
1876EIGEN_STRONG_INLINE Packet16c pabs(
const Packet16c& a) {
1880EIGEN_STRONG_INLINE Packet8bf pabs(
const Packet8bf& a) {
1881 EIGEN_DECLARE_CONST_FAST_Packet8us(abs_mask, 0x7FFF);
1882 return pand<Packet8us>(p8us_abs_mask, a);
1886EIGEN_STRONG_INLINE Packet8bf psignbit(
const Packet8bf& a) {
1887 return vec_sra(a.m_val, vec_splat_u16(15));
1890EIGEN_STRONG_INLINE Packet4f psignbit(
const Packet4f& a) {
1891 return (Packet4f)vec_sra((Packet4i)a, vec_splats((
unsigned int)(31)));
1895EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(
const Packet4i& a) {
1896 return vec_sra(a,
reinterpret_cast<Packet4ui
>(pset1<Packet4i>(N)));
1899EIGEN_STRONG_INLINE Packet4i plogical_shift_right(
const Packet4i& a) {
1900 return vec_sr(a,
reinterpret_cast<Packet4ui
>(pset1<Packet4i>(N)));
1903EIGEN_STRONG_INLINE Packet4i plogical_shift_left(
const Packet4i& a) {
1904 return vec_sl(a,
reinterpret_cast<Packet4ui
>(pset1<Packet4i>(N)));
1907EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(
const Packet16c& a) {
1908 return vec_sra(a, pset1<Packet16uc>(
static_cast<unsigned char>(N)));
1911EIGEN_STRONG_INLINE Packet16c plogical_shift_right(
const Packet16c& a) {
1912 return vec_sr(a, pset1<Packet16uc>(
static_cast<unsigned char>(N)));
1915EIGEN_STRONG_INLINE Packet16c plogical_shift_left(
const Packet16c& a) {
1916 return vec_sl(a, pset1<Packet16uc>(
static_cast<unsigned char>(N)));
1919EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(
const Packet16uc& a) {
1920 return vec_sr(a, pset1<Packet16uc>(
static_cast<unsigned char>(N)));
1923EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(
const Packet16uc& a) {
1924 return vec_sr(a, pset1<Packet16uc>(
static_cast<unsigned char>(N)));
1927EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(
const Packet16uc& a) {
1928 return vec_sl(a, pset1<Packet16uc>(
static_cast<unsigned char>(N)));
1931EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(
const Packet8s& a) {
1932 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1933 return vec_sra(a, p8us_mask);
1936EIGEN_STRONG_INLINE Packet8s plogical_shift_right(
const Packet8s& a) {
1937 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1938 return vec_sr(a, p8us_mask);
1941EIGEN_STRONG_INLINE Packet8s plogical_shift_left(
const Packet8s& a) {
1942 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1943 return vec_sl(a, p8us_mask);
1946EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(
const Packet8us& a) {
1947 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1948 return vec_sr(a, p8us_mask);
1951EIGEN_STRONG_INLINE Packet8us plogical_shift_right(
const Packet8us& a) {
1952 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1953 return vec_sr(a, p8us_mask);
1956EIGEN_STRONG_INLINE Packet8us plogical_shift_left(
const Packet8us& a) {
1957 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1958 return vec_sl(a, p8us_mask);
1961EIGEN_STRONG_INLINE Packet4f plogical_shift_left(
const Packet4f& a) {
1962 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1963 Packet4ui r = vec_sl(
reinterpret_cast<Packet4ui
>(a), p4ui_mask);
1964 return reinterpret_cast<Packet4f
>(r);
1968EIGEN_STRONG_INLINE Packet4f plogical_shift_right(
const Packet4f& a) {
1969 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1970 Packet4ui r = vec_sr(
reinterpret_cast<Packet4ui
>(a), p4ui_mask);
1971 return reinterpret_cast<Packet4f
>(r);
1975EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(
const Packet4ui& a) {
1976 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1977 return vec_sr(a, p4ui_mask);
1981EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(
const Packet4ui& a) {
1982 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1983 return vec_sr(a, p4ui_mask);
1987EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(
const Packet4ui& a) {
1988 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1989 return vec_sl(a, p4ui_mask);
1992EIGEN_STRONG_INLINE Packet4f Bf16ToF32Even(
const Packet8bf& bf) {
1993 return plogical_shift_left<16>(
reinterpret_cast<Packet4f
>(bf.m_val));
1996EIGEN_STRONG_INLINE Packet4f Bf16ToF32Odd(
const Packet8bf& bf) {
1997 const EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000);
1998 return pand<Packet4f>(
reinterpret_cast<Packet4f
>(bf.m_val),
reinterpret_cast<Packet4f
>(p4ui_high_mask));
2001EIGEN_ALWAYS_INLINE Packet8us pmerge(
const Packet4ui& even,
const Packet4ui& odd) {
2003 return vec_perm(
reinterpret_cast<Packet8us
>(odd),
reinterpret_cast<Packet8us
>(even), p16uc_MERGEO16);
2005 return vec_perm(
reinterpret_cast<Packet8us
>(even),
reinterpret_cast<Packet8us
>(odd), p16uc_MERGEE16);
2011EIGEN_STRONG_INLINE Packet8bf F32ToBf16Bool(
const Packet4f& even,
const Packet4f& odd) {
2012 return pmerge(
reinterpret_cast<Packet4ui
>(even),
reinterpret_cast<Packet4ui
>(odd));
2017#ifndef __VEC_CLASS_FP_NAN
2018#define __VEC_CLASS_FP_NAN (1 << 6)
2021#if defined(SUPPORT_BF16_SUBNORMALS) && !defined(__VEC_CLASS_FP_SUBNORMAL)
2022#define __VEC_CLASS_FP_SUBNORMAL_P (1 << 1)
2023#define __VEC_CLASS_FP_SUBNORMAL_N (1 << 0)
2025#define __VEC_CLASS_FP_SUBNORMAL (__VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N)
2028EIGEN_STRONG_INLINE Packet8bf F32ToBf16(
const Packet4f& p4f) {
2030 return reinterpret_cast<Packet8us
>(__builtin_vsx_xvcvspbf16(
reinterpret_cast<Packet16uc
>(p4f)));
2032 Packet4ui input =
reinterpret_cast<Packet4ui
>(p4f);
2033 Packet4ui lsb = plogical_shift_right<16>(input);
2034 lsb = pand<Packet4ui>(lsb,
reinterpret_cast<Packet4ui
>(p4i_ONE));
2036 EIGEN_DECLARE_CONST_FAST_Packet4ui(BIAS, 0x7FFFu);
2037 Packet4ui rounding_bias = padd<Packet4ui>(lsb, p4ui_BIAS);
2038 input = padd<Packet4ui>(input, rounding_bias);
2040 const EIGEN_DECLARE_CONST_FAST_Packet4ui(nan, 0x7FC00000);
2041#if defined(_ARCH_PWR9) && defined(EIGEN_VECTORIZE_VSX)
2042 Packet4bi nan_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_NAN);
2043 input = vec_sel(input, p4ui_nan, nan_selector);
2045#ifdef SUPPORT_BF16_SUBNORMALS
2046 Packet4bi subnormal_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_SUBNORMAL);
2047 input = vec_sel(input,
reinterpret_cast<Packet4ui
>(p4f), subnormal_selector);
2050#ifdef SUPPORT_BF16_SUBNORMALS
2052 const EIGEN_DECLARE_CONST_FAST_Packet4ui(exp_mask, 0x7F800000);
2053 Packet4ui exp = pand<Packet4ui>(p4ui_exp_mask,
reinterpret_cast<Packet4ui
>(p4f));
2055 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mantissa_mask, 0x7FFFFF);
2056 Packet4ui mantissa = pand<Packet4ui>(p4ui_mantissa_mask,
reinterpret_cast<Packet4ui
>(p4f));
2058 Packet4bi is_max_exp = vec_cmpeq(exp, p4ui_exp_mask);
2059 Packet4bi is_mant_zero = vec_cmpeq(mantissa,
reinterpret_cast<Packet4ui
>(p4i_ZERO));
2061 Packet4ui nan_selector =
2062 pandnot<Packet4ui>(
reinterpret_cast<Packet4ui
>(is_max_exp),
reinterpret_cast<Packet4ui
>(is_mant_zero));
2064 Packet4bi is_zero_exp = vec_cmpeq(exp,
reinterpret_cast<Packet4ui
>(p4i_ZERO));
2066 Packet4ui subnormal_selector =
2067 pandnot<Packet4ui>(
reinterpret_cast<Packet4ui
>(is_zero_exp),
reinterpret_cast<Packet4ui
>(is_mant_zero));
2069 input = vec_sel(input, p4ui_nan, nan_selector);
2070 input = vec_sel(input,
reinterpret_cast<Packet4ui
>(p4f), subnormal_selector);
2073 Packet4bi nan_selector = vec_cmpeq(p4f, p4f);
2075 input = vec_sel(p4ui_nan, input, nan_selector);
2079 input = plogical_shift_right<16>(input);
2080 return reinterpret_cast<Packet8us
>(input);
2091EIGEN_ALWAYS_INLINE Packet8bf Bf16PackHigh(
const Packet4f& lo,
const Packet4f& hi) {
2093 return vec_perm(
reinterpret_cast<Packet8us
>(lo),
reinterpret_cast<Packet8us
>(hi), p16uc_MERGEH16);
2095 return vec_perm(
reinterpret_cast<Packet8us
>(hi),
reinterpret_cast<Packet8us
>(lo), p16uc_MERGEE16);
2105EIGEN_ALWAYS_INLINE Packet8bf Bf16PackLow(
const Packet4f& lo,
const Packet4f& hi) {
2107 return vec_pack(
reinterpret_cast<Packet4ui
>(lo),
reinterpret_cast<Packet4ui
>(hi));
2109 return vec_perm(
reinterpret_cast<Packet8us
>(hi),
reinterpret_cast<Packet8us
>(lo), p16uc_MERGEO16);
2114EIGEN_ALWAYS_INLINE Packet8bf Bf16PackLow(
const Packet4f& hi,
const Packet4f& lo) {
2116 return vec_pack(
reinterpret_cast<Packet4ui
>(hi),
reinterpret_cast<Packet4ui
>(lo));
2118 return vec_perm(
reinterpret_cast<Packet8us
>(hi),
reinterpret_cast<Packet8us
>(lo), p16uc_MERGEE16);
2123EIGEN_ALWAYS_INLINE Packet8bf Bf16PackHigh(
const Packet4f& hi,
const Packet4f& lo) {
2125 return vec_perm(
reinterpret_cast<Packet8us
>(hi),
reinterpret_cast<Packet8us
>(lo), p16uc_MERGEL16);
2127 return vec_perm(
reinterpret_cast<Packet8us
>(hi),
reinterpret_cast<Packet8us
>(lo), p16uc_MERGEO16);
2137template <
bool lohi = true>
2138EIGEN_ALWAYS_INLINE Packet8bf F32ToBf16Two(
const Packet4f& lo,
const Packet4f& hi) {
2139 Packet8us p4f = Bf16PackHigh<lohi>(lo, hi);
2140 Packet8us p4f2 = Bf16PackLow<lohi>(lo, hi);
2142 Packet8us lsb = pand<Packet8us>(p4f, p8us_ONE);
2143 EIGEN_DECLARE_CONST_FAST_Packet8us(BIAS, 0x7FFFu);
2144 lsb = padd<Packet8us>(lsb, p8us_BIAS);
2145 lsb = padd<Packet8us>(lsb, p4f2);
2147 Packet8bi rounding_bias = vec_cmplt(lsb, p4f2);
2148 Packet8us input = psub<Packet8us>(p4f,
reinterpret_cast<Packet8us
>(rounding_bias));
2150#if defined(_ARCH_PWR9) && defined(EIGEN_VECTORIZE_VSX)
2151 Packet4bi nan_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_NAN);
2152 Packet4bi nan_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_NAN);
2153 Packet8us nan_selector =
2154 Bf16PackLow<lohi>(
reinterpret_cast<Packet4f
>(nan_selector_lo),
reinterpret_cast<Packet4f
>(nan_selector_hi));
2156 input = vec_sel(input, p8us_BIAS, nan_selector);
2158#ifdef SUPPORT_BF16_SUBNORMALS
2159 Packet4bi subnormal_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_SUBNORMAL);
2160 Packet4bi subnormal_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_SUBNORMAL);
2161 Packet8us subnormal_selector = Bf16PackLow<lohi>(
reinterpret_cast<Packet4f
>(subnormal_selector_lo),
2162 reinterpret_cast<Packet4f
>(subnormal_selector_hi));
2164 input = vec_sel(input,
reinterpret_cast<Packet8us
>(p4f), subnormal_selector);
2167#ifdef SUPPORT_BF16_SUBNORMALS
2169 const EIGEN_DECLARE_CONST_FAST_Packet8us(exp_mask, 0x7F80);
2170 Packet8us exp = pand<Packet8us>(p8us_exp_mask, p4f);
2172 const EIGEN_DECLARE_CONST_FAST_Packet8us(mantissa_mask, 0x7Fu);
2173 Packet8us mantissa = pand<Packet8us>(p8us_mantissa_mask, p4f);
2175 Packet8bi is_max_exp = vec_cmpeq(exp, p8us_exp_mask);
2176 Packet8bi is_mant_zero = vec_cmpeq(mantissa,
reinterpret_cast<Packet8us
>(p4i_ZERO));
2178 Packet8us nan_selector =
2179 pandnot<Packet8us>(
reinterpret_cast<Packet8us
>(is_max_exp),
reinterpret_cast<Packet8us
>(is_mant_zero));
2181 Packet8bi is_zero_exp = vec_cmpeq(exp,
reinterpret_cast<Packet8us
>(p4i_ZERO));
2183 Packet8us subnormal_selector =
2184 pandnot<Packet8us>(
reinterpret_cast<Packet8us
>(is_zero_exp),
reinterpret_cast<Packet8us
>(is_mant_zero));
2187 input = vec_sel(input, p8us_BIAS, nan_selector);
2188 input = vec_sel(input,
reinterpret_cast<Packet8us
>(p4f), subnormal_selector);
2191 Packet4bi nan_selector_lo = vec_cmpeq(lo, lo);
2192 Packet4bi nan_selector_hi = vec_cmpeq(hi, hi);
2193 Packet8us nan_selector =
2194 Bf16PackLow<lohi>(
reinterpret_cast<Packet4f
>(nan_selector_lo),
reinterpret_cast<Packet4f
>(nan_selector_hi));
2196 input = vec_sel(p8us_BIAS, input, nan_selector);
2206EIGEN_STRONG_INLINE Packet8bf F32ToBf16Both(
const Packet4f& lo,
const Packet4f& hi) {
2208 Packet8bf fp16_0 = F32ToBf16(lo);
2209 Packet8bf fp16_1 = F32ToBf16(hi);
2210 return vec_pack(
reinterpret_cast<Packet4ui
>(fp16_0.m_val),
reinterpret_cast<Packet4ui
>(fp16_1.m_val));
2212 return F32ToBf16Two(lo, hi);
2219EIGEN_STRONG_INLINE Packet8bf F32ToBf16(
const Packet4f& even,
const Packet4f& odd) {
2221 return pmerge(
reinterpret_cast<Packet4ui
>(F32ToBf16(even).m_val),
reinterpret_cast<Packet4ui
>(F32ToBf16(odd).m_val));
2223 return F32ToBf16Two<false>(even, odd);
2226#define BF16_TO_F32_UNARY_OP_WRAPPER(OP, A) \
2227 Packet4f a_even = Bf16ToF32Even(A); \
2228 Packet4f a_odd = Bf16ToF32Odd(A); \
2229 Packet4f op_even = OP(a_even); \
2230 Packet4f op_odd = OP(a_odd); \
2231 return F32ToBf16(op_even, op_odd);
2233#define BF16_TO_F32_BINARY_OP_WRAPPER(OP, A, B) \
2234 Packet4f a_even = Bf16ToF32Even(A); \
2235 Packet4f a_odd = Bf16ToF32Odd(A); \
2236 Packet4f b_even = Bf16ToF32Even(B); \
2237 Packet4f b_odd = Bf16ToF32Odd(B); \
2238 Packet4f op_even = OP(a_even, b_even); \
2239 Packet4f op_odd = OP(a_odd, b_odd); \
2240 return F32ToBf16(op_even, op_odd);
2242#define BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(OP, A, B) \
2243 Packet4f a_even = Bf16ToF32Even(A); \
2244 Packet4f a_odd = Bf16ToF32Odd(A); \
2245 Packet4f b_even = Bf16ToF32Even(B); \
2246 Packet4f b_odd = Bf16ToF32Odd(B); \
2247 Packet4f op_even = OP(a_even, b_even); \
2248 Packet4f op_odd = OP(a_odd, b_odd); \
2249 return F32ToBf16Bool(op_even, op_odd);
2252EIGEN_STRONG_INLINE Packet8bf padd<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
2253 BF16_TO_F32_BINARY_OP_WRAPPER(padd<Packet4f>, a, b);
2257EIGEN_STRONG_INLINE Packet8bf pmul<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
2258 BF16_TO_F32_BINARY_OP_WRAPPER(pmul<Packet4f>, a, b);
2262EIGEN_STRONG_INLINE Packet8bf pdiv<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
2263 BF16_TO_F32_BINARY_OP_WRAPPER(pdiv<Packet4f>, a, b);
2267EIGEN_STRONG_INLINE Packet8bf pnegate<Packet8bf>(
const Packet8bf& a) {
2268 EIGEN_DECLARE_CONST_FAST_Packet8us(neg_mask, 0x8000);
2269 return pxor<Packet8us>(p8us_neg_mask, a);
2273EIGEN_STRONG_INLINE Packet8bf psub<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
2274 BF16_TO_F32_BINARY_OP_WRAPPER(psub<Packet4f>, a, b);
2278EIGEN_STRONG_INLINE Packet8bf pexp<Packet8bf>(
const Packet8bf& a) {
2279 BF16_TO_F32_UNARY_OP_WRAPPER(pexp_float, a);
2283EIGEN_STRONG_INLINE Packet8bf pexp2<Packet8bf>(
const Packet8bf& a) {
2284 BF16_TO_F32_UNARY_OP_WRAPPER(generic_exp2, a);
2288EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(
const Packet4f& a,
const Packet4f& exponent) {
2289 return pldexp_generic(a, exponent);
2292EIGEN_STRONG_INLINE Packet8bf pldexp<Packet8bf>(
const Packet8bf& a,
const Packet8bf& exponent) {
2293 BF16_TO_F32_BINARY_OP_WRAPPER(pldexp<Packet4f>, a, exponent);
2297EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(
const Packet4f& a, Packet4f& exponent) {
2298 return pfrexp_generic(a, exponent);
2301EIGEN_STRONG_INLINE Packet8bf pfrexp<Packet8bf>(
const Packet8bf& a, Packet8bf& e) {
2302 Packet4f a_even = Bf16ToF32Even(a);
2303 Packet4f a_odd = Bf16ToF32Odd(a);
2306 Packet4f op_even = pfrexp<Packet4f>(a_even, e_even);
2307 Packet4f op_odd = pfrexp<Packet4f>(a_odd, e_odd);
2308 e = F32ToBf16(e_even, e_odd);
2309 return F32ToBf16(op_even, op_odd);
2313EIGEN_STRONG_INLINE Packet8bf psin<Packet8bf>(
const Packet8bf& a) {
2314 BF16_TO_F32_UNARY_OP_WRAPPER(psin_float, a);
2317EIGEN_STRONG_INLINE Packet8bf pcos<Packet8bf>(
const Packet8bf& a) {
2318 BF16_TO_F32_UNARY_OP_WRAPPER(pcos_float, a);
2321EIGEN_STRONG_INLINE Packet8bf plog<Packet8bf>(
const Packet8bf& a) {
2322 BF16_TO_F32_UNARY_OP_WRAPPER(plog_float, a);
2325EIGEN_STRONG_INLINE Packet8bf pfloor<Packet8bf>(
const Packet8bf& a) {
2326 BF16_TO_F32_UNARY_OP_WRAPPER(pfloor<Packet4f>, a);
2329EIGEN_STRONG_INLINE Packet8bf pceil<Packet8bf>(
const Packet8bf& a) {
2330 BF16_TO_F32_UNARY_OP_WRAPPER(pceil<Packet4f>, a);
2333EIGEN_STRONG_INLINE Packet8bf pround<Packet8bf>(
const Packet8bf& a) {
2334 BF16_TO_F32_UNARY_OP_WRAPPER(pround<Packet4f>, a);
2337EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(
const Packet8bf& a) {
2338 BF16_TO_F32_UNARY_OP_WRAPPER(ptrunc<Packet4f>, a);
2340#ifdef EIGEN_VECTORIZE_VSX
2342EIGEN_STRONG_INLINE Packet8bf print<Packet8bf>(
const Packet8bf& a) {
2343 BF16_TO_F32_UNARY_OP_WRAPPER(print<Packet4f>, a);
2347EIGEN_STRONG_INLINE Packet8bf pmadd(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
2348 Packet4f a_even = Bf16ToF32Even(a);
2349 Packet4f a_odd = Bf16ToF32Odd(a);
2350 Packet4f b_even = Bf16ToF32Even(b);
2351 Packet4f b_odd = Bf16ToF32Odd(b);
2352 Packet4f c_even = Bf16ToF32Even(c);
2353 Packet4f c_odd = Bf16ToF32Odd(c);
2354 Packet4f pmadd_even = pmadd<Packet4f>(a_even, b_even, c_even);
2355 Packet4f pmadd_odd = pmadd<Packet4f>(a_odd, b_odd, c_odd);
2356 return F32ToBf16(pmadd_even, pmadd_odd);
2360EIGEN_STRONG_INLINE Packet8bf pmsub(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
2361 Packet4f a_even = Bf16ToF32Even(a);
2362 Packet4f a_odd = Bf16ToF32Odd(a);
2363 Packet4f b_even = Bf16ToF32Even(b);
2364 Packet4f b_odd = Bf16ToF32Odd(b);
2365 Packet4f c_even = Bf16ToF32Even(c);
2366 Packet4f c_odd = Bf16ToF32Odd(c);
2367 Packet4f pmadd_even = pmsub<Packet4f>(a_even, b_even, c_even);
2368 Packet4f pmadd_odd = pmsub<Packet4f>(a_odd, b_odd, c_odd);
2369 return F32ToBf16(pmadd_even, pmadd_odd);
2372EIGEN_STRONG_INLINE Packet8bf pnmadd(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
2373 Packet4f a_even = Bf16ToF32Even(a);
2374 Packet4f a_odd = Bf16ToF32Odd(a);
2375 Packet4f b_even = Bf16ToF32Even(b);
2376 Packet4f b_odd = Bf16ToF32Odd(b);
2377 Packet4f c_even = Bf16ToF32Even(c);
2378 Packet4f c_odd = Bf16ToF32Odd(c);
2379 Packet4f pmadd_even = pnmadd<Packet4f>(a_even, b_even, c_even);
2380 Packet4f pmadd_odd = pnmadd<Packet4f>(a_odd, b_odd, c_odd);
2381 return F32ToBf16(pmadd_even, pmadd_odd);
2385EIGEN_STRONG_INLINE Packet8bf pnmsub(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
2386 Packet4f a_even = Bf16ToF32Even(a);
2387 Packet4f a_odd = Bf16ToF32Odd(a);
2388 Packet4f b_even = Bf16ToF32Even(b);
2389 Packet4f b_odd = Bf16ToF32Odd(b);
2390 Packet4f c_even = Bf16ToF32Even(c);
2391 Packet4f c_odd = Bf16ToF32Odd(c);
2392 Packet4f pmadd_even = pnmsub<Packet4f>(a_even, b_even, c_even);
2393 Packet4f pmadd_odd = pnmsub<Packet4f>(a_odd, b_odd, c_odd);
2394 return F32ToBf16(pmadd_even, pmadd_odd);
2398EIGEN_STRONG_INLINE Packet8bf pmin<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
2399 BF16_TO_F32_BINARY_OP_WRAPPER(pmin<Packet4f>, a, b);
2403EIGEN_STRONG_INLINE Packet8bf pmax<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
2404 BF16_TO_F32_BINARY_OP_WRAPPER(pmax<Packet4f>, a, b);
2408EIGEN_STRONG_INLINE Packet8bf pcmp_lt(
const Packet8bf& a,
const Packet8bf& b) {
2409 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt<Packet4f>, a, b);
2412EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan(
const Packet8bf& a,
const Packet8bf& b) {
2413 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt_or_nan<Packet4f>, a, b);
2416EIGEN_STRONG_INLINE Packet8bf pcmp_le(
const Packet8bf& a,
const Packet8bf& b) {
2417 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_le<Packet4f>, a, b);
2420EIGEN_STRONG_INLINE Packet8bf pcmp_eq(
const Packet8bf& a,
const Packet8bf& b) {
2421 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_eq<Packet4f>, a, b);
2426EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(
const Packet8bf& a) {
2427 const Packet8us magnitude = vec_and(a.m_val, vec_splats(
static_cast<unsigned short int>(0x7fff)));
2428 const Packet8bi is_zero = vec_cmpeq(magnitude, vec_splats(
static_cast<unsigned short int>(0)));
2429 const Packet8us keep =
2430 vec_or(
reinterpret_cast<Packet8us
>(vec_cmpgt(magnitude, vec_splats(
static_cast<unsigned short int>(0x7f80)))),
2431 vec_splats(
static_cast<unsigned short int>(0x8000)));
2432 const Packet8us value = vec_sel(vec_splats(
static_cast<unsigned short int>(0x3f80)), a.m_val, keep);
2433 return Packet8bf(vec_andc(value,
reinterpret_cast<Packet8us
>(is_zero)));
2437EIGEN_STRONG_INLINE bfloat16 pfirst(
const Packet8bf& a) {
2438 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16((pfirst<Packet8us>(a)));
2442EIGEN_STRONG_INLINE Packet8bf ploaddup<Packet8bf>(
const bfloat16* from) {
2443 return ploaddup<Packet8us>(
reinterpret_cast<const unsigned short int*
>(from));
2447EIGEN_STRONG_INLINE Packet8bf plset<Packet8bf>(
const bfloat16& a) {
2448 bfloat16 countdown[8] = {bfloat16(0), bfloat16(1), bfloat16(2), bfloat16(3),
2449 bfloat16(4), bfloat16(5), bfloat16(6), bfloat16(7)};
2450 return padd<Packet8bf>(pset1<Packet8bf>(a), pload<Packet8bf>(countdown));
2454EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a) {
2456 b = vec_sld(a, a, 8);
2458 b = vec_sld(sum, sum, 4);
2464EIGEN_STRONG_INLINE
int predux<Packet4i>(
const Packet4i& a) {
2466 b = vec_sld(a, a, 8);
2468 b = vec_sld(sum, sum, 4);
2474EIGEN_STRONG_INLINE bfloat16 predux<Packet8bf>(
const Packet8bf& a) {
2475 float redux_even = predux<Packet4f>(Bf16ToF32Even(a));
2476 float redux_odd = predux<Packet4f>(Bf16ToF32Odd(a));
2477 float f32_result = redux_even + redux_odd;
2478 return bfloat16(f32_result);
2480template <
typename Packet>
2481EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_size8(
const Packet& a) {
2484 __UNPACK_TYPE__(Packet) n[8];
2488 EIGEN_ALIGN16
int first_loader[4] = {vt.n[0], vt.n[1], vt.n[2], vt.n[3]};
2489 EIGEN_ALIGN16
int second_loader[4] = {vt.n[4], vt.n[5], vt.n[6], vt.n[7]};
2490 Packet4i first_half = pload<Packet4i>(first_loader);
2491 Packet4i second_half = pload<Packet4i>(second_loader);
2493 return static_cast<__UNPACK_TYPE__(Packet)
>(predux(first_half) + predux(second_half));
2497EIGEN_STRONG_INLINE
short int predux<Packet8s>(
const Packet8s& a) {
2498 return predux_size8<Packet8s>(a);
2502EIGEN_STRONG_INLINE
unsigned short int predux<Packet8us>(
const Packet8us& a) {
2503 return predux_size8<Packet8us>(a);
2506template <
typename Packet>
2507EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_size16(
const Packet& a) {
2510 __UNPACK_TYPE__(Packet) n[16];
2514 EIGEN_ALIGN16
int first_loader[4] = {vt.n[0], vt.n[1], vt.n[2], vt.n[3]};
2515 EIGEN_ALIGN16
int second_loader[4] = {vt.n[4], vt.n[5], vt.n[6], vt.n[7]};
2516 EIGEN_ALIGN16
int third_loader[4] = {vt.n[8], vt.n[9], vt.n[10], vt.n[11]};
2517 EIGEN_ALIGN16
int fourth_loader[4] = {vt.n[12], vt.n[13], vt.n[14], vt.n[15]};
2519 Packet4i first_quarter = pload<Packet4i>(first_loader);
2520 Packet4i second_quarter = pload<Packet4i>(second_loader);
2521 Packet4i third_quarter = pload<Packet4i>(third_loader);
2522 Packet4i fourth_quarter = pload<Packet4i>(fourth_loader);
2524 return static_cast<__UNPACK_TYPE__(Packet)
>(predux(first_quarter) + predux(second_quarter) + predux(third_quarter) +
2525 predux(fourth_quarter));
2529EIGEN_STRONG_INLINE
signed char predux<Packet16c>(
const Packet16c& a) {
2530 return predux_size16<Packet16c>(a);
2534EIGEN_STRONG_INLINE
unsigned char predux<Packet16uc>(
const Packet16uc& a) {
2535 return predux_size16<Packet16uc>(a);
2541EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a) {
2543 prod = pmul(a, vec_sld(a, a, 8));
2544 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
2548EIGEN_STRONG_INLINE
int predux_mul<Packet4i>(
const Packet4i& a) {
2549 EIGEN_ALIGN16
int aux[4];
2551 return aux[0] * aux[1] * aux[2] * aux[3];
2555EIGEN_STRONG_INLINE
short int predux_mul<Packet8s>(
const Packet8s& a) {
2556 Packet8s pair, quad, octo;
2558 pair = vec_mul(a, vec_sld(a, a, 8));
2559 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2560 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2562 return pfirst(octo);
2566EIGEN_STRONG_INLINE
unsigned short int predux_mul<Packet8us>(
const Packet8us& a) {
2567 Packet8us pair, quad, octo;
2569 pair = vec_mul(a, vec_sld(a, a, 8));
2570 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2571 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2573 return pfirst(octo);
2577EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet8bf>(
const Packet8bf& a) {
2578 float redux_even = predux_mul<Packet4f>(Bf16ToF32Even(a));
2579 float redux_odd = predux_mul<Packet4f>(Bf16ToF32Odd(a));
2580 float f32_result = redux_even * redux_odd;
2581 return bfloat16(f32_result);
2585EIGEN_STRONG_INLINE
signed char predux_mul<Packet16c>(
const Packet16c& a) {
2586 Packet16c pair, quad, octo, result;
2588 pair = vec_mul(a, vec_sld(a, a, 8));
2589 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2590 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2591 result = vec_mul(octo, vec_sld(octo, octo, 1));
2593 return pfirst(result);
2597EIGEN_STRONG_INLINE
unsigned char predux_mul<Packet16uc>(
const Packet16uc& a) {
2598 Packet16uc pair, quad, octo, result;
2600 pair = vec_mul(a, vec_sld(a, a, 8));
2601 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2602 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2603 result = vec_mul(octo, vec_sld(octo, octo, 1));
2605 return pfirst(result);
2609template <
typename Packet>
2610EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_min4(
const Packet& a) {
2612 b = vec_min(a, vec_sld(a, a, 8));
2613 res = vec_min(b, vec_sld(b, b, 4));
2618EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a) {
2619 return predux_min4<Packet4f>(a);
2623EIGEN_STRONG_INLINE
int predux_min<Packet4i>(
const Packet4i& a) {
2624 return predux_min4<Packet4i>(a);
2628EIGEN_STRONG_INLINE bfloat16 predux_min<Packet8bf>(
const Packet8bf& a) {
2629 float redux_even = predux_min<Packet4f>(Bf16ToF32Even(a));
2630 float redux_odd = predux_min<Packet4f>(Bf16ToF32Odd(a));
2631 float f32_result = (std::min)(redux_even, redux_odd);
2632 return bfloat16(f32_result);
2636EIGEN_STRONG_INLINE
short int predux_min<Packet8s>(
const Packet8s& a) {
2637 Packet8s pair, quad, octo;
2640 pair = vec_min(a, vec_sld(a, a, 8));
2643 quad = vec_min(pair, vec_sld(pair, pair, 4));
2646 octo = vec_min(quad, vec_sld(quad, quad, 2));
2647 return pfirst(octo);
2651EIGEN_STRONG_INLINE
unsigned short int predux_min<Packet8us>(
const Packet8us& a) {
2652 Packet8us pair, quad, octo;
2655 pair = vec_min(a, vec_sld(a, a, 8));
2658 quad = vec_min(pair, vec_sld(pair, pair, 4));
2661 octo = vec_min(quad, vec_sld(quad, quad, 2));
2662 return pfirst(octo);
2666EIGEN_STRONG_INLINE
signed char predux_min<Packet16c>(
const Packet16c& a) {
2667 Packet16c pair, quad, octo, result;
2669 pair = vec_min(a, vec_sld(a, a, 8));
2670 quad = vec_min(pair, vec_sld(pair, pair, 4));
2671 octo = vec_min(quad, vec_sld(quad, quad, 2));
2672 result = vec_min(octo, vec_sld(octo, octo, 1));
2674 return pfirst(result);
2678EIGEN_STRONG_INLINE
unsigned char predux_min<Packet16uc>(
const Packet16uc& a) {
2679 Packet16uc pair, quad, octo, result;
2681 pair = vec_min(a, vec_sld(a, a, 8));
2682 quad = vec_min(pair, vec_sld(pair, pair, 4));
2683 octo = vec_min(quad, vec_sld(quad, quad, 2));
2684 result = vec_min(octo, vec_sld(octo, octo, 1));
2686 return pfirst(result);
2689template <
typename Packet>
2690EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_max4(
const Packet& a) {
2692 b = vec_max(a, vec_sld(a, a, 8));
2693 res = vec_max(b, vec_sld(b, b, 4));
2698EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a) {
2699 return predux_max4<Packet4f>(a);
2703EIGEN_STRONG_INLINE
int predux_max<Packet4i>(
const Packet4i& a) {
2704 return predux_max4<Packet4i>(a);
2708EIGEN_STRONG_INLINE bfloat16 predux_max<Packet8bf>(
const Packet8bf& a) {
2709 float redux_even = predux_max<Packet4f>(Bf16ToF32Even(a));
2710 float redux_odd = predux_max<Packet4f>(Bf16ToF32Odd(a));
2711 float f32_result = (std::max)(redux_even, redux_odd);
2712 return bfloat16(f32_result);
2716EIGEN_STRONG_INLINE
short int predux_max<Packet8s>(
const Packet8s& a) {
2717 Packet8s pair, quad, octo;
2720 pair = vec_max(a, vec_sld(a, a, 8));
2723 quad = vec_max(pair, vec_sld(pair, pair, 4));
2726 octo = vec_max(quad, vec_sld(quad, quad, 2));
2727 return pfirst(octo);
2731EIGEN_STRONG_INLINE
unsigned short int predux_max<Packet8us>(
const Packet8us& a) {
2732 Packet8us pair, quad, octo;
2735 pair = vec_max(a, vec_sld(a, a, 8));
2738 quad = vec_max(pair, vec_sld(pair, pair, 4));
2741 octo = vec_max(quad, vec_sld(quad, quad, 2));
2742 return pfirst(octo);
2746EIGEN_STRONG_INLINE
signed char predux_max<Packet16c>(
const Packet16c& a) {
2747 Packet16c pair, quad, octo, result;
2749 pair = vec_max(a, vec_sld(a, a, 8));
2750 quad = vec_max(pair, vec_sld(pair, pair, 4));
2751 octo = vec_max(quad, vec_sld(quad, quad, 2));
2752 result = vec_max(octo, vec_sld(octo, octo, 1));
2754 return pfirst(result);
2758EIGEN_STRONG_INLINE
unsigned char predux_max<Packet16uc>(
const Packet16uc& a) {
2759 Packet16uc pair, quad, octo, result;
2761 pair = vec_max(a, vec_sld(a, a, 8));
2762 quad = vec_max(pair, vec_sld(pair, pair, 4));
2763 octo = vec_max(quad, vec_sld(quad, quad, 2));
2764 result = vec_max(octo, vec_sld(octo, octo, 1));
2766 return pfirst(result);
2770EIGEN_STRONG_INLINE
bool predux_any(
const Packet4f& x) {
2771 const Packet4ui zero = {0, 0, 0, 0};
2772 return vec_any_ne(
reinterpret_cast<Packet4ui
>(x), zero);
2776EIGEN_STRONG_INLINE
bool predux_any(
const Packet8bf& x) {
2777 const Packet8us zero = {0, 0, 0, 0, 0, 0, 0, 0};
2778 return vec_any_ne(x.m_val, zero);
2781template <
typename T>
2782EIGEN_DEVICE_FUNC
inline void ptranpose_common(PacketBlock<T, 4>& kernel) {
2784 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2785 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2786 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2787 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2788 kernel.packet[0] = vec_mergeh(t0, t2);
2789 kernel.packet[1] = vec_mergel(t0, t2);
2790 kernel.packet[2] = vec_mergeh(t1, t3);
2791 kernel.packet[3] = vec_mergel(t1, t3);
2794EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) { ptranpose_common<Packet4f>(kernel); }
2796EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) { ptranpose_common<Packet4i>(kernel); }
2798EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
2799 Packet8s t0, t1, t2, t3;
2800 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2801 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2802 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2803 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2804 kernel.packet[0] = vec_mergeh(t0, t2);
2805 kernel.packet[1] = vec_mergel(t0, t2);
2806 kernel.packet[2] = vec_mergeh(t1, t3);
2807 kernel.packet[3] = vec_mergel(t1, t3);
2810EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
2811 Packet8us t0, t1, t2, t3;
2812 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2813 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2814 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2815 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2816 kernel.packet[0] = vec_mergeh(t0, t2);
2817 kernel.packet[1] = vec_mergel(t0, t2);
2818 kernel.packet[2] = vec_mergeh(t1, t3);
2819 kernel.packet[3] = vec_mergel(t1, t3);
2822EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8bf, 4>& kernel) {
2823 Packet8us t0, t1, t2, t3;
2825 t0 = vec_mergeh(kernel.packet[0].m_val, kernel.packet[2].m_val);
2826 t1 = vec_mergel(kernel.packet[0].m_val, kernel.packet[2].m_val);
2827 t2 = vec_mergeh(kernel.packet[1].m_val, kernel.packet[3].m_val);
2828 t3 = vec_mergel(kernel.packet[1].m_val, kernel.packet[3].m_val);
2829 kernel.packet[0] = vec_mergeh(t0, t2);
2830 kernel.packet[1] = vec_mergel(t0, t2);
2831 kernel.packet[2] = vec_mergeh(t1, t3);
2832 kernel.packet[3] = vec_mergel(t1, t3);
2835EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
2836 Packet16c t0, t1, t2, t3;
2837 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2838 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2839 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2840 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2841 kernel.packet[0] = vec_mergeh(t0, t2);
2842 kernel.packet[1] = vec_mergel(t0, t2);
2843 kernel.packet[2] = vec_mergeh(t1, t3);
2844 kernel.packet[3] = vec_mergel(t1, t3);
2847EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
2848 Packet16uc t0, t1, t2, t3;
2849 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2850 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2851 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2852 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2853 kernel.packet[0] = vec_mergeh(t0, t2);
2854 kernel.packet[1] = vec_mergel(t0, t2);
2855 kernel.packet[2] = vec_mergeh(t1, t3);
2856 kernel.packet[3] = vec_mergel(t1, t3);
2859EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
2860 Packet8s v[8], sum[8];
2862 v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]);
2863 v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]);
2864 v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]);
2865 v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]);
2866 v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]);
2867 v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]);
2868 v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]);
2869 v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]);
2870 sum[0] = vec_mergeh(v[0], v[4]);
2871 sum[1] = vec_mergel(v[0], v[4]);
2872 sum[2] = vec_mergeh(v[1], v[5]);
2873 sum[3] = vec_mergel(v[1], v[5]);
2874 sum[4] = vec_mergeh(v[2], v[6]);
2875 sum[5] = vec_mergel(v[2], v[6]);
2876 sum[6] = vec_mergeh(v[3], v[7]);
2877 sum[7] = vec_mergel(v[3], v[7]);
2879 kernel.packet[0] = vec_mergeh(sum[0], sum[4]);
2880 kernel.packet[1] = vec_mergel(sum[0], sum[4]);
2881 kernel.packet[2] = vec_mergeh(sum[1], sum[5]);
2882 kernel.packet[3] = vec_mergel(sum[1], sum[5]);
2883 kernel.packet[4] = vec_mergeh(sum[2], sum[6]);
2884 kernel.packet[5] = vec_mergel(sum[2], sum[6]);
2885 kernel.packet[6] = vec_mergeh(sum[3], sum[7]);
2886 kernel.packet[7] = vec_mergel(sum[3], sum[7]);
2889EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
2890 Packet8us v[8], sum[8];
2892 v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]);
2893 v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]);
2894 v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]);
2895 v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]);
2896 v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]);
2897 v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]);
2898 v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]);
2899 v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]);
2900 sum[0] = vec_mergeh(v[0], v[4]);
2901 sum[1] = vec_mergel(v[0], v[4]);
2902 sum[2] = vec_mergeh(v[1], v[5]);
2903 sum[3] = vec_mergel(v[1], v[5]);
2904 sum[4] = vec_mergeh(v[2], v[6]);
2905 sum[5] = vec_mergel(v[2], v[6]);
2906 sum[6] = vec_mergeh(v[3], v[7]);
2907 sum[7] = vec_mergel(v[3], v[7]);
2909 kernel.packet[0] = vec_mergeh(sum[0], sum[4]);
2910 kernel.packet[1] = vec_mergel(sum[0], sum[4]);
2911 kernel.packet[2] = vec_mergeh(sum[1], sum[5]);
2912 kernel.packet[3] = vec_mergel(sum[1], sum[5]);
2913 kernel.packet[4] = vec_mergeh(sum[2], sum[6]);
2914 kernel.packet[5] = vec_mergel(sum[2], sum[6]);
2915 kernel.packet[6] = vec_mergeh(sum[3], sum[7]);
2916 kernel.packet[7] = vec_mergel(sum[3], sum[7]);
2919EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8bf, 8>& kernel) {
2920 Packet8bf v[8], sum[8];
2922 v[0] = vec_mergeh(kernel.packet[0].m_val, kernel.packet[4].m_val);
2923 v[1] = vec_mergel(kernel.packet[0].m_val, kernel.packet[4].m_val);
2924 v[2] = vec_mergeh(kernel.packet[1].m_val, kernel.packet[5].m_val);
2925 v[3] = vec_mergel(kernel.packet[1].m_val, kernel.packet[5].m_val);
2926 v[4] = vec_mergeh(kernel.packet[2].m_val, kernel.packet[6].m_val);
2927 v[5] = vec_mergel(kernel.packet[2].m_val, kernel.packet[6].m_val);
2928 v[6] = vec_mergeh(kernel.packet[3].m_val, kernel.packet[7].m_val);
2929 v[7] = vec_mergel(kernel.packet[3].m_val, kernel.packet[7].m_val);
2930 sum[0] = vec_mergeh(v[0].m_val, v[4].m_val);
2931 sum[1] = vec_mergel(v[0].m_val, v[4].m_val);
2932 sum[2] = vec_mergeh(v[1].m_val, v[5].m_val);
2933 sum[3] = vec_mergel(v[1].m_val, v[5].m_val);
2934 sum[4] = vec_mergeh(v[2].m_val, v[6].m_val);
2935 sum[5] = vec_mergel(v[2].m_val, v[6].m_val);
2936 sum[6] = vec_mergeh(v[3].m_val, v[7].m_val);
2937 sum[7] = vec_mergel(v[3].m_val, v[7].m_val);
2939 kernel.packet[0] = vec_mergeh(sum[0].m_val, sum[4].m_val);
2940 kernel.packet[1] = vec_mergel(sum[0].m_val, sum[4].m_val);
2941 kernel.packet[2] = vec_mergeh(sum[1].m_val, sum[5].m_val);
2942 kernel.packet[3] = vec_mergel(sum[1].m_val, sum[5].m_val);
2943 kernel.packet[4] = vec_mergeh(sum[2].m_val, sum[6].m_val);
2944 kernel.packet[5] = vec_mergel(sum[2].m_val, sum[6].m_val);
2945 kernel.packet[6] = vec_mergeh(sum[3].m_val, sum[7].m_val);
2946 kernel.packet[7] = vec_mergel(sum[3].m_val, sum[7].m_val);
2949EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
2950 Packet16c step1[16], step2[16], step3[16];
2952 step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]);
2953 step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]);
2954 step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]);
2955 step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]);
2956 step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]);
2957 step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]);
2958 step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]);
2959 step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]);
2960 step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]);
2961 step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]);
2962 step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]);
2963 step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]);
2964 step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]);
2965 step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]);
2966 step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]);
2967 step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]);
2969 step2[0] = vec_mergeh(step1[0], step1[8]);
2970 step2[1] = vec_mergel(step1[0], step1[8]);
2971 step2[2] = vec_mergeh(step1[1], step1[9]);
2972 step2[3] = vec_mergel(step1[1], step1[9]);
2973 step2[4] = vec_mergeh(step1[2], step1[10]);
2974 step2[5] = vec_mergel(step1[2], step1[10]);
2975 step2[6] = vec_mergeh(step1[3], step1[11]);
2976 step2[7] = vec_mergel(step1[3], step1[11]);
2977 step2[8] = vec_mergeh(step1[4], step1[12]);
2978 step2[9] = vec_mergel(step1[4], step1[12]);
2979 step2[10] = vec_mergeh(step1[5], step1[13]);
2980 step2[11] = vec_mergel(step1[5], step1[13]);
2981 step2[12] = vec_mergeh(step1[6], step1[14]);
2982 step2[13] = vec_mergel(step1[6], step1[14]);
2983 step2[14] = vec_mergeh(step1[7], step1[15]);
2984 step2[15] = vec_mergel(step1[7], step1[15]);
2986 step3[0] = vec_mergeh(step2[0], step2[8]);
2987 step3[1] = vec_mergel(step2[0], step2[8]);
2988 step3[2] = vec_mergeh(step2[1], step2[9]);
2989 step3[3] = vec_mergel(step2[1], step2[9]);
2990 step3[4] = vec_mergeh(step2[2], step2[10]);
2991 step3[5] = vec_mergel(step2[2], step2[10]);
2992 step3[6] = vec_mergeh(step2[3], step2[11]);
2993 step3[7] = vec_mergel(step2[3], step2[11]);
2994 step3[8] = vec_mergeh(step2[4], step2[12]);
2995 step3[9] = vec_mergel(step2[4], step2[12]);
2996 step3[10] = vec_mergeh(step2[5], step2[13]);
2997 step3[11] = vec_mergel(step2[5], step2[13]);
2998 step3[12] = vec_mergeh(step2[6], step2[14]);
2999 step3[13] = vec_mergel(step2[6], step2[14]);
3000 step3[14] = vec_mergeh(step2[7], step2[15]);
3001 step3[15] = vec_mergel(step2[7], step2[15]);
3003 kernel.packet[0] = vec_mergeh(step3[0], step3[8]);
3004 kernel.packet[1] = vec_mergel(step3[0], step3[8]);
3005 kernel.packet[2] = vec_mergeh(step3[1], step3[9]);
3006 kernel.packet[3] = vec_mergel(step3[1], step3[9]);
3007 kernel.packet[4] = vec_mergeh(step3[2], step3[10]);
3008 kernel.packet[5] = vec_mergel(step3[2], step3[10]);
3009 kernel.packet[6] = vec_mergeh(step3[3], step3[11]);
3010 kernel.packet[7] = vec_mergel(step3[3], step3[11]);
3011 kernel.packet[8] = vec_mergeh(step3[4], step3[12]);
3012 kernel.packet[9] = vec_mergel(step3[4], step3[12]);
3013 kernel.packet[10] = vec_mergeh(step3[5], step3[13]);
3014 kernel.packet[11] = vec_mergel(step3[5], step3[13]);
3015 kernel.packet[12] = vec_mergeh(step3[6], step3[14]);
3016 kernel.packet[13] = vec_mergel(step3[6], step3[14]);
3017 kernel.packet[14] = vec_mergeh(step3[7], step3[15]);
3018 kernel.packet[15] = vec_mergel(step3[7], step3[15]);
3021EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
3022 Packet16uc step1[16], step2[16], step3[16];
3024 step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]);
3025 step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]);
3026 step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]);
3027 step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]);
3028 step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]);
3029 step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]);
3030 step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]);
3031 step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]);
3032 step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]);
3033 step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]);
3034 step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]);
3035 step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]);
3036 step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]);
3037 step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]);
3038 step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]);
3039 step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]);
3041 step2[0] = vec_mergeh(step1[0], step1[8]);
3042 step2[1] = vec_mergel(step1[0], step1[8]);
3043 step2[2] = vec_mergeh(step1[1], step1[9]);
3044 step2[3] = vec_mergel(step1[1], step1[9]);
3045 step2[4] = vec_mergeh(step1[2], step1[10]);
3046 step2[5] = vec_mergel(step1[2], step1[10]);
3047 step2[6] = vec_mergeh(step1[3], step1[11]);
3048 step2[7] = vec_mergel(step1[3], step1[11]);
3049 step2[8] = vec_mergeh(step1[4], step1[12]);
3050 step2[9] = vec_mergel(step1[4], step1[12]);
3051 step2[10] = vec_mergeh(step1[5], step1[13]);
3052 step2[11] = vec_mergel(step1[5], step1[13]);
3053 step2[12] = vec_mergeh(step1[6], step1[14]);
3054 step2[13] = vec_mergel(step1[6], step1[14]);
3055 step2[14] = vec_mergeh(step1[7], step1[15]);
3056 step2[15] = vec_mergel(step1[7], step1[15]);
3058 step3[0] = vec_mergeh(step2[0], step2[8]);
3059 step3[1] = vec_mergel(step2[0], step2[8]);
3060 step3[2] = vec_mergeh(step2[1], step2[9]);
3061 step3[3] = vec_mergel(step2[1], step2[9]);
3062 step3[4] = vec_mergeh(step2[2], step2[10]);
3063 step3[5] = vec_mergel(step2[2], step2[10]);
3064 step3[6] = vec_mergeh(step2[3], step2[11]);
3065 step3[7] = vec_mergel(step2[3], step2[11]);
3066 step3[8] = vec_mergeh(step2[4], step2[12]);
3067 step3[9] = vec_mergel(step2[4], step2[12]);
3068 step3[10] = vec_mergeh(step2[5], step2[13]);
3069 step3[11] = vec_mergel(step2[5], step2[13]);
3070 step3[12] = vec_mergeh(step2[6], step2[14]);
3071 step3[13] = vec_mergel(step2[6], step2[14]);
3072 step3[14] = vec_mergeh(step2[7], step2[15]);
3073 step3[15] = vec_mergel(step2[7], step2[15]);
3075 kernel.packet[0] = vec_mergeh(step3[0], step3[8]);
3076 kernel.packet[1] = vec_mergel(step3[0], step3[8]);
3077 kernel.packet[2] = vec_mergeh(step3[1], step3[9]);
3078 kernel.packet[3] = vec_mergel(step3[1], step3[9]);
3079 kernel.packet[4] = vec_mergeh(step3[2], step3[10]);
3080 kernel.packet[5] = vec_mergel(step3[2], step3[10]);
3081 kernel.packet[6] = vec_mergeh(step3[3], step3[11]);
3082 kernel.packet[7] = vec_mergel(step3[3], step3[11]);
3083 kernel.packet[8] = vec_mergeh(step3[4], step3[12]);
3084 kernel.packet[9] = vec_mergel(step3[4], step3[12]);
3085 kernel.packet[10] = vec_mergeh(step3[5], step3[13]);
3086 kernel.packet[11] = vec_mergel(step3[5], step3[13]);
3087 kernel.packet[12] = vec_mergeh(step3[6], step3[14]);
3088 kernel.packet[13] = vec_mergel(step3[6], step3[14]);
3089 kernel.packet[14] = vec_mergeh(step3[7], step3[15]);
3090 kernel.packet[15] = vec_mergel(step3[7], step3[15]);
3094#ifdef EIGEN_VECTORIZE_VSX
3095using Packet2d = __vector double;
3096using Packet2ul = __vector
unsigned long long;
3097using Packet2l = __vector
long long;
3099using Packet2bl = Packet2ul;
3101using Packet2bl = __vector __bool long;
3104static Packet2l p2l_ZERO =
reinterpret_cast<Packet2l
>(p4i_ZERO);
3105static Packet2ul p2ul_SIGN = {0x8000000000000000ull, 0x8000000000000000ull};
3106static Packet2ul p2ul_PREV0DOT5 = {0x3FDFFFFFFFFFFFFFull, 0x3FDFFFFFFFFFFFFFull};
3107static Packet2d p2d_ONE = {1.0, 1.0};
3108static Packet2d p2d_ZERO =
reinterpret_cast<Packet2d
>(p4f_ZERO);
3109static Packet2d p2d_MZERO = {numext::bit_cast<double>(0x8000000000000000ull),
3110 numext::bit_cast<double>(0x8000000000000000ull)};
3113static Packet2d p2d_COUNTDOWN =
3114 reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(p2d_ZERO),
reinterpret_cast<Packet4f
>(p2d_ONE), 8));
3116static Packet2d p2d_COUNTDOWN =
3117 reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(p2d_ONE),
reinterpret_cast<Packet4f
>(p2d_ZERO), 8));
3121Packet2d vec_splat_dbl(
const Packet2d& a) {
3122 return vec_splat(a, index);
3126struct packet_traits<double> : default_packet_traits {
3127 using type = Packet2d;
3128 using half = Packet2d;
3131 AlignedOnScalar = 1,
3141 HasSin = EIGEN_FAST_MATH,
3142 HasCos = EIGEN_FAST_MATH,
3143 HasTan = EIGEN_FAST_MATH,
3144 HasTanh = EIGEN_FAST_MATH,
3145 HasErf = EIGEN_FAST_MATH,
3146 HasErfc = EIGEN_FAST_MATH,
3156#if !EIGEN_COMP_CLANG
3166struct unpacket_traits<Packet2d> {
3167 using type = double;
3168 using integer_packet = Packet2l;
3172 vectorizable =
true,
3173 masked_load_available =
false,
3174 masked_store_available =
false
3176 using half = Packet2d;
3179struct unpacket_traits<Packet2l> {
3180 using type = int64_t;
3181 using half = Packet2l;
3185 vectorizable =
false,
3186 masked_load_available =
false,
3187 masked_store_available =
false
3191inline std::ostream& operator<<(std::ostream& s,
const Packet2l& v) {
3197 s << vt.n[0] <<
", " << vt.n[1];
3201inline std::ostream& operator<<(std::ostream& s,
const Packet2d& v) {
3207 s << vt.n[0] <<
", " << vt.n[1];
3213EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from) {
3214 EIGEN_DEBUG_ALIGNED_LOAD
3215 return vec_xl(0,
const_cast<double*
>(from));
3219EIGEN_ALWAYS_INLINE Packet2d pload_partial<Packet2d>(
const double* from,
const Index n,
const Index offset) {
3220 return pload_partial_common<Packet2d>(from, n, offset);
3224EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from) {
3225 EIGEN_DEBUG_ALIGNED_STORE
3226 vec_xst(from, 0, to);
3230EIGEN_ALWAYS_INLINE
void pstore_partial<double>(
double* to,
const Packet2d& from,
const Index n,
const Index offset) {
3231 pstore_partial_common<Packet2d>(to, from, n, offset);
3235EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
3236 Packet2d v = {from, from};
3240EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(
const int64_t& from) {
3241 Packet2l v = {from, from};
3246EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(
unsigned long from) {
3247 Packet2l v = {
static_cast<long long>(from),
static_cast<long long>(from)};
3248 return reinterpret_cast<Packet2d
>(v);
3252EIGEN_STRONG_INLINE
void pbroadcast4<Packet2d>(
const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2,
3255 a0 = pset1<Packet2d>(a[0]);
3256 a1 = pset1<Packet2d>(a[1]);
3257 a2 = pset1<Packet2d>(a[2]);
3258 a3 = pset1<Packet2d>(a[3]);
3262EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2d pgather<double, Packet2d>(
const double* from, Index stride) {
3263 return pgather_common<Packet2d>(from, stride);
3266EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2d pgather_partial<double, Packet2d>(
const double* from, Index stride,
3268 return pgather_common<Packet2d>(from, stride, n);
3271EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter<double, Packet2d>(
double* to,
const Packet2d& from, Index stride) {
3272 pscatter_common<Packet2d>(to, from, stride);
3275EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pscatter_partial<double, Packet2d>(
double* to,
const Packet2d& from,
3276 Index stride,
const Index n) {
3277 pscatter_common<Packet2d>(to, from, stride, n);
3281EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a) {
3282 return pset1<Packet2d>(a) + p2d_COUNTDOWN;
3286EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3291EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3296EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
3297#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3300 return vec_xor(a, p2d_MZERO);
3305EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3306 return vec_madd(a, b, p2d_MZERO);
3309EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3310 return vec_div(a, b);
3314struct is_fused_madd<Packet2d> : true_type {};
3316EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
3317 return vec_madd(a, b, c);
3320EIGEN_STRONG_INLINE Packet2d pmsub(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
3321 return vec_msub(a, b, c);
3324EIGEN_STRONG_INLINE Packet2d pnmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
3325 return vec_nmsub(a, b, c);
3328EIGEN_STRONG_INLINE Packet2d pnmsub(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
3329 return vec_nmadd(a, b, c);
3333EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3336 __asm__(
"xvcmpgedp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa"(ret) :
"wa"(a),
"wa"(b));
3341EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3344 __asm__(
"xvcmpgtdp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" :
"=&wa"(ret) :
"wa"(a),
"wa"(b));
3349EIGEN_STRONG_INLINE Packet2d pcmp_le(
const Packet2d& a,
const Packet2d& b) {
3350 return reinterpret_cast<Packet2d
>(vec_cmple(a, b));
3353EIGEN_STRONG_INLINE Packet2d pcmp_lt(
const Packet2d& a,
const Packet2d& b) {
3354 return reinterpret_cast<Packet2d
>(vec_cmplt(a, b));
3357EIGEN_STRONG_INLINE Packet2d pcmp_eq(
const Packet2d& a,
const Packet2d& b) {
3358 return reinterpret_cast<Packet2d
>(vec_cmpeq(a, b));
3361#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3362EIGEN_STRONG_INLINE Packet2l pcmp_eq(
const Packet2l& a,
const Packet2l& b) {
3363 return reinterpret_cast<Packet2l
>(vec_cmpeq(a, b));
3366EIGEN_STRONG_INLINE Packet2l pcmp_eq(
const Packet2l& a,
const Packet2l& b) {
3367 Packet4i halves =
reinterpret_cast<Packet4i
>(vec_cmpeq(
reinterpret_cast<Packet4i
>(a),
reinterpret_cast<Packet4i
>(b)));
3368 Packet4i flipped = vec_perm(halves, halves, p16uc_COMPLEX32_REV);
3369 return reinterpret_cast<Packet2l
>(pand(halves, flipped));
3375#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3376EIGEN_STRONG_INLINE Packet2l pcmp_lt(
const Packet2l& a,
const Packet2l& b) {
3377 return reinterpret_cast<Packet2l
>(vec_cmplt(a, b));
3380EIGEN_STRONG_INLINE Packet2l pcmp_lt(
const Packet2l& a,
const Packet2l& b) {
3381 const Packet2l ret = {a[0] < b[0] ? -1 : 0, a[1] < b[1] ? -1 : 0};
3386EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(
const Packet2d& a,
const Packet2d& b) {
3387 Packet2d c =
reinterpret_cast<Packet2d
>(vec_cmpge(a, b));
3388 return vec_nor(c, c);
3392EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3393 return vec_and(a, b);
3397EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3398 return vec_or(a, b);
3402EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3403 return vec_xor(a, b);
3407EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
3408 return vec_and(a, vec_nor(b, b));
3412EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(
const Packet2d& a) {
3413 Packet2d t = vec_add(
3414 reinterpret_cast<Packet2d
>(vec_or(vec_and(
reinterpret_cast<Packet2ul
>(a), p2ul_SIGN), p2ul_PREV0DOT5)), a);
3417 __asm__(
"xvrdpiz %x0, %x1\n\t" :
"=&wa"(res) :
"wa"(t));
3422EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(
const Packet2d& a) {
3426EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(
const Packet2d& a) {
3427 return vec_floor(a);
3430EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(
const Packet2d& a) {
3431 return vec_trunc(a);
3434EIGEN_STRONG_INLINE Packet2d print<Packet2d>(
const Packet2d& a) {
3437 __asm__(
"xvrdpic %x0, %x1\n\t" :
"=&wa"(res) :
"wa"(a));
3443EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from) {
3444 EIGEN_DEBUG_UNALIGNED_LOAD
3445 return vec_xl(0,
const_cast<double*
>(from));
3449EIGEN_ALWAYS_INLINE Packet2d ploadu_partial<Packet2d>(
const double* from,
const Index n,
const Index offset) {
3450 return ploadu_partial_common<Packet2d>(from, n, offset);
3454EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(
const double* from) {
3455 return pset1<Packet2d>(from[0]);
3459EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from) {
3460 EIGEN_DEBUG_UNALIGNED_STORE
3461 vec_xst(from, 0, to);
3465EIGEN_ALWAYS_INLINE
void pstoreu_partial<double>(
double* to,
const Packet2d& from,
const Index n,
const Index offset) {
3466 pstoreu_partial_common<Packet2d>(to, from, n, offset);
3470EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) {
3471 EIGEN_PPC_PREFETCH(addr);
3475EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
3476 EIGEN_ALIGN16
double x[2];
3477 pstore<double>(x, a);
3482EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a) {
3483 return vec_sld(a, a, 8);
3486EIGEN_STRONG_INLINE Packet2d pabs(
const Packet2d& a) {
3489#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3491EIGEN_STRONG_INLINE Packet2d psignbit(
const Packet2d& a) {
3492 return (Packet2d)vec_sra((Packet2l)a, vec_splats((
unsigned long long)(63)));
3496static Packet16uc p16uc_DUPSIGN = {0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8};
3498static Packet16uc p16uc_DUPSIGN = {7, 7, 7, 7, 7, 7, 7, 7, 15, 15, 15, 15, 15, 15, 15, 15};
3502EIGEN_STRONG_INLINE Packet2d psignbit(
const Packet2d& a) {
3503 Packet16c tmp = vec_sra(
reinterpret_cast<Packet16c
>(a), vec_splats((
unsigned char)(7)));
3504 return reinterpret_cast<Packet2d
>(vec_perm(tmp, tmp, p16uc_DUPSIGN));
3509inline Packet2l pcast<Packet2d, Packet2l>(
const Packet2d& x);
3512inline Packet2d pcast<Packet2l, Packet2d>(
const Packet2l& x);
3520#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3523EIGEN_STRONG_INLINE Packet2l plogical_shift_left(
const Packet2l& a) {
3524 const Packet2ul shift = {N, N};
3525 return vec_sl(a, shift);
3529EIGEN_STRONG_INLINE Packet2l plogical_shift_right(
const Packet2l& a) {
3530 const Packet2ul shift = {N, N};
3531 return vec_sr(a, shift);
3538EIGEN_ALWAYS_INLINE Packet4i shift_even_left(
const Packet4i& a) {
3539 static const Packet16uc perm = {0x14, 0x15, 0x16, 0x17, 0x00, 0x01, 0x02, 0x03,
3540 0x1c, 0x1d, 0x1e, 0x1f, 0x08, 0x09, 0x0a, 0x0b};
3542 return vec_perm(p4i_ZERO, a, perm);
3544 return vec_perm(a, p4i_ZERO, perm);
3550EIGEN_ALWAYS_INLINE Packet4i shift_odd_right(
const Packet4i& a) {
3551 static const Packet16uc perm = {0x04, 0x05, 0x06, 0x07, 0x10, 0x11, 0x12, 0x13,
3552 0x0c, 0x0d, 0x0e, 0x0f, 0x18, 0x19, 0x1a, 0x1b};
3554 return vec_perm(p4i_ZERO, a, perm);
3556 return vec_perm(a, p4i_ZERO, perm);
3560template <
int N,
typename EnableIf =
void>
3561struct plogical_shift_left_impl;
3564struct plogical_shift_left_impl<N, std::enable_if_t<(N < 32) && (N >= 0)> > {
3565 static EIGEN_STRONG_INLINE Packet2l run(
const Packet2l& a) {
3566 static const unsigned n =
static_cast<unsigned>(N);
3567 const Packet4ui shift = {n, n, n, n};
3568 const Packet4i ai =
reinterpret_cast<Packet4i
>(a);
3569 static const unsigned m =
static_cast<unsigned>(32 - N);
3570 const Packet4ui shift_right = {m, m, m, m};
3571 const Packet4i out_hi = vec_sl(ai, shift);
3572 const Packet4i out_lo = shift_even_left(vec_sr(ai, shift_right));
3573 return reinterpret_cast<Packet2l
>(por<Packet4i>(out_hi, out_lo));
3578struct plogical_shift_left_impl<N, std::enable_if_t<(N >= 32)> > {
3579 static EIGEN_STRONG_INLINE Packet2l run(
const Packet2l& a) {
3580 static const unsigned m =
static_cast<unsigned>(N - 32);
3581 const Packet4ui shift = {m, m, m, m};
3582 const Packet4i ai =
reinterpret_cast<Packet4i
>(a);
3583 return reinterpret_cast<Packet2l
>(shift_even_left(vec_sl(ai, shift)));
3588EIGEN_STRONG_INLINE Packet2l plogical_shift_left(
const Packet2l& a) {
3589 return plogical_shift_left_impl<N>::run(a);
3592template <
int N,
typename EnableIf =
void>
3593struct plogical_shift_right_impl;
3596struct plogical_shift_right_impl<N, std::enable_if_t<(N < 32) && (N >= 0)> > {
3597 static EIGEN_STRONG_INLINE Packet2l run(
const Packet2l& a) {
3598 static const unsigned n =
static_cast<unsigned>(N);
3599 const Packet4ui shift = {n, n, n, n};
3600 const Packet4i ai =
reinterpret_cast<Packet4i
>(a);
3601 static const unsigned m =
static_cast<unsigned>(32 - N);
3602 const Packet4ui shift_left = {m, m, m, m};
3603 const Packet4i out_lo = vec_sr(ai, shift);
3604 const Packet4i out_hi = shift_odd_right(vec_sl(ai, shift_left));
3605 return reinterpret_cast<Packet2l
>(por<Packet4i>(out_hi, out_lo));
3610struct plogical_shift_right_impl<N, std::enable_if_t<(N >= 32)> > {
3611 static EIGEN_STRONG_INLINE Packet2l run(
const Packet2l& a) {
3612 static const unsigned m =
static_cast<unsigned>(N - 32);
3613 const Packet4ui shift = {m, m, m, m};
3614 const Packet4i ai =
reinterpret_cast<Packet4i
>(a);
3615 return reinterpret_cast<Packet2l
>(shift_odd_right(vec_sr(ai, shift)));
3620EIGEN_STRONG_INLINE Packet2l plogical_shift_right(
const Packet2l& a) {
3621 return plogical_shift_right_impl<N>::run(a);
3626EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(
const Packet2d& a,
const Packet2d& exponent) {
3628 const Packet2d max_exponent = pset1<Packet2d>(2099.0);
3629 const Packet2d last_max = pset1<Packet2d>(1022.0);
3630 const Packet2l e = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
3631 const Packet2l b = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(last_max)), last_max));
3632 const Packet2l bias = {1023, 1023};
3633 const Packet2l even = {-2, -2};
3634 const Packet2l t = psub(e, b) & even;
3635 const Packet2d c1 =
reinterpret_cast<Packet2d
>(plogical_shift_left<51>(t + bias + bias));
3636 const Packet2d c2 =
reinterpret_cast<Packet2d
>(plogical_shift_left<52>(psub(e, t) + bias));
3637 return pldexp_apply_factors(a, c1, c2);
3642EIGEN_STRONG_INLINE Packet2d pfrexp_generic_get_biased_exponent(
const Packet2d& a) {
3643 return pcast<Packet2l, Packet2d>(plogical_shift_right<52>(
reinterpret_cast<Packet2l
>(pabs(a))));
3647EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(
const Packet2d& a, Packet2d& exponent) {
3648 return pfrexp_generic(a, exponent);
3652EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a) {
3654 b =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(a),
reinterpret_cast<Packet4f
>(a), 8));
3656 return pfirst<Packet2d>(sum);
3660EIGEN_STRONG_INLINE
bool predux_any(
const Packet2d& a) {
3662 const Packet4ui zero = {0, 0, 0, 0};
3663 return vec_any_ne(
reinterpret_cast<Packet4ui
>(a), zero);
3669EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a) {
3671 pmul(a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(a),
reinterpret_cast<Packet4ui
>(a), 8))));
3676EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a) {
3678 pmin(a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(a),
reinterpret_cast<Packet4ui
>(a), 8))));
3683EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a) {
3685 pmax(a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(a),
reinterpret_cast<Packet4ui
>(a), 8))));
3688EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
3690 t0 = vec_mergeh(kernel.packet[0], kernel.packet[1]);
3691 t1 = vec_mergel(kernel.packet[0], kernel.packet[1]);
3692 kernel.packet[0] = t0;
3693 kernel.packet[1] = t1;
@ Aligned16
Definition Constants.h:238