Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2016 Konstantinos Margaritis <markos@freevec.org>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_PACKET_MATH_ALTIVEC_H
12#define EIGEN_PACKET_MATH_ALTIVEC_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4
23#endif
24
25#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
26#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
27#endif
28
29// NOTE Altivec has 32 registers, but Eigen only accepts a value of 8 or 16
30#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
31#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
32#endif
33
34using Packet4f = __vector float;
35using Packet4i = __vector int;
36using Packet4ui = __vector unsigned int;
37using Packet4bi = __vector __bool int;
38using Packet8s = __vector short int;
39using Packet8us = __vector unsigned short int;
40using Packet8bi = __vector __bool short;
41using Packet16c = __vector signed char;
42using Packet16uc = __vector unsigned char;
43using Packet8bf = eigen_packet_wrapper<__vector unsigned short int, 0>;
44
45// To avoid repeating the same code, but we need to reuse the constants
46// and it doesn't really work to declare them global, so we define macros instead
47#define EIGEN_DECLARE_CONST_FAST_Packet4f(NAME, X) Packet4f p4f_##NAME = {X, X, X, X}
48
49#define EIGEN_DECLARE_CONST_FAST_Packet4i(NAME, X) Packet4i p4i_##NAME = vec_splat_s32(X)
50
51#define EIGEN_DECLARE_CONST_FAST_Packet4ui(NAME, X) Packet4ui p4ui_##NAME = {X, X, X, X}
52
53#define EIGEN_DECLARE_CONST_FAST_Packet8us(NAME, X) Packet8us p8us_##NAME = {X, X, X, X, X, X, X, X}
54
55#define EIGEN_DECLARE_CONST_FAST_Packet16uc(NAME, X) \
56 Packet16uc p16uc_##NAME = {X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X}
57
58#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) Packet4f p4f_##NAME = pset1<Packet4f>(X)
59
60#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) Packet4i p4i_##NAME = pset1<Packet4i>(X)
61
62#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) Packet2d p2d_##NAME = pset1<Packet2d>(X)
63
64#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) Packet2l p2l_##NAME = pset1<Packet2l>(X)
65
66#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
67 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
68
69#define DST_CHAN 1
70#define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride))
71#define __UNPACK_TYPE__(PACKETNAME) typename unpacket_traits<PACKETNAME>::type
72
73// These constants are endian-agnostic
74static EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0}
75static EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,}
76static EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1}
77static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16, -16); //{ -16, -16, -16, -16}
78static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1, -1); //{ -1, -1, -1, -1}
79static EIGEN_DECLARE_CONST_FAST_Packet4ui(SIGN, 0x80000000u);
80static EIGEN_DECLARE_CONST_FAST_Packet4ui(PREV0DOT5, 0x3EFFFFFFu);
81static EIGEN_DECLARE_CONST_FAST_Packet8us(ONE, 1); //{ 1, 1, 1, 1, 1, 1, 1, 1}
82static Packet4f p4f_MZERO =
83 (Packet4f)vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000}
84#ifndef __VSX__
85static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0}
86#endif
87
88static Packet4f p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0};
89static Packet4i p4i_COUNTDOWN = {0, 1, 2, 3};
90static Packet8s p8s_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7};
91static Packet8us p8us_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7};
92
93static Packet16c p16c_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
94static Packet16uc p16uc_COUNTDOWN = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
95
96static Packet16uc p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3};
97static Packet16uc p16uc_REVERSE16 = {14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1};
98static Packet16uc p16uc_REVERSE8 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
99
100#ifdef _BIG_ENDIAN
101static Packet16uc p16uc_DUPLICATE32_HI = {0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7};
102#endif
103static const Packet16uc p16uc_DUPLICATE16_EVEN = {0, 1, 0, 1, 4, 5, 4, 5, 8, 9, 8, 9, 12, 13, 12, 13};
104static const Packet16uc p16uc_DUPLICATE16_ODD = {2, 3, 2, 3, 6, 7, 6, 7, 10, 11, 10, 11, 14, 15, 14, 15};
105
106static Packet16uc p16uc_QUADRUPLICATE16_HI = {0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3};
107static Packet16uc p16uc_QUADRUPLICATE16 = {0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3};
108
109static Packet16uc p16uc_MERGEE16 = {0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29};
110static Packet16uc p16uc_MERGEO16 = {2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31};
111#ifdef _BIG_ENDIAN
112static Packet16uc p16uc_MERGEH16 = {0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29};
113#else
114static Packet16uc p16uc_MERGEL16 = {2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31};
115#endif
116
117// Handle endianness properly while loading constants
118// Define global static constants:
119#ifdef _BIG_ENDIAN
120static Packet16uc p16uc_FORWARD = vec_lvsl(0, (float*)0);
121static Packet16uc p16uc_PSET32_WODD =
122 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
123 8); //{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
124static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
125 8); //{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
126static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc)vec_abs(p4i_MINUS16), 3),
127 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
128#else
129static Packet16uc p16uc_FORWARD = p16uc_REVERSE32;
130static Packet16uc p16uc_PSET32_WODD =
131 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
132 8); //{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
133static Packet16uc p16uc_PSET32_WEVEN =
134 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
135 8); //{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
136static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc)vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO,
137 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
138#endif // _BIG_ENDIAN
139
140static Packet16uc p16uc_PSET64_HI = (Packet16uc)vec_mergeh(
141 (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };
142static Packet16uc p16uc_PSET64_LO = (Packet16uc)vec_mergel(
143 (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
144static Packet16uc p16uc_TRANSPOSE64_HI =
145 p16uc_PSET64_HI + p16uc_HALF64_0_16; //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
146static Packet16uc p16uc_TRANSPOSE64_LO =
147 p16uc_PSET64_LO + p16uc_HALF64_0_16; //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};
148
149static Packet16uc p16uc_COMPLEX32_REV =
150 vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
151
152#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
153#define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR);
154#else
155#define EIGEN_PPC_PREFETCH(ADDR) asm(" dcbt [%[addr]]\n" ::[addr] "r"(ADDR) : "cc");
156#endif
157
158#if EIGEN_COMP_LLVM
159#define LOAD_STORE_UNROLL_16 _Pragma("unroll 16")
160#else
161#define LOAD_STORE_UNROLL_16 _Pragma("GCC unroll(16)")
162#endif
163
164template <>
165struct packet_traits<float> : default_packet_traits {
166 using type = Packet4f;
167 using half = Packet4f;
168 enum {
169 Vectorizable = 1,
170 AlignedOnScalar = 1,
171 size = 4,
172
173 HasAdd = 1,
174 HasSub = 1,
175 HasMul = 1,
176 HasDiv = 1,
177 HasMin = 1,
178 HasMax = 1,
179 HasAbs = 1,
180 HasSin = EIGEN_FAST_MATH,
181 HasCos = EIGEN_FAST_MATH,
182 HasTan = EIGEN_FAST_MATH,
183 HasACos = 1,
184 HasASin = 1,
185 HasATan = 1,
186 HasATanh = 1,
187 HasLog = 1,
188 HasExp = 1,
189 HasLog1p = 1,
190 HasExpm1 = 1,
191#ifdef EIGEN_VECTORIZE_VSX
192 HasCmp = 1,
193 HasPow = 1,
194 HasSqrt = 1,
195 HasCbrt = 1,
196#if !EIGEN_COMP_CLANG
197 HasRsqrt = 1,
198#else
199 HasRsqrt = 0,
200#endif
201 HasTanh = EIGEN_FAST_MATH,
202 HasErf = EIGEN_FAST_MATH,
203 HasErfc = EIGEN_FAST_MATH,
204#else
205 HasSqrt = 0,
206 HasRsqrt = 0,
207 HasTanh = 0,
208 HasErf = 0,
209#endif
210 HasNegate = 1,
211 };
212};
213template <>
214struct packet_traits<bfloat16> : default_packet_traits {
215 using type = Packet8bf;
216 using half = Packet8bf;
217 enum {
218 Vectorizable = 1,
219 AlignedOnScalar = 1,
220 size = 8,
221
222 HasAdd = 1,
223 HasSub = 1,
224 HasMul = 1,
225 HasDiv = 1,
226 HasMin = 1,
227 HasMax = 1,
228 HasAbs = 1,
229 HasSin = EIGEN_FAST_MATH,
230 HasCos = EIGEN_FAST_MATH,
231 HasLog = 1,
232 HasExp = 1,
233#ifdef EIGEN_VECTORIZE_VSX
234 HasSqrt = 1,
235#if !EIGEN_COMP_CLANG
236 HasRsqrt = 1,
237#else
238 HasRsqrt = 0,
239#endif
240#else
241 HasSqrt = 0,
242 HasRsqrt = 0,
243#endif
244 HasTanh = 0,
245 HasErf = 0,
246 HasNegate = 1,
247 };
248};
249
250template <>
251struct packet_traits<int> : default_packet_traits {
252 using type = Packet4i;
253 using half = Packet4i;
254 enum {
255 Vectorizable = 1,
256 AlignedOnScalar = 1,
257 size = 4,
258
259 HasAdd = 1,
260 HasSub = 1,
261 HasShift = 1,
262 HasMul = 1,
263#if defined(_ARCH_PWR10) && (EIGEN_COMP_LLVM || EIGEN_GNUC_STRICT_AT_LEAST(11, 0, 0))
264 HasDiv = 1,
265#else
266 HasDiv = 0,
267#endif
268 HasCmp = 1
269 };
270};
271
272template <>
273struct packet_traits<short int> : default_packet_traits {
274 using type = Packet8s;
275 using half = Packet8s;
276 enum {
277 Vectorizable = 1,
278 AlignedOnScalar = 1,
279 size = 8,
280
281 HasAdd = 1,
282 HasSub = 1,
283 HasMul = 1,
284 HasDiv = 0,
285 HasCmp = 1
286 };
287};
288
289template <>
290struct packet_traits<unsigned short int> : default_packet_traits {
291 using type = Packet8us;
292 using half = Packet8us;
293 enum {
294 Vectorizable = 1,
295 AlignedOnScalar = 1,
296 size = 8,
297
298 HasAdd = 1,
299 HasSub = 1,
300 HasMul = 1,
301 HasDiv = 0,
302 HasCmp = 1
303 };
304};
305
306template <>
307struct packet_traits<signed char> : default_packet_traits {
308 using type = Packet16c;
309 using half = Packet16c;
310 enum {
311 Vectorizable = 1,
312 AlignedOnScalar = 1,
313 size = 16,
314
315 HasAdd = 1,
316 HasSub = 1,
317 HasMul = 1,
318 HasDiv = 0,
319 HasCmp = 1
320 };
321};
322
323template <>
324struct packet_traits<unsigned char> : default_packet_traits {
325 using type = Packet16uc;
326 using half = Packet16uc;
327 enum {
328 Vectorizable = 1,
329 AlignedOnScalar = 1,
330 size = 16,
331
332 HasAdd = 1,
333 HasSub = 1,
334 HasMul = 1,
335 HasDiv = 0,
336 HasCmp = 1
337 };
338};
339
340template <>
341struct unpacket_traits<Packet4f> {
342 using type = float;
343 using half = Packet4f;
344 using integer_packet = Packet4i;
345 enum {
346 size = 4,
347 alignment = Aligned16,
348 vectorizable = true,
349 masked_load_available = false,
350 masked_store_available = false
351 };
352};
353template <>
354struct unpacket_traits<Packet4i> {
355 using type = int;
356 using half = Packet4i;
357 enum {
358 size = 4,
359 alignment = Aligned16,
360 vectorizable = true,
361 masked_load_available = false,
362 masked_store_available = false
363 };
364};
365template <>
366struct unpacket_traits<Packet8s> {
367 using type = short int;
368 using half = Packet8s;
369 enum {
370 size = 8,
371 alignment = Aligned16,
372 vectorizable = true,
373 masked_load_available = false,
374 masked_store_available = false
375 };
376};
377template <>
378struct unpacket_traits<Packet8us> {
379 using type = unsigned short int;
380 using half = Packet8us;
381 enum {
382 size = 8,
383 alignment = Aligned16,
384 vectorizable = true,
385 masked_load_available = false,
386 masked_store_available = false
387 };
388};
389
390template <>
391struct unpacket_traits<Packet16c> {
392 using type = signed char;
393 using half = Packet16c;
394 enum {
395 size = 16,
396 alignment = Aligned16,
397 vectorizable = true,
398 masked_load_available = false,
399 masked_store_available = false
400 };
401};
402template <>
403struct unpacket_traits<Packet16uc> {
404 using type = unsigned char;
405 using half = Packet16uc;
406 enum {
407 size = 16,
408 alignment = Aligned16,
409 vectorizable = true,
410 masked_load_available = false,
411 masked_store_available = false
412 };
413};
414
415template <>
416struct unpacket_traits<Packet8bf> {
417 using type = bfloat16;
418 using half = Packet8bf;
419 enum {
420 size = 8,
421 alignment = Aligned16,
422 vectorizable = true,
423 masked_load_available = false,
424 masked_store_available = false
425 };
426};
427
428template <typename Packet>
429EIGEN_STRONG_INLINE Packet pload_common(const __UNPACK_TYPE__(Packet) * from) {
430 // some versions of GCC throw "unused-but-set-parameter".
431 // ignoring these warnings for now.
432 EIGEN_UNUSED_VARIABLE(from);
433 EIGEN_DEBUG_ALIGNED_LOAD
434#ifdef EIGEN_VECTORIZE_VSX
435 return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from));
436#else
437 return vec_ld(0, from);
438#endif
439}
440
441// Need to define them first or we get specialization after instantiation errors
442template <>
443EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
444 return pload_common<Packet4f>(from);
445}
446
447template <>
448EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) {
449 return pload_common<Packet4i>(from);
450}
451
452template <>
453EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(const short int* from) {
454 return pload_common<Packet8s>(from);
455}
456
457template <>
458EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(const unsigned short int* from) {
459 return pload_common<Packet8us>(from);
460}
461
462template <>
463EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(const signed char* from) {
464 return pload_common<Packet16c>(from);
465}
466
467template <>
468EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(const unsigned char* from) {
469 return pload_common<Packet16uc>(from);
470}
471
472template <>
473EIGEN_STRONG_INLINE Packet8bf pload<Packet8bf>(const bfloat16* from) {
474 return pload_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
475}
476
477template <typename Packet>
478EIGEN_ALWAYS_INLINE Packet pload_ignore(const __UNPACK_TYPE__(Packet) * from) {
479 // some versions of GCC throw "unused-but-set-parameter".
480 // ignoring these warnings for now.
481 EIGEN_UNUSED_VARIABLE(from);
482 EIGEN_DEBUG_ALIGNED_LOAD
483 // Ignore partial input memory initialized
484#if !EIGEN_COMP_LLVM
485#pragma GCC diagnostic push
486#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
487#endif
488#ifdef EIGEN_VECTORIZE_VSX
489 return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from));
490#else
491 return vec_ld(0, from);
492#endif
493#if !EIGEN_COMP_LLVM
494#pragma GCC diagnostic pop
495#endif
496}
497
498template <>
499EIGEN_ALWAYS_INLINE Packet8bf pload_ignore<Packet8bf>(const bfloat16* from) {
500 return pload_ignore<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
501}
502
503template <typename Packet>
504EIGEN_ALWAYS_INLINE Packet pload_partial_common(const __UNPACK_TYPE__(Packet) * from, const Index n,
505 const Index offset) {
506 // some versions of GCC throw "unused-but-set-parameter".
507 // ignoring these warnings for now.
508 const Index packet_size = unpacket_traits<Packet>::size;
509 eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
510 const Index size = sizeof(__UNPACK_TYPE__(Packet));
511#ifdef _ARCH_PWR9
512 EIGEN_UNUSED_VARIABLE(packet_size);
513 EIGEN_DEBUG_ALIGNED_LOAD
514 EIGEN_UNUSED_VARIABLE(from);
515 Packet load = vec_xl_len(const_cast<__UNPACK_TYPE__(Packet)*>(from), n * size);
516 if (offset) {
517 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
518#ifdef _BIG_ENDIAN
519 load = Packet(vec_sro(Packet16uc(load), shift));
520#else
521 load = Packet(vec_slo(Packet16uc(load), shift));
522#endif
523 }
524 return load;
525#else
526 if (n) {
527 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size];
528 unsigned char* load2 = reinterpret_cast<unsigned char*>(load + offset);
529 unsigned char* from2 = reinterpret_cast<unsigned char*>(const_cast<__UNPACK_TYPE__(Packet)*>(from));
530 Index n2 = n * size;
531 if (16 <= n2) {
532 pstoreu(load2, ploadu<Packet16uc>(from2));
533 } else {
534 memcpy((void*)load2, (void*)from2, n2);
535 }
536 return pload_ignore<Packet>(load);
537 } else {
538 return Packet(pset1<Packet16uc>(0));
539 }
540#endif
541}
542
543template <>
544EIGEN_ALWAYS_INLINE Packet4f pload_partial<Packet4f>(const float* from, const Index n, const Index offset) {
545 return pload_partial_common<Packet4f>(from, n, offset);
546}
547
548template <>
549EIGEN_ALWAYS_INLINE Packet4i pload_partial<Packet4i>(const int* from, const Index n, const Index offset) {
550 return pload_partial_common<Packet4i>(from, n, offset);
551}
552
553template <>
554EIGEN_ALWAYS_INLINE Packet8s pload_partial<Packet8s>(const short int* from, const Index n, const Index offset) {
555 return pload_partial_common<Packet8s>(from, n, offset);
556}
557
558template <>
559EIGEN_ALWAYS_INLINE Packet8us pload_partial<Packet8us>(const unsigned short int* from, const Index n,
560 const Index offset) {
561 return pload_partial_common<Packet8us>(from, n, offset);
562}
563
564template <>
565EIGEN_ALWAYS_INLINE Packet8bf pload_partial<Packet8bf>(const bfloat16* from, const Index n, const Index offset) {
566 return pload_partial_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from), n, offset);
567}
568
569template <>
570EIGEN_ALWAYS_INLINE Packet16c pload_partial<Packet16c>(const signed char* from, const Index n, const Index offset) {
571 return pload_partial_common<Packet16c>(from, n, offset);
572}
573
574template <>
575EIGEN_ALWAYS_INLINE Packet16uc pload_partial<Packet16uc>(const unsigned char* from, const Index n, const Index offset) {
576 return pload_partial_common<Packet16uc>(from, n, offset);
577}
578
579template <typename Packet>
580EIGEN_STRONG_INLINE void pstore_common(__UNPACK_TYPE__(Packet) * to, const Packet& from) {
581 // some versions of GCC throw "unused-but-set-parameter" (float *to).
582 // ignoring these warnings for now.
583 EIGEN_UNUSED_VARIABLE(to);
584 EIGEN_DEBUG_ALIGNED_STORE
585#ifdef EIGEN_VECTORIZE_VSX
586 vec_xst(from, 0, to);
587#else
588 vec_st(from, 0, to);
589#endif
590}
591
592template <>
593EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
594 pstore_common<Packet4f>(to, from);
595}
596
597template <>
598EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) {
599 pstore_common<Packet4i>(to, from);
600}
601
602template <>
603EIGEN_STRONG_INLINE void pstore<short int>(short int* to, const Packet8s& from) {
604 pstore_common<Packet8s>(to, from);
605}
606
607template <>
608EIGEN_STRONG_INLINE void pstore<unsigned short int>(unsigned short int* to, const Packet8us& from) {
609 pstore_common<Packet8us>(to, from);
610}
611
612template <>
613EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet8bf& from) {
614 pstore_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from.m_val);
615}
616
617template <>
618EIGEN_STRONG_INLINE void pstore<signed char>(signed char* to, const Packet16c& from) {
619 pstore_common<Packet16c>(to, from);
620}
621
622template <>
623EIGEN_STRONG_INLINE void pstore<unsigned char>(unsigned char* to, const Packet16uc& from) {
624 pstore_common<Packet16uc>(to, from);
625}
626
627template <typename Packet>
628EIGEN_ALWAYS_INLINE void pstore_partial_common(__UNPACK_TYPE__(Packet) * to, const Packet& from, const Index n,
629 const Index offset) {
630 // some versions of GCC throw "unused-but-set-parameter" (float *to).
631 // ignoring these warnings for now.
632 const Index packet_size = unpacket_traits<Packet>::size;
633 eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
634 const Index size = sizeof(__UNPACK_TYPE__(Packet));
635#ifdef _ARCH_PWR9
636 EIGEN_UNUSED_VARIABLE(packet_size);
637 EIGEN_UNUSED_VARIABLE(to);
638 EIGEN_DEBUG_ALIGNED_STORE
639 Packet store = from;
640 if (offset) {
641 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
642#ifdef _BIG_ENDIAN
643 store = Packet(vec_slo(Packet16uc(store), shift));
644#else
645 store = Packet(vec_sro(Packet16uc(store), shift));
646#endif
647 }
648 vec_xst_len(store, to, n * size);
649#else
650 if (n) {
651 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size];
652 pstore(store, from);
653 unsigned char* store2 = reinterpret_cast<unsigned char*>(store + offset);
654 unsigned char* to2 = reinterpret_cast<unsigned char*>(to);
655 Index n2 = n * size;
656 if (16 <= n2) {
657 pstore(to2, ploadu<Packet16uc>(store2));
658 } else {
659 memcpy((void*)to2, (void*)store2, n2);
660 }
661 }
662#endif
663}
664
665template <>
666EIGEN_ALWAYS_INLINE void pstore_partial<float>(float* to, const Packet4f& from, const Index n, const Index offset) {
667 pstore_partial_common<Packet4f>(to, from, n, offset);
668}
669
670template <>
671EIGEN_ALWAYS_INLINE void pstore_partial<int>(int* to, const Packet4i& from, const Index n, const Index offset) {
672 pstore_partial_common<Packet4i>(to, from, n, offset);
673}
674
675template <>
676EIGEN_ALWAYS_INLINE void pstore_partial<short int>(short int* to, const Packet8s& from, const Index n,
677 const Index offset) {
678 pstore_partial_common<Packet8s>(to, from, n, offset);
679}
680
681template <>
682EIGEN_ALWAYS_INLINE void pstore_partial<unsigned short int>(unsigned short int* to, const Packet8us& from,
683 const Index n, const Index offset) {
684 pstore_partial_common<Packet8us>(to, from, n, offset);
685}
686
687template <>
688EIGEN_ALWAYS_INLINE void pstore_partial<bfloat16>(bfloat16* to, const Packet8bf& from, const Index n,
689 const Index offset) {
690 pstore_partial_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from.m_val, n, offset);
691}
692
693template <>
694EIGEN_ALWAYS_INLINE void pstore_partial<signed char>(signed char* to, const Packet16c& from, const Index n,
695 const Index offset) {
696 pstore_partial_common<Packet16c>(to, from, n, offset);
697}
698
699template <>
700EIGEN_ALWAYS_INLINE void pstore_partial<unsigned char>(unsigned char* to, const Packet16uc& from, const Index n,
701 const Index offset) {
702 pstore_partial_common<Packet16uc>(to, from, n, offset);
703}
704
705template <typename Packet>
706EIGEN_STRONG_INLINE Packet pset1_size4(const __UNPACK_TYPE__(Packet) & from) {
707 Packet v = {from, from, from, from};
708 return v;
709}
710
711template <typename Packet>
712EIGEN_STRONG_INLINE Packet pset1_size8(const __UNPACK_TYPE__(Packet) & from) {
713 Packet v = {from, from, from, from, from, from, from, from};
714 return v;
715}
716
717template <typename Packet>
718EIGEN_STRONG_INLINE Packet pset1_size16(const __UNPACK_TYPE__(Packet) & from) {
719 Packet v = {from, from, from, from, from, from, from, from, from, from, from, from, from, from, from, from};
720 return v;
721}
722
723template <>
724EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
725 return pset1_size4<Packet4f>(from);
726}
727
728template <>
729EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) {
730 return pset1_size4<Packet4i>(from);
731}
732
733template <>
734EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const short int& from) {
735 return pset1_size8<Packet8s>(from);
736}
737
738template <>
739EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(const unsigned short int& from) {
740 return pset1_size8<Packet8us>(from);
741}
742
743template <>
744EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(const signed char& from) {
745 return pset1_size16<Packet16c>(from);
746}
747
748template <>
749EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(const unsigned char& from) {
750 return pset1_size16<Packet16uc>(from);
751}
752
753template <>
754EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(unsigned int from) {
755 return reinterpret_cast<Packet4f>(pset1<Packet4i>(from));
756}
757
758template <>
759EIGEN_STRONG_INLINE Packet8bf pset1<Packet8bf>(const bfloat16& from) {
760 return pset1_size8<Packet8us>(reinterpret_cast<const unsigned short int&>(from));
761}
762
763template <typename Packet>
764EIGEN_STRONG_INLINE void pbroadcast4_common(const __UNPACK_TYPE__(Packet) * a, Packet& a0, Packet& a1, Packet& a2,
765 Packet& a3) {
766 a3 = pload<Packet>(a);
767 a0 = vec_splat(a3, 0);
768 a1 = vec_splat(a3, 1);
769 a2 = vec_splat(a3, 2);
770 a3 = vec_splat(a3, 3);
771}
772
773template <>
774EIGEN_STRONG_INLINE void pbroadcast4<Packet4f>(const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
775 pbroadcast4_common<Packet4f>(a, a0, a1, a2, a3);
776}
777template <>
778EIGEN_STRONG_INLINE void pbroadcast4<Packet4i>(const int* a, Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3) {
779 pbroadcast4_common<Packet4i>(a, a0, a1, a2, a3);
780}
781
782template <typename Packet>
783EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pgather_common(const __UNPACK_TYPE__(Packet) * from, Index stride,
784 const Index n = unpacket_traits<Packet>::size) {
785 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits<Packet>::size];
786 eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will gather past end of packet");
787 if (stride == 1) {
788 if (n == unpacket_traits<Packet>::size) {
789 return ploadu<Packet>(from);
790 } else {
791 return ploadu_partial<Packet>(from, n);
792 }
793 } else {
794 LOAD_STORE_UNROLL_16
795 for (Index i = 0; i < n; i++) {
796 a[i] = from[i * stride];
797 }
798 // Leave rest of the array uninitialized
799 return pload_ignore<Packet>(a);
800 }
801}
802
803template <>
804EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
805 return pgather_common<Packet4f>(from, stride);
806}
807
808template <>
809EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i pgather<int, Packet4i>(const int* from, Index stride) {
810 return pgather_common<Packet4i>(from, stride);
811}
812
813template <>
814EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s pgather<short int, Packet8s>(const short int* from, Index stride) {
815 return pgather_common<Packet8s>(from, stride);
816}
817
818template <>
819EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us pgather<unsigned short int, Packet8us>(const unsigned short int* from,
820 Index stride) {
821 return pgather_common<Packet8us>(from, stride);
822}
823
824template <>
825EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf pgather<bfloat16, Packet8bf>(const bfloat16* from, Index stride) {
826 return pgather_common<Packet8bf>(from, stride);
827}
828
829template <>
830EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c pgather<signed char, Packet16c>(const signed char* from, Index stride) {
831 return pgather_common<Packet16c>(from, stride);
832}
833
834template <>
835EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc pgather<unsigned char, Packet16uc>(const unsigned char* from,
836 Index stride) {
837 return pgather_common<Packet16uc>(from, stride);
838}
839
840template <>
841EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4f pgather_partial<float, Packet4f>(const float* from, Index stride,
842 const Index n) {
843 return pgather_common<Packet4f>(from, stride, n);
844}
845
846template <>
847EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4i pgather_partial<int, Packet4i>(const int* from, Index stride,
848 const Index n) {
849 return pgather_common<Packet4i>(from, stride, n);
850}
851
852template <>
853EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8s pgather_partial<short int, Packet8s>(const short int* from, Index stride,
854 const Index n) {
855 return pgather_common<Packet8s>(from, stride, n);
856}
857
858template <>
859EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8us
860pgather_partial<unsigned short int, Packet8us>(const unsigned short int* from, Index stride, const Index n) {
861 return pgather_common<Packet8us>(from, stride, n);
862}
863
864template <>
865EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet8bf pgather_partial<bfloat16, Packet8bf>(const bfloat16* from, Index stride,
866 const Index n) {
867 return pgather_common<Packet8bf>(from, stride, n);
868}
869
870template <>
871EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16c pgather_partial<signed char, Packet16c>(const signed char* from,
872 Index stride, const Index n) {
873 return pgather_common<Packet16c>(from, stride, n);
874}
875
876template <>
877EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet16uc pgather_partial<unsigned char, Packet16uc>(const unsigned char* from,
878 Index stride,
879 const Index n) {
880 return pgather_common<Packet16uc>(from, stride, n);
881}
882
883template <typename Packet>
884EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_common(__UNPACK_TYPE__(Packet) * to, const Packet& from,
885 Index stride,
886 const Index n = unpacket_traits<Packet>::size) {
887 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[unpacket_traits<Packet>::size];
888 eigen_internal_assert(n <= unpacket_traits<Packet>::size && "number of elements will scatter past end of packet");
889 if (stride == 1) {
890 if (n == unpacket_traits<Packet>::size) {
891 return pstoreu(to, from);
892 } else {
893 return pstoreu_partial(to, from, n);
894 }
895 } else {
896 pstore<__UNPACK_TYPE__(Packet)>(a, from);
897 LOAD_STORE_UNROLL_16
898 for (Index i = 0; i < n; i++) {
899 to[i * stride] = a[i];
900 }
901 }
902}
903
904template <>
905EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
906 pscatter_common<Packet4f>(to, from, stride);
907}
908
909template <>
910EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride) {
911 pscatter_common<Packet4i>(to, from, stride);
912}
913
914template <>
915EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<short int, Packet8s>(short int* to, const Packet8s& from,
916 Index stride) {
917 pscatter_common<Packet8s>(to, from, stride);
918}
919
920template <>
921EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<unsigned short int, Packet8us>(unsigned short int* to,
922 const Packet8us& from,
923 Index stride) {
924 pscatter_common<Packet8us>(to, from, stride);
925}
926
927template <>
928EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<bfloat16, Packet8bf>(bfloat16* to, const Packet8bf& from,
929 Index stride) {
930 pscatter_common<Packet8bf>(to, from, stride);
931}
932
933template <>
934EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<signed char, Packet16c>(signed char* to, const Packet16c& from,
935 Index stride) {
936 pscatter_common<Packet16c>(to, from, stride);
937}
938
939template <>
940EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<unsigned char, Packet16uc>(unsigned char* to,
941 const Packet16uc& from, Index stride) {
942 pscatter_common<Packet16uc>(to, from, stride);
943}
944
945template <>
946EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<float, Packet4f>(float* to, const Packet4f& from,
947 Index stride, const Index n) {
948 pscatter_common<Packet4f>(to, from, stride, n);
949}
950
951template <>
952EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<int, Packet4i>(int* to, const Packet4i& from, Index stride,
953 const Index n) {
954 pscatter_common<Packet4i>(to, from, stride, n);
955}
956
957template <>
958EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<short int, Packet8s>(short int* to, const Packet8s& from,
959 Index stride, const Index n) {
960 pscatter_common<Packet8s>(to, from, stride, n);
961}
962
963template <>
964EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<unsigned short int, Packet8us>(unsigned short int* to,
965 const Packet8us& from,
966 Index stride,
967 const Index n) {
968 pscatter_common<Packet8us>(to, from, stride, n);
969}
970
971template <>
972EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<bfloat16, Packet8bf>(bfloat16* to, const Packet8bf& from,
973 Index stride, const Index n) {
974 pscatter_common<Packet8bf>(to, from, stride, n);
975}
976
977template <>
978EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<signed char, Packet16c>(signed char* to,
979 const Packet16c& from, Index stride,
980 const Index n) {
981 pscatter_common<Packet16c>(to, from, stride, n);
982}
983
984template <>
985EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<unsigned char, Packet16uc>(unsigned char* to,
986 const Packet16uc& from,
987 Index stride, const Index n) {
988 pscatter_common<Packet16uc>(to, from, stride, n);
989}
990
991template <>
992EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
993 return pset1<Packet4f>(a) + p4f_COUNTDOWN;
994}
995template <>
996EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) {
997 return pset1<Packet4i>(a) + p4i_COUNTDOWN;
998}
999template <>
1000EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(const short int& a) {
1001 return pset1<Packet8s>(a) + p8s_COUNTDOWN;
1002}
1003template <>
1004EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(const unsigned short int& a) {
1005 return pset1<Packet8us>(a) + p8us_COUNTDOWN;
1006}
1007template <>
1008EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(const signed char& a) {
1009 return pset1<Packet16c>(a) + p16c_COUNTDOWN;
1010}
1011template <>
1012EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(const unsigned char& a) {
1013 return pset1<Packet16uc>(a) + p16uc_COUNTDOWN;
1014}
1015
1016template <>
1017EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
1018 return a + b;
1019}
1020template <>
1021EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
1022 return a + b;
1023}
1024template <>
1025EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1026 return a + b;
1027}
1028template <>
1029EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(const Packet8s& a, const Packet8s& b) {
1030 return a + b;
1031}
1032template <>
1033EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(const Packet8us& a, const Packet8us& b) {
1034 return a + b;
1035}
1036template <>
1037EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(const Packet16c& a, const Packet16c& b) {
1038 return a + b;
1039}
1040template <>
1041EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1042 return a + b;
1043}
1044
1045template <>
1046EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
1047 return a - b;
1048}
1049template <>
1050EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
1051 return a - b;
1052}
1053template <>
1054EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(const Packet8s& a, const Packet8s& b) {
1055 return a - b;
1056}
1057template <>
1058EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(const Packet8us& a, const Packet8us& b) {
1059 return a - b;
1060}
1061template <>
1062EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(const Packet16c& a, const Packet16c& b) {
1063 return a - b;
1064}
1065template <>
1066EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1067 return a - b;
1068}
1069
1070template <>
1071EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
1072#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1073 return vec_neg(a);
1074#else
1075 return vec_xor(a, p4f_MZERO);
1076#endif
1077}
1078template <>
1079EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) {
1080#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1081 return vec_neg(a);
1082#else
1083 return reinterpret_cast<Packet16c>(p4i_ZERO) - a;
1084#endif
1085}
1086template <>
1087EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
1088#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1089 return vec_neg(a);
1090#else
1091 return reinterpret_cast<Packet8s>(p4i_ZERO) - a;
1092#endif
1093}
1094template <>
1095EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
1096#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1097 return vec_neg(a);
1098#else
1099 return p4i_ZERO - a;
1100#endif
1101}
1102
1103template <>
1104EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
1105 return vec_madd(a, b, p4f_MZERO);
1106}
1107template <>
1108EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
1109 return a * b;
1110}
1111template <>
1112EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(const Packet8s& a, const Packet8s& b) {
1113 return vec_mul(a, b);
1114}
1115template <>
1116EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(const Packet8us& a, const Packet8us& b) {
1117 return vec_mul(a, b);
1118}
1119template <>
1120EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(const Packet16c& a, const Packet16c& b) {
1121 return vec_mul(a, b);
1122}
1123template <>
1124EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1125 return vec_mul(a, b);
1126}
1127
1128template <>
1129EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
1130#ifndef __VSX__ // VSX actually provides a div instruction
1131 Packet4f t, y_0, y_1;
1132
1133 // Altivec does not offer a divide instruction, we have to do a reciprocal approximation
1134 y_0 = vec_re(b);
1135
1136 // Do one Newton-Raphson iteration to get the needed accuracy
1137 t = vec_nmsub(y_0, b, p4f_ONE);
1138 y_1 = vec_madd(y_0, t, y_0);
1139 // For b = +-0 or +-inf the step computes 0 * inf = NaN, while the estimate +-inf or +-0 is exact.
1140 y_1 = vec_sel(y_0, y_1, vec_cmpeq(t, t));
1141
1142 return vec_madd(a, y_1, p4f_MZERO);
1143#else
1144 return vec_div(a, b);
1145#endif
1146}
1147
1148template <>
1149EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) {
1150#if defined(_ARCH_PWR10) && (EIGEN_COMP_LLVM || EIGEN_GNUC_STRICT_AT_LEAST(11, 0, 0))
1151 return vec_div(a, b);
1152#else
1153 EIGEN_UNUSED_VARIABLE(a);
1154 EIGEN_UNUSED_VARIABLE(b);
1155 eigen_assert(false && "packet integer division are not supported by AltiVec");
1156 return pset1<Packet4i>(0);
1157#endif
1158}
1159
1160template <>
1161struct is_fused_madd<Packet4f> : true_type {};
1162template <>
1163EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1164 return vec_madd(a, b, c);
1165}
1166// This overload is required for integer packet types.
1167template <>
1168EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
1169 return a * b + c;
1170}
1171template <>
1172EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
1173 return vec_madd(a, b, c);
1174}
1175template <>
1176EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) {
1177 return vec_madd(a, b, c);
1178}
1179
1180#ifdef EIGEN_VECTORIZE_VSX
1181template <>
1182EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1183 return vec_msub(a, b, c);
1184}
1185template <>
1186EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1187 return vec_nmsub(a, b, c);
1188}
1189template <>
1190EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
1191 return vec_nmadd(a, b, c);
1192}
1193#endif
1194
1195template <>
1196EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
1197#ifdef EIGEN_VECTORIZE_VSX
1198 // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN
1199 Packet4f ret;
1200 __asm__("xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa"(ret) : "wa"(a), "wa"(b));
1201 return ret;
1202#else
1203 return vec_min(a, b);
1204#endif
1205}
1206template <>
1207EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
1208 return vec_min(a, b);
1209}
1210template <>
1211EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(const Packet8s& a, const Packet8s& b) {
1212 return vec_min(a, b);
1213}
1214template <>
1215EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(const Packet8us& a, const Packet8us& b) {
1216 return vec_min(a, b);
1217}
1218template <>
1219EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(const Packet16c& a, const Packet16c& b) {
1220 return vec_min(a, b);
1221}
1222template <>
1223EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1224 return vec_min(a, b);
1225}
1226
1227template <>
1228EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
1229#ifdef EIGEN_VECTORIZE_VSX
1230 // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN
1231 Packet4f ret;
1232 __asm__("xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa"(ret) : "wa"(a), "wa"(b));
1233 return ret;
1234#else
1235 return vec_max(a, b);
1236#endif
1237}
1238template <>
1239EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
1240 return vec_max(a, b);
1241}
1242template <>
1243EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(const Packet8s& a, const Packet8s& b) {
1244 return vec_max(a, b);
1245}
1246template <>
1247EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(const Packet8us& a, const Packet8us& b) {
1248 return vec_max(a, b);
1249}
1250template <>
1251EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(const Packet16c& a, const Packet16c& b) {
1252 return vec_max(a, b);
1253}
1254template <>
1255EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1256 return vec_max(a, b);
1257}
1258
1259template <>
1260EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) {
1261 return reinterpret_cast<Packet4f>(vec_cmple(a, b));
1262}
1263// To fix bug with vec_cmplt on older versions. Clang needs it: for AltiVec vectors its a < b is one bool, true when
1264// every lane compares less, which breaks the generic pcmp_lt.
1265#if defined(EIGEN_VECTORIZE_VSX) || EIGEN_COMP_CLANG
1266template <>
1267EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) {
1268 return reinterpret_cast<Packet4f>(vec_cmplt(a, b));
1269}
1270#endif
1271template <>
1272EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) {
1273 return reinterpret_cast<Packet4f>(vec_cmpeq(a, b));
1274}
1275template <>
1276EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) {
1277 Packet4f c = reinterpret_cast<Packet4f>(vec_cmpge(a, b));
1278 return vec_nor(c, c);
1279}
1280
1281#ifdef EIGEN_VECTORIZE_VSX
1282template <>
1283EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) {
1284 return reinterpret_cast<Packet4i>(vec_cmple(a, b));
1285}
1286#endif
1287template <>
1288EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) {
1289 return reinterpret_cast<Packet4i>(vec_cmplt(a, b));
1290}
1291template <>
1292EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) {
1293 return reinterpret_cast<Packet4i>(vec_cmpeq(a, b));
1294}
1295#ifdef EIGEN_VECTORIZE_VSX
1296template <>
1297EIGEN_STRONG_INLINE Packet8s pcmp_le(const Packet8s& a, const Packet8s& b) {
1298 return reinterpret_cast<Packet8s>(vec_cmple(a, b));
1299}
1300#endif
1301template <>
1302EIGEN_STRONG_INLINE Packet8s pcmp_lt(const Packet8s& a, const Packet8s& b) {
1303 return reinterpret_cast<Packet8s>(vec_cmplt(a, b));
1304}
1305template <>
1306EIGEN_STRONG_INLINE Packet8s pcmp_eq(const Packet8s& a, const Packet8s& b) {
1307 return reinterpret_cast<Packet8s>(vec_cmpeq(a, b));
1308}
1309#ifdef EIGEN_VECTORIZE_VSX
1310template <>
1311EIGEN_STRONG_INLINE Packet8us pcmp_le(const Packet8us& a, const Packet8us& b) {
1312 return reinterpret_cast<Packet8us>(vec_cmple(a, b));
1313}
1314#endif
1315template <>
1316EIGEN_STRONG_INLINE Packet8us pcmp_lt(const Packet8us& a, const Packet8us& b) {
1317 return reinterpret_cast<Packet8us>(vec_cmplt(a, b));
1318}
1319template <>
1320EIGEN_STRONG_INLINE Packet8us pcmp_eq(const Packet8us& a, const Packet8us& b) {
1321 return reinterpret_cast<Packet8us>(vec_cmpeq(a, b));
1322}
1323#ifdef EIGEN_VECTORIZE_VSX
1324template <>
1325EIGEN_STRONG_INLINE Packet16c pcmp_le(const Packet16c& a, const Packet16c& b) {
1326 return reinterpret_cast<Packet16c>(vec_cmple(a, b));
1327}
1328#endif
1329template <>
1330EIGEN_STRONG_INLINE Packet16c pcmp_lt(const Packet16c& a, const Packet16c& b) {
1331 return reinterpret_cast<Packet16c>(vec_cmplt(a, b));
1332}
1333template <>
1334EIGEN_STRONG_INLINE Packet16c pcmp_eq(const Packet16c& a, const Packet16c& b) {
1335 return reinterpret_cast<Packet16c>(vec_cmpeq(a, b));
1336}
1337#ifdef EIGEN_VECTORIZE_VSX
1338template <>
1339EIGEN_STRONG_INLINE Packet16uc pcmp_le(const Packet16uc& a, const Packet16uc& b) {
1340 return reinterpret_cast<Packet16uc>(vec_cmple(a, b));
1341}
1342#endif
1343template <>
1344EIGEN_STRONG_INLINE Packet16uc pcmp_lt(const Packet16uc& a, const Packet16uc& b) {
1345 return reinterpret_cast<Packet16uc>(vec_cmplt(a, b));
1346}
1347template <>
1348EIGEN_STRONG_INLINE Packet16uc pcmp_eq(const Packet16uc& a, const Packet16uc& b) {
1349 return reinterpret_cast<Packet16uc>(vec_cmpeq(a, b));
1350}
1351
1352template <>
1353EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
1354 return vec_and(a, b);
1355}
1356template <>
1357EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
1358 return vec_and(a, b);
1359}
1360template <>
1361EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1362 return vec_and(a, b);
1363}
1364template <>
1365EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(const Packet8us& a, const Packet8us& b) {
1366 return vec_and(a, b);
1367}
1368template <>
1369EIGEN_STRONG_INLINE Packet8bf pand<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
1370 return pand<Packet8us>(a, b);
1371}
1372
1373template <>
1374EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
1375 return vec_or(a, b);
1376}
1377template <>
1378EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
1379 return vec_or(a, b);
1380}
1381template <>
1382EIGEN_STRONG_INLINE Packet8s por<Packet8s>(const Packet8s& a, const Packet8s& b) {
1383 return vec_or(a, b);
1384}
1385template <>
1386EIGEN_STRONG_INLINE Packet8us por<Packet8us>(const Packet8us& a, const Packet8us& b) {
1387 return vec_or(a, b);
1388}
1389template <>
1390EIGEN_STRONG_INLINE Packet8bf por<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
1391 return por<Packet8us>(a, b);
1392}
1393
1394template <>
1395EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
1396 return vec_xor(a, b);
1397}
1398template <>
1399EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
1400 return vec_xor(a, b);
1401}
1402template <>
1403EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(const Packet8us& a, const Packet8us& b) {
1404 return vec_xor(a, b);
1405}
1406template <>
1407EIGEN_STRONG_INLINE Packet8bf pxor<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
1408 return pxor<Packet8us>(a, b);
1409}
1410
1411template <>
1412EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
1413 return vec_andc(a, b);
1414}
1415template <>
1416EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
1417 return vec_andc(a, b);
1418}
1419
1420template <>
1421EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) {
1422 return vec_sel(b, a, reinterpret_cast<Packet4ui>(mask));
1423}
1424
1425template <>
1426EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) {
1427 Packet4f t = vec_add(
1428 reinterpret_cast<Packet4f>(vec_or(vec_and(reinterpret_cast<Packet4ui>(a), p4ui_SIGN), p4ui_PREV0DOT5)), a);
1429 Packet4f res;
1430
1431#ifdef EIGEN_VECTORIZE_VSX
1432 __asm__("xvrspiz %x0, %x1\n\t" : "=&wa"(res) : "wa"(t));
1433#else
1434 __asm__("vrfiz %0, %1\n\t" : "=v"(res) : "v"(t));
1435#endif
1436
1437 return res;
1438}
1439template <>
1440EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
1441 return vec_ceil(a);
1442}
1443template <>
1444EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
1445 return vec_floor(a);
1446}
1447template <>
1448EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
1449 return vec_trunc(a);
1450}
1451#ifdef EIGEN_VECTORIZE_VSX
1452template <>
1453EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
1454 Packet4f res;
1455
1456 __asm__("xvrspic %x0, %x1\n\t" : "=&wa"(res) : "wa"(a));
1457
1458 return res;
1459}
1460#endif
1461
1462template <typename Packet>
1463EIGEN_STRONG_INLINE Packet ploadu_common(const __UNPACK_TYPE__(Packet) * from) {
1464 EIGEN_DEBUG_UNALIGNED_LOAD
1465#if defined(EIGEN_VECTORIZE_VSX)
1466 return vec_xl(0, const_cast<__UNPACK_TYPE__(Packet)*>(from));
1467#else
1468 Packet16uc MSQ = vec_ld(0, (unsigned char*)from); // most significant quadword
1469 Packet16uc LSQ = vec_ld(15, (unsigned char*)from); // least significant quadword
1470 Packet16uc mask = vec_lvsl(0, from); // create the permute mask
1471 // TODO: Add static_cast here
1472 return (Packet)vec_perm(MSQ, LSQ, mask); // align the data
1473#endif
1474}
1475
1476template <>
1477EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
1478 return ploadu_common<Packet4f>(from);
1479}
1480template <>
1481EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) {
1482 return ploadu_common<Packet4i>(from);
1483}
1484template <>
1485EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(const short int* from) {
1486 return ploadu_common<Packet8s>(from);
1487}
1488template <>
1489EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(const unsigned short int* from) {
1490 return ploadu_common<Packet8us>(from);
1491}
1492template <>
1493EIGEN_STRONG_INLINE Packet8bf ploadu<Packet8bf>(const bfloat16* from) {
1494 return ploadu_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
1495}
1496template <>
1497EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(const signed char* from) {
1498 return ploadu_common<Packet16c>(from);
1499}
1500template <>
1501EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(const unsigned char* from) {
1502 return ploadu_common<Packet16uc>(from);
1503}
1504
1505template <typename Packet>
1506EIGEN_ALWAYS_INLINE Packet ploadu_partial_common(const __UNPACK_TYPE__(Packet) * from, const Index n,
1507 const Index offset) {
1508 const Index packet_size = unpacket_traits<Packet>::size;
1509 eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
1510 const Index size = sizeof(__UNPACK_TYPE__(Packet));
1511#ifdef _ARCH_PWR9
1512 EIGEN_UNUSED_VARIABLE(packet_size);
1513 EIGEN_DEBUG_ALIGNED_LOAD
1514 EIGEN_DEBUG_UNALIGNED_LOAD
1515 Packet load = vec_xl_len(const_cast<__UNPACK_TYPE__(Packet)*>(from), n * size);
1516 if (offset) {
1517 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
1518#ifdef _BIG_ENDIAN
1519 load = Packet(vec_sro(Packet16uc(load), shift));
1520#else
1521 load = Packet(vec_slo(Packet16uc(load), shift));
1522#endif
1523 }
1524 return load;
1525#else
1526 if (n) {
1527 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) load[packet_size];
1528 unsigned char* load2 = reinterpret_cast<unsigned char*>(load + offset);
1529 unsigned char* from2 = reinterpret_cast<unsigned char*>(const_cast<__UNPACK_TYPE__(Packet)*>(from));
1530 Index n2 = n * size;
1531 if (16 <= n2) {
1532 pstoreu(load2, ploadu<Packet16uc>(from2));
1533 } else {
1534 memcpy((void*)load2, (void*)from2, n2);
1535 }
1536 return pload_ignore<Packet>(load);
1537 } else {
1538 return Packet(pset1<Packet16uc>(0));
1539 }
1540#endif
1541}
1542
1543template <>
1544EIGEN_ALWAYS_INLINE Packet4f ploadu_partial<Packet4f>(const float* from, const Index n, const Index offset) {
1545 return ploadu_partial_common<Packet4f>(from, n, offset);
1546}
1547template <>
1548EIGEN_ALWAYS_INLINE Packet4i ploadu_partial<Packet4i>(const int* from, const Index n, const Index offset) {
1549 return ploadu_partial_common<Packet4i>(from, n, offset);
1550}
1551template <>
1552EIGEN_ALWAYS_INLINE Packet8s ploadu_partial<Packet8s>(const short int* from, const Index n, const Index offset) {
1553 return ploadu_partial_common<Packet8s>(from, n, offset);
1554}
1555template <>
1556EIGEN_ALWAYS_INLINE Packet8us ploadu_partial<Packet8us>(const unsigned short int* from, const Index n,
1557 const Index offset) {
1558 return ploadu_partial_common<Packet8us>(from, n, offset);
1559}
1560template <>
1561EIGEN_ALWAYS_INLINE Packet8bf ploadu_partial<Packet8bf>(const bfloat16* from, const Index n, const Index offset) {
1562 return ploadu_partial_common<Packet8us>(reinterpret_cast<const unsigned short int*>(from), n, offset);
1563}
1564template <>
1565EIGEN_ALWAYS_INLINE Packet16c ploadu_partial<Packet16c>(const signed char* from, const Index n, const Index offset) {
1566 return ploadu_partial_common<Packet16c>(from, n, offset);
1567}
1568template <>
1569EIGEN_ALWAYS_INLINE Packet16uc ploadu_partial<Packet16uc>(const unsigned char* from, const Index n,
1570 const Index offset) {
1571 return ploadu_partial_common<Packet16uc>(from, n, offset);
1572}
1573
1574template <typename Packet>
1575EIGEN_STRONG_INLINE Packet ploaddup_common(const __UNPACK_TYPE__(Packet) * from) {
1576 // Only the first half of a packet is readable.
1577#ifdef EIGEN_VECTORIZE_VSX
1578 unsigned long long bits;
1579 memcpy(&bits, from, sizeof(bits));
1580 const Packet p = reinterpret_cast<Packet>(vec_splats(bits));
1581#else
1582 Packet p = reinterpret_cast<Packet>(vec_splat_s32(0));
1583 memcpy(&p, from, sizeof(Packet) / 2);
1584#endif
1585 return vec_mergeh(p, p);
1586}
1587template <>
1588EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
1589 return ploaddup_common<Packet4f>(from);
1590}
1591template <>
1592EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) {
1593 return ploaddup_common<Packet4i>(from);
1594}
1595
1596template <>
1597EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(const short int* from) {
1598 return ploaddup_common<Packet8s>(from);
1599}
1600
1601template <>
1602EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(const unsigned short int* from) {
1603 return ploaddup_common<Packet8us>(from);
1604}
1605
1606template <>
1607EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(const short int* from) {
1608 Packet8s p;
1609 if ((std::ptrdiff_t(from) % 16) == 0)
1610 p = pload<Packet8s>(from);
1611 else
1612 p = ploadu<Packet8s>(from);
1613 return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI);
1614}
1615
1616template <>
1617EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(const unsigned short int* from) {
1618 Packet8us p;
1619 if ((std::ptrdiff_t(from) % 16) == 0)
1620 p = pload<Packet8us>(from);
1621 else
1622 p = ploadu<Packet8us>(from);
1623 return vec_perm(p, p, p16uc_QUADRUPLICATE16_HI);
1624}
1625
1626template <>
1627EIGEN_STRONG_INLINE Packet8bf ploadquad<Packet8bf>(const bfloat16* from) {
1628 return ploadquad<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
1629}
1630
1631template <>
1632EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(const signed char* from) {
1633 return ploaddup_common<Packet16c>(from);
1634}
1635
1636template <>
1637EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(const unsigned char* from) {
1638 return ploaddup_common<Packet16uc>(from);
1639}
1640
1641template <>
1642EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(const signed char* from) {
1643 Packet16c p;
1644 if ((std::ptrdiff_t(from) % 16) == 0)
1645 p = pload<Packet16c>(from);
1646 else
1647 p = ploadu<Packet16c>(from);
1648 return vec_perm(p, p, p16uc_QUADRUPLICATE16);
1649}
1650
1651template <>
1652EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(const unsigned char* from) {
1653 Packet16uc p;
1654 if ((std::ptrdiff_t(from) % 16) == 0)
1655 p = pload<Packet16uc>(from);
1656 else
1657 p = ploadu<Packet16uc>(from);
1658 return vec_perm(p, p, p16uc_QUADRUPLICATE16);
1659}
1660
1661template <typename Packet>
1662EIGEN_STRONG_INLINE void pstoreu_common(__UNPACK_TYPE__(Packet) * to, const Packet& from) {
1663 EIGEN_DEBUG_UNALIGNED_STORE
1664#if defined(EIGEN_VECTORIZE_VSX)
1665 vec_xst(from, 0, to);
1666#else
1667 // Taken from http://developer.apple.com/hardwaredrivers/ve/alignment.html
1668 // Warning: not thread safe!
1669 Packet16uc MSQ, LSQ, edges;
1670 Packet16uc edgeAlign, align;
1671
1672 MSQ = vec_ld(0, (unsigned char*)to); // most significant quadword
1673 LSQ = vec_ld(15, (unsigned char*)to); // least significant quadword
1674 edgeAlign = vec_lvsl(0, to); // permute map to extract edges
1675 edges = vec_perm(LSQ, MSQ, edgeAlign); // extract the edges
1676 align = vec_lvsr(0, to); // permute map to misalign data
1677 MSQ = vec_perm(edges, (Packet16uc)from, align); // misalign the data (MSQ)
1678 LSQ = vec_perm((Packet16uc)from, edges, align); // misalign the data (LSQ)
1679 vec_st(LSQ, 15, (unsigned char*)to); // Store the LSQ part first
1680 vec_st(MSQ, 0, (unsigned char*)to); // Store the MSQ part second
1681#endif
1682}
1683template <>
1684EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
1685 pstoreu_common<Packet4f>(to, from);
1686}
1687template <>
1688EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) {
1689 pstoreu_common<Packet4i>(to, from);
1690}
1691template <>
1692EIGEN_STRONG_INLINE void pstoreu<short int>(short int* to, const Packet8s& from) {
1693 pstoreu_common<Packet8s>(to, from);
1694}
1695template <>
1696EIGEN_STRONG_INLINE void pstoreu<unsigned short int>(unsigned short int* to, const Packet8us& from) {
1697 pstoreu_common<Packet8us>(to, from);
1698}
1699template <>
1700EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet8bf& from) {
1701 pstoreu_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from.m_val);
1702}
1703template <>
1704EIGEN_STRONG_INLINE void pstoreu<signed char>(signed char* to, const Packet16c& from) {
1705 pstoreu_common<Packet16c>(to, from);
1706}
1707template <>
1708EIGEN_STRONG_INLINE void pstoreu<unsigned char>(unsigned char* to, const Packet16uc& from) {
1709 pstoreu_common<Packet16uc>(to, from);
1710}
1711
1712template <typename Packet>
1713EIGEN_ALWAYS_INLINE void pstoreu_partial_common(__UNPACK_TYPE__(Packet) * to, const Packet& from, const Index n,
1714 const Index offset) {
1715 const Index packet_size = unpacket_traits<Packet>::size;
1716 eigen_internal_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
1717 const Index size = sizeof(__UNPACK_TYPE__(Packet));
1718#ifdef _ARCH_PWR9
1719 EIGEN_UNUSED_VARIABLE(packet_size);
1720 EIGEN_DEBUG_UNALIGNED_STORE
1721 Packet store = from;
1722 if (offset) {
1723 Packet16uc shift = pset1<Packet16uc>(offset * 8 * size);
1724#ifdef _BIG_ENDIAN
1725 store = Packet(vec_slo(Packet16uc(store), shift));
1726#else
1727 store = Packet(vec_sro(Packet16uc(store), shift));
1728#endif
1729 }
1730 vec_xst_len(store, to, n * size);
1731#else
1732 if (n) {
1733 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) store[packet_size];
1734 pstore(store, from);
1735 unsigned char* store2 = reinterpret_cast<unsigned char*>(store + offset);
1736 unsigned char* to2 = reinterpret_cast<unsigned char*>(to);
1737 Index n2 = n * size;
1738 if (16 <= n2) {
1739 pstoreu(to2, ploadu<Packet16uc>(store2));
1740 } else {
1741 memcpy((void*)to2, (void*)store2, n2);
1742 }
1743 }
1744#endif
1745}
1746
1747template <>
1748EIGEN_ALWAYS_INLINE void pstoreu_partial<float>(float* to, const Packet4f& from, const Index n, const Index offset) {
1749 pstoreu_partial_common<Packet4f>(to, from, n, offset);
1750}
1751template <>
1752EIGEN_ALWAYS_INLINE void pstoreu_partial<int>(int* to, const Packet4i& from, const Index n, const Index offset) {
1753 pstoreu_partial_common<Packet4i>(to, from, n, offset);
1754}
1755template <>
1756EIGEN_ALWAYS_INLINE void pstoreu_partial<short int>(short int* to, const Packet8s& from, const Index n,
1757 const Index offset) {
1758 pstoreu_partial_common<Packet8s>(to, from, n, offset);
1759}
1760template <>
1761EIGEN_ALWAYS_INLINE void pstoreu_partial<unsigned short int>(unsigned short int* to, const Packet8us& from,
1762 const Index n, const Index offset) {
1763 pstoreu_partial_common<Packet8us>(to, from, n, offset);
1764}
1765template <>
1766EIGEN_ALWAYS_INLINE void pstoreu_partial<bfloat16>(bfloat16* to, const Packet8bf& from, const Index n,
1767 const Index offset) {
1768 pstoreu_partial_common<Packet8us>(reinterpret_cast<unsigned short int*>(to), from, n, offset);
1769}
1770template <>
1771EIGEN_ALWAYS_INLINE void pstoreu_partial<signed char>(signed char* to, const Packet16c& from, const Index n,
1772 const Index offset) {
1773 pstoreu_partial_common<Packet16c>(to, from, n, offset);
1774}
1775template <>
1776EIGEN_ALWAYS_INLINE void pstoreu_partial<unsigned char>(unsigned char* to, const Packet16uc& from, const Index n,
1777 const Index offset) {
1778 pstoreu_partial_common<Packet16uc>(to, from, n, offset);
1779}
1780
1781template <>
1782EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1783 EIGEN_PPC_PREFETCH(addr);
1784}
1785template <>
1786EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
1787 EIGEN_PPC_PREFETCH(addr);
1788}
1789
1790template <>
1791EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
1792 EIGEN_ALIGN16 float x;
1793 vec_ste(a, 0, &x);
1794 return x;
1795}
1796template <>
1797EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) {
1798 EIGEN_ALIGN16 int x;
1799 vec_ste(a, 0, &x);
1800 return x;
1801}
1802
1803template <typename Packet>
1804EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) pfirst_common(const Packet& a) {
1805 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) x;
1806 vec_ste(a, 0, &x);
1807 return x;
1808}
1809
1810template <>
1811EIGEN_STRONG_INLINE short int pfirst<Packet8s>(const Packet8s& a) {
1812 return pfirst_common<Packet8s>(a);
1813}
1814
1815template <>
1816EIGEN_STRONG_INLINE unsigned short int pfirst<Packet8us>(const Packet8us& a) {
1817 return pfirst_common<Packet8us>(a);
1818}
1819
1820template <>
1821EIGEN_STRONG_INLINE signed char pfirst<Packet16c>(const Packet16c& a) {
1822 return pfirst_common<Packet16c>(a);
1823}
1824
1825template <>
1826EIGEN_STRONG_INLINE unsigned char pfirst<Packet16uc>(const Packet16uc& a) {
1827 return pfirst_common<Packet16uc>(a);
1828}
1829
1830template <>
1831EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
1832 return reinterpret_cast<Packet4f>(
1833 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
1834}
1835template <>
1836EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
1837 return reinterpret_cast<Packet4i>(
1838 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
1839}
1840template <>
1841EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a) {
1842 return reinterpret_cast<Packet8s>(
1843 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE16));
1844}
1845template <>
1846EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) {
1847 return reinterpret_cast<Packet8us>(
1848 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE16));
1849}
1850template <>
1851EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) {
1852 return vec_perm(a, a, p16uc_REVERSE8);
1853}
1854template <>
1855EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) {
1856 return vec_perm(a, a, p16uc_REVERSE8);
1857}
1858template <>
1859EIGEN_STRONG_INLINE Packet8bf preverse(const Packet8bf& a) {
1860 return preverse<Packet8us>(a);
1861}
1862
1863template <>
1864EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) {
1865 return vec_abs(a);
1866}
1867template <>
1868EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) {
1869 return vec_abs(a);
1870}
1871template <>
1872EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) {
1873 return vec_abs(a);
1874}
1875template <>
1876EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) {
1877 return vec_abs(a);
1878}
1879template <>
1880EIGEN_STRONG_INLINE Packet8bf pabs(const Packet8bf& a) {
1881 EIGEN_DECLARE_CONST_FAST_Packet8us(abs_mask, 0x7FFF);
1882 return pand<Packet8us>(p8us_abs_mask, a);
1883}
1884
1885template <>
1886EIGEN_STRONG_INLINE Packet8bf psignbit(const Packet8bf& a) {
1887 return vec_sra(a.m_val, vec_splat_u16(15));
1888}
1889template <>
1890EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) {
1891 return (Packet4f)vec_sra((Packet4i)a, vec_splats((unsigned int)(31)));
1892}
1893
1894template <int N>
1895EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) {
1896 return vec_sra(a, reinterpret_cast<Packet4ui>(pset1<Packet4i>(N)));
1897}
1898template <int N>
1899EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) {
1900 return vec_sr(a, reinterpret_cast<Packet4ui>(pset1<Packet4i>(N)));
1901}
1902template <int N>
1903EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) {
1904 return vec_sl(a, reinterpret_cast<Packet4ui>(pset1<Packet4i>(N)));
1905}
1906template <int N>
1907EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(const Packet16c& a) {
1908 return vec_sra(a, pset1<Packet16uc>(static_cast<unsigned char>(N)));
1909}
1910template <int N>
1911EIGEN_STRONG_INLINE Packet16c plogical_shift_right(const Packet16c& a) {
1912 return vec_sr(a, pset1<Packet16uc>(static_cast<unsigned char>(N)));
1913}
1914template <int N>
1915EIGEN_STRONG_INLINE Packet16c plogical_shift_left(const Packet16c& a) {
1916 return vec_sl(a, pset1<Packet16uc>(static_cast<unsigned char>(N)));
1917}
1918template <int N>
1919EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(const Packet16uc& a) {
1920 return vec_sr(a, pset1<Packet16uc>(static_cast<unsigned char>(N)));
1921}
1922template <int N>
1923EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(const Packet16uc& a) {
1924 return vec_sr(a, pset1<Packet16uc>(static_cast<unsigned char>(N)));
1925}
1926template <int N>
1927EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(const Packet16uc& a) {
1928 return vec_sl(a, pset1<Packet16uc>(static_cast<unsigned char>(N)));
1929}
1930template <int N>
1931EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(const Packet8s& a) {
1932 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1933 return vec_sra(a, p8us_mask);
1934}
1935template <int N>
1936EIGEN_STRONG_INLINE Packet8s plogical_shift_right(const Packet8s& a) {
1937 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1938 return vec_sr(a, p8us_mask);
1939}
1940template <int N>
1941EIGEN_STRONG_INLINE Packet8s plogical_shift_left(const Packet8s& a) {
1942 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1943 return vec_sl(a, p8us_mask);
1944}
1945template <int N>
1946EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(const Packet8us& a) {
1947 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1948 return vec_sr(a, p8us_mask);
1949}
1950template <int N>
1951EIGEN_STRONG_INLINE Packet8us plogical_shift_right(const Packet8us& a) {
1952 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1953 return vec_sr(a, p8us_mask);
1954}
1955template <int N>
1956EIGEN_STRONG_INLINE Packet8us plogical_shift_left(const Packet8us& a) {
1957 const EIGEN_DECLARE_CONST_FAST_Packet8us(mask, N);
1958 return vec_sl(a, p8us_mask);
1959}
1960template <int N>
1961EIGEN_STRONG_INLINE Packet4f plogical_shift_left(const Packet4f& a) {
1962 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1963 Packet4ui r = vec_sl(reinterpret_cast<Packet4ui>(a), p4ui_mask);
1964 return reinterpret_cast<Packet4f>(r);
1965}
1966
1967template <int N>
1968EIGEN_STRONG_INLINE Packet4f plogical_shift_right(const Packet4f& a) {
1969 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1970 Packet4ui r = vec_sr(reinterpret_cast<Packet4ui>(a), p4ui_mask);
1971 return reinterpret_cast<Packet4f>(r);
1972}
1973
1974template <int N>
1975EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(const Packet4ui& a) {
1976 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1977 return vec_sr(a, p4ui_mask);
1978}
1979
1980template <int N>
1981EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) {
1982 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1983 return vec_sr(a, p4ui_mask);
1984}
1985
1986template <int N>
1987EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) {
1988 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mask, N);
1989 return vec_sl(a, p4ui_mask);
1990}
1991
1992EIGEN_STRONG_INLINE Packet4f Bf16ToF32Even(const Packet8bf& bf) {
1993 return plogical_shift_left<16>(reinterpret_cast<Packet4f>(bf.m_val));
1994}
1995
1996EIGEN_STRONG_INLINE Packet4f Bf16ToF32Odd(const Packet8bf& bf) {
1997 const EIGEN_DECLARE_CONST_FAST_Packet4ui(high_mask, 0xFFFF0000);
1998 return pand<Packet4f>(reinterpret_cast<Packet4f>(bf.m_val), reinterpret_cast<Packet4f>(p4ui_high_mask));
1999}
2000
2001EIGEN_ALWAYS_INLINE Packet8us pmerge(const Packet4ui& even, const Packet4ui& odd) {
2002#ifdef _BIG_ENDIAN
2003 return vec_perm(reinterpret_cast<Packet8us>(odd), reinterpret_cast<Packet8us>(even), p16uc_MERGEO16);
2004#else
2005 return vec_perm(reinterpret_cast<Packet8us>(even), reinterpret_cast<Packet8us>(odd), p16uc_MERGEE16);
2006#endif
2007}
2008
2009// Simple interleaving of bool masks, prevents true values from being
2010// converted to NaNs.
2011EIGEN_STRONG_INLINE Packet8bf F32ToBf16Bool(const Packet4f& even, const Packet4f& odd) {
2012 return pmerge(reinterpret_cast<Packet4ui>(even), reinterpret_cast<Packet4ui>(odd));
2013}
2014
2015// #define SUPPORT_BF16_SUBNORMALS
2016
2017#ifndef __VEC_CLASS_FP_NAN
2018#define __VEC_CLASS_FP_NAN (1 << 6)
2019#endif
2020
2021#if defined(SUPPORT_BF16_SUBNORMALS) && !defined(__VEC_CLASS_FP_SUBNORMAL)
2022#define __VEC_CLASS_FP_SUBNORMAL_P (1 << 1)
2023#define __VEC_CLASS_FP_SUBNORMAL_N (1 << 0)
2024
2025#define __VEC_CLASS_FP_SUBNORMAL (__VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N)
2026#endif
2027
2028EIGEN_STRONG_INLINE Packet8bf F32ToBf16(const Packet4f& p4f) {
2029#ifdef _ARCH_PWR10
2030 return reinterpret_cast<Packet8us>(__builtin_vsx_xvcvspbf16(reinterpret_cast<Packet16uc>(p4f)));
2031#else
2032 Packet4ui input = reinterpret_cast<Packet4ui>(p4f);
2033 Packet4ui lsb = plogical_shift_right<16>(input);
2034 lsb = pand<Packet4ui>(lsb, reinterpret_cast<Packet4ui>(p4i_ONE));
2035
2036 EIGEN_DECLARE_CONST_FAST_Packet4ui(BIAS, 0x7FFFu);
2037 Packet4ui rounding_bias = padd<Packet4ui>(lsb, p4ui_BIAS);
2038 input = padd<Packet4ui>(input, rounding_bias);
2039
2040 const EIGEN_DECLARE_CONST_FAST_Packet4ui(nan, 0x7FC00000);
2041#if defined(_ARCH_PWR9) && defined(EIGEN_VECTORIZE_VSX)
2042 Packet4bi nan_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_NAN);
2043 input = vec_sel(input, p4ui_nan, nan_selector);
2044
2045#ifdef SUPPORT_BF16_SUBNORMALS
2046 Packet4bi subnormal_selector = vec_test_data_class(p4f, __VEC_CLASS_FP_SUBNORMAL);
2047 input = vec_sel(input, reinterpret_cast<Packet4ui>(p4f), subnormal_selector);
2048#endif
2049#else
2050#ifdef SUPPORT_BF16_SUBNORMALS
2051 // Test NaN and Subnormal
2052 const EIGEN_DECLARE_CONST_FAST_Packet4ui(exp_mask, 0x7F800000);
2053 Packet4ui exp = pand<Packet4ui>(p4ui_exp_mask, reinterpret_cast<Packet4ui>(p4f));
2054
2055 const EIGEN_DECLARE_CONST_FAST_Packet4ui(mantissa_mask, 0x7FFFFF);
2056 Packet4ui mantissa = pand<Packet4ui>(p4ui_mantissa_mask, reinterpret_cast<Packet4ui>(p4f));
2057
2058 Packet4bi is_max_exp = vec_cmpeq(exp, p4ui_exp_mask);
2059 Packet4bi is_mant_zero = vec_cmpeq(mantissa, reinterpret_cast<Packet4ui>(p4i_ZERO));
2060
2061 Packet4ui nan_selector =
2062 pandnot<Packet4ui>(reinterpret_cast<Packet4ui>(is_max_exp), reinterpret_cast<Packet4ui>(is_mant_zero));
2063
2064 Packet4bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast<Packet4ui>(p4i_ZERO));
2065
2066 Packet4ui subnormal_selector =
2067 pandnot<Packet4ui>(reinterpret_cast<Packet4ui>(is_zero_exp), reinterpret_cast<Packet4ui>(is_mant_zero));
2068
2069 input = vec_sel(input, p4ui_nan, nan_selector);
2070 input = vec_sel(input, reinterpret_cast<Packet4ui>(p4f), subnormal_selector);
2071#else
2072 // Test only NaN
2073 Packet4bi nan_selector = vec_cmpeq(p4f, p4f);
2074
2075 input = vec_sel(p4ui_nan, input, nan_selector);
2076#endif
2077#endif
2078
2079 input = plogical_shift_right<16>(input);
2080 return reinterpret_cast<Packet8us>(input);
2081#endif
2082}
2083
2084#ifdef _BIG_ENDIAN
2090template <bool lohi>
2091EIGEN_ALWAYS_INLINE Packet8bf Bf16PackHigh(const Packet4f& lo, const Packet4f& hi) {
2092 if (lohi) {
2093 return vec_perm(reinterpret_cast<Packet8us>(lo), reinterpret_cast<Packet8us>(hi), p16uc_MERGEH16);
2094 } else {
2095 return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEE16);
2096 }
2097}
2098
2104template <bool lohi>
2105EIGEN_ALWAYS_INLINE Packet8bf Bf16PackLow(const Packet4f& lo, const Packet4f& hi) {
2106 if (lohi) {
2107 return vec_pack(reinterpret_cast<Packet4ui>(lo), reinterpret_cast<Packet4ui>(hi));
2108 } else {
2109 return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEO16);
2110 }
2111}
2112#else
2113template <bool lohi>
2114EIGEN_ALWAYS_INLINE Packet8bf Bf16PackLow(const Packet4f& hi, const Packet4f& lo) {
2115 if (lohi) {
2116 return vec_pack(reinterpret_cast<Packet4ui>(hi), reinterpret_cast<Packet4ui>(lo));
2117 } else {
2118 return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEE16);
2119 }
2120}
2121
2122template <bool lohi>
2123EIGEN_ALWAYS_INLINE Packet8bf Bf16PackHigh(const Packet4f& hi, const Packet4f& lo) {
2124 if (lohi) {
2125 return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEL16);
2126 } else {
2127 return vec_perm(reinterpret_cast<Packet8us>(hi), reinterpret_cast<Packet8us>(lo), p16uc_MERGEO16);
2128 }
2129}
2130#endif
2131
2137template <bool lohi = true>
2138EIGEN_ALWAYS_INLINE Packet8bf F32ToBf16Two(const Packet4f& lo, const Packet4f& hi) {
2139 Packet8us p4f = Bf16PackHigh<lohi>(lo, hi);
2140 Packet8us p4f2 = Bf16PackLow<lohi>(lo, hi);
2141
2142 Packet8us lsb = pand<Packet8us>(p4f, p8us_ONE);
2143 EIGEN_DECLARE_CONST_FAST_Packet8us(BIAS, 0x7FFFu);
2144 lsb = padd<Packet8us>(lsb, p8us_BIAS);
2145 lsb = padd<Packet8us>(lsb, p4f2);
2146
2147 Packet8bi rounding_bias = vec_cmplt(lsb, p4f2);
2148 Packet8us input = psub<Packet8us>(p4f, reinterpret_cast<Packet8us>(rounding_bias));
2149
2150#if defined(_ARCH_PWR9) && defined(EIGEN_VECTORIZE_VSX)
2151 Packet4bi nan_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_NAN);
2152 Packet4bi nan_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_NAN);
2153 Packet8us nan_selector =
2154 Bf16PackLow<lohi>(reinterpret_cast<Packet4f>(nan_selector_lo), reinterpret_cast<Packet4f>(nan_selector_hi));
2155
2156 input = vec_sel(input, p8us_BIAS, nan_selector);
2157
2158#ifdef SUPPORT_BF16_SUBNORMALS
2159 Packet4bi subnormal_selector_lo = vec_test_data_class(lo, __VEC_CLASS_FP_SUBNORMAL);
2160 Packet4bi subnormal_selector_hi = vec_test_data_class(hi, __VEC_CLASS_FP_SUBNORMAL);
2161 Packet8us subnormal_selector = Bf16PackLow<lohi>(reinterpret_cast<Packet4f>(subnormal_selector_lo),
2162 reinterpret_cast<Packet4f>(subnormal_selector_hi));
2163
2164 input = vec_sel(input, reinterpret_cast<Packet8us>(p4f), subnormal_selector);
2165#endif
2166#else
2167#ifdef SUPPORT_BF16_SUBNORMALS
2168 // Test NaN and Subnormal
2169 const EIGEN_DECLARE_CONST_FAST_Packet8us(exp_mask, 0x7F80);
2170 Packet8us exp = pand<Packet8us>(p8us_exp_mask, p4f);
2171
2172 const EIGEN_DECLARE_CONST_FAST_Packet8us(mantissa_mask, 0x7Fu);
2173 Packet8us mantissa = pand<Packet8us>(p8us_mantissa_mask, p4f);
2174
2175 Packet8bi is_max_exp = vec_cmpeq(exp, p8us_exp_mask);
2176 Packet8bi is_mant_zero = vec_cmpeq(mantissa, reinterpret_cast<Packet8us>(p4i_ZERO));
2177
2178 Packet8us nan_selector =
2179 pandnot<Packet8us>(reinterpret_cast<Packet8us>(is_max_exp), reinterpret_cast<Packet8us>(is_mant_zero));
2180
2181 Packet8bi is_zero_exp = vec_cmpeq(exp, reinterpret_cast<Packet8us>(p4i_ZERO));
2182
2183 Packet8us subnormal_selector =
2184 pandnot<Packet8us>(reinterpret_cast<Packet8us>(is_zero_exp), reinterpret_cast<Packet8us>(is_mant_zero));
2185
2186 // Using BIAS as NaN (since any or all of the last 7 bits can be set)
2187 input = vec_sel(input, p8us_BIAS, nan_selector);
2188 input = vec_sel(input, reinterpret_cast<Packet8us>(p4f), subnormal_selector);
2189#else
2190 // Test only NaN
2191 Packet4bi nan_selector_lo = vec_cmpeq(lo, lo);
2192 Packet4bi nan_selector_hi = vec_cmpeq(hi, hi);
2193 Packet8us nan_selector =
2194 Bf16PackLow<lohi>(reinterpret_cast<Packet4f>(nan_selector_lo), reinterpret_cast<Packet4f>(nan_selector_hi));
2195
2196 input = vec_sel(p8us_BIAS, input, nan_selector);
2197#endif
2198#endif
2199
2200 return input;
2201}
2202
2206EIGEN_STRONG_INLINE Packet8bf F32ToBf16Both(const Packet4f& lo, const Packet4f& hi) {
2207#ifdef _ARCH_PWR10
2208 Packet8bf fp16_0 = F32ToBf16(lo);
2209 Packet8bf fp16_1 = F32ToBf16(hi);
2210 return vec_pack(reinterpret_cast<Packet4ui>(fp16_0.m_val), reinterpret_cast<Packet4ui>(fp16_1.m_val));
2211#else
2212 return F32ToBf16Two(lo, hi);
2213#endif
2214}
2215
2219EIGEN_STRONG_INLINE Packet8bf F32ToBf16(const Packet4f& even, const Packet4f& odd) {
2220#ifdef _ARCH_PWR10
2221 return pmerge(reinterpret_cast<Packet4ui>(F32ToBf16(even).m_val), reinterpret_cast<Packet4ui>(F32ToBf16(odd).m_val));
2222#else
2223 return F32ToBf16Two<false>(even, odd);
2224#endif
2225}
2226#define BF16_TO_F32_UNARY_OP_WRAPPER(OP, A) \
2227 Packet4f a_even = Bf16ToF32Even(A); \
2228 Packet4f a_odd = Bf16ToF32Odd(A); \
2229 Packet4f op_even = OP(a_even); \
2230 Packet4f op_odd = OP(a_odd); \
2231 return F32ToBf16(op_even, op_odd);
2232
2233#define BF16_TO_F32_BINARY_OP_WRAPPER(OP, A, B) \
2234 Packet4f a_even = Bf16ToF32Even(A); \
2235 Packet4f a_odd = Bf16ToF32Odd(A); \
2236 Packet4f b_even = Bf16ToF32Even(B); \
2237 Packet4f b_odd = Bf16ToF32Odd(B); \
2238 Packet4f op_even = OP(a_even, b_even); \
2239 Packet4f op_odd = OP(a_odd, b_odd); \
2240 return F32ToBf16(op_even, op_odd);
2241
2242#define BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(OP, A, B) \
2243 Packet4f a_even = Bf16ToF32Even(A); \
2244 Packet4f a_odd = Bf16ToF32Odd(A); \
2245 Packet4f b_even = Bf16ToF32Even(B); \
2246 Packet4f b_odd = Bf16ToF32Odd(B); \
2247 Packet4f op_even = OP(a_even, b_even); \
2248 Packet4f op_odd = OP(a_odd, b_odd); \
2249 return F32ToBf16Bool(op_even, op_odd);
2250
2251template <>
2252EIGEN_STRONG_INLINE Packet8bf padd<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2253 BF16_TO_F32_BINARY_OP_WRAPPER(padd<Packet4f>, a, b);
2254}
2255
2256template <>
2257EIGEN_STRONG_INLINE Packet8bf pmul<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2258 BF16_TO_F32_BINARY_OP_WRAPPER(pmul<Packet4f>, a, b);
2259}
2260
2261template <>
2262EIGEN_STRONG_INLINE Packet8bf pdiv<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2263 BF16_TO_F32_BINARY_OP_WRAPPER(pdiv<Packet4f>, a, b);
2264}
2265
2266template <>
2267EIGEN_STRONG_INLINE Packet8bf pnegate<Packet8bf>(const Packet8bf& a) {
2268 EIGEN_DECLARE_CONST_FAST_Packet8us(neg_mask, 0x8000);
2269 return pxor<Packet8us>(p8us_neg_mask, a);
2270}
2271
2272template <>
2273EIGEN_STRONG_INLINE Packet8bf psub<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2274 BF16_TO_F32_BINARY_OP_WRAPPER(psub<Packet4f>, a, b);
2275}
2276
2277template <>
2278EIGEN_STRONG_INLINE Packet8bf pexp<Packet8bf>(const Packet8bf& a) {
2279 BF16_TO_F32_UNARY_OP_WRAPPER(pexp_float, a);
2280}
2281
2282template <>
2283EIGEN_STRONG_INLINE Packet8bf pexp2<Packet8bf>(const Packet8bf& a) {
2284 BF16_TO_F32_UNARY_OP_WRAPPER(generic_exp2, a);
2285}
2286
2287template <>
2288EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
2289 return pldexp_generic(a, exponent);
2290}
2291template <>
2292EIGEN_STRONG_INLINE Packet8bf pldexp<Packet8bf>(const Packet8bf& a, const Packet8bf& exponent) {
2293 BF16_TO_F32_BINARY_OP_WRAPPER(pldexp<Packet4f>, a, exponent);
2294}
2295
2296template <>
2297EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) {
2298 return pfrexp_generic(a, exponent);
2299}
2300template <>
2301EIGEN_STRONG_INLINE Packet8bf pfrexp<Packet8bf>(const Packet8bf& a, Packet8bf& e) {
2302 Packet4f a_even = Bf16ToF32Even(a);
2303 Packet4f a_odd = Bf16ToF32Odd(a);
2304 Packet4f e_even;
2305 Packet4f e_odd;
2306 Packet4f op_even = pfrexp<Packet4f>(a_even, e_even);
2307 Packet4f op_odd = pfrexp<Packet4f>(a_odd, e_odd);
2308 e = F32ToBf16(e_even, e_odd);
2309 return F32ToBf16(op_even, op_odd);
2310}
2311
2312template <>
2313EIGEN_STRONG_INLINE Packet8bf psin<Packet8bf>(const Packet8bf& a) {
2314 BF16_TO_F32_UNARY_OP_WRAPPER(psin_float, a);
2315}
2316template <>
2317EIGEN_STRONG_INLINE Packet8bf pcos<Packet8bf>(const Packet8bf& a) {
2318 BF16_TO_F32_UNARY_OP_WRAPPER(pcos_float, a);
2319}
2320template <>
2321EIGEN_STRONG_INLINE Packet8bf plog<Packet8bf>(const Packet8bf& a) {
2322 BF16_TO_F32_UNARY_OP_WRAPPER(plog_float, a);
2323}
2324template <>
2325EIGEN_STRONG_INLINE Packet8bf pfloor<Packet8bf>(const Packet8bf& a) {
2326 BF16_TO_F32_UNARY_OP_WRAPPER(pfloor<Packet4f>, a);
2327}
2328template <>
2329EIGEN_STRONG_INLINE Packet8bf pceil<Packet8bf>(const Packet8bf& a) {
2330 BF16_TO_F32_UNARY_OP_WRAPPER(pceil<Packet4f>, a);
2331}
2332template <>
2333EIGEN_STRONG_INLINE Packet8bf pround<Packet8bf>(const Packet8bf& a) {
2334 BF16_TO_F32_UNARY_OP_WRAPPER(pround<Packet4f>, a);
2335}
2336template <>
2337EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(const Packet8bf& a) {
2338 BF16_TO_F32_UNARY_OP_WRAPPER(ptrunc<Packet4f>, a);
2339}
2340#ifdef EIGEN_VECTORIZE_VSX
2341template <>
2342EIGEN_STRONG_INLINE Packet8bf print<Packet8bf>(const Packet8bf& a) {
2343 BF16_TO_F32_UNARY_OP_WRAPPER(print<Packet4f>, a);
2344}
2345#endif
2346template <>
2347EIGEN_STRONG_INLINE Packet8bf pmadd(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2348 Packet4f a_even = Bf16ToF32Even(a);
2349 Packet4f a_odd = Bf16ToF32Odd(a);
2350 Packet4f b_even = Bf16ToF32Even(b);
2351 Packet4f b_odd = Bf16ToF32Odd(b);
2352 Packet4f c_even = Bf16ToF32Even(c);
2353 Packet4f c_odd = Bf16ToF32Odd(c);
2354 Packet4f pmadd_even = pmadd<Packet4f>(a_even, b_even, c_even);
2355 Packet4f pmadd_odd = pmadd<Packet4f>(a_odd, b_odd, c_odd);
2356 return F32ToBf16(pmadd_even, pmadd_odd);
2357}
2358
2359template <>
2360EIGEN_STRONG_INLINE Packet8bf pmsub(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2361 Packet4f a_even = Bf16ToF32Even(a);
2362 Packet4f a_odd = Bf16ToF32Odd(a);
2363 Packet4f b_even = Bf16ToF32Even(b);
2364 Packet4f b_odd = Bf16ToF32Odd(b);
2365 Packet4f c_even = Bf16ToF32Even(c);
2366 Packet4f c_odd = Bf16ToF32Odd(c);
2367 Packet4f pmadd_even = pmsub<Packet4f>(a_even, b_even, c_even);
2368 Packet4f pmadd_odd = pmsub<Packet4f>(a_odd, b_odd, c_odd);
2369 return F32ToBf16(pmadd_even, pmadd_odd);
2370}
2371template <>
2372EIGEN_STRONG_INLINE Packet8bf pnmadd(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2373 Packet4f a_even = Bf16ToF32Even(a);
2374 Packet4f a_odd = Bf16ToF32Odd(a);
2375 Packet4f b_even = Bf16ToF32Even(b);
2376 Packet4f b_odd = Bf16ToF32Odd(b);
2377 Packet4f c_even = Bf16ToF32Even(c);
2378 Packet4f c_odd = Bf16ToF32Odd(c);
2379 Packet4f pmadd_even = pnmadd<Packet4f>(a_even, b_even, c_even);
2380 Packet4f pmadd_odd = pnmadd<Packet4f>(a_odd, b_odd, c_odd);
2381 return F32ToBf16(pmadd_even, pmadd_odd);
2382}
2383
2384template <>
2385EIGEN_STRONG_INLINE Packet8bf pnmsub(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2386 Packet4f a_even = Bf16ToF32Even(a);
2387 Packet4f a_odd = Bf16ToF32Odd(a);
2388 Packet4f b_even = Bf16ToF32Even(b);
2389 Packet4f b_odd = Bf16ToF32Odd(b);
2390 Packet4f c_even = Bf16ToF32Even(c);
2391 Packet4f c_odd = Bf16ToF32Odd(c);
2392 Packet4f pmadd_even = pnmsub<Packet4f>(a_even, b_even, c_even);
2393 Packet4f pmadd_odd = pnmsub<Packet4f>(a_odd, b_odd, c_odd);
2394 return F32ToBf16(pmadd_even, pmadd_odd);
2395}
2396
2397template <>
2398EIGEN_STRONG_INLINE Packet8bf pmin<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2399 BF16_TO_F32_BINARY_OP_WRAPPER(pmin<Packet4f>, a, b);
2400}
2401
2402template <>
2403EIGEN_STRONG_INLINE Packet8bf pmax<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2404 BF16_TO_F32_BINARY_OP_WRAPPER(pmax<Packet4f>, a, b);
2405}
2406
2407template <>
2408EIGEN_STRONG_INLINE Packet8bf pcmp_lt(const Packet8bf& a, const Packet8bf& b) {
2409 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt<Packet4f>, a, b);
2410}
2411template <>
2412EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan(const Packet8bf& a, const Packet8bf& b) {
2413 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_lt_or_nan<Packet4f>, a, b);
2414}
2415template <>
2416EIGEN_STRONG_INLINE Packet8bf pcmp_le(const Packet8bf& a, const Packet8bf& b) {
2417 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_le<Packet4f>, a, b);
2418}
2419template <>
2420EIGEN_STRONG_INLINE Packet8bf pcmp_eq(const Packet8bf& a, const Packet8bf& b) {
2421 BF16_TO_F32_BINARY_OP_WRAPPER_BOOL(pcmp_eq<Packet4f>, a, b);
2422}
2423
2424// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ.
2425template <>
2426EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(const Packet8bf& a) {
2427 const Packet8us magnitude = vec_and(a.m_val, vec_splats(static_cast<unsigned short int>(0x7fff)));
2428 const Packet8bi is_zero = vec_cmpeq(magnitude, vec_splats(static_cast<unsigned short int>(0)));
2429 const Packet8us keep =
2430 vec_or(reinterpret_cast<Packet8us>(vec_cmpgt(magnitude, vec_splats(static_cast<unsigned short int>(0x7f80)))),
2431 vec_splats(static_cast<unsigned short int>(0x8000)));
2432 const Packet8us value = vec_sel(vec_splats(static_cast<unsigned short int>(0x3f80)), a.m_val, keep);
2433 return Packet8bf(vec_andc(value, reinterpret_cast<Packet8us>(is_zero)));
2434}
2435
2436template <>
2437EIGEN_STRONG_INLINE bfloat16 pfirst(const Packet8bf& a) {
2438 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16((pfirst<Packet8us>(a)));
2439}
2440
2441template <>
2442EIGEN_STRONG_INLINE Packet8bf ploaddup<Packet8bf>(const bfloat16* from) {
2443 return ploaddup<Packet8us>(reinterpret_cast<const unsigned short int*>(from));
2444}
2445
2446template <>
2447EIGEN_STRONG_INLINE Packet8bf plset<Packet8bf>(const bfloat16& a) {
2448 bfloat16 countdown[8] = {bfloat16(0), bfloat16(1), bfloat16(2), bfloat16(3),
2449 bfloat16(4), bfloat16(5), bfloat16(6), bfloat16(7)};
2450 return padd<Packet8bf>(pset1<Packet8bf>(a), pload<Packet8bf>(countdown));
2451}
2452
2453template <>
2454EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
2455 Packet4f b, sum;
2456 b = vec_sld(a, a, 8);
2457 sum = a + b;
2458 b = vec_sld(sum, sum, 4);
2459 sum += b;
2460 return pfirst(sum);
2461}
2462
2463template <>
2464EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) {
2465 Packet4i b, sum;
2466 b = vec_sld(a, a, 8);
2467 sum = a + b;
2468 b = vec_sld(sum, sum, 4);
2469 sum += b;
2470 return pfirst(sum);
2471}
2472
2473template <>
2474EIGEN_STRONG_INLINE bfloat16 predux<Packet8bf>(const Packet8bf& a) {
2475 float redux_even = predux<Packet4f>(Bf16ToF32Even(a));
2476 float redux_odd = predux<Packet4f>(Bf16ToF32Odd(a));
2477 float f32_result = redux_even + redux_odd;
2478 return bfloat16(f32_result);
2479}
2480template <typename Packet>
2481EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_size8(const Packet& a) {
2482 union {
2483 Packet v;
2484 __UNPACK_TYPE__(Packet) n[8];
2485 } vt;
2486 vt.v = a;
2487
2488 EIGEN_ALIGN16 int first_loader[4] = {vt.n[0], vt.n[1], vt.n[2], vt.n[3]};
2489 EIGEN_ALIGN16 int second_loader[4] = {vt.n[4], vt.n[5], vt.n[6], vt.n[7]};
2490 Packet4i first_half = pload<Packet4i>(first_loader);
2491 Packet4i second_half = pload<Packet4i>(second_loader);
2492
2493 return static_cast<__UNPACK_TYPE__(Packet)>(predux(first_half) + predux(second_half));
2494}
2495
2496template <>
2497EIGEN_STRONG_INLINE short int predux<Packet8s>(const Packet8s& a) {
2498 return predux_size8<Packet8s>(a);
2499}
2500
2501template <>
2502EIGEN_STRONG_INLINE unsigned short int predux<Packet8us>(const Packet8us& a) {
2503 return predux_size8<Packet8us>(a);
2504}
2505
2506template <typename Packet>
2507EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_size16(const Packet& a) {
2508 union {
2509 Packet v;
2510 __UNPACK_TYPE__(Packet) n[16];
2511 } vt;
2512 vt.v = a;
2513
2514 EIGEN_ALIGN16 int first_loader[4] = {vt.n[0], vt.n[1], vt.n[2], vt.n[3]};
2515 EIGEN_ALIGN16 int second_loader[4] = {vt.n[4], vt.n[5], vt.n[6], vt.n[7]};
2516 EIGEN_ALIGN16 int third_loader[4] = {vt.n[8], vt.n[9], vt.n[10], vt.n[11]};
2517 EIGEN_ALIGN16 int fourth_loader[4] = {vt.n[12], vt.n[13], vt.n[14], vt.n[15]};
2518
2519 Packet4i first_quarter = pload<Packet4i>(first_loader);
2520 Packet4i second_quarter = pload<Packet4i>(second_loader);
2521 Packet4i third_quarter = pload<Packet4i>(third_loader);
2522 Packet4i fourth_quarter = pload<Packet4i>(fourth_loader);
2523
2524 return static_cast<__UNPACK_TYPE__(Packet)>(predux(first_quarter) + predux(second_quarter) + predux(third_quarter) +
2525 predux(fourth_quarter));
2526}
2527
2528template <>
2529EIGEN_STRONG_INLINE signed char predux<Packet16c>(const Packet16c& a) {
2530 return predux_size16<Packet16c>(a);
2531}
2532
2533template <>
2534EIGEN_STRONG_INLINE unsigned char predux<Packet16uc>(const Packet16uc& a) {
2535 return predux_size16<Packet16uc>(a);
2536}
2537
2538// Other reduction functions:
2539// mul
2540template <>
2541EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
2542 Packet4f prod;
2543 prod = pmul(a, vec_sld(a, a, 8));
2544 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
2545}
2546
2547template <>
2548EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) {
2549 EIGEN_ALIGN16 int aux[4];
2550 pstore(aux, a);
2551 return aux[0] * aux[1] * aux[2] * aux[3];
2552}
2553
2554template <>
2555EIGEN_STRONG_INLINE short int predux_mul<Packet8s>(const Packet8s& a) {
2556 Packet8s pair, quad, octo;
2557
2558 pair = vec_mul(a, vec_sld(a, a, 8));
2559 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2560 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2561
2562 return pfirst(octo);
2563}
2564
2565template <>
2566EIGEN_STRONG_INLINE unsigned short int predux_mul<Packet8us>(const Packet8us& a) {
2567 Packet8us pair, quad, octo;
2568
2569 pair = vec_mul(a, vec_sld(a, a, 8));
2570 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2571 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2572
2573 return pfirst(octo);
2574}
2575
2576template <>
2577EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet8bf>(const Packet8bf& a) {
2578 float redux_even = predux_mul<Packet4f>(Bf16ToF32Even(a));
2579 float redux_odd = predux_mul<Packet4f>(Bf16ToF32Odd(a));
2580 float f32_result = redux_even * redux_odd;
2581 return bfloat16(f32_result);
2582}
2583
2584template <>
2585EIGEN_STRONG_INLINE signed char predux_mul<Packet16c>(const Packet16c& a) {
2586 Packet16c pair, quad, octo, result;
2587
2588 pair = vec_mul(a, vec_sld(a, a, 8));
2589 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2590 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2591 result = vec_mul(octo, vec_sld(octo, octo, 1));
2592
2593 return pfirst(result);
2594}
2595
2596template <>
2597EIGEN_STRONG_INLINE unsigned char predux_mul<Packet16uc>(const Packet16uc& a) {
2598 Packet16uc pair, quad, octo, result;
2599
2600 pair = vec_mul(a, vec_sld(a, a, 8));
2601 quad = vec_mul(pair, vec_sld(pair, pair, 4));
2602 octo = vec_mul(quad, vec_sld(quad, quad, 2));
2603 result = vec_mul(octo, vec_sld(octo, octo, 1));
2604
2605 return pfirst(result);
2606}
2607
2608// min
2609template <typename Packet>
2610EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_min4(const Packet& a) {
2611 Packet b, res;
2612 b = vec_min(a, vec_sld(a, a, 8));
2613 res = vec_min(b, vec_sld(b, b, 4));
2614 return pfirst(res);
2615}
2616
2617template <>
2618EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
2619 return predux_min4<Packet4f>(a);
2620}
2621
2622template <>
2623EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) {
2624 return predux_min4<Packet4i>(a);
2625}
2626
2627template <>
2628EIGEN_STRONG_INLINE bfloat16 predux_min<Packet8bf>(const Packet8bf& a) {
2629 float redux_even = predux_min<Packet4f>(Bf16ToF32Even(a));
2630 float redux_odd = predux_min<Packet4f>(Bf16ToF32Odd(a));
2631 float f32_result = (std::min)(redux_even, redux_odd);
2632 return bfloat16(f32_result);
2633}
2634
2635template <>
2636EIGEN_STRONG_INLINE short int predux_min<Packet8s>(const Packet8s& a) {
2637 Packet8s pair, quad, octo;
2638
2639 // pair = { Min(a0,a4), Min(a1,a5), Min(a2,a6), Min(a3,a7) }
2640 pair = vec_min(a, vec_sld(a, a, 8));
2641
2642 // quad = { Min(a0, a4, a2, a6), Min(a1, a5, a3, a7) }
2643 quad = vec_min(pair, vec_sld(pair, pair, 4));
2644
2645 // octo = { Min(a0, a4, a2, a6, a1, a5, a3, a7) }
2646 octo = vec_min(quad, vec_sld(quad, quad, 2));
2647 return pfirst(octo);
2648}
2649
2650template <>
2651EIGEN_STRONG_INLINE unsigned short int predux_min<Packet8us>(const Packet8us& a) {
2652 Packet8us pair, quad, octo;
2653
2654 // pair = { Min(a0,a4), Min(a1,a5), Min(a2,a6), Min(a3,a7) }
2655 pair = vec_min(a, vec_sld(a, a, 8));
2656
2657 // quad = { Min(a0, a4, a2, a6), Min(a1, a5, a3, a7) }
2658 quad = vec_min(pair, vec_sld(pair, pair, 4));
2659
2660 // octo = { Min(a0, a4, a2, a6, a1, a5, a3, a7) }
2661 octo = vec_min(quad, vec_sld(quad, quad, 2));
2662 return pfirst(octo);
2663}
2664
2665template <>
2666EIGEN_STRONG_INLINE signed char predux_min<Packet16c>(const Packet16c& a) {
2667 Packet16c pair, quad, octo, result;
2668
2669 pair = vec_min(a, vec_sld(a, a, 8));
2670 quad = vec_min(pair, vec_sld(pair, pair, 4));
2671 octo = vec_min(quad, vec_sld(quad, quad, 2));
2672 result = vec_min(octo, vec_sld(octo, octo, 1));
2673
2674 return pfirst(result);
2675}
2676
2677template <>
2678EIGEN_STRONG_INLINE unsigned char predux_min<Packet16uc>(const Packet16uc& a) {
2679 Packet16uc pair, quad, octo, result;
2680
2681 pair = vec_min(a, vec_sld(a, a, 8));
2682 quad = vec_min(pair, vec_sld(pair, pair, 4));
2683 octo = vec_min(quad, vec_sld(quad, quad, 2));
2684 result = vec_min(octo, vec_sld(octo, octo, 1));
2685
2686 return pfirst(result);
2687}
2688// max
2689template <typename Packet>
2690EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_max4(const Packet& a) {
2691 Packet b, res;
2692 b = vec_max(a, vec_sld(a, a, 8));
2693 res = vec_max(b, vec_sld(b, b, 4));
2694 return pfirst(res);
2695}
2696
2697template <>
2698EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
2699 return predux_max4<Packet4f>(a);
2700}
2701
2702template <>
2703EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) {
2704 return predux_max4<Packet4i>(a);
2705}
2706
2707template <>
2708EIGEN_STRONG_INLINE bfloat16 predux_max<Packet8bf>(const Packet8bf& a) {
2709 float redux_even = predux_max<Packet4f>(Bf16ToF32Even(a));
2710 float redux_odd = predux_max<Packet4f>(Bf16ToF32Odd(a));
2711 float f32_result = (std::max)(redux_even, redux_odd);
2712 return bfloat16(f32_result);
2713}
2714
2715template <>
2716EIGEN_STRONG_INLINE short int predux_max<Packet8s>(const Packet8s& a) {
2717 Packet8s pair, quad, octo;
2718
2719 // pair = { Max(a0,a4), Max(a1,a5), Max(a2,a6), Max(a3,a7) }
2720 pair = vec_max(a, vec_sld(a, a, 8));
2721
2722 // quad = { Max(a0, a4, a2, a6), Max(a1, a5, a3, a7) }
2723 quad = vec_max(pair, vec_sld(pair, pair, 4));
2724
2725 // octo = { Max(a0, a4, a2, a6, a1, a5, a3, a7) }
2726 octo = vec_max(quad, vec_sld(quad, quad, 2));
2727 return pfirst(octo);
2728}
2729
2730template <>
2731EIGEN_STRONG_INLINE unsigned short int predux_max<Packet8us>(const Packet8us& a) {
2732 Packet8us pair, quad, octo;
2733
2734 // pair = { Max(a0,a4), Max(a1,a5), Max(a2,a6), Max(a3,a7) }
2735 pair = vec_max(a, vec_sld(a, a, 8));
2736
2737 // quad = { Max(a0, a4, a2, a6), Max(a1, a5, a3, a7) }
2738 quad = vec_max(pair, vec_sld(pair, pair, 4));
2739
2740 // octo = { Max(a0, a4, a2, a6, a1, a5, a3, a7) }
2741 octo = vec_max(quad, vec_sld(quad, quad, 2));
2742 return pfirst(octo);
2743}
2744
2745template <>
2746EIGEN_STRONG_INLINE signed char predux_max<Packet16c>(const Packet16c& a) {
2747 Packet16c pair, quad, octo, result;
2748
2749 pair = vec_max(a, vec_sld(a, a, 8));
2750 quad = vec_max(pair, vec_sld(pair, pair, 4));
2751 octo = vec_max(quad, vec_sld(quad, quad, 2));
2752 result = vec_max(octo, vec_sld(octo, octo, 1));
2753
2754 return pfirst(result);
2755}
2756
2757template <>
2758EIGEN_STRONG_INLINE unsigned char predux_max<Packet16uc>(const Packet16uc& a) {
2759 Packet16uc pair, quad, octo, result;
2760
2761 pair = vec_max(a, vec_sld(a, a, 8));
2762 quad = vec_max(pair, vec_sld(pair, pair, 4));
2763 octo = vec_max(quad, vec_sld(quad, quad, 2));
2764 result = vec_max(octo, vec_sld(octo, octo, 1));
2765
2766 return pfirst(result);
2767}
2768
2769template <>
2770EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x) {
2771 const Packet4ui zero = {0, 0, 0, 0};
2772 return vec_any_ne(reinterpret_cast<Packet4ui>(x), zero);
2773}
2774
2775template <>
2776EIGEN_STRONG_INLINE bool predux_any(const Packet8bf& x) {
2777 const Packet8us zero = {0, 0, 0, 0, 0, 0, 0, 0};
2778 return vec_any_ne(x.m_val, zero);
2779}
2780
2781template <typename T>
2782EIGEN_DEVICE_FUNC inline void ptranpose_common(PacketBlock<T, 4>& kernel) {
2783 T t0, t1, t2, t3;
2784 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2785 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2786 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2787 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2788 kernel.packet[0] = vec_mergeh(t0, t2);
2789 kernel.packet[1] = vec_mergel(t0, t2);
2790 kernel.packet[2] = vec_mergeh(t1, t3);
2791 kernel.packet[3] = vec_mergel(t1, t3);
2792}
2793
2794EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) { ptranpose_common<Packet4f>(kernel); }
2795
2796EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) { ptranpose_common<Packet4i>(kernel); }
2797
2798EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
2799 Packet8s t0, t1, t2, t3;
2800 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2801 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2802 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2803 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2804 kernel.packet[0] = vec_mergeh(t0, t2);
2805 kernel.packet[1] = vec_mergel(t0, t2);
2806 kernel.packet[2] = vec_mergeh(t1, t3);
2807 kernel.packet[3] = vec_mergel(t1, t3);
2808}
2809
2810EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
2811 Packet8us t0, t1, t2, t3;
2812 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2813 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2814 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2815 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2816 kernel.packet[0] = vec_mergeh(t0, t2);
2817 kernel.packet[1] = vec_mergel(t0, t2);
2818 kernel.packet[2] = vec_mergeh(t1, t3);
2819 kernel.packet[3] = vec_mergel(t1, t3);
2820}
2821
2822EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8bf, 4>& kernel) {
2823 Packet8us t0, t1, t2, t3;
2824
2825 t0 = vec_mergeh(kernel.packet[0].m_val, kernel.packet[2].m_val);
2826 t1 = vec_mergel(kernel.packet[0].m_val, kernel.packet[2].m_val);
2827 t2 = vec_mergeh(kernel.packet[1].m_val, kernel.packet[3].m_val);
2828 t3 = vec_mergel(kernel.packet[1].m_val, kernel.packet[3].m_val);
2829 kernel.packet[0] = vec_mergeh(t0, t2);
2830 kernel.packet[1] = vec_mergel(t0, t2);
2831 kernel.packet[2] = vec_mergeh(t1, t3);
2832 kernel.packet[3] = vec_mergel(t1, t3);
2833}
2834
2835EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
2836 Packet16c t0, t1, t2, t3;
2837 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2838 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2839 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2840 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2841 kernel.packet[0] = vec_mergeh(t0, t2);
2842 kernel.packet[1] = vec_mergel(t0, t2);
2843 kernel.packet[2] = vec_mergeh(t1, t3);
2844 kernel.packet[3] = vec_mergel(t1, t3);
2845}
2846
2847EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
2848 Packet16uc t0, t1, t2, t3;
2849 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
2850 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
2851 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
2852 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
2853 kernel.packet[0] = vec_mergeh(t0, t2);
2854 kernel.packet[1] = vec_mergel(t0, t2);
2855 kernel.packet[2] = vec_mergeh(t1, t3);
2856 kernel.packet[3] = vec_mergel(t1, t3);
2857}
2858
2859EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
2860 Packet8s v[8], sum[8];
2861
2862 v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]);
2863 v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]);
2864 v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]);
2865 v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]);
2866 v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]);
2867 v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]);
2868 v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]);
2869 v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]);
2870 sum[0] = vec_mergeh(v[0], v[4]);
2871 sum[1] = vec_mergel(v[0], v[4]);
2872 sum[2] = vec_mergeh(v[1], v[5]);
2873 sum[3] = vec_mergel(v[1], v[5]);
2874 sum[4] = vec_mergeh(v[2], v[6]);
2875 sum[5] = vec_mergel(v[2], v[6]);
2876 sum[6] = vec_mergeh(v[3], v[7]);
2877 sum[7] = vec_mergel(v[3], v[7]);
2878
2879 kernel.packet[0] = vec_mergeh(sum[0], sum[4]);
2880 kernel.packet[1] = vec_mergel(sum[0], sum[4]);
2881 kernel.packet[2] = vec_mergeh(sum[1], sum[5]);
2882 kernel.packet[3] = vec_mergel(sum[1], sum[5]);
2883 kernel.packet[4] = vec_mergeh(sum[2], sum[6]);
2884 kernel.packet[5] = vec_mergel(sum[2], sum[6]);
2885 kernel.packet[6] = vec_mergeh(sum[3], sum[7]);
2886 kernel.packet[7] = vec_mergel(sum[3], sum[7]);
2887}
2888
2889EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
2890 Packet8us v[8], sum[8];
2891
2892 v[0] = vec_mergeh(kernel.packet[0], kernel.packet[4]);
2893 v[1] = vec_mergel(kernel.packet[0], kernel.packet[4]);
2894 v[2] = vec_mergeh(kernel.packet[1], kernel.packet[5]);
2895 v[3] = vec_mergel(kernel.packet[1], kernel.packet[5]);
2896 v[4] = vec_mergeh(kernel.packet[2], kernel.packet[6]);
2897 v[5] = vec_mergel(kernel.packet[2], kernel.packet[6]);
2898 v[6] = vec_mergeh(kernel.packet[3], kernel.packet[7]);
2899 v[7] = vec_mergel(kernel.packet[3], kernel.packet[7]);
2900 sum[0] = vec_mergeh(v[0], v[4]);
2901 sum[1] = vec_mergel(v[0], v[4]);
2902 sum[2] = vec_mergeh(v[1], v[5]);
2903 sum[3] = vec_mergel(v[1], v[5]);
2904 sum[4] = vec_mergeh(v[2], v[6]);
2905 sum[5] = vec_mergel(v[2], v[6]);
2906 sum[6] = vec_mergeh(v[3], v[7]);
2907 sum[7] = vec_mergel(v[3], v[7]);
2908
2909 kernel.packet[0] = vec_mergeh(sum[0], sum[4]);
2910 kernel.packet[1] = vec_mergel(sum[0], sum[4]);
2911 kernel.packet[2] = vec_mergeh(sum[1], sum[5]);
2912 kernel.packet[3] = vec_mergel(sum[1], sum[5]);
2913 kernel.packet[4] = vec_mergeh(sum[2], sum[6]);
2914 kernel.packet[5] = vec_mergel(sum[2], sum[6]);
2915 kernel.packet[6] = vec_mergeh(sum[3], sum[7]);
2916 kernel.packet[7] = vec_mergel(sum[3], sum[7]);
2917}
2918
2919EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8bf, 8>& kernel) {
2920 Packet8bf v[8], sum[8];
2921
2922 v[0] = vec_mergeh(kernel.packet[0].m_val, kernel.packet[4].m_val);
2923 v[1] = vec_mergel(kernel.packet[0].m_val, kernel.packet[4].m_val);
2924 v[2] = vec_mergeh(kernel.packet[1].m_val, kernel.packet[5].m_val);
2925 v[3] = vec_mergel(kernel.packet[1].m_val, kernel.packet[5].m_val);
2926 v[4] = vec_mergeh(kernel.packet[2].m_val, kernel.packet[6].m_val);
2927 v[5] = vec_mergel(kernel.packet[2].m_val, kernel.packet[6].m_val);
2928 v[6] = vec_mergeh(kernel.packet[3].m_val, kernel.packet[7].m_val);
2929 v[7] = vec_mergel(kernel.packet[3].m_val, kernel.packet[7].m_val);
2930 sum[0] = vec_mergeh(v[0].m_val, v[4].m_val);
2931 sum[1] = vec_mergel(v[0].m_val, v[4].m_val);
2932 sum[2] = vec_mergeh(v[1].m_val, v[5].m_val);
2933 sum[3] = vec_mergel(v[1].m_val, v[5].m_val);
2934 sum[4] = vec_mergeh(v[2].m_val, v[6].m_val);
2935 sum[5] = vec_mergel(v[2].m_val, v[6].m_val);
2936 sum[6] = vec_mergeh(v[3].m_val, v[7].m_val);
2937 sum[7] = vec_mergel(v[3].m_val, v[7].m_val);
2938
2939 kernel.packet[0] = vec_mergeh(sum[0].m_val, sum[4].m_val);
2940 kernel.packet[1] = vec_mergel(sum[0].m_val, sum[4].m_val);
2941 kernel.packet[2] = vec_mergeh(sum[1].m_val, sum[5].m_val);
2942 kernel.packet[3] = vec_mergel(sum[1].m_val, sum[5].m_val);
2943 kernel.packet[4] = vec_mergeh(sum[2].m_val, sum[6].m_val);
2944 kernel.packet[5] = vec_mergel(sum[2].m_val, sum[6].m_val);
2945 kernel.packet[6] = vec_mergeh(sum[3].m_val, sum[7].m_val);
2946 kernel.packet[7] = vec_mergel(sum[3].m_val, sum[7].m_val);
2947}
2948
2949EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
2950 Packet16c step1[16], step2[16], step3[16];
2951
2952 step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]);
2953 step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]);
2954 step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]);
2955 step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]);
2956 step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]);
2957 step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]);
2958 step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]);
2959 step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]);
2960 step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]);
2961 step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]);
2962 step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]);
2963 step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]);
2964 step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]);
2965 step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]);
2966 step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]);
2967 step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]);
2968
2969 step2[0] = vec_mergeh(step1[0], step1[8]);
2970 step2[1] = vec_mergel(step1[0], step1[8]);
2971 step2[2] = vec_mergeh(step1[1], step1[9]);
2972 step2[3] = vec_mergel(step1[1], step1[9]);
2973 step2[4] = vec_mergeh(step1[2], step1[10]);
2974 step2[5] = vec_mergel(step1[2], step1[10]);
2975 step2[6] = vec_mergeh(step1[3], step1[11]);
2976 step2[7] = vec_mergel(step1[3], step1[11]);
2977 step2[8] = vec_mergeh(step1[4], step1[12]);
2978 step2[9] = vec_mergel(step1[4], step1[12]);
2979 step2[10] = vec_mergeh(step1[5], step1[13]);
2980 step2[11] = vec_mergel(step1[5], step1[13]);
2981 step2[12] = vec_mergeh(step1[6], step1[14]);
2982 step2[13] = vec_mergel(step1[6], step1[14]);
2983 step2[14] = vec_mergeh(step1[7], step1[15]);
2984 step2[15] = vec_mergel(step1[7], step1[15]);
2985
2986 step3[0] = vec_mergeh(step2[0], step2[8]);
2987 step3[1] = vec_mergel(step2[0], step2[8]);
2988 step3[2] = vec_mergeh(step2[1], step2[9]);
2989 step3[3] = vec_mergel(step2[1], step2[9]);
2990 step3[4] = vec_mergeh(step2[2], step2[10]);
2991 step3[5] = vec_mergel(step2[2], step2[10]);
2992 step3[6] = vec_mergeh(step2[3], step2[11]);
2993 step3[7] = vec_mergel(step2[3], step2[11]);
2994 step3[8] = vec_mergeh(step2[4], step2[12]);
2995 step3[9] = vec_mergel(step2[4], step2[12]);
2996 step3[10] = vec_mergeh(step2[5], step2[13]);
2997 step3[11] = vec_mergel(step2[5], step2[13]);
2998 step3[12] = vec_mergeh(step2[6], step2[14]);
2999 step3[13] = vec_mergel(step2[6], step2[14]);
3000 step3[14] = vec_mergeh(step2[7], step2[15]);
3001 step3[15] = vec_mergel(step2[7], step2[15]);
3002
3003 kernel.packet[0] = vec_mergeh(step3[0], step3[8]);
3004 kernel.packet[1] = vec_mergel(step3[0], step3[8]);
3005 kernel.packet[2] = vec_mergeh(step3[1], step3[9]);
3006 kernel.packet[3] = vec_mergel(step3[1], step3[9]);
3007 kernel.packet[4] = vec_mergeh(step3[2], step3[10]);
3008 kernel.packet[5] = vec_mergel(step3[2], step3[10]);
3009 kernel.packet[6] = vec_mergeh(step3[3], step3[11]);
3010 kernel.packet[7] = vec_mergel(step3[3], step3[11]);
3011 kernel.packet[8] = vec_mergeh(step3[4], step3[12]);
3012 kernel.packet[9] = vec_mergel(step3[4], step3[12]);
3013 kernel.packet[10] = vec_mergeh(step3[5], step3[13]);
3014 kernel.packet[11] = vec_mergel(step3[5], step3[13]);
3015 kernel.packet[12] = vec_mergeh(step3[6], step3[14]);
3016 kernel.packet[13] = vec_mergel(step3[6], step3[14]);
3017 kernel.packet[14] = vec_mergeh(step3[7], step3[15]);
3018 kernel.packet[15] = vec_mergel(step3[7], step3[15]);
3019}
3020
3021EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
3022 Packet16uc step1[16], step2[16], step3[16];
3023
3024 step1[0] = vec_mergeh(kernel.packet[0], kernel.packet[8]);
3025 step1[1] = vec_mergel(kernel.packet[0], kernel.packet[8]);
3026 step1[2] = vec_mergeh(kernel.packet[1], kernel.packet[9]);
3027 step1[3] = vec_mergel(kernel.packet[1], kernel.packet[9]);
3028 step1[4] = vec_mergeh(kernel.packet[2], kernel.packet[10]);
3029 step1[5] = vec_mergel(kernel.packet[2], kernel.packet[10]);
3030 step1[6] = vec_mergeh(kernel.packet[3], kernel.packet[11]);
3031 step1[7] = vec_mergel(kernel.packet[3], kernel.packet[11]);
3032 step1[8] = vec_mergeh(kernel.packet[4], kernel.packet[12]);
3033 step1[9] = vec_mergel(kernel.packet[4], kernel.packet[12]);
3034 step1[10] = vec_mergeh(kernel.packet[5], kernel.packet[13]);
3035 step1[11] = vec_mergel(kernel.packet[5], kernel.packet[13]);
3036 step1[12] = vec_mergeh(kernel.packet[6], kernel.packet[14]);
3037 step1[13] = vec_mergel(kernel.packet[6], kernel.packet[14]);
3038 step1[14] = vec_mergeh(kernel.packet[7], kernel.packet[15]);
3039 step1[15] = vec_mergel(kernel.packet[7], kernel.packet[15]);
3040
3041 step2[0] = vec_mergeh(step1[0], step1[8]);
3042 step2[1] = vec_mergel(step1[0], step1[8]);
3043 step2[2] = vec_mergeh(step1[1], step1[9]);
3044 step2[3] = vec_mergel(step1[1], step1[9]);
3045 step2[4] = vec_mergeh(step1[2], step1[10]);
3046 step2[5] = vec_mergel(step1[2], step1[10]);
3047 step2[6] = vec_mergeh(step1[3], step1[11]);
3048 step2[7] = vec_mergel(step1[3], step1[11]);
3049 step2[8] = vec_mergeh(step1[4], step1[12]);
3050 step2[9] = vec_mergel(step1[4], step1[12]);
3051 step2[10] = vec_mergeh(step1[5], step1[13]);
3052 step2[11] = vec_mergel(step1[5], step1[13]);
3053 step2[12] = vec_mergeh(step1[6], step1[14]);
3054 step2[13] = vec_mergel(step1[6], step1[14]);
3055 step2[14] = vec_mergeh(step1[7], step1[15]);
3056 step2[15] = vec_mergel(step1[7], step1[15]);
3057
3058 step3[0] = vec_mergeh(step2[0], step2[8]);
3059 step3[1] = vec_mergel(step2[0], step2[8]);
3060 step3[2] = vec_mergeh(step2[1], step2[9]);
3061 step3[3] = vec_mergel(step2[1], step2[9]);
3062 step3[4] = vec_mergeh(step2[2], step2[10]);
3063 step3[5] = vec_mergel(step2[2], step2[10]);
3064 step3[6] = vec_mergeh(step2[3], step2[11]);
3065 step3[7] = vec_mergel(step2[3], step2[11]);
3066 step3[8] = vec_mergeh(step2[4], step2[12]);
3067 step3[9] = vec_mergel(step2[4], step2[12]);
3068 step3[10] = vec_mergeh(step2[5], step2[13]);
3069 step3[11] = vec_mergel(step2[5], step2[13]);
3070 step3[12] = vec_mergeh(step2[6], step2[14]);
3071 step3[13] = vec_mergel(step2[6], step2[14]);
3072 step3[14] = vec_mergeh(step2[7], step2[15]);
3073 step3[15] = vec_mergel(step2[7], step2[15]);
3074
3075 kernel.packet[0] = vec_mergeh(step3[0], step3[8]);
3076 kernel.packet[1] = vec_mergel(step3[0], step3[8]);
3077 kernel.packet[2] = vec_mergeh(step3[1], step3[9]);
3078 kernel.packet[3] = vec_mergel(step3[1], step3[9]);
3079 kernel.packet[4] = vec_mergeh(step3[2], step3[10]);
3080 kernel.packet[5] = vec_mergel(step3[2], step3[10]);
3081 kernel.packet[6] = vec_mergeh(step3[3], step3[11]);
3082 kernel.packet[7] = vec_mergel(step3[3], step3[11]);
3083 kernel.packet[8] = vec_mergeh(step3[4], step3[12]);
3084 kernel.packet[9] = vec_mergel(step3[4], step3[12]);
3085 kernel.packet[10] = vec_mergeh(step3[5], step3[13]);
3086 kernel.packet[11] = vec_mergel(step3[5], step3[13]);
3087 kernel.packet[12] = vec_mergeh(step3[6], step3[14]);
3088 kernel.packet[13] = vec_mergel(step3[6], step3[14]);
3089 kernel.packet[14] = vec_mergeh(step3[7], step3[15]);
3090 kernel.packet[15] = vec_mergel(step3[7], step3[15]);
3091}
3092
3093//---------- double ----------
3094#ifdef EIGEN_VECTORIZE_VSX
3095using Packet2d = __vector double;
3096using Packet2ul = __vector unsigned long long;
3097using Packet2l = __vector long long;
3098#if EIGEN_COMP_CLANG
3099using Packet2bl = Packet2ul;
3100#else
3101using Packet2bl = __vector __bool long;
3102#endif
3103
3104static Packet2l p2l_ZERO = reinterpret_cast<Packet2l>(p4i_ZERO);
3105static Packet2ul p2ul_SIGN = {0x8000000000000000ull, 0x8000000000000000ull};
3106static Packet2ul p2ul_PREV0DOT5 = {0x3FDFFFFFFFFFFFFFull, 0x3FDFFFFFFFFFFFFFull};
3107static Packet2d p2d_ONE = {1.0, 1.0};
3108static Packet2d p2d_ZERO = reinterpret_cast<Packet2d>(p4f_ZERO);
3109static Packet2d p2d_MZERO = {numext::bit_cast<double>(0x8000000000000000ull),
3110 numext::bit_cast<double>(0x8000000000000000ull)};
3111
3112#ifdef _BIG_ENDIAN
3113static Packet2d p2d_COUNTDOWN =
3114 reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(p2d_ZERO), reinterpret_cast<Packet4f>(p2d_ONE), 8));
3115#else
3116static Packet2d p2d_COUNTDOWN =
3117 reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(p2d_ONE), reinterpret_cast<Packet4f>(p2d_ZERO), 8));
3118#endif
3119
3120template <int index>
3121Packet2d vec_splat_dbl(const Packet2d& a) {
3122 return vec_splat(a, index);
3123}
3124
3125template <>
3126struct packet_traits<double> : default_packet_traits {
3127 using type = Packet2d;
3128 using half = Packet2d;
3129 enum {
3130 Vectorizable = 1,
3131 AlignedOnScalar = 1,
3132 size = 2,
3133
3134 HasAdd = 1,
3135 HasSub = 1,
3136 HasMul = 1,
3137 HasDiv = 1,
3138 HasMin = 1,
3139 HasMax = 1,
3140 HasAbs = 1,
3141 HasSin = EIGEN_FAST_MATH,
3142 HasCos = EIGEN_FAST_MATH,
3143 HasTan = EIGEN_FAST_MATH,
3144 HasTanh = EIGEN_FAST_MATH,
3145 HasErf = EIGEN_FAST_MATH,
3146 HasErfc = EIGEN_FAST_MATH,
3147 HasATanh = 1,
3148 HasATan = 0,
3149 HasCmp = 1,
3150 HasLog = 1,
3151 HasExp = 1,
3152 HasLog1p = 1,
3153 HasExpm1 = 1,
3154 HasSqrt = 1,
3155 HasCbrt = 1,
3156#if !EIGEN_COMP_CLANG
3157 HasRsqrt = 1,
3158#else
3159 HasRsqrt = 0,
3160#endif
3161 HasNegate = 1,
3162 };
3163};
3164
3165template <>
3166struct unpacket_traits<Packet2d> {
3167 using type = double;
3168 using integer_packet = Packet2l;
3169 enum {
3170 size = 2,
3171 alignment = Aligned16,
3172 vectorizable = true,
3173 masked_load_available = false,
3174 masked_store_available = false
3175 };
3176 using half = Packet2d;
3177};
3178template <>
3179struct unpacket_traits<Packet2l> {
3180 using type = int64_t;
3181 using half = Packet2l;
3182 enum {
3183 size = 2,
3184 alignment = Aligned16,
3185 vectorizable = false,
3186 masked_load_available = false,
3187 masked_store_available = false
3188 };
3189};
3190
3191inline std::ostream& operator<<(std::ostream& s, const Packet2l& v) {
3192 union {
3193 Packet2l v;
3194 int64_t n[2];
3195 } vt;
3196 vt.v = v;
3197 s << vt.n[0] << ", " << vt.n[1];
3198 return s;
3199}
3200
3201inline std::ostream& operator<<(std::ostream& s, const Packet2d& v) {
3202 union {
3203 Packet2d v;
3204 double n[2];
3205 } vt;
3206 vt.v = v;
3207 s << vt.n[0] << ", " << vt.n[1];
3208 return s;
3209}
3210
3211// Need to define them first or we get specialization after instantiation errors
3212template <>
3213EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
3214 EIGEN_DEBUG_ALIGNED_LOAD
3215 return vec_xl(0, const_cast<double*>(from)); // cast needed by Clang
3216}
3217
3218template <>
3219EIGEN_ALWAYS_INLINE Packet2d pload_partial<Packet2d>(const double* from, const Index n, const Index offset) {
3220 return pload_partial_common<Packet2d>(from, n, offset);
3221}
3222
3223template <>
3224EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
3225 EIGEN_DEBUG_ALIGNED_STORE
3226 vec_xst(from, 0, to);
3227}
3228
3229template <>
3230EIGEN_ALWAYS_INLINE void pstore_partial<double>(double* to, const Packet2d& from, const Index n, const Index offset) {
3231 pstore_partial_common<Packet2d>(to, from, n, offset);
3232}
3233
3234template <>
3235EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
3236 Packet2d v = {from, from};
3237 return v;
3238}
3239template <>
3240EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) {
3241 Packet2l v = {from, from};
3242 return v;
3243}
3244
3245template <>
3246EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(unsigned long from) {
3247 Packet2l v = {static_cast<long long>(from), static_cast<long long>(from)};
3248 return reinterpret_cast<Packet2d>(v);
3249}
3250
3251template <>
3252EIGEN_STRONG_INLINE void pbroadcast4<Packet2d>(const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2,
3253 Packet2d& a3) {
3254 // This way is faster than vec_splat (at least for doubles in Power 9)
3255 a0 = pset1<Packet2d>(a[0]);
3256 a1 = pset1<Packet2d>(a[1]);
3257 a2 = pset1<Packet2d>(a[2]);
3258 a3 = pset1<Packet2d>(a[3]);
3259}
3260
3261template <>
3262EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
3263 return pgather_common<Packet2d>(from, stride);
3264}
3265template <>
3266EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2d pgather_partial<double, Packet2d>(const double* from, Index stride,
3267 const Index n) {
3268 return pgather_common<Packet2d>(from, stride, n);
3269}
3270template <>
3271EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
3272 pscatter_common<Packet2d>(to, from, stride);
3273}
3274template <>
3275EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter_partial<double, Packet2d>(double* to, const Packet2d& from,
3276 Index stride, const Index n) {
3277 pscatter_common<Packet2d>(to, from, stride, n);
3278}
3279
3280template <>
3281EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
3282 return pset1<Packet2d>(a) + p2d_COUNTDOWN;
3283}
3284
3285template <>
3286EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
3287 return a + b;
3288}
3289
3290template <>
3291EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
3292 return a - b;
3293}
3294
3295template <>
3296EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
3297#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3298 return vec_neg(a);
3299#else
3300 return vec_xor(a, p2d_MZERO);
3301#endif
3302}
3303
3304template <>
3305EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
3306 return vec_madd(a, b, p2d_MZERO);
3307}
3308template <>
3309EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
3310 return vec_div(a, b);
3311}
3312
3313template <>
3314struct is_fused_madd<Packet2d> : true_type {};
3315template <>
3316EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
3317 return vec_madd(a, b, c);
3318}
3319template <>
3320EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
3321 return vec_msub(a, b, c);
3322}
3323template <>
3324EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
3325 return vec_nmsub(a, b, c);
3326}
3327template <>
3328EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
3329 return vec_nmadd(a, b, c);
3330}
3331
3332template <>
3333EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
3334 // NOTE: about 10% slower than vec_min, but consistent with std::min and SSE regarding NaN
3335 Packet2d ret;
3336 __asm__("xvcmpgedp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa"(ret) : "wa"(a), "wa"(b));
3337 return ret;
3338}
3339
3340template <>
3341EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
3342 // NOTE: about 10% slower than vec_max, but consistent with std::max and SSE regarding NaN
3343 Packet2d ret;
3344 __asm__("xvcmpgtdp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa"(ret) : "wa"(a), "wa"(b));
3345 return ret;
3346}
3347
3348template <>
3349EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b) {
3350 return reinterpret_cast<Packet2d>(vec_cmple(a, b));
3351}
3352template <>
3353EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b) {
3354 return reinterpret_cast<Packet2d>(vec_cmplt(a, b));
3355}
3356template <>
3357EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) {
3358 return reinterpret_cast<Packet2d>(vec_cmpeq(a, b));
3359}
3360template <>
3361#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3362EIGEN_STRONG_INLINE Packet2l pcmp_eq(const Packet2l& a, const Packet2l& b) {
3363 return reinterpret_cast<Packet2l>(vec_cmpeq(a, b));
3364}
3365#else
3366EIGEN_STRONG_INLINE Packet2l pcmp_eq(const Packet2l& a, const Packet2l& b) {
3367 Packet4i halves = reinterpret_cast<Packet4i>(vec_cmpeq(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(b)));
3368 Packet4i flipped = vec_perm(halves, halves, p16uc_COMPLEX32_REV);
3369 return reinterpret_cast<Packet2l>(pand(halves, flipped));
3370}
3371#endif
3372// Not the generic a < b ? ptrue(a) : pzero(a): under Clang's default -faltivec-src-compat=mixed,
3373// a < b on vector long long is a scalar all-lanes predicate, not a lane mask.
3374template <>
3375#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3376EIGEN_STRONG_INLINE Packet2l pcmp_lt(const Packet2l& a, const Packet2l& b) {
3377 return reinterpret_cast<Packet2l>(vec_cmplt(a, b));
3378}
3379#else
3380EIGEN_STRONG_INLINE Packet2l pcmp_lt(const Packet2l& a, const Packet2l& b) {
3381 const Packet2l ret = {a[0] < b[0] ? -1 : 0, a[1] < b[1] ? -1 : 0};
3382 return ret;
3383}
3384#endif
3385template <>
3386EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) {
3387 Packet2d c = reinterpret_cast<Packet2d>(vec_cmpge(a, b));
3388 return vec_nor(c, c);
3389}
3390
3391template <>
3392EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
3393 return vec_and(a, b);
3394}
3395
3396template <>
3397EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
3398 return vec_or(a, b);
3399}
3400
3401template <>
3402EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
3403 return vec_xor(a, b);
3404}
3405
3406template <>
3407EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
3408 return vec_and(a, vec_nor(b, b));
3409}
3410
3411template <>
3412EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) {
3413 Packet2d t = vec_add(
3414 reinterpret_cast<Packet2d>(vec_or(vec_and(reinterpret_cast<Packet2ul>(a), p2ul_SIGN), p2ul_PREV0DOT5)), a);
3415 Packet2d res;
3416
3417 __asm__("xvrdpiz %x0, %x1\n\t" : "=&wa"(res) : "wa"(t));
3418
3419 return res;
3420}
3421template <>
3422EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
3423 return vec_ceil(a);
3424}
3425template <>
3426EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
3427 return vec_floor(a);
3428}
3429template <>
3430EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
3431 return vec_trunc(a);
3432}
3433template <>
3434EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
3435 Packet2d res;
3436
3437 __asm__("xvrdpic %x0, %x1\n\t" : "=&wa"(res) : "wa"(a));
3438
3439 return res;
3440}
3441
3442template <>
3443EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
3444 EIGEN_DEBUG_UNALIGNED_LOAD
3445 return vec_xl(0, const_cast<double*>(from));
3446}
3447
3448template <>
3449EIGEN_ALWAYS_INLINE Packet2d ploadu_partial<Packet2d>(const double* from, const Index n, const Index offset) {
3450 return ploadu_partial_common<Packet2d>(from, n, offset);
3451}
3452
3453template <>
3454EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) {
3455 return pset1<Packet2d>(from[0]);
3456}
3457
3458template <>
3459EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
3460 EIGEN_DEBUG_UNALIGNED_STORE
3461 vec_xst(from, 0, to);
3462}
3463
3464template <>
3465EIGEN_ALWAYS_INLINE void pstoreu_partial<double>(double* to, const Packet2d& from, const Index n, const Index offset) {
3466 pstoreu_partial_common<Packet2d>(to, from, n, offset);
3467}
3468
3469template <>
3470EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
3471 EIGEN_PPC_PREFETCH(addr);
3472}
3473
3474template <>
3475EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
3476 EIGEN_ALIGN16 double x[2];
3477 pstore<double>(x, a);
3478 return x[0];
3479}
3480
3481template <>
3482EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
3483 return vec_sld(a, a, 8);
3484}
3485template <>
3486EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) {
3487 return vec_abs(a);
3488}
3489#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3490template <>
3491EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) {
3492 return (Packet2d)vec_sra((Packet2l)a, vec_splats((unsigned long long)(63)));
3493}
3494#else
3495#ifdef _BIG_ENDIAN
3496static Packet16uc p16uc_DUPSIGN = {0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8};
3497#else
3498static Packet16uc p16uc_DUPSIGN = {7, 7, 7, 7, 7, 7, 7, 7, 15, 15, 15, 15, 15, 15, 15, 15};
3499#endif
3500
3501template <>
3502EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) {
3503 Packet16c tmp = vec_sra(reinterpret_cast<Packet16c>(a), vec_splats((unsigned char)(7)));
3504 return reinterpret_cast<Packet2d>(vec_perm(tmp, tmp, p16uc_DUPSIGN));
3505}
3506#endif
3507
3508template <>
3509inline Packet2l pcast<Packet2d, Packet2l>(const Packet2d& x);
3510
3511template <>
3512inline Packet2d pcast<Packet2l, Packet2d>(const Packet2l& x);
3513
3514// Packet2l shifts.
3515// For POWER8 we simply use vec_sr/l.
3516//
3517// Things are more complicated for POWER7. There is actually a
3518// vec_xxsxdi intrinsic but it is not supported by some gcc versions.
3519// So we need to shift by N % 32 and rearrange bytes.
3520#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
3521
3522template <int N>
3523EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
3524 const Packet2ul shift = {N, N};
3525 return vec_sl(a, shift);
3526}
3527
3528template <int N>
3529EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
3530 const Packet2ul shift = {N, N};
3531 return vec_sr(a, shift);
3532}
3533
3534#else
3535
3536// Shifts [A, B, C, D] to [B, 0, D, 0].
3537// Used to implement left shifts for Packet2l.
3538EIGEN_ALWAYS_INLINE Packet4i shift_even_left(const Packet4i& a) {
3539 static const Packet16uc perm = {0x14, 0x15, 0x16, 0x17, 0x00, 0x01, 0x02, 0x03,
3540 0x1c, 0x1d, 0x1e, 0x1f, 0x08, 0x09, 0x0a, 0x0b};
3541#ifdef _BIG_ENDIAN
3542 return vec_perm(p4i_ZERO, a, perm);
3543#else
3544 return vec_perm(a, p4i_ZERO, perm);
3545#endif
3546}
3547
3548// Shifts [A, B, C, D] to [0, A, 0, C].
3549// Used to implement right shifts for Packet2l.
3550EIGEN_ALWAYS_INLINE Packet4i shift_odd_right(const Packet4i& a) {
3551 static const Packet16uc perm = {0x04, 0x05, 0x06, 0x07, 0x10, 0x11, 0x12, 0x13,
3552 0x0c, 0x0d, 0x0e, 0x0f, 0x18, 0x19, 0x1a, 0x1b};
3553#ifdef _BIG_ENDIAN
3554 return vec_perm(p4i_ZERO, a, perm);
3555#else
3556 return vec_perm(a, p4i_ZERO, perm);
3557#endif
3558}
3559
3560template <int N, typename EnableIf = void>
3561struct plogical_shift_left_impl;
3562
3563template <int N>
3564struct plogical_shift_left_impl<N, std::enable_if_t<(N < 32) && (N >= 0)> > {
3565 static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) {
3566 static const unsigned n = static_cast<unsigned>(N);
3567 const Packet4ui shift = {n, n, n, n};
3568 const Packet4i ai = reinterpret_cast<Packet4i>(a);
3569 static const unsigned m = static_cast<unsigned>(32 - N);
3570 const Packet4ui shift_right = {m, m, m, m};
3571 const Packet4i out_hi = vec_sl(ai, shift);
3572 const Packet4i out_lo = shift_even_left(vec_sr(ai, shift_right));
3573 return reinterpret_cast<Packet2l>(por<Packet4i>(out_hi, out_lo));
3574 }
3575};
3576
3577template <int N>
3578struct plogical_shift_left_impl<N, std::enable_if_t<(N >= 32)> > {
3579 static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) {
3580 static const unsigned m = static_cast<unsigned>(N - 32);
3581 const Packet4ui shift = {m, m, m, m};
3582 const Packet4i ai = reinterpret_cast<Packet4i>(a);
3583 return reinterpret_cast<Packet2l>(shift_even_left(vec_sl(ai, shift)));
3584 }
3585};
3586
3587template <int N>
3588EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
3589 return plogical_shift_left_impl<N>::run(a);
3590}
3591
3592template <int N, typename EnableIf = void>
3593struct plogical_shift_right_impl;
3594
3595template <int N>
3596struct plogical_shift_right_impl<N, std::enable_if_t<(N < 32) && (N >= 0)> > {
3597 static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) {
3598 static const unsigned n = static_cast<unsigned>(N);
3599 const Packet4ui shift = {n, n, n, n};
3600 const Packet4i ai = reinterpret_cast<Packet4i>(a);
3601 static const unsigned m = static_cast<unsigned>(32 - N);
3602 const Packet4ui shift_left = {m, m, m, m};
3603 const Packet4i out_lo = vec_sr(ai, shift);
3604 const Packet4i out_hi = shift_odd_right(vec_sl(ai, shift_left));
3605 return reinterpret_cast<Packet2l>(por<Packet4i>(out_hi, out_lo));
3606 }
3607};
3608
3609template <int N>
3610struct plogical_shift_right_impl<N, std::enable_if_t<(N >= 32)> > {
3611 static EIGEN_STRONG_INLINE Packet2l run(const Packet2l& a) {
3612 static const unsigned m = static_cast<unsigned>(N - 32);
3613 const Packet4ui shift = {m, m, m, m};
3614 const Packet4i ai = reinterpret_cast<Packet4i>(a);
3615 return reinterpret_cast<Packet2l>(shift_odd_right(vec_sr(ai, shift)));
3616 }
3617};
3618
3619template <int N>
3620EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
3621 return plogical_shift_right_impl<N>::run(a);
3622}
3623#endif
3624
3625template <>
3626EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
3627 // The single-rounding split of pldexp_generic on int64 exponents.
3628 const Packet2d max_exponent = pset1<Packet2d>(2099.0);
3629 const Packet2d last_max = pset1<Packet2d>(1022.0);
3630 const Packet2l e = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
3631 const Packet2l b = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(last_max)), last_max));
3632 const Packet2l bias = {1023, 1023};
3633 const Packet2l even = {-2, -2};
3634 const Packet2l t = psub(e, b) & even;
3635 const Packet2d c1 = reinterpret_cast<Packet2d>(plogical_shift_left<51>(t + bias + bias)); // 2^(t/2)
3636 const Packet2d c2 = reinterpret_cast<Packet2d>(plogical_shift_left<52>(psub(e, t) + bias)); // 2^(e - t)
3637 return pldexp_apply_factors(a, c1, c2); // a * 2^e
3638}
3639
3640// Extract exponent without existence of Packet2l.
3641template <>
3642EIGEN_STRONG_INLINE Packet2d pfrexp_generic_get_biased_exponent(const Packet2d& a) {
3643 return pcast<Packet2l, Packet2d>(plogical_shift_right<52>(reinterpret_cast<Packet2l>(pabs(a))));
3644}
3645
3646template <>
3647EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent) {
3648 return pfrexp_generic(a, exponent);
3649}
3650
3651template <>
3652EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
3653 Packet2d b, sum;
3654 b = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(a), reinterpret_cast<Packet4f>(a), 8));
3655 sum = a + b;
3656 return pfirst<Packet2d>(sum);
3657}
3658
3659template <>
3660EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) {
3661 // A 64-bit lane is nonzero iff one of its 32-bit halves is; the doubleword compare needs POWER8.
3662 const Packet4ui zero = {0, 0, 0, 0};
3663 return vec_any_ne(reinterpret_cast<Packet4ui>(a), zero);
3664}
3665
3666// Other reduction functions:
3667// mul
3668template <>
3669EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
3670 return pfirst(
3671 pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
3672}
3673
3674// min
3675template <>
3676EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
3677 return pfirst(
3678 pmin(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
3679}
3680
3681// max
3682template <>
3683EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
3684 return pfirst(
3685 pmax(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
3686}
3687
3688EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
3689 Packet2d t0, t1;
3690 t0 = vec_mergeh(kernel.packet[0], kernel.packet[1]);
3691 t1 = vec_mergel(kernel.packet[0], kernel.packet[1]);
3692 kernel.packet[0] = t0;
3693 kernel.packet[1] = t1;
3694}
3695
3696#endif // __VSX__
3697} // end namespace internal
3698
3699} // end namespace Eigen
3700
3701#endif // EIGEN_PACKET_MATH_ALTIVEC_H
@ Aligned16
Definition Constants.h:238