Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Konstantinos Margaritis <markos@freevec.org>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_PACKET_MATH_ZVECTOR_H
12#define EIGEN_PACKET_MATH_ZVECTOR_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 16
23#endif
24
25#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
26#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
27#endif
28
29#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
30#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
31#endif
32
33using Packet4i = __vector int;
34using Packet4ui = __vector unsigned int;
35using Packet4bi = __vector __bool int;
36using Packet8i = __vector short int;
37using Packet16uc = __vector unsigned char;
38using Packet2d = __vector double;
39using Packet2ul = __vector unsigned long long;
40using Packet2l = __vector long long;
41
42// Z14 has builtin support for float vectors
43using Packet4f = __vector float;
44
45typedef union {
46 numext::int32_t i[4];
47 numext::uint32_t ui[4];
48 numext::int64_t l[2];
49 numext::uint64_t ul[2];
50 double d[2];
51 float f[4];
52 Packet4i v4i;
53 Packet4ui v4ui;
54 Packet2l v2l;
55 Packet2ul v2ul;
56 Packet2d v2d;
57 Packet4f v4f;
58} Packet;
59
60// We don't want to write the same code all the time, but we need to reuse the constants
61// and it doesn't really work to declare them global, so we define macros instead
62
63#define EIGEN_DECLARE_CONST_FAST_Packet4i(NAME, X) Packet4i p4i_##NAME = reinterpret_cast<Packet4i>(vec_splat_s32(X))
64
65#define EIGEN_DECLARE_CONST_FAST_Packet2d(NAME, X) Packet2d p2d_##NAME = reinterpret_cast<Packet2d>(vec_splat_s64(X))
66
67#define EIGEN_DECLARE_CONST_FAST_Packet2l(NAME, X) Packet2l p2l_##NAME = reinterpret_cast<Packet2l>(vec_splat_s64(X))
68
69#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) Packet4i p4i_##NAME = pset1<Packet4i>(X)
70
71#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) Packet2d p2d_##NAME = pset1<Packet2d>(X)
72
73#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) Packet2l p2l_##NAME = pset1<Packet2l>(X)
74
75// These constants are endian-agnostic
76static EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,}
77static EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1}
78
79static EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0);
80static EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0);
81static EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1);
82
83static Packet2d p2d_ONE = {1.0, 1.0};
84static Packet2d p2d_ZERO_ = {numext::bit_cast<double>(0x8000000000000000ull),
85 numext::bit_cast<double>(0x8000000000000000ull)};
86
87#define EIGEN_DECLARE_CONST_FAST_Packet4f(NAME, X) Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X))
88
89#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) Packet4f p4f_##NAME = pset1<Packet4f>(X)
90
91#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) \
92 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
93
94static EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0}
95static EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1, -1); //{ -1, -1, -1, -1}
96static Packet4f p4f_MZERO = {0x80000000, 0x80000000, 0x80000000, 0x80000000};
97
98static Packet4i p4i_COUNTDOWN = {0, 1, 2, 3};
99static Packet4f p4f_COUNTDOWN = {0.0, 1.0, 2.0, 3.0};
100static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(
101 vec_sld(reinterpret_cast<Packet16uc>(p2d_ZERO), reinterpret_cast<Packet16uc>(p2d_ONE), 8));
102
103static Packet16uc p16uc_PSET64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7};
104static Packet16uc p16uc_DUPLICATE32_HI = {0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7};
105
106// Mask alignment
107#define EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0
108
109#define EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & EIGEN_MASK_ALIGNMENT)
110
111// Handle endianness properly while loading constants
112// Define global static constants:
113
114static Packet16uc p16uc_FORWARD = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
115static Packet16uc p16uc_REVERSE32 = {12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3};
116static Packet16uc p16uc_REVERSE64 = {8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7};
117
118static Packet16uc p16uc_PSET32_WODD =
119 vec_sld((Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 2),
120 8); //{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
121static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc)vec_splat((Packet4ui)p16uc_FORWARD, 3),
122 8); //{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
123static Packet16uc p16uc_PSET64_LO = (Packet16uc)vec_mergel(
124 (Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
125static Packet16uc p16uc_TRANSPOSE64_HI = {0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23};
126static Packet16uc p16uc_TRANSPOSE64_LO = {8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31};
127
128static Packet16uc p16uc_COMPLEX32_REV =
129 vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
130
131static Packet16uc p16uc_COMPLEX32_REV2 =
132 vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
133
134#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
135#define EIGEN_ZVECTOR_PREFETCH(ADDR) __builtin_prefetch(ADDR);
136#else
137#define EIGEN_ZVECTOR_PREFETCH(ADDR) asm(" pfd [%[addr]]\n" ::[addr] "r"(ADDR) : "cc");
138#endif
139
140template <>
141struct packet_traits<int> : default_packet_traits {
142 using type = Packet4i;
143 using half = Packet4i;
144 enum {
145 Vectorizable = 1,
146 AlignedOnScalar = 1,
147 size = 4,
148
149 HasAdd = 1,
150 HasSub = 1,
151 HasMul = 1,
152 HasDiv = 1,
153 };
154};
155
156template <>
157struct packet_traits<float> : default_packet_traits {
158 using type = Packet4f;
159 using half = Packet4f;
160 enum {
161 Vectorizable = 1,
162 AlignedOnScalar = 1,
163 size = 4,
164
165 HasCmp = 1,
166 HasAdd = 1,
167 HasSub = 1,
168 HasMul = 1,
169 HasDiv = 1,
170 HasMin = 1,
171 HasMax = 1,
172 HasAbs = 1,
173 HasSin = 0,
174 HasCos = 0,
175 HasLog = 0,
176 HasExp = 1,
177 HasSqrt = 1,
178 HasRsqrt = 1,
179 HasTanh = 1,
180 HasErf = 1,
181 HasNegate = 1,
182 };
183};
184
185template <>
186struct packet_traits<double> : default_packet_traits {
187 using type = Packet2d;
188 using half = Packet2d;
189 enum {
190 Vectorizable = 1,
191 AlignedOnScalar = 1,
192 size = 2,
193
194 HasAdd = 1,
195 HasSub = 1,
196 HasMul = 1,
197 HasDiv = 1,
198 HasMin = 1,
199 HasMax = 1,
200 HasAbs = 1,
201 HasSin = 0,
202 HasCos = 0,
203 HasLog = 0,
204 HasExp = 1,
205 HasSqrt = 1,
206 HasRsqrt = 1,
207 HasNegate = 1,
208 };
209};
210
211template <>
212struct unpacket_traits<Packet4i> {
213 using type = int;
214 enum {
215 size = 4,
216 alignment = Aligned16,
217 vectorizable = true,
218 masked_load_available = false,
219 masked_store_available = false
220 };
221 using half = Packet4i;
222};
223template <>
224struct unpacket_traits<Packet4f> {
225 using type = float;
226 enum {
227 size = 4,
228 alignment = Aligned16,
229 vectorizable = true,
230 masked_load_available = false,
231 masked_store_available = false
232 };
233 using half = Packet4f;
234 using integer_packet = Packet4i;
235};
236template <>
237struct unpacket_traits<Packet2d> {
238 using type = double;
239 enum {
240 size = 2,
241 alignment = Aligned16,
242 vectorizable = true,
243 masked_load_available = false,
244 masked_store_available = false
245 };
246 using half = Packet2d;
247 using integer_packet = Packet2l;
248};
249// The integer_packet of Packet2d, for the generic bit-manipulation paths; int64 is not vectorized.
250template <>
251struct unpacket_traits<Packet2l> {
252 using type = numext::int64_t;
253 using half = Packet2l;
254 static constexpr int size = 2;
255 static constexpr int alignment = Aligned16;
256 static constexpr bool vectorizable = false;
257 static constexpr bool masked_load_available = false;
258 static constexpr bool masked_store_available = false;
259};
260
261/* Forward declaration */
262EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel);
263
264inline std::ostream& operator<<(std::ostream& s, const Packet4i& v) {
265 Packet vt;
266 vt.v4i = v;
267 s << vt.i[0] << ", " << vt.i[1] << ", " << vt.i[2] << ", " << vt.i[3];
268 return s;
269}
270
271inline std::ostream& operator<<(std::ostream& s, const Packet4ui& v) {
272 Packet vt;
273 vt.v4ui = v;
274 s << vt.ui[0] << ", " << vt.ui[1] << ", " << vt.ui[2] << ", " << vt.ui[3];
275 return s;
276}
277
278inline std::ostream& operator<<(std::ostream& s, const Packet2l& v) {
279 Packet vt;
280 vt.v2l = v;
281 s << vt.l[0] << ", " << vt.l[1];
282 return s;
283}
284
285inline std::ostream& operator<<(std::ostream& s, const Packet2ul& v) {
286 Packet vt;
287 vt.v2ul = v;
288 s << vt.ul[0] << ", " << vt.ul[1];
289 return s;
290}
291
292inline std::ostream& operator<<(std::ostream& s, const Packet2d& v) {
293 Packet vt;
294 vt.v2d = v;
295 s << vt.d[0] << ", " << vt.d[1];
296 return s;
297}
298
299inline std::ostream& operator<<(std::ostream& s, const Packet4f& v) {
300 Packet vt;
301 vt.v4f = v;
302 s << vt.f[0] << ", " << vt.f[1] << ", " << vt.f[2] << ", " << vt.f[3];
303 return s;
304}
305
306template <>
307EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) {
308 EIGEN_DEBUG_ALIGNED_LOAD
309 return vec_xl(0, from);
310}
311
312template <>
313EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
314 EIGEN_DEBUG_ALIGNED_LOAD
315 return vec_xl(0, from);
316}
317
318template <>
319EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) {
320 EIGEN_DEBUG_ALIGNED_STORE
321 vec_xst(from, 0, to);
322}
323
324template <>
325EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
326 EIGEN_DEBUG_ALIGNED_STORE
327 vec_xst(from, 0, to);
328}
329
330template <>
331EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) {
332 return pfrexp_generic(a, exponent);
333}
334
335template <>
336EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent) {
337 return pfrexp_generic(a, exponent);
338}
339
340template <>
341EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) {
342 return vec_splats(from);
343}
344template <>
345EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
346 return vec_splats(from);
347}
348template <>
349EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const numext::int64_t& from) {
350 return vec_splats(static_cast<long long>(from));
351}
352
353template <>
354EIGEN_STRONG_INLINE void pbroadcast4<Packet4i>(const int* a, Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3) {
355 a3 = pload<Packet4i>(a);
356 a0 = vec_splat(a3, 0);
357 a1 = vec_splat(a3, 1);
358 a2 = vec_splat(a3, 2);
359 a3 = vec_splat(a3, 3);
360}
361
362template <>
363EIGEN_STRONG_INLINE void pbroadcast4<Packet2d>(const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2,
364 Packet2d& a3) {
365 a1 = pload<Packet2d>(a);
366 a0 = vec_splat(a1, 0);
367 a1 = vec_splat(a1, 1);
368 a3 = pload<Packet2d>(a + 2);
369 a2 = vec_splat(a3, 0);
370 a3 = vec_splat(a3, 1);
371}
372
373template <>
374EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride) {
375 EIGEN_ALIGN16 int ai[4];
376 ai[0] = from[0 * stride];
377 ai[1] = from[1 * stride];
378 ai[2] = from[2 * stride];
379 ai[3] = from[3 * stride];
380 return pload<Packet4i>(ai);
381}
382
383template <>
384EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
385 EIGEN_ALIGN16 double af[2];
386 af[0] = from[0 * stride];
387 af[1] = from[1 * stride];
388 return pload<Packet2d>(af);
389}
390
391template <>
392EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride) {
393 EIGEN_ALIGN16 int ai[4];
394 pstore<int>((int*)ai, from);
395 to[0 * stride] = ai[0];
396 to[1 * stride] = ai[1];
397 to[2 * stride] = ai[2];
398 to[3 * stride] = ai[3];
399}
400
401template <>
402EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
403 EIGEN_ALIGN16 double af[2];
404 pstore<double>(af, from);
405 to[0 * stride] = af[0];
406 to[1 * stride] = af[1];
407}
408
409template <>
410EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
411 return (a + b);
412}
413template <>
414EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
415 return (a + b);
416}
417
418template <>
419EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
420 return (a - b);
421}
422template <>
423EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
424 return (a - b);
425}
426
427template <>
428EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
429 return (a * b);
430}
431template <>
432EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
433 return (a * b);
434}
435
436template <>
437EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) {
438 return (a / b);
439}
440template <>
441EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
442 return (a / b);
443}
444
445template <>
446EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
447 return (-a);
448}
449template <>
450EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
451 return (-a);
452}
453
454template <>
455EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
456 return padd<Packet4i>(pmul<Packet4i>(a, b), c);
457}
458template <>
459struct is_fused_madd<Packet2d> : true_type {};
460template <>
461EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
462 return vec_madd(a, b, c);
463}
464
465template <>
466EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) {
467 return padd<Packet4i>(pset1<Packet4i>(a), p4i_COUNTDOWN);
468}
469template <>
470EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
471 return padd<Packet2d>(pset1<Packet2d>(a), p2d_COUNTDOWN);
472}
473
474template <>
475EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
476 return vec_min(a, b);
477}
478// pmin/pmax follow std::min/std::max: they return a whenever either operand is NaN, so a NaN in a propagates through
479// clamps such as pmax(pmin(x, hi), lo). vec_min/vec_max (VFMIN/VFMAX mode 0 on z14) drop it.
480template <>
481EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
482 return __builtin_s390_vfmindb(a, b, 3); // mode 3: std::min
483}
484
485template <>
486EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
487 return vec_max(a, b);
488}
489template <>
490EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
491 return __builtin_s390_vfmaxdb(a, b, 3); // mode 3: std::max
492}
493
494template <>
495EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
496 return vec_and(a, b);
497}
498template <>
499EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
500 return vec_and(a, b);
501}
502
503template <>
504EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
505 return vec_or(a, b);
506}
507template <>
508EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
509 return vec_or(a, b);
510}
511
512template <>
513EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
514 return vec_xor(a, b);
515}
516template <>
517EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
518 return vec_xor(a, b);
519}
520
521template <>
522EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
523 return pand<Packet4i>(a, vec_nor(b, b));
524}
525template <>
526EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
527 return vec_and(a, vec_nor(b, b));
528}
529
530template <>
531EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) {
532 return (a + b);
533}
534template <>
535EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) {
536 return (a - b);
537}
538template <>
539EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) {
540 return vec_and(a, b);
541}
542template <>
543EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b) {
544 return vec_or(a, b);
545}
546template <>
547EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b) {
548 return vec_xor(a, b);
549}
550template <>
551EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b) {
552 return vec_and(a, vec_nor(b, b));
553}
554template <>
555EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(const Packet2l& a, const Packet2l& b) {
556 return reinterpret_cast<Packet2l>(vec_cmpeq(a, b));
557}
558template <>
559EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(const Packet2l& a, const Packet2l& b) {
560 return reinterpret_cast<Packet2l>(vec_cmplt(a, b));
561}
562template <>
563EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(const Packet2l& a, const Packet2l& b) {
564 return reinterpret_cast<Packet2l>(vec_cmple(a, b));
565}
566template <>
567EIGEN_STRONG_INLINE Packet2l pselect<Packet2l>(const Packet2l& mask, const Packet2l& a, const Packet2l& b) {
568 return vec_sel(b, a, reinterpret_cast<Packet2ul>(mask));
569}
570
571template <>
572struct cast_impl<Packet2l, Packet2d> {
573 EIGEN_DEVICE_FUNC static inline Packet2d run(const Packet2l& a) { return vec_double(a); }
574};
575
576template <>
577struct cast_impl<Packet2d, Packet2l> {
578 EIGEN_DEVICE_FUNC static inline Packet2l run(const Packet2d& a) {
579 return vec_signed(vec_sel(p2d_ZERO, a, vec_cmpeq(a, a))); // A NaN lane converts to 0.
580 }
581};
582
583template <>
584EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) {
585 /* Uses non-default rounding for vec_round */
586 return __builtin_s390_vfidb(a, 0, 1);
587}
588template <>
589EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
590 return vec_ceil(a);
591}
592template <>
593EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
594 return vec_floor(a);
595}
596template <>
597EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
598 // M5 = 4: round to nearest, ties to even.
599 return __builtin_s390_vfidb(a, 4, 4);
600}
601template <>
602EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
603 // M5 = 5: round toward zero.
604 return __builtin_s390_vfidb(a, 4, 5);
605}
606template <>
607EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan<Packet2d>(const Packet2d& a, const Packet2d& b) {
608 return pnot(pcmp_le(b, a));
609}
610
611template <>
612EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) {
613 return pload<Packet4i>(from);
614}
615template <>
616EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
617 return pload<Packet2d>(from);
618}
619
620template <>
621EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) {
622 unsigned long long bits;
623 memcpy(&bits, from, sizeof(bits));
624 const Packet4i p = reinterpret_cast<Packet4i>(vec_splats(bits));
625 return vec_mergeh(p, p);
626}
627
628template <>
629EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) {
630 return pset1<Packet2d>(from[0]);
631}
632
633template <>
634EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) {
635 pstore<int>(to, from);
636}
637template <>
638EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
639 pstore<double>(to, from);
640}
641
642template <>
643EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
644 EIGEN_ZVECTOR_PREFETCH(addr);
645}
646template <>
647EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
648 EIGEN_ZVECTOR_PREFETCH(addr);
649}
650
651template <int N>
652EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(const Packet2l& a) {
653 return Packet2l{parithmetic_shift_right<N>(a[0]), parithmetic_shift_right<N>(a[1])};
654}
655template <int N>
656EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) {
657 return Packet4i{parithmetic_shift_right<N>(a[0]), parithmetic_shift_right<N>(a[1]), parithmetic_shift_right<N>(a[2]),
658 parithmetic_shift_right<N>(a[3])};
659}
660
661template <int N>
662EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
663 return Packet2l{plogical_shift_right<N>(a[0]), plogical_shift_right<N>(a[1])};
664}
665template <int N>
666EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) {
667 return Packet4i{plogical_shift_right<N>(a[0]), plogical_shift_right<N>(a[1]), plogical_shift_right<N>(a[2]),
668 plogical_shift_right<N>(a[3])};
669}
670
671template <int N>
672EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
673 return Packet2l{plogical_shift_left<N>(a[0]), plogical_shift_left<N>(a[1])};
674}
675template <int N>
676EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) {
677 return Packet4i{plogical_shift_left<N>(a[0]), plogical_shift_left<N>(a[1]), plogical_shift_left<N>(a[2]),
678 plogical_shift_left<N>(a[3])};
679}
680
681template <>
682EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) {
683 EIGEN_ALIGN16 int x[4];
684 pstore(x, a);
685 return x[0];
686}
687template <>
688EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
689 EIGEN_ALIGN16 double x[2];
690 pstore(x, a);
691 return x[0];
692}
693
694template <>
695EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
696 return reinterpret_cast<Packet4i>(
697 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
698}
699
700template <>
701EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
702 return reinterpret_cast<Packet2d>(
703 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64));
704}
705
706template <>
707EIGEN_STRONG_INLINE Packet4i pabs<Packet4i>(const Packet4i& a) {
708 return vec_abs(a);
709}
710template <>
711EIGEN_STRONG_INLINE Packet2d pabs<Packet2d>(const Packet2d& a) {
712 return vec_abs(a);
713}
714
715template <>
716EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a) {
717 Packet4i b, sum;
718 b = vec_sld(a, a, 8);
719 sum = padd<Packet4i>(a, b);
720 b = vec_sld(sum, sum, 4);
721 sum = padd<Packet4i>(sum, b);
722 return pfirst(sum);
723}
724
725template <>
726EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
727 Packet2d b, sum;
728 b = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8));
729 sum = padd<Packet2d>(a, b);
730 return pfirst(sum);
731}
732
733template <>
734EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) {
735 const Packet2ul zero = {0, 0};
736 return vec_any_ne(reinterpret_cast<Packet2ul>(a), zero);
737}
738
739// Other reduction functions:
740// mul
741template <>
742EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a) {
743 EIGEN_ALIGN16 int aux[4];
744 pstore(aux, a);
745 return aux[0] * aux[1] * aux[2] * aux[3];
746}
747
748template <>
749EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
750 return pfirst(
751 pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
752}
753
754// min
755template <>
756EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a) {
757 Packet4i b, res;
758 b = pmin<Packet4i>(a, vec_sld(a, a, 8));
759 res = pmin<Packet4i>(b, vec_sld(b, b, 4));
760 return pfirst(res);
761}
762
763template <>
764EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
765 return pfirst(pmin<Packet2d>(
766 a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
767}
768
769// max
770template <>
771EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a) {
772 Packet4i b, res;
773 b = pmax<Packet4i>(a, vec_sld(a, a, 8));
774 res = pmax<Packet4i>(b, vec_sld(b, b, 4));
775 return pfirst(res);
776}
777
778// max
779template <>
780EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
781 return pfirst(pmax<Packet2d>(
782 a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
783}
784
785EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
786 Packet4i t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
787 Packet4i t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
788 Packet4i t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
789 Packet4i t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
790 kernel.packet[0] = vec_mergeh(t0, t2);
791 kernel.packet[1] = vec_mergel(t0, t2);
792 kernel.packet[2] = vec_mergeh(t1, t3);
793 kernel.packet[3] = vec_mergel(t1, t3);
794}
795
796EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
797 Packet2d t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
798 Packet2d t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
799 kernel.packet[0] = t0;
800 kernel.packet[1] = t1;
801}
802
803template <>
804EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
805 EIGEN_DEBUG_ALIGNED_LOAD
806 return vec_xl(0, from);
807}
808
809template <>
810EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
811 EIGEN_DEBUG_ALIGNED_STORE
812 vec_xst(from, 0, to);
813}
814
815template <>
816EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
817 return vec_splats(from);
818}
819
820template <>
821EIGEN_STRONG_INLINE void pbroadcast4<Packet4f>(const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
822 a3 = pload<Packet4f>(a);
823 a0 = vec_splat(a3, 0);
824 a1 = vec_splat(a3, 1);
825 a2 = vec_splat(a3, 2);
826 a3 = vec_splat(a3, 3);
827}
828
829template <>
830EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
831 EIGEN_ALIGN16 float af[4];
832 af[0] = from[0 * stride];
833 af[1] = from[1 * stride];
834 af[2] = from[2 * stride];
835 af[3] = from[3 * stride];
836 return pload<Packet4f>(af);
837}
838
839template <>
840EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
841 EIGEN_ALIGN16 float af[4];
842 pstore<float>((float*)af, from);
843 to[0 * stride] = af[0];
844 to[1 * stride] = af[1];
845 to[2 * stride] = af[2];
846 to[3 * stride] = af[3];
847}
848
849template <>
850EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
851 return (a + b);
852}
853template <>
854EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
855 return (a - b);
856}
857template <>
858EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
859 return (a * b);
860}
861template <>
862EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
863 return (a / b);
864}
865template <>
866EIGEN_STRONG_INLINE Packet4f pnegate<Packet4f>(const Packet4f& a) {
867 return (-a);
868}
869template <>
870struct is_fused_madd<Packet4f> : true_type {};
871template <>
872EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
873 return vec_madd(a, b, c);
874}
875
876template <>
877EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
878 return __builtin_s390_vfminsb(a, b, 3); // mode 3: std::min
879}
880template <>
881EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
882 return __builtin_s390_vfmaxsb(a, b, 3); // mode 3: std::max
883}
884template <>
885EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
886 return vec_and(a, b);
887}
888template <>
889EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
890 return vec_or(a, b);
891}
892template <>
893EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
894 return vec_xor(a, b);
895}
896template <>
897EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
898 return vec_and(a, vec_nor(b, b));
899}
900template <>
901EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) {
902 /* Uses non-default rounding for vec_round */
903 return __builtin_s390_vfisb(a, 0, 1);
904}
905template <>
906EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
907 return vec_ceil(a);
908}
909template <>
910EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
911 return vec_floor(a);
912}
913template <>
914EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
915 // M5 = 4: round to nearest, ties to even.
916 return __builtin_s390_vfisb(a, 4, 4);
917}
918template <>
919EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
920 // M5 = 5: round toward zero.
921 return __builtin_s390_vfisb(a, 4, 5);
922}
923template <>
924EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b) {
925 return pnot(pcmp_le(b, a));
926}
927template <>
928EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>(const Packet4f& a) {
929 return vec_abs(a);
930}
931template <>
932EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
933 EIGEN_ALIGN16 float x[4];
934 pstore(x, a);
935 return x[0];
936}
937
938template <>
939EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
940 unsigned long long bits;
941 memcpy(&bits, from, sizeof(bits));
942 const Packet4f p = reinterpret_cast<Packet4f>(vec_splats(bits));
943 return vec_mergeh(p, p);
944}
945
946template <>
947EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
948 return reinterpret_cast<Packet4f>(
949 vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
950}
951
952template <>
953EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
954 Packet4f b, sum;
955 b = vec_sld(a, a, 8);
956 sum = padd<Packet4f>(a, b);
957 b = vec_sld(sum, sum, 4);
958 sum = padd<Packet4f>(sum, b);
959 return pfirst(sum);
960}
961
962template <>
963EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) {
964 const Packet4ui zero = {0, 0, 0, 0};
965 return vec_any_ne(reinterpret_cast<Packet4ui>(a), zero);
966}
967
968// Other reduction functions:
969// mul
970template <>
971EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
972 Packet4f prod;
973 prod = pmul(a, vec_sld(a, a, 8));
974 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
975}
976
977// min
978template <>
979EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
980 Packet4f b, res;
981 b = pmin<Packet4f>(a, vec_sld(a, a, 8));
982 res = pmin<Packet4f>(b, vec_sld(b, b, 4));
983 return pfirst(res);
984}
985
986// max
987template <>
988EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
989 Packet4f b, res;
990 b = pmax<Packet4f>(a, vec_sld(a, a, 8));
991 res = pmax<Packet4f>(b, vec_sld(b, b, 4));
992 return pfirst(res);
993}
994
995EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
996 Packet4f t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
997 Packet4f t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
998 Packet4f t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
999 Packet4f t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
1000 kernel.packet[0] = vec_mergeh(t0, t2);
1001 kernel.packet[1] = vec_mergel(t0, t2);
1002 kernel.packet[2] = vec_mergeh(t1, t3);
1003 kernel.packet[3] = vec_mergel(t1, t3);
1004}
1005
1006template <>
1007EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
1008 return pldexp_generic(a, exponent);
1009}
1010
1011template <>
1012EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
1013 // The single-rounding split of pldexp_generic on int64 exponents.
1014 const Packet2d max_exponent = pset1<Packet2d>(2099.0);
1015 const Packet2d last_max = pset1<Packet2d>(1022.0);
1016 const Packet2l e = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
1017 const Packet2l b = pcast<Packet2d, Packet2l>(pmin(pmax(exponent, pnegate(last_max)), last_max));
1018 const Packet2l bias = {1023, 1023};
1019 const Packet2l even = {-2, -2};
1020 const Packet2l t = psub(e, b) & even;
1021 const Packet2d c1 = reinterpret_cast<Packet2d>(plogical_shift_left<51>(t + bias + bias)); // 2^(t/2)
1022 const Packet2d c2 = reinterpret_cast<Packet2d>(plogical_shift_left<52>(psub(e, t) + bias)); // 2^(e - t)
1023 return pldexp_apply_factors(a, c1, c2); // a * 2^e
1024}
1025
1026template <>
1027EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1028 EIGEN_ZVECTOR_PREFETCH(addr);
1029}
1030template <>
1031EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
1032 return pload<Packet4f>(from);
1033}
1034template <>
1035EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
1036 pstore<float>(to, from);
1037}
1038template <>
1039EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
1040 return padd<Packet4f>(pset1<Packet4f>(a), p4f_COUNTDOWN);
1041}
1042
1043#if !defined(vec_float) || !defined(__ARCH__) || (defined(__ARCH__) && __ARCH__ < 13)
1044#pragma GCC warning "float->int and int->float conversion is simulated. compile for z15 for improved performance"
1045template <>
1046struct cast_impl<Packet4i, Packet4f> {
1047 EIGEN_DEVICE_FUNC static inline Packet4f run(const Packet4i& a) {
1048 return Packet4f{float(a[0]), float(a[1]), float(a[2]), float(a[3])};
1049 }
1050};
1051
1052template <>
1053struct cast_impl<Packet4f, Packet4i> {
1054 // A NaN lane converts to 0: the scalar conversion would be undefined, and pexp's clamp lets NaN through.
1055 EIGEN_DEVICE_FUNC static inline Packet4i run(const Packet4f& a) {
1056 return Packet4i{to_int(a[0]), to_int(a[1]), to_int(a[2]), to_int(a[3])};
1057 }
1058 EIGEN_DEVICE_FUNC static inline int to_int(float x) { return x == x ? int(x) : 0; }
1059};
1060#else
1061template <>
1062struct cast_impl<Packet4i, Packet4f> {
1063 EIGEN_DEVICE_FUNC static inline Packet4f run(const Packet4i& a) { return vec_float(a); }
1064};
1065
1066template <>
1067struct cast_impl<Packet4f, Packet4i> {
1068 EIGEN_DEVICE_FUNC static inline Packet4i run(const Packet4f& a) {
1069 return vec_signed(vec_sel(p4f_ZERO, a, vec_cmpeq(a, a))); // A NaN lane converts to 0.
1070 }
1071};
1072#endif
1073
1074template <>
1075EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
1076 return pset1<Packet4f>(Eigen::numext::bit_cast<float>(from));
1077}
1078template <>
1079EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
1080 return pset1<Packet2d>(Eigen::numext::bit_cast<double>(from));
1081}
1082
1083} // end namespace internal
1084
1085} // end namespace Eigen
1086
1087#endif // EIGEN_PACKET_MATH_ZVECTOR_H
@ Aligned16
Definition Constants.h:238