Eigen  3.3.9
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Konstantinos Margaritis <markos@freevec.org>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_PACKET_MATH_ZVECTOR_H
11#define EIGEN_PACKET_MATH_ZVECTOR_H
12
13namespace Eigen {
14
15namespace internal {
16
17#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
18#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4
19#endif
20
21#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
22#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
23#endif
24
25#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD
26#define EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD
27#endif
28
29#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
30#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16
31#endif
32
33typedef __vector int Packet4i;
34typedef __vector unsigned int Packet4ui;
35typedef __vector __bool int Packet4bi;
36typedef __vector short int Packet8i;
37typedef __vector unsigned char Packet16uc;
38typedef __vector double Packet2d;
39typedef __vector unsigned long long Packet2ul;
40typedef __vector long long Packet2l;
41
42typedef struct {
43 Packet2d v4f[2];
44} Packet4f;
45
46typedef union {
47 numext::int32_t i[4];
48 numext::uint32_t ui[4];
49 numext::int64_t l[2];
50 numext::uint64_t ul[2];
51 double d[2];
52 Packet4i v4i;
53 Packet4ui v4ui;
54 Packet2l v2l;
55 Packet2ul v2ul;
56 Packet2d v2d;
57} Packet;
58
59// We don't want to write the same code all the time, but we need to reuse the constants
60// and it doesn't really work to declare them global, so we define macros instead
61
62#define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \
63 Packet4i p4i_##NAME = reinterpret_cast<Packet4i>(vec_splat_s32(X))
64
65#define _EIGEN_DECLARE_CONST_FAST_Packet2d(NAME,X) \
66 Packet2d p2d_##NAME = reinterpret_cast<Packet2d>(vec_splat_s64(X))
67
68#define _EIGEN_DECLARE_CONST_FAST_Packet2l(NAME,X) \
69 Packet2l p2l_##NAME = reinterpret_cast<Packet2l>(vec_splat_s64(X))
70
71#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
72 Packet4i p4i_##NAME = pset1<Packet4i>(X)
73
74#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \
75 Packet2d p2d_##NAME = pset1<Packet2d>(X)
76
77#define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \
78 Packet2l p2l_##NAME = pset1<Packet2l>(X)
79
80// These constants are endian-agnostic
81//static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,}
82static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE, 1); //{ 1, 1, 1, 1}
83
84static _EIGEN_DECLARE_CONST_FAST_Packet2d(ZERO, 0);
85static _EIGEN_DECLARE_CONST_FAST_Packet2l(ZERO, 0);
86static _EIGEN_DECLARE_CONST_FAST_Packet2l(ONE, 1);
87
88static Packet2d p2d_ONE = { 1.0, 1.0 };
89static Packet2d p2d_ZERO_ = { -0.0, -0.0 };
90
91static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 };
92static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 };
93static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet16uc>(p2d_ZERO), reinterpret_cast<Packet16uc>(p2d_ONE), 8));
94
95static Packet16uc p16uc_PSET64_HI = { 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };
96static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 };
97
98// Mask alignment
99#define _EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0
100
101#define _EIGEN_ALIGNED_PTR(x) ((std::ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT)
102
103// Handle endianness properly while loading constants
104// Define global static constants:
105
106static Packet16uc p16uc_FORWARD = { 0,1,2,3, 4,5,6,7, 8,9,10,11, 12,13,14,15 };
107static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 };
108static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
109
110static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
111static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
112/*static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
113
114static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };*/
115static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
116/*static Packet16uc p16uc_TRANSPOSE64_HI = vec_add(p16uc_PSET64_HI, p16uc_HALF64_0_16); //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
117static Packet16uc p16uc_TRANSPOSE64_LO = vec_add(p16uc_PSET64_LO, p16uc_HALF64_0_16); //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};*/
118static Packet16uc p16uc_TRANSPOSE64_HI = { 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
119static Packet16uc p16uc_TRANSPOSE64_LO = { 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};
120
121//static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
122
123//static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
124
125
126#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
127 #define EIGEN_ZVECTOR_PREFETCH(ADDR) __builtin_prefetch(ADDR);
128#else
129 #define EIGEN_ZVECTOR_PREFETCH(ADDR) asm( " pfd [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" );
130#endif
131
132template<> struct packet_traits<int> : default_packet_traits
133{
134 typedef Packet4i type;
135 typedef Packet4i half;
136 enum {
137 Vectorizable = 1,
138 AlignedOnScalar = 1,
139 size = 4,
140 HasHalfPacket = 0,
141
142 HasAdd = 1,
143 HasSub = 1,
144 HasMul = 1,
145 HasDiv = 1,
146 HasBlend = 1
147 };
148};
149
150template<> struct packet_traits<float> : default_packet_traits
151{
152 typedef Packet4f type;
153 typedef Packet4f half;
154 enum {
155 Vectorizable = 1,
156 AlignedOnScalar = 1,
157 size=4,
158 HasHalfPacket = 0,
159
160 HasAdd = 1,
161 HasSub = 1,
162 HasMul = 1,
163 HasDiv = 1,
164 HasMin = 1,
165 HasMax = 1,
166 HasAbs = 1,
167 HasSin = 0,
168 HasCos = 0,
169 HasLog = 0,
170 HasExp = 1,
171 HasSqrt = 1,
172 HasRsqrt = 1,
173 HasRound = 1,
174 HasFloor = 1,
175 HasCeil = 1,
176 HasNegate = 1,
177 HasBlend = 1
178 };
179};
180
181template<> struct packet_traits<double> : default_packet_traits
182{
183 typedef Packet2d type;
184 typedef Packet2d half;
185 enum {
186 Vectorizable = 1,
187 AlignedOnScalar = 1,
188 size=2,
189 HasHalfPacket = 1,
190
191 HasAdd = 1,
192 HasSub = 1,
193 HasMul = 1,
194 HasDiv = 1,
195 HasMin = 1,
196 HasMax = 1,
197 HasAbs = 1,
198 HasSin = 0,
199 HasCos = 0,
200 HasLog = 0,
201 HasExp = 1,
202 HasSqrt = 1,
203 HasRsqrt = 1,
204 HasRound = 1,
205 HasFloor = 1,
206 HasCeil = 1,
207 HasNegate = 1,
208 HasBlend = 1
209 };
210};
211
212template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; };
213template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16}; typedef Packet4f half; };
214template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; };
215
216/* Forward declaration */
217EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f,4>& kernel);
218
219inline std::ostream & operator <<(std::ostream & s, const Packet4i & v)
220{
221 Packet vt;
222 vt.v4i = v;
223 s << vt.i[0] << ", " << vt.i[1] << ", " << vt.i[2] << ", " << vt.i[3];
224 return s;
225}
226
227inline std::ostream & operator <<(std::ostream & s, const Packet4ui & v)
228{
229 Packet vt;
230 vt.v4ui = v;
231 s << vt.ui[0] << ", " << vt.ui[1] << ", " << vt.ui[2] << ", " << vt.ui[3];
232 return s;
233}
234
235inline std::ostream & operator <<(std::ostream & s, const Packet2l & v)
236{
237 Packet vt;
238 vt.v2l = v;
239 s << vt.l[0] << ", " << vt.l[1];
240 return s;
241}
242
243inline std::ostream & operator <<(std::ostream & s, const Packet2ul & v)
244{
245 Packet vt;
246 vt.v2ul = v;
247 s << vt.ul[0] << ", " << vt.ul[1] ;
248 return s;
249}
250
251inline std::ostream & operator <<(std::ostream & s, const Packet2d & v)
252{
253 Packet vt;
254 vt.v2d = v;
255 s << vt.d[0] << ", " << vt.d[1];
256 return s;
257}
258
259/* Helper function to simulate a vec_splat_packet4f
260 */
261template<int element> EIGEN_STRONG_INLINE Packet4f vec_splat_packet4f(const Packet4f& from)
262{
263 Packet4f splat;
264 switch (element) {
265 case 0:
266 splat.v4f[0] = vec_splat(from.v4f[0], 0);
267 splat.v4f[1] = splat.v4f[0];
268 break;
269 case 1:
270 splat.v4f[0] = vec_splat(from.v4f[0], 1);
271 splat.v4f[1] = splat.v4f[0];
272 break;
273 case 2:
274 splat.v4f[0] = vec_splat(from.v4f[1], 0);
275 splat.v4f[1] = splat.v4f[0];
276 break;
277 case 3:
278 splat.v4f[0] = vec_splat(from.v4f[1], 1);
279 splat.v4f[1] = splat.v4f[0];
280 break;
281 }
282 return splat;
283}
284
285template<int Offset>
286struct palign_impl<Offset,Packet4i>
287{
288 static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second)
289 {
290 switch (Offset % 4) {
291 case 1:
292 first = vec_sld(first, second, 4); break;
293 case 2:
294 first = vec_sld(first, second, 8); break;
295 case 3:
296 first = vec_sld(first, second, 12); break;
297 }
298 }
299};
300
301/* This is a tricky one, we have to translate float alignment to vector elements of sizeof double
302 */
303template<int Offset>
304struct palign_impl<Offset,Packet4f>
305{
306 static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second)
307 {
308 switch (Offset % 4) {
309 case 1:
310 first.v4f[0] = vec_sld(first.v4f[0], first.v4f[1], 8);
311 first.v4f[1] = vec_sld(first.v4f[1], second.v4f[0], 8);
312 break;
313 case 2:
314 first.v4f[0] = first.v4f[1];
315 first.v4f[1] = second.v4f[0];
316 break;
317 case 3:
318 first.v4f[0] = vec_sld(first.v4f[1], second.v4f[0], 8);
319 first.v4f[1] = vec_sld(second.v4f[0], second.v4f[1], 8);
320 break;
321 }
322 }
323};
324
325
326template<int Offset>
327struct palign_impl<Offset,Packet2d>
328{
329 static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second)
330 {
331 if (Offset == 1)
332 first = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(first), reinterpret_cast<Packet4i>(second), 8));
333 }
334};
335
336template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from)
337{
338 // FIXME: No intrinsic yet
339 EIGEN_DEBUG_ALIGNED_LOAD
340 Packet *vfrom;
341 vfrom = (Packet *) from;
342 return vfrom->v4i;
343}
344
345template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from)
346{
347 // FIXME: No intrinsic yet
348 EIGEN_DEBUG_ALIGNED_LOAD
349 Packet4f vfrom;
350 vfrom.v4f[0] = vec_ld2f(&from[0]);
351 vfrom.v4f[1] = vec_ld2f(&from[2]);
352 return vfrom;
353}
354
355template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from)
356{
357 // FIXME: No intrinsic yet
358 EIGEN_DEBUG_ALIGNED_LOAD
359 Packet *vfrom;
360 vfrom = (Packet *) from;
361 return vfrom->v2d;
362}
363
364template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from)
365{
366 // FIXME: No intrinsic yet
367 EIGEN_DEBUG_ALIGNED_STORE
368 Packet *vto;
369 vto = (Packet *) to;
370 vto->v4i = from;
371}
372
373template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from)
374{
375 // FIXME: No intrinsic yet
376 EIGEN_DEBUG_ALIGNED_STORE
377 vec_st2f(from.v4f[0], &to[0]);
378 vec_st2f(from.v4f[1], &to[2]);
379}
380
381
382template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from)
383{
384 // FIXME: No intrinsic yet
385 EIGEN_DEBUG_ALIGNED_STORE
386 Packet *vto;
387 vto = (Packet *) to;
388 vto->v2d = from;
389}
390
391template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from)
392{
393 return vec_splats(from);
394}
395template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
396 return vec_splats(from);
397}
398template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from)
399{
400 Packet4f to;
401 to.v4f[0] = pset1<Packet2d>(static_cast<const double&>(from));
402 to.v4f[1] = to.v4f[0];
403 return to;
404}
405
406template<> EIGEN_STRONG_INLINE void
407pbroadcast4<Packet4i>(const int *a,
408 Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3)
409{
410 a3 = pload<Packet4i>(a);
411 a0 = vec_splat(a3, 0);
412 a1 = vec_splat(a3, 1);
413 a2 = vec_splat(a3, 2);
414 a3 = vec_splat(a3, 3);
415}
416
417template<> EIGEN_STRONG_INLINE void
418pbroadcast4<Packet4f>(const float *a,
419 Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3)
420{
421 a3 = pload<Packet4f>(a);
422 a0 = vec_splat_packet4f<0>(a3);
423 a1 = vec_splat_packet4f<1>(a3);
424 a2 = vec_splat_packet4f<2>(a3);
425 a3 = vec_splat_packet4f<3>(a3);
426}
427
428template<> EIGEN_STRONG_INLINE void
429pbroadcast4<Packet2d>(const double *a,
430 Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3)
431{
432 a1 = pload<Packet2d>(a);
433 a0 = vec_splat(a1, 0);
434 a1 = vec_splat(a1, 1);
435 a3 = pload<Packet2d>(a+2);
436 a2 = vec_splat(a3, 0);
437 a3 = vec_splat(a3, 1);
438}
439
440template<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride)
441{
442 int EIGEN_ALIGN16 ai[4];
443 ai[0] = from[0*stride];
444 ai[1] = from[1*stride];
445 ai[2] = from[2*stride];
446 ai[3] = from[3*stride];
447 return pload<Packet4i>(ai);
448}
449
450template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride)
451{
452 float EIGEN_ALIGN16 ai[4];
453 ai[0] = from[0*stride];
454 ai[1] = from[1*stride];
455 ai[2] = from[2*stride];
456 ai[3] = from[3*stride];
457 return pload<Packet4f>(ai);
458}
459
460template<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride)
461{
462 double EIGEN_ALIGN16 af[2];
463 af[0] = from[0*stride];
464 af[1] = from[1*stride];
465 return pload<Packet2d>(af);
466}
467
468template<> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride)
469{
470 int EIGEN_ALIGN16 ai[4];
471 pstore<int>((int *)ai, from);
472 to[0*stride] = ai[0];
473 to[1*stride] = ai[1];
474 to[2*stride] = ai[2];
475 to[3*stride] = ai[3];
476}
477
478template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride)
479{
480 float EIGEN_ALIGN16 ai[4];
481 pstore<float>((float *)ai, from);
482 to[0*stride] = ai[0];
483 to[1*stride] = ai[1];
484 to[2*stride] = ai[2];
485 to[3*stride] = ai[3];
486}
487
488template<> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride)
489{
490 double EIGEN_ALIGN16 af[2];
491 pstore<double>(af, from);
492 to[0*stride] = af[0];
493 to[1*stride] = af[1];
494}
495
496template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a + b); }
497template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b)
498{
499 Packet4f c;
500 c.v4f[0] = a.v4f[0] + b.v4f[0];
501 c.v4f[1] = a.v4f[1] + b.v4f[1];
502 return c;
503}
504template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a + b); }
505
506template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a - b); }
507template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b)
508{
509 Packet4f c;
510 c.v4f[0] = a.v4f[0] - b.v4f[0];
511 c.v4f[1] = a.v4f[1] - b.v4f[1];
512 return c;
513}
514template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a - b); }
515
516template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a * b); }
517template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b)
518{
519 Packet4f c;
520 c.v4f[0] = a.v4f[0] * b.v4f[0];
521 c.v4f[1] = a.v4f[1] * b.v4f[1];
522 return c;
523}
524template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a * b); }
525
526template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) { return (a / b); }
527template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b)
528{
529 Packet4f c;
530 c.v4f[0] = a.v4f[0] / b.v4f[0];
531 c.v4f[1] = a.v4f[1] / b.v4f[1];
532 return c;
533}
534template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return (a / b); }
535
536template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return (-a); }
537template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a)
538{
539 Packet4f c;
540 c.v4f[0] = -a.v4f[0];
541 c.v4f[1] = -a.v4f[1];
542 return c;
543}
544template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return (-a); }
545
546template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
547template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; }
548template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; }
549
550template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return padd<Packet4i>(pmul<Packet4i>(a, b), c); }
551template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
552{
553 Packet4f res;
554 res.v4f[0] = vec_madd(a.v4f[0], b.v4f[0], c.v4f[0]);
555 res.v4f[1] = vec_madd(a.v4f[1], b.v4f[1], c.v4f[1]);
556 return res;
557}
558template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_madd(a, b, c); }
559
560template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) { return padd<Packet4i>(pset1<Packet4i>(a), p4i_COUNTDOWN); }
561template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { return padd<Packet4f>(pset1<Packet4f>(a), p4f_COUNTDOWN); }
562template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return padd<Packet2d>(pset1<Packet2d>(a), p2d_COUNTDOWN); }
563
564template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_min(a, b); }
565template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_min(a, b); }
566template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b)
567{
568 Packet4f res;
569 res.v4f[0] = pmin(a.v4f[0], b.v4f[0]);
570 res.v4f[1] = pmin(a.v4f[1], b.v4f[1]);
571 return res;
572}
573
574template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_max(a, b); }
575template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_max(a, b); }
576template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b)
577{
578 Packet4f res;
579 res.v4f[0] = pmax(a.v4f[0], b.v4f[0]);
580 res.v4f[1] = pmax(a.v4f[1], b.v4f[1]);
581 return res;
582}
583
584template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_and(a, b); }
585template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, b); }
586template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b)
587{
588 Packet4f res;
589 res.v4f[0] = pand(a.v4f[0], b.v4f[0]);
590 res.v4f[1] = pand(a.v4f[1], b.v4f[1]);
591 return res;
592}
593
594template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_or(a, b); }
595template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_or(a, b); }
596template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b)
597{
598 Packet4f res;
599 res.v4f[0] = pand(a.v4f[0], b.v4f[0]);
600 res.v4f[1] = pand(a.v4f[1], b.v4f[1]);
601 return res;
602}
603
604template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_xor(a, b); }
605template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_xor(a, b); }
606template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b)
607{
608 Packet4f res;
609 res.v4f[0] = pand(a.v4f[0], b.v4f[0]);
610 res.v4f[1] = pand(a.v4f[1], b.v4f[1]);
611 return res;
612}
613
614template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return pand<Packet4i>(a, vec_nor(b, b)); }
615template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); }
616template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b)
617{
618 Packet4f res;
619 res.v4f[0] = pandnot(a.v4f[0], b.v4f[0]);
620 res.v4f[1] = pandnot(a.v4f[1], b.v4f[1]);
621 return res;
622}
623
624template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a)
625{
626 Packet4f res;
627 res.v4f[0] = vec_round(a.v4f[0]);
628 res.v4f[1] = vec_round(a.v4f[1]);
629 return res;
630}
631template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) { return vec_round(a); }
632template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a)
633{
634 Packet4f res;
635 res.v4f[0] = vec_ceil(a.v4f[0]);
636 res.v4f[1] = vec_ceil(a.v4f[1]);
637 return res;
638}
639template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { return vec_ceil(a); }
640template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a)
641{
642 Packet4f res;
643 res.v4f[0] = vec_floor(a.v4f[0]);
644 res.v4f[1] = vec_floor(a.v4f[1]);
645 return res;
646}
647template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return vec_floor(a); }
648
649template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) { return pload<Packet4i>(from); }
650template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { return pload<Packet4f>(from); }
651template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) { return pload<Packet2d>(from); }
652
653
654template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
655{
656 Packet4i p = pload<Packet4i>(from);
657 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
658}
659
660template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
661{
662 Packet4f p = pload<Packet4f>(from);
663 p.v4f[1] = vec_splat(p.v4f[0], 1);
664 p.v4f[0] = vec_splat(p.v4f[0], 0);
665 return p;
666}
667
668template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from)
669{
670 Packet2d p = pload<Packet2d>(from);
671 return vec_perm(p, p, p16uc_PSET64_HI);
672}
673
674template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { pstore<int>(to, from); }
675template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { pstore<float>(to, from); }
676template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) { pstore<double>(to, from); }
677
678template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { EIGEN_ZVECTOR_PREFETCH(addr); }
679template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_ZVECTOR_PREFETCH(addr); }
680template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { EIGEN_ZVECTOR_PREFETCH(addr); }
681
682template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int EIGEN_ALIGN16 x[4]; pstore(x, a); return x[0]; }
683template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[2]; vec_st2f(a.v4f[0], &x[0]); return x[0]; }
684template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { double EIGEN_ALIGN16 x[2]; pstore(x, a); return x[0]; }
685
686template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
687{
688 return reinterpret_cast<Packet4i>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
689}
690
691template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
692{
693 return reinterpret_cast<Packet2d>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64));
694}
695
696template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
697{
698 Packet4f rev;
699 rev.v4f[0] = preverse<Packet2d>(a.v4f[1]);
700 rev.v4f[1] = preverse<Packet2d>(a.v4f[0]);
701 return rev;
702}
703
704template<> EIGEN_STRONG_INLINE Packet4i pabs<Packet4i>(const Packet4i& a) { return vec_abs(a); }
705template<> EIGEN_STRONG_INLINE Packet2d pabs<Packet2d>(const Packet2d& a) { return vec_abs(a); }
706template<> EIGEN_STRONG_INLINE Packet4f pabs<Packet4f>(const Packet4f& a)
707{
708 Packet4f res;
709 res.v4f[0] = pabs(a.v4f[0]);
710 res.v4f[1] = pabs(a.v4f[1]);
711 return res;
712}
713
714template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
715{
716 Packet4i b, sum;
717 b = vec_sld(a, a, 8);
718 sum = padd<Packet4i>(a, b);
719 b = vec_sld(sum, sum, 4);
720 sum = padd<Packet4i>(sum, b);
721 return pfirst(sum);
722}
723
724template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
725{
726 Packet2d b, sum;
727 b = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8));
728 sum = padd<Packet2d>(a, b);
729 return pfirst(sum);
730}
731template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
732{
733 Packet2d sum;
734 sum = padd<Packet2d>(a.v4f[0], a.v4f[1]);
735 double first = predux<Packet2d>(sum);
736 return static_cast<float>(first);
737}
738
739template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
740{
741 Packet4i v[4], sum[4];
742
743 // It's easier and faster to transpose then add as columns
744 // Check: http://www.freevec.org/function/matrix_4x4_transpose_floats for explanation
745 // Do the transpose, first set of moves
746 v[0] = vec_mergeh(vecs[0], vecs[2]);
747 v[1] = vec_mergel(vecs[0], vecs[2]);
748 v[2] = vec_mergeh(vecs[1], vecs[3]);
749 v[3] = vec_mergel(vecs[1], vecs[3]);
750 // Get the resulting vectors
751 sum[0] = vec_mergeh(v[0], v[2]);
752 sum[1] = vec_mergel(v[0], v[2]);
753 sum[2] = vec_mergeh(v[1], v[3]);
754 sum[3] = vec_mergel(v[1], v[3]);
755
756 // Now do the summation:
757 // Lines 0+1
758 sum[0] = padd<Packet4i>(sum[0], sum[1]);
759 // Lines 2+3
760 sum[1] = padd<Packet4i>(sum[2], sum[3]);
761 // Add the results
762 sum[0] = padd<Packet4i>(sum[0], sum[1]);
763
764 return sum[0];
765}
766
767template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
768{
769 Packet2d v[2], sum;
770 v[0] = padd<Packet2d>(vecs[0], reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(vecs[0]), reinterpret_cast<Packet4ui>(vecs[0]), 8)));
771 v[1] = padd<Packet2d>(vecs[1], reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(vecs[1]), reinterpret_cast<Packet4ui>(vecs[1]), 8)));
772
773 sum = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(v[0]), reinterpret_cast<Packet4ui>(v[1]), 8));
774
775 return sum;
776}
777
778template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
779{
780 PacketBlock<Packet4f,4> transpose;
781 transpose.packet[0] = vecs[0];
782 transpose.packet[1] = vecs[1];
783 transpose.packet[2] = vecs[2];
784 transpose.packet[3] = vecs[3];
785 ptranspose(transpose);
786
787 Packet4f sum = padd(transpose.packet[0], transpose.packet[1]);
788 sum = padd(sum, transpose.packet[2]);
789 sum = padd(sum, transpose.packet[3]);
790 return sum;
791}
792
793// Other reduction functions:
794// mul
795template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
796{
797 EIGEN_ALIGN16 int aux[4];
798 pstore(aux, a);
799 return aux[0] * aux[1] * aux[2] * aux[3];
800}
801
802template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
803{
804 return pfirst(pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
805}
806
807template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
808{
809 // Return predux_mul<Packet2d> of the subvectors product
810 return static_cast<float>(pfirst(predux_mul(pmul(a.v4f[0], a.v4f[1]))));
811}
812
813// min
814template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
815{
816 Packet4i b, res;
817 b = pmin<Packet4i>(a, vec_sld(a, a, 8));
818 res = pmin<Packet4i>(b, vec_sld(b, b, 4));
819 return pfirst(res);
820}
821
822template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
823{
824 return pfirst(pmin<Packet2d>(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
825}
826
827template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
828{
829 Packet2d b, res;
830 b = pmin<Packet2d>(a.v4f[0], a.v4f[1]);
831 res = pmin<Packet2d>(b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8)));
832 return static_cast<float>(pfirst(res));
833}
834
835// max
836template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
837{
838 Packet4i b, res;
839 b = pmax<Packet4i>(a, vec_sld(a, a, 8));
840 res = pmax<Packet4i>(b, vec_sld(b, b, 4));
841 return pfirst(res);
842}
843
844// max
845template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
846{
847 return pfirst(pmax<Packet2d>(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(a), reinterpret_cast<Packet4i>(a), 8))));
848}
849
850template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
851{
852 Packet2d b, res;
853 b = pmax<Packet2d>(a.v4f[0], a.v4f[1]);
854 res = pmax<Packet2d>(b, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4i>(b), reinterpret_cast<Packet4i>(b), 8)));
855 return static_cast<float>(pfirst(res));
856}
857
858EIGEN_DEVICE_FUNC inline void
859ptranspose(PacketBlock<Packet4i,4>& kernel) {
860 Packet4i t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
861 Packet4i t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
862 Packet4i t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
863 Packet4i t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
864 kernel.packet[0] = vec_mergeh(t0, t2);
865 kernel.packet[1] = vec_mergel(t0, t2);
866 kernel.packet[2] = vec_mergeh(t1, t3);
867 kernel.packet[3] = vec_mergel(t1, t3);
868}
869
870EIGEN_DEVICE_FUNC inline void
871ptranspose(PacketBlock<Packet2d,2>& kernel) {
872 Packet2d t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
873 Packet2d t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
874 kernel.packet[0] = t0;
875 kernel.packet[1] = t1;
876}
877
878/* Split the Packet4f PacketBlock into 4 Packet2d PacketBlocks and transpose each one
879 */
880EIGEN_DEVICE_FUNC inline void
881ptranspose(PacketBlock<Packet4f,4>& kernel) {
882 PacketBlock<Packet2d,2> t0,t1,t2,t3;
883 // copy top-left 2x2 Packet2d block
884 t0.packet[0] = kernel.packet[0].v4f[0];
885 t0.packet[1] = kernel.packet[1].v4f[0];
886
887 // copy top-right 2x2 Packet2d block
888 t1.packet[0] = kernel.packet[0].v4f[1];
889 t1.packet[1] = kernel.packet[1].v4f[1];
890
891 // copy bottom-left 2x2 Packet2d block
892 t2.packet[0] = kernel.packet[2].v4f[0];
893 t2.packet[1] = kernel.packet[3].v4f[0];
894
895 // copy bottom-right 2x2 Packet2d block
896 t3.packet[0] = kernel.packet[2].v4f[1];
897 t3.packet[1] = kernel.packet[3].v4f[1];
898
899 // Transpose all 2x2 blocks
900 ptranspose(t0);
901 ptranspose(t1);
902 ptranspose(t2);
903 ptranspose(t3);
904
905 // Copy back transposed blocks, but exchange t1 and t2 due to transposition
906 kernel.packet[0].v4f[0] = t0.packet[0];
907 kernel.packet[0].v4f[1] = t2.packet[0];
908 kernel.packet[1].v4f[0] = t0.packet[1];
909 kernel.packet[1].v4f[1] = t2.packet[1];
910 kernel.packet[2].v4f[0] = t1.packet[0];
911 kernel.packet[2].v4f[1] = t3.packet[0];
912 kernel.packet[3].v4f[0] = t1.packet[1];
913 kernel.packet[3].v4f[1] = t3.packet[1];
914}
915
916template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) {
917 Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
918 Packet4ui mask = vec_cmpeq(select, reinterpret_cast<Packet4ui>(p4i_ONE));
919 return vec_sel(elsePacket, thenPacket, mask);
920}
921
922template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) {
923 Packet2ul select_hi = { ifPacket.select[0], ifPacket.select[1] };
924 Packet2ul select_lo = { ifPacket.select[2], ifPacket.select[3] };
925 Packet2ul mask_hi = vec_cmpeq(select_hi, reinterpret_cast<Packet2ul>(p2l_ONE));
926 Packet2ul mask_lo = vec_cmpeq(select_lo, reinterpret_cast<Packet2ul>(p2l_ONE));
927 Packet4f result;
928 result.v4f[0] = vec_sel(elsePacket.v4f[0], thenPacket.v4f[0], mask_hi);
929 result.v4f[1] = vec_sel(elsePacket.v4f[1], thenPacket.v4f[1], mask_lo);
930 return result;
931}
932
933template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) {
934 Packet2ul select = { ifPacket.select[0], ifPacket.select[1] };
935 Packet2ul mask = vec_cmpeq(select, reinterpret_cast<Packet2ul>(p2l_ONE));
936 return vec_sel(elsePacket, thenPacket, mask);
937}
938
939} // end namespace internal
940
941} // end namespace Eigen
942
943#endif // EIGEN_PACKET_MATH_ZVECTOR_H
Eigen::Aligned16
@ Aligned16
Definition Constants.h:230
Eigen
Namespace containing all symbols from the Eigen library.
Definition A05_PortingFrom2To3.dox:1
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:65