Eigen  3.3.9
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2016 Konstantinos Margaritis <markos@freevec.org>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_PACKET_MATH_ALTIVEC_H
11#define EIGEN_PACKET_MATH_ALTIVEC_H
12
13namespace Eigen {
14
15namespace internal {
16
17#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
18#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4
19#endif
20
21#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
22#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
23#endif
24
25// NOTE Altivec has 32 registers, but Eigen only accepts a value of 8 or 16
26#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
27#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
28#endif
29
30typedef __vector float Packet4f;
31typedef __vector int Packet4i;
32typedef __vector unsigned int Packet4ui;
33typedef __vector __bool int Packet4bi;
34typedef __vector short int Packet8i;
35typedef __vector unsigned char Packet16uc;
36
37// We don't want to write the same code all the time, but we need to reuse the constants
38// and it doesn't really work to declare them global, so we define macros instead
39#define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \
40 Packet4f p4f_##NAME = {X, X, X, X}
41
42#define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \
43 Packet4i p4i_##NAME = vec_splat_s32(X)
44
45#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
46 Packet4f p4f_##NAME = pset1<Packet4f>(X)
47
48#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
49 Packet4i p4i_##NAME = pset1<Packet4i>(X)
50
51#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \
52 Packet2d p2d_##NAME = pset1<Packet2d>(X)
53
54#define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \
55 Packet2l p2l_##NAME = pset1<Packet2l>(X)
56
57#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
58 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
59
60#define DST_CHAN 1
61#define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride))
62#define __UNPACK_TYPE__(PACKETNAME) typename unpacket_traits<PACKETNAME>::type
63
64// These constants are endian-agnostic
65static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0); //{ 0.0, 0.0, 0.0, 0.0}
66static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0); //{ 0, 0, 0, 0,}
67static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE,1); //{ 1, 1, 1, 1}
68static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16,-16); //{ -16, -16, -16, -16}
69static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1); //{ -1, -1, -1, -1}
70static Packet4f p4f_MZERO = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1); //{ 0x80000000, 0x80000000, 0x80000000, 0x80000000}
71#ifndef __VSX__
72static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0); //{ 1.0, 1.0, 1.0, 1.0}
73#endif
74
75static Packet4ui p4ui_SIGN = {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u};
76static Packet4ui p4ui_PREV0DOT5 = {0x3EFFFFFFu, 0x3EFFFFFFu, 0x3EFFFFFFu, 0x3EFFFFFFu};
77
78static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 };
79static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 };
80
81static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 };
82static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 };
83
84// Handle endianness properly while loading constants
85// Define global static constants:
86#ifdef _BIG_ENDIAN
87static Packet16uc p16uc_FORWARD = vec_lvsl(0, (float*)0);
88#ifdef __VSX__
89static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
90#endif
91static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
92static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
93static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
94#else
95static Packet16uc p16uc_FORWARD = p16uc_REVERSE32;
96static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
97static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);//{ 0,1,2,3, 0,1,2,3, 8,9,10,11, 8,9,10,11 };
98static Packet16uc p16uc_PSET32_WEVEN = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);//{ 4,5,6,7, 4,5,6,7, 12,13,14,15, 12,13,14,15 };
99static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO, 8); //{ 0,0,0,0, 0,0,0,0, 16,16,16,16, 16,16,16,16};
100#endif // _BIG_ENDIAN
101
102static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 0,1,2,3, 4,5,6,7, 0,1,2,3, 4,5,6,7 };
103static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN); //{ 8,9,10,11, 12,13,14,15, 8,9,10,11, 12,13,14,15 };
104static Packet16uc p16uc_TRANSPOSE64_HI = p16uc_PSET64_HI + p16uc_HALF64_0_16; //{ 0,1,2,3, 4,5,6,7, 16,17,18,19, 20,21,22,23};
105static Packet16uc p16uc_TRANSPOSE64_LO = p16uc_PSET64_LO + p16uc_HALF64_0_16; //{ 8,9,10,11, 12,13,14,15, 24,25,26,27, 28,29,30,31};
106
107static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8); //{ 4,5,6,7, 0,1,2,3, 12,13,14,15, 8,9,10,11 };
108
109#ifdef _BIG_ENDIAN
110static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
111#else
112static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_PSET64_HI, p16uc_PSET64_LO, 8); //{ 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
113#endif // _BIG_ENDIAN
114
115#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
116 #define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR);
117#else
118 #define EIGEN_PPC_PREFETCH(ADDR) asm( " dcbt [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" );
119#endif
120
121template<> struct packet_traits<float> : default_packet_traits
122{
123 typedef Packet4f type;
124 typedef Packet4f half;
125 enum {
126 Vectorizable = 1,
127 AlignedOnScalar = 1,
128 size=4,
129 HasHalfPacket = 1,
130
131 HasAdd = 1,
132 HasSub = 1,
133 HasMul = 1,
134 HasDiv = 1,
135 HasMin = 1,
136 HasMax = 1,
137 HasAbs = 1,
138 HasSin = 0,
139 HasCos = 0,
140 HasLog = 0,
141 HasExp = 1,
142#ifdef __VSX__
143 HasSqrt = 1,
144#if !EIGEN_COMP_CLANG
145 HasRsqrt = 1,
146#else
147 HasRsqrt = 0,
148#endif
149#else
150 HasSqrt = 0,
151 HasRsqrt = 0,
152#endif
153 HasRound = 1,
154 HasFloor = 1,
155 HasCeil = 1,
156 HasNegate = 1,
157 HasBlend = 1
158 };
159};
160template<> struct packet_traits<int> : default_packet_traits
161{
162 typedef Packet4i type;
163 typedef Packet4i half;
164 enum {
165 Vectorizable = 1,
166 AlignedOnScalar = 1,
167 size = 4,
168 HasHalfPacket = 0,
169
170 HasAdd = 1,
171 HasSub = 1,
172 HasMul = 1,
173 HasDiv = 0,
174 HasBlend = 1
175 };
176};
177
178
179template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4, alignment=Aligned16}; typedef Packet4f half; };
180template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4, alignment=Aligned16}; typedef Packet4i half; };
181
182inline std::ostream & operator <<(std::ostream & s, const Packet16uc & v)
183{
184 union {
185 Packet16uc v;
186 unsigned char n[16];
187 } vt;
188 vt.v = v;
189 for (int i=0; i< 16; i++)
190 s << (int)vt.n[i] << ", ";
191 return s;
192}
193
194inline std::ostream & operator <<(std::ostream & s, const Packet4f & v)
195{
196 union {
197 Packet4f v;
198 float n[4];
199 } vt;
200 vt.v = v;
201 s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
202 return s;
203}
204
205inline std::ostream & operator <<(std::ostream & s, const Packet4i & v)
206{
207 union {
208 Packet4i v;
209 int n[4];
210 } vt;
211 vt.v = v;
212 s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
213 return s;
214}
215
216inline std::ostream & operator <<(std::ostream & s, const Packet4ui & v)
217{
218 union {
219 Packet4ui v;
220 unsigned int n[4];
221 } vt;
222 vt.v = v;
223 s << vt.n[0] << ", " << vt.n[1] << ", " << vt.n[2] << ", " << vt.n[3];
224 return s;
225}
226
227template <typename Packet, typename Scalar>
228EIGEN_STRONG_INLINE Packet pload_common(const Scalar* from)
229{
230 // some versions of GCC throw "unused-but-set-parameter".
231 // ignoring these warnings for now.
232 EIGEN_UNUSED_VARIABLE(from);
233 EIGEN_DEBUG_ALIGNED_LOAD
234 return vec_ld(0, from);
235}
236
237// Need to define them first or we get specialization after instantiation errors
238template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from)
239{
240 return pload_common<Packet4f, float>(from);
241}
242
243template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from)
244{
245 return pload_common<Packet4i, int>(from);
246}
247
248template <typename Packet>
249EIGEN_STRONG_INLINE void pstore_common(__UNPACK_TYPE__(Packet)* to, const Packet& from){
250 // some versions of GCC throw "unused-but-set-parameter" (float *to).
251 // ignoring these warnings for now.
252 EIGEN_UNUSED_VARIABLE(to);
253 EIGEN_DEBUG_ALIGNED_STORE
254#ifdef __VSX__
255 vec_xst(from, 0, to);
256#else
257 vec_st(from, 0, to);
258#endif
259}
260
261template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from)
262{
263 pstore_common<Packet4f>(to, from);
264}
265
266template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from)
267{
268 pstore_common<Packet4i>(to, from);
269}
270
271template<typename Packet>
272EIGEN_STRONG_INLINE Packet pset1_size4(const __UNPACK_TYPE__(Packet)& from)
273{
274 Packet v = {from, from, from, from};
275 return v;
276}
277
278template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
279 return pset1_size4<Packet4f>(from);
280}
281
282template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) {
283 return pset1_size4<Packet4i>(from);
284}
285
286template<typename Packet> EIGEN_STRONG_INLINE void
287pbroadcast4_common(const __UNPACK_TYPE__(Packet) *a,
288 Packet& a0, Packet& a1, Packet& a2, Packet& a3)
289{
290 a3 = pload<Packet>(a);
291 a0 = vec_splat(a3, 0);
292 a1 = vec_splat(a3, 1);
293 a2 = vec_splat(a3, 2);
294 a3 = vec_splat(a3, 3);
295}
296
297template<> EIGEN_STRONG_INLINE void
298pbroadcast4<Packet4f>(const float *a,
299 Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3)
300{
301 pbroadcast4_common<Packet4f>(a, a0, a1, a2, a3);
302}
303template<> EIGEN_STRONG_INLINE void
304pbroadcast4<Packet4i>(const int *a,
305 Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3)
306{
307 pbroadcast4_common<Packet4i>(a, a0, a1, a2, a3);
308}
309
310template<typename Packet> EIGEN_DEVICE_FUNC inline Packet pgather_common(const __UNPACK_TYPE__(Packet)* from, Index stride)
311{
312 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[4];
313 a[0] = from[0*stride];
314 a[1] = from[1*stride];
315 a[2] = from[2*stride];
316 a[3] = from[3*stride];
317 return pload<Packet>(a);
318}
319
320template<> EIGEN_DEVICE_FUNC inline Packet4f pgather<float, Packet4f>(const float* from, Index stride)
321{
322 return pgather_common<Packet4f>(from, stride);
323}
324
325template<> EIGEN_DEVICE_FUNC inline Packet4i pgather<int, Packet4i>(const int* from, Index stride)
326{
327 return pgather_common<Packet4i>(from, stride);
328}
329
330template<typename Packet> EIGEN_DEVICE_FUNC inline void pscatter_size4(__UNPACK_TYPE__(Packet)* to, const Packet& from, Index stride)
331{
332 EIGEN_ALIGN16 __UNPACK_TYPE__(Packet) a[4];
333 pstore<__UNPACK_TYPE__(Packet)>(a, from);
334 to[0*stride] = a[0];
335 to[1*stride] = a[1];
336 to[2*stride] = a[2];
337 to[3*stride] = a[3];
338}
339
340template<> EIGEN_DEVICE_FUNC inline void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride)
341{
342 pscatter_size4<Packet4f>(to, from, stride);
343}
344
345template<> EIGEN_DEVICE_FUNC inline void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride)
346{
347 pscatter_size4<Packet4i>(to, from, stride);
348}
349
350template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) { return pset1<Packet4f>(a) + p4f_COUNTDOWN; }
351template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) { return pset1<Packet4i>(a) + p4i_COUNTDOWN; }
352
353template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return a + b; }
354template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return a + b; }
355
356template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return a - b; }
357template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return a - b; }
358
359template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return p4f_ZERO - a; }
360template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return p4i_ZERO - a; }
361
362template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; }
363template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
364
365template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_madd(a,b, p4f_MZERO); }
366template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return a * b; }
367
368template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b)
369{
370#ifndef __VSX__ // VSX actually provides a div instruction
371 Packet4f t, y_0, y_1;
372
373 // Altivec does not offer a divide instruction, we have to do a reciprocal approximation
374 y_0 = vec_re(b);
375
376 // Do one Newton-Raphson iteration to get the needed accuracy
377 t = vec_nmsub(y_0, b, p4f_ONE);
378 y_1 = vec_madd(y_0, t, y_0);
379
380 return vec_madd(a, y_1, p4f_MZERO);
381#else
382 return vec_div(a, b);
383#endif
384}
385
386template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
387{ eigen_assert(false && "packet integer division are not supported by AltiVec");
388 return pset1<Packet4i>(0);
389}
390
391// for some weird raisons, it has to be overloaded for packet of integers
392template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vec_madd(a,b,c); }
393template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return a*b + c; }
394
395template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b)
396{
397 #ifdef __VSX__
398 Packet4f ret;
399 __asm__ ("xvcmpgesp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b));
400 return ret;
401 #else
402 return vec_min(a, b);
403 #endif
404}
405template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_min(a, b); }
406
407template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b)
408{
409 #ifdef __VSX__
410 Packet4f ret;
411 __asm__ ("xvcmpgtsp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b));
412 return ret;
413 #else
414 return vec_max(a, b);
415 #endif
416}
417template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_max(a, b); }
418
419template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_and(a, b); }
420template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_and(a, b); }
421
422template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_or(a, b); }
423template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_or(a, b); }
424
425template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_xor(a, b); }
426template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_xor(a, b); }
427
428template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) { return vec_and(a, vec_nor(b, b)); }
429template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return vec_and(a, vec_nor(b, b)); }
430
431template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a)
432{
433 Packet4f t = vec_add(reinterpret_cast<Packet4f>(vec_or(vec_and(reinterpret_cast<Packet4ui>(a), p4ui_SIGN), p4ui_PREV0DOT5)), a);
434 Packet4f res;
435
436#ifdef __VSX__
437 __asm__("xvrspiz %x0, %x1\n\t"
438 : "=&wa" (res)
439 : "wa" (t));
440#else
441 __asm__("vrfiz %0, %1\n\t"
442 : "=v" (res)
443 : "v" (t));
444#endif
445
446 return res;
447}
448template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) { return vec_ceil(a); }
449template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) { return vec_floor(a); }
450
451template<typename Packet> EIGEN_STRONG_INLINE Packet ploadu_common(const __UNPACK_TYPE__(Packet)* from)
452{
453 EIGEN_DEBUG_UNALIGNED_LOAD
454 Packet16uc mask = vec_lvsl(0, from); // create the permute mask
455 Packet16uc MSQ = vec_ld(0, (unsigned char *)from); // most significant quadword
456 Packet16uc LSQ = vec_ld(15, (unsigned char *)from); // least significant quadword
457 //TODO: Add static_cast here
458 return (Packet) vec_perm(MSQ, LSQ, mask); // align the data
459}
460
461template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
462{
463 return ploadu_common<Packet4f>(from);
464}
465template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from)
466{
467 return ploadu_common<Packet4i>(from);
468}
469
470template<typename Packet> EIGEN_STRONG_INLINE Packet ploaddup_common(const __UNPACK_TYPE__(Packet)* from)
471{
472 Packet p;
473 if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet>(from);
474 else p = ploadu<Packet>(from);
475 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
476}
477template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
478{
479 return ploaddup_common<Packet4f>(from);
480}
481template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
482{
483 return ploaddup_common<Packet4i>(from);
484}
485
486template<typename Packet> EIGEN_STRONG_INLINE void pstoreu_common(__UNPACK_TYPE__(Packet)* to, const Packet& from)
487{
488 EIGEN_DEBUG_UNALIGNED_STORE
489#ifdef __VSX__
490 vec_xst(from, 0, to);
491#else
492 // Taken from http://developer.apple.com/hardwaredrivers/ve/alignment.html
493 // Warning: not thread safe!
494 Packet16uc MSQ, LSQ, edges;
495 Packet16uc edgeAlign, align;
496
497 MSQ = vec_ld(0, (unsigned char *)to); // most significant quadword
498 LSQ = vec_ld(15, (unsigned char *)to); // least significant quadword
499 edgeAlign = vec_lvsl(0, to); // permute map to extract edges
500 edges=vec_perm(LSQ,MSQ,edgeAlign); // extract the edges
501 align = vec_lvsr( 0, to ); // permute map to misalign data
502 MSQ = vec_perm(edges,(Packet16uc)from,align); // misalign the data (MSQ)
503 LSQ = vec_perm((Packet16uc)from,edges,align); // misalign the data (LSQ)
504 vec_st( LSQ, 15, (unsigned char *)to ); // Store the LSQ part first
505 vec_st( MSQ, 0, (unsigned char *)to ); // Store the MSQ part second
506#endif
507}
508template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from)
509{
510 pstoreu_common<Packet4f>(to, from);
511}
512template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from)
513{
514 pstoreu_common<Packet4i>(to, from);
515}
516
517template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_PPC_PREFETCH(addr); }
518template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { EIGEN_PPC_PREFETCH(addr); }
519
520template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { EIGEN_ALIGN16 float x; vec_ste(a, 0, &x); return x; }
521template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { EIGEN_ALIGN16 int x; vec_ste(a, 0, &x); return x; }
522
523template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
524{
525 return reinterpret_cast<Packet4f>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32));
526}
527template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
528{
529 return reinterpret_cast<Packet4i>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE32)); }
530
531template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vec_abs(a); }
532template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vec_abs(a); }
533
534template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
535{
536 Packet4f b, sum;
537 b = vec_sld(a, a, 8);
538 sum = a + b;
539 b = vec_sld(sum, sum, 4);
540 sum += b;
541 return pfirst(sum);
542}
543
544template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
545{
546 Packet4f v[4], sum[4];
547
548 // It's easier and faster to transpose then add as columns
549 // Check: http://www.freevec.org/function/matrix_4x4_transpose_floats for explanation
550 // Do the transpose, first set of moves
551 v[0] = vec_mergeh(vecs[0], vecs[2]);
552 v[1] = vec_mergel(vecs[0], vecs[2]);
553 v[2] = vec_mergeh(vecs[1], vecs[3]);
554 v[3] = vec_mergel(vecs[1], vecs[3]);
555 // Get the resulting vectors
556 sum[0] = vec_mergeh(v[0], v[2]);
557 sum[1] = vec_mergel(v[0], v[2]);
558 sum[2] = vec_mergeh(v[1], v[3]);
559 sum[3] = vec_mergel(v[1], v[3]);
560
561 // Now do the summation:
562 // Lines 0+1
563 sum[0] = sum[0] + sum[1];
564 // Lines 2+3
565 sum[1] = sum[2] + sum[3];
566 // Add the results
567 sum[0] = sum[0] + sum[1];
568
569 return sum[0];
570}
571
572template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
573{
574 Packet4i sum;
575 sum = vec_sums(a, p4i_ZERO);
576#ifdef _BIG_ENDIAN
577 sum = vec_sld(sum, p4i_ZERO, 12);
578#else
579 sum = vec_sld(p4i_ZERO, sum, 4);
580#endif
581 return pfirst(sum);
582}
583
584template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
585{
586 Packet4i v[4], sum[4];
587
588 // It's easier and faster to transpose then add as columns
589 // Check: http://www.freevec.org/function/matrix_4x4_transpose_floats for explanation
590 // Do the transpose, first set of moves
591 v[0] = vec_mergeh(vecs[0], vecs[2]);
592 v[1] = vec_mergel(vecs[0], vecs[2]);
593 v[2] = vec_mergeh(vecs[1], vecs[3]);
594 v[3] = vec_mergel(vecs[1], vecs[3]);
595 // Get the resulting vectors
596 sum[0] = vec_mergeh(v[0], v[2]);
597 sum[1] = vec_mergel(v[0], v[2]);
598 sum[2] = vec_mergeh(v[1], v[3]);
599 sum[3] = vec_mergel(v[1], v[3]);
600
601 // Now do the summation:
602 // Lines 0+1
603 sum[0] = sum[0] + sum[1];
604 // Lines 2+3
605 sum[1] = sum[2] + sum[3];
606 // Add the results
607 sum[0] = sum[0] + sum[1];
608
609 return sum[0];
610}
611
612// Other reduction functions:
613// mul
614template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
615{
616 Packet4f prod;
617 prod = pmul(a, vec_sld(a, a, 8));
618 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
619}
620
621template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
622{
623 EIGEN_ALIGN16 int aux[4];
624 pstore(aux, a);
625 return aux[0] * aux[1] * aux[2] * aux[3];
626}
627
628// min
629template<typename Packet> EIGEN_STRONG_INLINE
630__UNPACK_TYPE__(Packet) predux_min4(const Packet& a)
631{
632 Packet b, res;
633 b = vec_min(a, vec_sld(a, a, 8));
634 res = vec_min(b, vec_sld(b, b, 4));
635 return pfirst(res);
636}
637
638
639template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
640{
641 return predux_min4<Packet4f>(a);
642}
643
644template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
645{
646 return predux_min4<Packet4i>(a);
647}
648// max
649template<typename Packet> EIGEN_STRONG_INLINE __UNPACK_TYPE__(Packet) predux_max4(const Packet& a)
650{
651 Packet b, res;
652 b = vec_max(a, vec_sld(a, a, 8));
653 res = vec_max(b, vec_sld(b, b, 4));
654 return pfirst(res);
655}
656
657template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
658{
659 return predux_max4<Packet4f>(a);
660}
661
662template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
663{
664 return predux_max4<Packet4i>(a);
665}
666
667template<int Offset>
668struct palign_impl<Offset,Packet4f>
669{
670 static EIGEN_STRONG_INLINE void run(Packet4f& first, const Packet4f& second)
671 {
672#ifdef _BIG_ENDIAN
673 switch (Offset % 4) {
674 case 1:
675 first = vec_sld(first, second, 4); break;
676 case 2:
677 first = vec_sld(first, second, 8); break;
678 case 3:
679 first = vec_sld(first, second, 12); break;
680 }
681#else
682 switch (Offset % 4) {
683 case 1:
684 first = vec_sld(second, first, 12); break;
685 case 2:
686 first = vec_sld(second, first, 8); break;
687 case 3:
688 first = vec_sld(second, first, 4); break;
689 }
690#endif
691 }
692};
693
694template<int Offset>
695struct palign_impl<Offset,Packet4i>
696{
697 static EIGEN_STRONG_INLINE void run(Packet4i& first, const Packet4i& second)
698 {
699#ifdef _BIG_ENDIAN
700 switch (Offset % 4) {
701 case 1:
702 first = vec_sld(first, second, 4); break;
703 case 2:
704 first = vec_sld(first, second, 8); break;
705 case 3:
706 first = vec_sld(first, second, 12); break;
707 }
708#else
709 switch (Offset % 4) {
710 case 1:
711 first = vec_sld(second, first, 12); break;
712 case 2:
713 first = vec_sld(second, first, 8); break;
714 case 3:
715 first = vec_sld(second, first, 4); break;
716 }
717#endif
718 }
719};
720
721template <typename T>
722EIGEN_DEVICE_FUNC inline void ptranpose_common(PacketBlock<T, 4>& kernel) {
723 T t0, t1, t2, t3;
724 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
725 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
726 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
727 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
728 kernel.packet[0] = vec_mergeh(t0, t2);
729 kernel.packet[1] = vec_mergel(t0, t2);
730 kernel.packet[2] = vec_mergeh(t1, t3);
731 kernel.packet[3] = vec_mergel(t1, t3);
732}
733
734EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) { ptranpose_common<Packet4f>(kernel); }
735
736EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) { ptranpose_common<Packet4i>(kernel); }
737
738template<typename Packet> EIGEN_STRONG_INLINE
739Packet pblend4(const Selector<4>& ifPacket, const Packet& thenPacket, const Packet& elsePacket) {
740 Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
741 Packet4ui mask = reinterpret_cast<Packet4ui>(vec_cmpeq(reinterpret_cast<Packet4ui>(select), reinterpret_cast<Packet4ui>(p4i_ONE)));
742 return vec_sel(elsePacket, thenPacket, mask);
743}
744
745template<> EIGEN_STRONG_INLINE Packet4i pblend(const Selector<4>& ifPacket, const Packet4i& thenPacket, const Packet4i& elsePacket) {
746 return pblend4<Packet4i>(ifPacket, thenPacket, elsePacket);
747}
748
749template<> EIGEN_STRONG_INLINE Packet4f pblend(const Selector<4>& ifPacket, const Packet4f& thenPacket, const Packet4f& elsePacket) {
750 return pblend4<Packet4f>(ifPacket, thenPacket, elsePacket);
751}
752
753
754//---------- double ----------
755#ifdef __VSX__
756typedef __vector double Packet2d;
757typedef __vector unsigned long long Packet2ul;
758typedef __vector long long Packet2l;
759#if EIGEN_COMP_CLANG
760typedef Packet2ul Packet2bl;
761#else
762typedef __vector __bool long Packet2bl;
763#endif
764
765static Packet2l p2l_ONE = { 1, 1 };
766static Packet2l p2l_ZERO = reinterpret_cast<Packet2l>(p4i_ZERO);
767static Packet2d p2d_ONE = { 1.0, 1.0 };
768static Packet2d p2d_ZERO = reinterpret_cast<Packet2d>(p4f_ZERO);
769static Packet2d p2d_MZERO = { -0.0, -0.0 };
770static Packet2ul p2ul_SIGN = {0x8000000000000000ull, 0x8000000000000000ull};
771static Packet2ul p2ul_PREV0DOT5 = {0x3FDFFFFFFFFFFFFFull, 0x3FDFFFFFFFFFFFFFull};
772
773#ifdef _BIG_ENDIAN
774static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(p2d_ZERO), reinterpret_cast<Packet4f>(p2d_ONE), 8));
775#else
776static Packet2d p2d_COUNTDOWN = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(p2d_ONE), reinterpret_cast<Packet4f>(p2d_ZERO), 8));
777#endif
778
779template<int index> Packet2d vec_splat_dbl(Packet2d& a)
780{
781 return vec_splat(a, index);
782}
783
784template<> struct packet_traits<double> : default_packet_traits
785{
786 typedef Packet2d type;
787 typedef Packet2d half;
788 enum {
789 Vectorizable = 1,
790 AlignedOnScalar = 1,
791 size=2,
792 HasHalfPacket = 1,
793
794 HasAdd = 1,
795 HasSub = 1,
796 HasMul = 1,
797 HasDiv = 1,
798 HasMin = 1,
799 HasMax = 1,
800 HasAbs = 1,
801 HasSin = 0,
802 HasCos = 0,
803 HasLog = 0,
804 HasExp = 1,
805 HasSqrt = 1,
806#if !EIGEN_COMP_CLANG
807 HasRsqrt = 1,
808#else
809 HasRsqrt = 0,
810#endif
811 HasRound = 1,
812 HasFloor = 1,
813 HasCeil = 1,
814 HasNegate = 1,
815 HasBlend = 1
816 };
817};
818
819template<> struct unpacket_traits<Packet2d> { typedef double type; enum {size=2, alignment=Aligned16}; typedef Packet2d half; };
820
821inline std::ostream & operator <<(std::ostream & s, const Packet2l & v)
822{
823 union {
824 Packet2l v;
825 int64_t n[2];
826 } vt;
827 vt.v = v;
828 s << vt.n[0] << ", " << vt.n[1];
829 return s;
830}
831
832inline std::ostream & operator <<(std::ostream & s, const Packet2d & v)
833{
834 union {
835 Packet2d v;
836 double n[2];
837 } vt;
838 vt.v = v;
839 s << vt.n[0] << ", " << vt.n[1];
840 return s;
841}
842
843// Need to define them first or we get specialization after instantiation errors
844template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from)
845{
846 EIGEN_DEBUG_ALIGNED_LOAD
847 return vec_xl(0, const_cast<double *>(from)); // cast needed by Clang
848}
849
850template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from)
851{
852 EIGEN_DEBUG_ALIGNED_STORE
853 vec_xst(from, 0, to);
854}
855
856template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
857 Packet2d v = {from, from};
858 return v;
859}
860
861template<> EIGEN_STRONG_INLINE void
862pbroadcast4<Packet2d>(const double *a,
863 Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3)
864{
865 //This way is faster than vec_splat (at least for doubles in Power 9)
866 a0 = pset1<Packet2d>(a[0]);
867 a1 = pset1<Packet2d>(a[1]);
868 a2 = pset1<Packet2d>(a[2]);
869 a3 = pset1<Packet2d>(a[3]);
870}
871
872template<> EIGEN_DEVICE_FUNC inline Packet2d pgather<double, Packet2d>(const double* from, Index stride)
873{
874 EIGEN_ALIGN16 double af[2];
875 af[0] = from[0*stride];
876 af[1] = from[1*stride];
877 return pload<Packet2d>(af);
878}
879template<> EIGEN_DEVICE_FUNC inline void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride)
880{
881 EIGEN_ALIGN16 double af[2];
882 pstore<double>(af, from);
883 to[0*stride] = af[0];
884 to[1*stride] = af[1];
885}
886
887template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) { return pset1<Packet2d>(a) + p2d_COUNTDOWN; }
888
889template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return a + b; }
890
891template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return a - b; }
892
893template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a)
894{
895#ifdef __POWER8_VECTOR__
896 return vec_neg(a);
897#else
898 return vec_xor(a, p2d_MZERO);
899#endif
900}
901
902template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; }
903
904template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_madd(a,b,p2d_MZERO); }
905template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_div(a,b); }
906
907// for some weird raisons, it has to be overloaded for packet of integers
908template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { return vec_madd(a, b, c); }
909
910template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b)
911{
912 Packet2d ret;
913 __asm__ ("xvcmpgedp %x0,%x1,%x2\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b));
914 return ret;
915 }
916
917template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b)
918{
919 Packet2d ret;
920 __asm__ ("xvcmpgtdp %x0,%x2,%x1\n\txxsel %x0,%x1,%x2,%x0" : "=&wa" (ret) : "wa" (a), "wa" (b));
921 return ret;
922}
923
924template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, b); }
925
926template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_or(a, b); }
927
928template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_xor(a, b); }
929
930template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) { return vec_and(a, vec_nor(b, b)); }
931
932template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a)
933{
934 Packet2d t = vec_add(reinterpret_cast<Packet2d>(vec_or(vec_and(reinterpret_cast<Packet2ul>(a), p2ul_SIGN), p2ul_PREV0DOT5)), a);
935 Packet2d res;
936
937 __asm__("xvrdpiz %x0, %x1\n\t"
938 : "=&wa" (res)
939 : "wa" (t));
940
941 return res;
942}
943template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) { return vec_ceil(a); }
944template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) { return vec_floor(a); }
945
946template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
947{
948 EIGEN_DEBUG_UNALIGNED_LOAD
949 return vec_xl(0, const_cast<double*>(from));
950}
951
952template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from)
953{
954 Packet2d p;
955 if((std::ptrdiff_t(from) % 16) == 0) p = pload<Packet2d>(from);
956 else p = ploadu<Packet2d>(from);
957 return vec_splat_dbl<0>(p);
958}
959
960template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from)
961{
962 EIGEN_DEBUG_UNALIGNED_STORE
963 vec_xst(from, 0, to);
964}
965
966template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { EIGEN_PPC_PREFETCH(addr); }
967
968template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { EIGEN_ALIGN16 double x[2]; pstore<double>(x, a); return x[0]; }
969
970template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
971{
972 return reinterpret_cast<Packet2d>(vec_perm(reinterpret_cast<Packet16uc>(a), reinterpret_cast<Packet16uc>(a), p16uc_REVERSE64));
973}
974template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vec_abs(a); }
975
976template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
977{
978 Packet2d b, sum;
979 b = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(a), reinterpret_cast<Packet4f>(a), 8));
980 sum = a + b;
981 return pfirst<Packet2d>(sum);
982}
983
984template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(const Packet2d* vecs)
985{
986 Packet2d v[2], sum;
987 v[0] = vecs[0] + reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(vecs[0]), reinterpret_cast<Packet4f>(vecs[0]), 8));
988 v[1] = vecs[1] + reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(vecs[1]), reinterpret_cast<Packet4f>(vecs[1]), 8));
989
990#ifdef _BIG_ENDIAN
991 sum = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(v[0]), reinterpret_cast<Packet4f>(v[1]), 8));
992#else
993 sum = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4f>(v[1]), reinterpret_cast<Packet4f>(v[0]), 8));
994#endif
995
996 return sum;
997}
998// Other reduction functions:
999// mul
1000template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
1001{
1002 return pfirst(pmul(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
1003}
1004
1005// min
1006template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
1007{
1008 return pfirst(pmin(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
1009}
1010
1011// max
1012template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
1013{
1014 return pfirst(pmax(a, reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(a), reinterpret_cast<Packet4ui>(a), 8))));
1015}
1016
1017template<int Offset>
1018struct palign_impl<Offset,Packet2d>
1019{
1020 static EIGEN_STRONG_INLINE void run(Packet2d& first, const Packet2d& second)
1021 {
1022 if (Offset == 1)
1023#ifdef _BIG_ENDIAN
1024 first = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(first), reinterpret_cast<Packet4ui>(second), 8));
1025#else
1026 first = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(second), reinterpret_cast<Packet4ui>(first), 8));
1027#endif
1028 }
1029};
1030
1031EIGEN_DEVICE_FUNC inline void
1032ptranspose(PacketBlock<Packet2d,2>& kernel) {
1033 Packet2d t0, t1;
1034 t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
1035 t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
1036 kernel.packet[0] = t0;
1037 kernel.packet[1] = t1;
1038}
1039
1040template<> EIGEN_STRONG_INLINE Packet2d pblend(const Selector<2>& ifPacket, const Packet2d& thenPacket, const Packet2d& elsePacket) {
1041 Packet2l select = { ifPacket.select[0], ifPacket.select[1] };
1042 Packet2bl mask = reinterpret_cast<Packet2bl>( vec_cmpeq(reinterpret_cast<Packet2d>(select), reinterpret_cast<Packet2d>(p2l_ONE)) );
1043 return vec_sel(elsePacket, thenPacket, mask);
1044}
1045#endif // __VSX__
1046} // end namespace internal
1047
1048} // end namespace Eigen
1049
1050#endif // EIGEN_PACKET_MATH_ALTIVEC_H
Eigen::Aligned16
@ Aligned16
Definition Constants.h:230
Eigen
Namespace containing all symbols from the Eigen library.
Definition A05_PortingFrom2To3.dox:1
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:65