NEON/PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2010 Konstantinos Margaritis <markos@codex.gr>
6// Heavily based on Gael's SSE version.
7//
8// This Source Code Form is subject to the terms of the Mozilla
9// Public License v. 2.0. If a copy of the MPL was not distributed
10// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11
12#ifndef EIGEN_PACKET_MATH_NEON_H
13#define EIGEN_PACKET_MATH_NEON_H
14
15namespace Eigen {
16
17namespace internal {
18
19#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
21#endif
22
23// FIXME NEON has 16 quad registers, but since the current register allocator
24// is so bad, it is much better to reduce it to 8
25#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
26#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 8
27#endif
28
29typedef float32x4_t Packet4f;
30typedef int32x4_t Packet4i;
31typedef uint32x4_t Packet4ui;
32
33#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
34 const Packet4f p4f_##NAME = pset1<Packet4f>(X)
35
36#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
37 const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int>(X))
38
39#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
40 const Packet4i p4i_##NAME = pset1<Packet4i>(X)
41
42#if defined(__llvm__) && !defined(__clang__)
43 //Special treatment for Apple's llvm-gcc, its NEON packet types are unions
44 #define EIGEN_INIT_NEON_PACKET2(X, Y) {{X, Y}}
45 #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {{X, Y, Z, W}}
46#else
47 //Default initializer for packets
48 #define EIGEN_INIT_NEON_PACKET2(X, Y) {X, Y}
49 #define EIGEN_INIT_NEON_PACKET4(X, Y, Z, W) {X, Y, Z, W}
50#endif
51
52#ifndef __pld
53#define __pld(x) asm volatile ( " pld [%[addr]]\n" :: [addr] "r" (x) : "cc" );
54#endif
55
56template<> struct packet_traits<float> : default_packet_traits
57{
58 typedef Packet4f type;
59 enum {
60 Vectorizable = 1,
61 AlignedOnScalar = 1,
62 size = 4,
63
64 HasDiv = 1,
65 // FIXME check the Has*
66 HasSin = 0,
67 HasCos = 0,
68 HasLog = 0,
69 HasExp = 0,
70 HasSqrt = 0
71 };
72};
73template<> struct packet_traits<int> : default_packet_traits
74{
75 typedef Packet4i type;
76 enum {
77 Vectorizable = 1,
78 AlignedOnScalar = 1,
79 size=4
80 // FIXME check the Has*
81 };
82};
83
84#if EIGEN_GNUC_AT_MOST(4,4) && !defined(__llvm__)
85// workaround gcc 4.2, 4.3 and 4.4 compilatin issue
86EIGEN_STRONG_INLINE float32x4_t vld1q_f32(const float* x) { return ::vld1q_f32((const float32_t*)x); }
87EIGEN_STRONG_INLINE float32x2_t vld1_f32 (const float* x) { return ::vld1_f32 ((const float32_t*)x); }
88EIGEN_STRONG_INLINE void vst1q_f32(float* to, float32x4_t from) { ::vst1q_f32((float32_t*)to,from); }
89EIGEN_STRONG_INLINE void vst1_f32 (float* to, float32x2_t from) { ::vst1_f32 ((float32_t*)to,from); }
90#endif
91
92template<> struct unpacket_traits<Packet4f> { typedef float type; enum {size=4}; };
93template<> struct unpacket_traits<Packet4i> { typedef int type; enum {size=4}; };
94
95template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return vdupq_n_f32(from); }
96template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) { return vdupq_n_s32(from); }
97
98template<> EIGEN_STRONG_INLINE Packet4f plset<float>(const float& a)
99{
100 Packet4f countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
101 return vaddq_f32(pset1<Packet4f>(a), countdown);
102}
103template<> EIGEN_STRONG_INLINE Packet4i plset<int>(const int& a)
104{
105 Packet4i countdown = EIGEN_INIT_NEON_PACKET4(0, 1, 2, 3);
106 return vaddq_s32(pset1<Packet4i>(a), countdown);
107}
108
109template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return vaddq_f32(a,b); }
110template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return vaddq_s32(a,b); }
111
112template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return vsubq_f32(a,b); }
113template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return vsubq_s32(a,b); }
114
115template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return vnegq_f32(a); }
116template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return vnegq_s32(a); }
117
118template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmulq_f32(a,b); }
119template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmulq_s32(a,b); }
120
121template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b)
122{
123 Packet4f inv, restep, div;
124
125 // NEON does not offer a divide instruction, we have to do a reciprocal approximation
126 // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers
127 // a reciprocal estimate AND a reciprocal step -which saves a few instructions
128 // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with
129 // Newton-Raphson and vrecpsq_f32()
130 inv = vrecpeq_f32(b);
131
132 // This returns a differential, by which we will have to multiply inv to get a better
133 // approximation of 1/b.
134 restep = vrecpsq_f32(b, inv);
135 inv = vmulq_f32(restep, inv);
136
137 // Finally, multiply a by 1/b and get the wanted result of the division.
138 div = vmulq_f32(a, inv);
139
140 return div;
141}
142template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
143{ eigen_assert(false && "packet integer division are not supported by NEON");
144 return pset1<Packet4i>(0);
145}
146
147// for some weird raisons, it has to be overloaded for packet of integers
148template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { return vmlaq_f32(c,a,b); }
149template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { return vmlaq_s32(c,a,b); }
150
151template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return vminq_f32(a,b); }
152template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vminq_s32(a,b); }
153
154template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxq_f32(a,b); }
155template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmaxq_s32(a,b); }
156
157// Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
158template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b)
159{
160 return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
161}
162template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vandq_s32(a,b); }
163
164template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b)
165{
166 return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
167}
168template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vorrq_s32(a,b); }
169
170template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b)
171{
172 return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
173}
174template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return veorq_s32(a,b); }
175
176template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b)
177{
178 return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b)));
179}
180template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) { return vbicq_s32(a,b); }
181
182template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); }
183template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); }
184
185template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from); }
186template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from); }
187
188template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
189{
190 float32x2_t lo, hi;
191 lo = vdup_n_f32(*from);
192 hi = vdup_n_f32(*(from+1));
193 return vcombine_f32(lo, hi);
194}
195template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from)
196{
197 int32x2_t lo, hi;
198 lo = vdup_n_s32(*from);
199 hi = vdup_n_s32(*(from+1));
200 return vcombine_s32(lo, hi);
201}
202
203template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to, from); }
204template<> EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to, from); }
205
206template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to, from); }
207template<> EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to, from); }
208
209template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { __pld(addr); }
210template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { __pld(addr); }
211
212// FIXME only store the 2 first elements ?
213template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { float EIGEN_ALIGN16 x[4]; vst1q_f32(x, a); return x[0]; }
214template<> EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) { int EIGEN_ALIGN16 x[4]; vst1q_s32(x, a); return x[0]; }
215
216template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
217 float32x2_t a_lo, a_hi;
218 Packet4f a_r64;
219
220 a_r64 = vrev64q_f32(a);
221 a_lo = vget_low_f32(a_r64);
222 a_hi = vget_high_f32(a_r64);
223 return vcombine_f32(a_hi, a_lo);
224}
225template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
226 int32x2_t a_lo, a_hi;
227 Packet4i a_r64;
228
229 a_r64 = vrev64q_s32(a);
230 a_lo = vget_low_s32(a_r64);
231 a_hi = vget_high_s32(a_r64);
232 return vcombine_s32(a_hi, a_lo);
233}
234template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vabsq_f32(a); }
235template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s32(a); }
236
237template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
238{
239 float32x2_t a_lo, a_hi, sum;
240 float s[2];
241
242 a_lo = vget_low_f32(a);
243 a_hi = vget_high_f32(a);
244 sum = vpadd_f32(a_lo, a_hi);
245 sum = vpadd_f32(sum, sum);
246 vst1_f32(s, sum);
247
248 return s[0];
249}
250
251template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(const Packet4f* vecs)
252{
253 float32x4x2_t vtrn1, vtrn2, res1, res2;
254 Packet4f sum1, sum2, sum;
255
256 // NEON zip performs interleaving of the supplied vectors.
257 // We perform two interleaves in a row to acquire the transposed vector
258 vtrn1 = vzipq_f32(vecs[0], vecs[2]);
259 vtrn2 = vzipq_f32(vecs[1], vecs[3]);
260 res1 = vzipq_f32(vtrn1.val[0], vtrn2.val[0]);
261 res2 = vzipq_f32(vtrn1.val[1], vtrn2.val[1]);
262
263 // Do the addition of the resulting vectors
264 sum1 = vaddq_f32(res1.val[0], res1.val[1]);
265 sum2 = vaddq_f32(res2.val[0], res2.val[1]);
266 sum = vaddq_f32(sum1, sum2);
267
268 return sum;
269}
270
271template<> EIGEN_STRONG_INLINE int predux<Packet4i>(const Packet4i& a)
272{
273 int32x2_t a_lo, a_hi, sum;
274 int32_t s[2];
275
276 a_lo = vget_low_s32(a);
277 a_hi = vget_high_s32(a);
278 sum = vpadd_s32(a_lo, a_hi);
279 sum = vpadd_s32(sum, sum);
280 vst1_s32(s, sum);
281
282 return s[0];
283}
284
285template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(const Packet4i* vecs)
286{
287 int32x4x2_t vtrn1, vtrn2, res1, res2;
288 Packet4i sum1, sum2, sum;
289
290 // NEON zip performs interleaving of the supplied vectors.
291 // We perform two interleaves in a row to acquire the transposed vector
292 vtrn1 = vzipq_s32(vecs[0], vecs[2]);
293 vtrn2 = vzipq_s32(vecs[1], vecs[3]);
294 res1 = vzipq_s32(vtrn1.val[0], vtrn2.val[0]);
295 res2 = vzipq_s32(vtrn1.val[1], vtrn2.val[1]);
296
297 // Do the addition of the resulting vectors
298 sum1 = vaddq_s32(res1.val[0], res1.val[1]);
299 sum2 = vaddq_s32(res2.val[0], res2.val[1]);
300 sum = vaddq_s32(sum1, sum2);
301
302 return sum;
303}
304
305// Other reduction functions:
306// mul
307template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
308{
309 float32x2_t a_lo, a_hi, prod;
310 float s[2];
311
312 // Get a_lo = |a1|a2| and a_hi = |a3|a4|
313 a_lo = vget_low_f32(a);
314 a_hi = vget_high_f32(a);
315 // Get the product of a_lo * a_hi -> |a1*a3|a2*a4|
316 prod = vmul_f32(a_lo, a_hi);
317 // Multiply prod with its swapped value |a2*a4|a1*a3|
318 prod = vmul_f32(prod, vrev64_f32(prod));
319 vst1_f32(s, prod);
320
321 return s[0];
322}
323template<> EIGEN_STRONG_INLINE int predux_mul<Packet4i>(const Packet4i& a)
324{
325 int32x2_t a_lo, a_hi, prod;
326 int32_t s[2];
327
328 // Get a_lo = |a1|a2| and a_hi = |a3|a4|
329 a_lo = vget_low_s32(a);
330 a_hi = vget_high_s32(a);
331 // Get the product of a_lo * a_hi -> |a1*a3|a2*a4|
332 prod = vmul_s32(a_lo, a_hi);
333 // Multiply prod with its swapped value |a2*a4|a1*a3|
334 prod = vmul_s32(prod, vrev64_s32(prod));
335 vst1_s32(s, prod);
336
337 return s[0];
338}
339
340// min
341template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
342{
343 float32x2_t a_lo, a_hi, min;
344 float s[2];
345
346 a_lo = vget_low_f32(a);
347 a_hi = vget_high_f32(a);
348 min = vpmin_f32(a_lo, a_hi);
349 min = vpmin_f32(min, min);
350 vst1_f32(s, min);
351
352 return s[0];
353}
354template<> EIGEN_STRONG_INLINE int predux_min<Packet4i>(const Packet4i& a)
355{
356 int32x2_t a_lo, a_hi, min;
357 int32_t s[2];
358
359 a_lo = vget_low_s32(a);
360 a_hi = vget_high_s32(a);
361 min = vpmin_s32(a_lo, a_hi);
362 min = vpmin_s32(min, min);
363 vst1_s32(s, min);
364
365 return s[0];
366}
367
368// max
369template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
370{
371 float32x2_t a_lo, a_hi, max;
372 float s[2];
373
374 a_lo = vget_low_f32(a);
375 a_hi = vget_high_f32(a);
376 max = vpmax_f32(a_lo, a_hi);
377 max = vpmax_f32(max, max);
378 vst1_f32(s, max);
379
380 return s[0];
381}
382template<> EIGEN_STRONG_INLINE int predux_max<Packet4i>(const Packet4i& a)
383{
384 int32x2_t a_lo, a_hi, max;
385 int32_t s[2];
386
387 a_lo = vget_low_s32(a);
388 a_hi = vget_high_s32(a);
389 max = vpmax_s32(a_lo, a_hi);
390 max = vpmax_s32(max, max);
391 vst1_s32(s, max);
392
393 return s[0];
394}
395
396// this PALIGN_NEON business is to work around a bug in LLVM Clang 3.0 causing incorrect compilation errors,
397// see bug 347 and this LLVM bug: http://llvm.org/bugs/show_bug.cgi?id=11074
398#define PALIGN_NEON(Offset,Type,Command) \
399template<>\
400struct palign_impl<Offset,Type>\
401{\
402 EIGEN_STRONG_INLINE static void run(Type& first, const Type& second)\
403 {\
404 if (Offset!=0)\
405 first = Command(first, second, Offset);\
406 }\
407};\
408
409PALIGN_NEON(0,Packet4f,vextq_f32)
410PALIGN_NEON(1,Packet4f,vextq_f32)
411PALIGN_NEON(2,Packet4f,vextq_f32)
412PALIGN_NEON(3,Packet4f,vextq_f32)
413PALIGN_NEON(0,Packet4i,vextq_s32)
414PALIGN_NEON(1,Packet4i,vextq_s32)
415PALIGN_NEON(2,Packet4i,vextq_s32)
416PALIGN_NEON(3,Packet4i,vextq_s32)
417
418#undef PALIGN_NEON
419
420} // end namespace internal
421
422} // end namespace Eigen
423
424#endif // EIGEN_PACKET_MATH_NEON_H
Eigen::internal
Definition LDLT.h:18