Eigen  3.4.1 (git rev 28ded8800c26864e537852658428ab44c8399e87)
 
Loading...
Searching...
No Matches
GenericPacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_GENERIC_PACKET_MATH_H
12#define EIGEN_GENERIC_PACKET_MATH_H
13
14namespace Eigen {
15
16namespace internal {
17
25
26#ifndef EIGEN_DEBUG_ALIGNED_LOAD
27#define EIGEN_DEBUG_ALIGNED_LOAD
28#endif
29
30#ifndef EIGEN_DEBUG_UNALIGNED_LOAD
31#define EIGEN_DEBUG_UNALIGNED_LOAD
32#endif
33
34#ifndef EIGEN_DEBUG_ALIGNED_STORE
35#define EIGEN_DEBUG_ALIGNED_STORE
36#endif
37
38#ifndef EIGEN_DEBUG_UNALIGNED_STORE
39#define EIGEN_DEBUG_UNALIGNED_STORE
40#endif
41
42struct default_packet_traits
43{
44 enum {
45 HasHalfPacket = 0,
46
47 HasAdd = 1,
48 HasSub = 1,
49 HasShift = 1,
50 HasMul = 1,
51 HasNegate = 1,
52 HasAbs = 1,
53 HasArg = 0,
54 HasAbs2 = 1,
55 HasAbsDiff = 0,
56 HasMin = 1,
57 HasMax = 1,
58 HasConj = 1,
59 HasSetLinear = 1,
60 HasBlend = 0,
61 // This flag is used to indicate whether packet comparison is supported.
62 // pcmp_eq, pcmp_lt and pcmp_le should be defined for it to be true.
63 HasCmp = 0,
64
65 HasDiv = 0,
66 HasSqrt = 0,
67 HasRsqrt = 0,
68 HasExp = 0,
69 HasExpm1 = 0,
70 HasLog = 0,
71 HasLog1p = 0,
72 HasLog10 = 0,
73 HasPow = 0,
74
75 HasSin = 0,
76 HasCos = 0,
77 HasTan = 0,
78 HasASin = 0,
79 HasACos = 0,
80 HasATan = 0,
81 HasSinh = 0,
82 HasCosh = 0,
83 HasTanh = 0,
84 HasLGamma = 0,
85 HasDiGamma = 0,
86 HasZeta = 0,
87 HasPolygamma = 0,
88 HasErf = 0,
89 HasErfc = 0,
90 HasNdtri = 0,
91 HasBessel = 0,
92 HasIGamma = 0,
93 HasIGammaDerA = 0,
94 HasGammaSampleDerAlpha = 0,
95 HasIGammac = 0,
96 HasBetaInc = 0,
97
98 HasRound = 0,
99 HasRint = 0,
100 HasFloor = 0,
101 HasCeil = 0,
102 HasSign = 0
103 };
104};
105
106template<typename T> struct packet_traits : default_packet_traits
107{
108 typedef T type;
109 typedef T half;
110 enum {
111 Vectorizable = 0,
112 size = 1,
113 AlignedOnScalar = 0,
114 HasHalfPacket = 0
115 };
116 enum {
117 HasAdd = 0,
118 HasSub = 0,
119 HasMul = 0,
120 HasNegate = 0,
121 HasAbs = 0,
122 HasAbs2 = 0,
123 HasMin = 0,
124 HasMax = 0,
125 HasConj = 0,
126 HasSetLinear = 0
127 };
128};
129
130template<typename T> struct packet_traits<const T> : packet_traits<T> { };
131
132template<typename T> struct unpacket_traits
133{
134 typedef T type;
135 typedef T half;
136 enum
137 {
138 size = 1,
139 alignment = 1,
140 vectorizable = false,
141 masked_load_available=false,
142 masked_store_available=false
143 };
144};
145
146template<typename T> struct unpacket_traits<const T> : unpacket_traits<T> { };
147
148template <typename Src, typename Tgt> struct type_casting_traits {
149 enum {
150 VectorizedCast = 0,
151 SrcCoeffRatio = 1,
152 TgtCoeffRatio = 1
153 };
154};
155
158template<typename T, int unique_id = 0>
159struct eigen_packet_wrapper
160{
161 EIGEN_ALWAYS_INLINE operator T&() { return m_val; }
162 EIGEN_ALWAYS_INLINE operator const T&() const { return m_val; }
163#if EIGEN_HAS_CXX11
164 EIGEN_ALWAYS_INLINE eigen_packet_wrapper() = default;
165#else
166 EIGEN_ALWAYS_INLINE eigen_packet_wrapper() {};
167#endif
168 EIGEN_ALWAYS_INLINE eigen_packet_wrapper(const T &v) : m_val(v) {}
169 EIGEN_ALWAYS_INLINE eigen_packet_wrapper& operator=(const T &v) {
170 m_val = v;
171 return *this;
172 }
173
174 T m_val;
175};
176
177
181template<typename Packet>
182struct is_scalar {
183 typedef typename unpacket_traits<Packet>::type Scalar;
184 enum {
185 value = internal::is_same<Packet, Scalar>::value
186 };
187};
188
190template <typename SrcPacket, typename TgtPacket>
191EIGEN_DEVICE_FUNC inline TgtPacket
192pcast(const SrcPacket& a) {
193 return static_cast<TgtPacket>(a);
194}
195template <typename SrcPacket, typename TgtPacket>
196EIGEN_DEVICE_FUNC inline TgtPacket
197pcast(const SrcPacket& a, const SrcPacket& /*b*/) {
198 return static_cast<TgtPacket>(a);
199}
200template <typename SrcPacket, typename TgtPacket>
201EIGEN_DEVICE_FUNC inline TgtPacket
202pcast(const SrcPacket& a, const SrcPacket& /*b*/, const SrcPacket& /*c*/, const SrcPacket& /*d*/) {
203 return static_cast<TgtPacket>(a);
204}
205template <typename SrcPacket, typename TgtPacket>
206EIGEN_DEVICE_FUNC inline TgtPacket
207pcast(const SrcPacket& a, const SrcPacket& /*b*/, const SrcPacket& /*c*/, const SrcPacket& /*d*/,
208 const SrcPacket& /*e*/, const SrcPacket& /*f*/, const SrcPacket& /*g*/, const SrcPacket& /*h*/) {
209 return static_cast<TgtPacket>(a);
210}
211
213template <typename Target, typename Packet>
214EIGEN_DEVICE_FUNC inline Target
215preinterpret(const Packet& a); /* { return reinterpret_cast<const Target&>(a); } */
216
218template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
219padd(const Packet& a, const Packet& b) { return a+b; }
220// Avoid compiler warning for boolean algebra.
221template<> EIGEN_DEVICE_FUNC inline bool
222padd(const bool& a, const bool& b) { return a || b; }
223
225template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
226psub(const Packet& a, const Packet& b) { return a-b; }
227
229template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
230pnegate(const Packet& a) { return -a; }
231
232template<> EIGEN_DEVICE_FUNC inline bool
233pnegate(const bool& a) { return !a; }
234
236template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
237pconj(const Packet& a) { return numext::conj(a); }
238
240template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
241pmul(const Packet& a, const Packet& b) { return a*b; }
242// Avoid compiler warning for boolean algebra.
243template<> EIGEN_DEVICE_FUNC inline bool
244pmul(const bool& a, const bool& b) { return a && b; }
245
247template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
248pdiv(const Packet& a, const Packet& b) { return a/b; }
249
250// In the generic case, memset to all one bits.
251template<typename Packet, typename EnableIf = void>
252struct ptrue_impl {
253 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& /*a*/){
254 Packet b;
255 memset(static_cast<void*>(&b), 0xff, sizeof(Packet));
256 return b;
257 }
258};
259
260// For non-trivial scalars, set to Scalar(1) (i.e. a non-zero value).
261// Although this is technically not a valid bitmask, the scalar path for pselect
262// uses a comparison to zero, so this should still work in most cases. We don't
263// have another option, since the scalar type requires initialization.
264template<typename T>
265struct ptrue_impl<T,
266 typename internal::enable_if<is_scalar<T>::value && NumTraits<T>::RequireInitialization>::type > {
267 static EIGEN_DEVICE_FUNC inline T run(const T& /*a*/){
268 return T(1);
269 }
270};
271
273template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
274ptrue(const Packet& a) {
275 return ptrue_impl<Packet>::run(a);
276}
277
278// In the general case, memset to zero.
279template<typename Packet, typename EnableIf = void>
280struct pzero_impl {
281 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& /*a*/) {
282 Packet b;
283 memset(static_cast<void*>(&b), 0x00, sizeof(Packet));
284 return b;
285 }
286};
287
288// For scalars, explicitly set to Scalar(0), since the underlying representation
289// for zero may not consist of all-zero bits.
290template<typename T>
291struct pzero_impl<T,
292 typename internal::enable_if<is_scalar<T>::value>::type> {
293 static EIGEN_DEVICE_FUNC inline T run(const T& /*a*/) {
294 return T(0);
295 }
296};
297
299template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
300pzero(const Packet& a) {
301 return pzero_impl<Packet>::run(a);
302}
303
305template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
306pcmp_le(const Packet& a, const Packet& b) { return a<=b ? ptrue(a) : pzero(a); }
307
309template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
310pcmp_lt(const Packet& a, const Packet& b) { return a<b ? ptrue(a) : pzero(a); }
311
313template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
314pcmp_eq(const Packet& a, const Packet& b) { return a==b ? ptrue(a) : pzero(a); }
315
317template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
318pcmp_lt_or_nan(const Packet& a, const Packet& b) { return a>=b ? pzero(a) : ptrue(a); }
319
320template<typename T>
321struct bit_and {
322 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR EIGEN_ALWAYS_INLINE T operator()(const T& a, const T& b) const {
323 return a & b;
324 }
325};
326
327template<typename T>
328struct bit_or {
329 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR EIGEN_ALWAYS_INLINE T operator()(const T& a, const T& b) const {
330 return a | b;
331 }
332};
333
334template<typename T>
335struct bit_xor {
336 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR EIGEN_ALWAYS_INLINE T operator()(const T& a, const T& b) const {
337 return a ^ b;
338 }
339};
340
341template<typename T>
342struct bit_not {
343 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR EIGEN_ALWAYS_INLINE T operator()(const T& a) const {
344 return ~a;
345 }
346};
347
348// Use operators &, |, ^, ~.
349template<typename T>
350struct operator_bitwise_helper {
351 EIGEN_DEVICE_FUNC static inline T bitwise_and(const T& a, const T& b) { return bit_and<T>()(a, b); }
352 EIGEN_DEVICE_FUNC static inline T bitwise_or(const T& a, const T& b) { return bit_or<T>()(a, b); }
353 EIGEN_DEVICE_FUNC static inline T bitwise_xor(const T& a, const T& b) { return bit_xor<T>()(a, b); }
354 EIGEN_DEVICE_FUNC static inline T bitwise_not(const T& a) { return bit_not<T>()(a); }
355};
356
357// Apply binary operations byte-by-byte
358template<typename T>
359struct bytewise_bitwise_helper {
360 EIGEN_DEVICE_FUNC static inline T bitwise_and(const T& a, const T& b) {
361 return binary(a, b, bit_and<unsigned char>());
362 }
363 EIGEN_DEVICE_FUNC static inline T bitwise_or(const T& a, const T& b) {
364 return binary(a, b, bit_or<unsigned char>());
365 }
366 EIGEN_DEVICE_FUNC static inline T bitwise_xor(const T& a, const T& b) {
367 return binary(a, b, bit_xor<unsigned char>());
368 }
369 EIGEN_DEVICE_FUNC static inline T bitwise_not(const T& a) {
370 return unary(a,bit_not<unsigned char>());
371 }
372
373 private:
374 template<typename Op>
375 EIGEN_DEVICE_FUNC static inline T unary(const T& a, Op op) {
376 const unsigned char* a_ptr = reinterpret_cast<const unsigned char*>(&a);
377 T c;
378 unsigned char* c_ptr = reinterpret_cast<unsigned char*>(&c);
379 for (size_t i = 0; i < sizeof(T); ++i) {
380 *c_ptr++ = op(*a_ptr++);
381 }
382 return c;
383 }
384
385 template<typename Op>
386 EIGEN_DEVICE_FUNC static inline T binary(const T& a, const T& b, Op op) {
387 const unsigned char* a_ptr = reinterpret_cast<const unsigned char*>(&a);
388 const unsigned char* b_ptr = reinterpret_cast<const unsigned char*>(&b);
389 T c;
390 unsigned char* c_ptr = reinterpret_cast<unsigned char*>(&c);
391 for (size_t i = 0; i < sizeof(T); ++i) {
392 *c_ptr++ = op(*a_ptr++, *b_ptr++);
393 }
394 return c;
395 }
396};
397
398// In the general case, use byte-by-byte manipulation.
399template<typename T, typename EnableIf = void>
400struct bitwise_helper : public bytewise_bitwise_helper<T> {};
401
402// For integers or non-trivial scalars, use binary operators.
403template<typename T>
404struct bitwise_helper<T,
405 typename internal::enable_if<
406 is_scalar<T>::value && (NumTraits<T>::IsInteger || NumTraits<T>::RequireInitialization)>::type
407 > : public operator_bitwise_helper<T> {};
408
410template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
411pand(const Packet& a, const Packet& b) {
412 return bitwise_helper<Packet>::bitwise_and(a, b);
413}
414
416template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
417por(const Packet& a, const Packet& b) {
418 return bitwise_helper<Packet>::bitwise_or(a, b);
419}
420
422template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
423pxor(const Packet& a, const Packet& b) {
424 return bitwise_helper<Packet>::bitwise_xor(a, b);
425}
426
428template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
429pnot(const Packet& a) {
430 return bitwise_helper<Packet>::bitwise_not(a);
431}
432
434template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
435pandnot(const Packet& a, const Packet& b) { return pand(a, pnot(b)); }
436
437// In the general case, use bitwise select.
438template<typename Packet, typename EnableIf = void>
439struct pselect_impl {
440 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& mask, const Packet& a, const Packet& b) {
441 return por(pand(a,mask),pandnot(b,mask));
442 }
443};
444
445// For scalars, use ternary select.
446template<typename Packet>
447struct pselect_impl<Packet,
448 typename internal::enable_if<is_scalar<Packet>::value>::type > {
449 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& mask, const Packet& a, const Packet& b) {
450 return numext::equal_strict(mask, Packet(0)) ? b : a;
451 }
452};
453
455template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
456pselect(const Packet& mask, const Packet& a, const Packet& b) {
457 return pselect_impl<Packet>::run(mask, a, b);
458}
459
460template<> EIGEN_DEVICE_FUNC inline bool pselect<bool>(
461 const bool& cond, const bool& a, const bool& b) {
462 return cond ? a : b;
463}
464
467template<int NaNPropagation>
468struct pminmax_impl {
469 template <typename Packet, typename Op>
470 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a, const Packet& b, Op op) {
471 return op(a,b);
472 }
473};
474
477template<>
478struct pminmax_impl<PropagateNaN> {
479 template <typename Packet, typename Op>
480 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a, const Packet& b, Op op) {
481 Packet not_nan_mask_a = pcmp_eq(a, a);
482 Packet not_nan_mask_b = pcmp_eq(b, b);
483 return pselect(not_nan_mask_a,
484 pselect(not_nan_mask_b, op(a, b), b),
485 a);
486 }
487};
488
492template<>
493struct pminmax_impl<PropagateNumbers> {
494 template <typename Packet, typename Op>
495 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a, const Packet& b, Op op) {
496 Packet not_nan_mask_a = pcmp_eq(a, a);
497 Packet not_nan_mask_b = pcmp_eq(b, b);
498 return pselect(not_nan_mask_a,
499 pselect(not_nan_mask_b, op(a, b), a),
500 b);
501 }
502};
503
504
505#ifndef SYCL_DEVICE_ONLY
506#define EIGEN_BINARY_OP_NAN_PROPAGATION(Type, Func) Func
507#else
508#define EIGEN_BINARY_OP_NAN_PROPAGATION(Type, Func) \
509[](const Type& a, const Type& b) { \
510 return Func(a, b);}
511#endif
512
515template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
516pmin(const Packet& a, const Packet& b) { return numext::mini(a,b); }
517
520template <int NaNPropagation, typename Packet>
521EIGEN_DEVICE_FUNC inline Packet pmin(const Packet& a, const Packet& b) {
522 return pminmax_impl<NaNPropagation>::run(a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet, (pmin<Packet>)));
523}
524
527template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
528pmax(const Packet& a, const Packet& b) { return numext::maxi(a, b); }
529
532template <int NaNPropagation, typename Packet>
533EIGEN_DEVICE_FUNC inline Packet pmax(const Packet& a, const Packet& b) {
534 return pminmax_impl<NaNPropagation>::run(a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet,(pmax<Packet>)));
535}
536
538template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
539pabs(const Packet& a) { return numext::abs(a); }
540template<> EIGEN_DEVICE_FUNC inline unsigned int
541pabs(const unsigned int& a) { return a; }
542template<> EIGEN_DEVICE_FUNC inline unsigned long
543pabs(const unsigned long& a) { return a; }
544template<> EIGEN_DEVICE_FUNC inline unsigned long long
545pabs(const unsigned long long& a) { return a; }
546
548template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
549paddsub(const Packet& a, const Packet& b) {
550 return pselect(peven_mask(a), padd(a, b), psub(a, b));
551 }
552
554template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
555parg(const Packet& a) { using numext::arg; return arg(a); }
556
557
559template<int N> EIGEN_DEVICE_FUNC inline int
560parithmetic_shift_right(const int& a) { return a >> N; }
561template<int N> EIGEN_DEVICE_FUNC inline long int
562parithmetic_shift_right(const long int& a) { return a >> N; }
563
565template<int N> EIGEN_DEVICE_FUNC inline int
566plogical_shift_right(const int& a) { return static_cast<int>(static_cast<unsigned int>(a) >> N); }
567template<int N> EIGEN_DEVICE_FUNC inline long int
568plogical_shift_right(const long int& a) { return static_cast<long>(static_cast<unsigned long>(a) >> N); }
569
571template<int N> EIGEN_DEVICE_FUNC inline int
572plogical_shift_left(const int& a) { return a << N; }
573template<int N> EIGEN_DEVICE_FUNC inline long int
574plogical_shift_left(const long int& a) { return a << N; }
575
579template <typename Packet>
580EIGEN_DEVICE_FUNC inline Packet pfrexp(const Packet& a, Packet& exponent) {
581 int exp;
582 EIGEN_USING_STD(frexp);
583 Packet result = static_cast<Packet>(frexp(a, &exp));
584 exponent = static_cast<Packet>(exp);
585 return result;
586}
587
591template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
592pldexp(const Packet &a, const Packet &exponent) {
593 EIGEN_USING_STD(ldexp)
594 return static_cast<Packet>(ldexp(a, static_cast<int>(exponent)));
595}
596
598template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
599pabsdiff(const Packet& a, const Packet& b) { return pselect(pcmp_lt(a, b), psub(b, a), psub(a, b)); }
600
602template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
603pload(const typename unpacket_traits<Packet>::type* from) { return *from; }
604
606template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
607ploadu(const typename unpacket_traits<Packet>::type* from) { return *from; }
608
613template<typename Packet> EIGEN_DEVICE_FUNC inline
614typename enable_if<unpacket_traits<Packet>::masked_load_available, Packet>::type
615ploadu(const typename unpacket_traits<Packet>::type* from, typename unpacket_traits<Packet>::mask_t umask);
616
618template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
619pset1(const typename unpacket_traits<Packet>::type& a) { return a; }
620
622template<typename Packet,typename BitsType> EIGEN_DEVICE_FUNC inline Packet
623pset1frombits(BitsType a);
624
626template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
627pload1(const typename unpacket_traits<Packet>::type *a) { return pset1<Packet>(*a); }
628
634template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
635ploaddup(const typename unpacket_traits<Packet>::type* from) { return *from; }
636
643template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
644ploadquad(const typename unpacket_traits<Packet>::type* from)
645{ return pload1<Packet>(from); }
646
656template<typename Packet> EIGEN_DEVICE_FUNC
657inline void pbroadcast4(const typename unpacket_traits<Packet>::type *a,
658 Packet& a0, Packet& a1, Packet& a2, Packet& a3)
659{
660 a0 = pload1<Packet>(a+0);
661 a1 = pload1<Packet>(a+1);
662 a2 = pload1<Packet>(a+2);
663 a3 = pload1<Packet>(a+3);
664}
665
673template<typename Packet> EIGEN_DEVICE_FUNC
674inline void pbroadcast2(const typename unpacket_traits<Packet>::type *a,
675 Packet& a0, Packet& a1)
676{
677 a0 = pload1<Packet>(a+0);
678 a1 = pload1<Packet>(a+1);
679}
680
682template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
683plset(const typename unpacket_traits<Packet>::type& a) { return a; }
684
687template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
688peven_mask(const Packet& /*a*/) {
689 typedef typename unpacket_traits<Packet>::type Scalar;
690 const size_t n = unpacket_traits<Packet>::size;
691 EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar elements[n];
692 for(size_t i = 0; i < n; ++i) {
693 memset(elements+i, ((i & 1) == 0 ? 0xff : 0), sizeof(Scalar));
694 }
695 return ploadu<Packet>(elements);
696}
697
698
700template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pstore(Scalar* to, const Packet& from)
701{ (*to) = from; }
702
704template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pstoreu(Scalar* to, const Packet& from)
705{ (*to) = from; }
706
711template<typename Scalar, typename Packet>
712EIGEN_DEVICE_FUNC inline
713typename enable_if<unpacket_traits<Packet>::masked_store_available, void>::type
714pstoreu(Scalar* to, const Packet& from, typename unpacket_traits<Packet>::mask_t umask);
715
716 template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline Packet pgather(const Scalar* from, Index /*stride*/)
717 { return ploadu<Packet>(from); }
718
719 template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index /*stride*/)
720 { pstore(to, from); }
721
723template<typename Scalar> EIGEN_DEVICE_FUNC inline void prefetch(const Scalar* addr)
724{
725#if defined(EIGEN_HIP_DEVICE_COMPILE)
726 // do nothing
727#elif defined(EIGEN_CUDA_ARCH)
728#if defined(__LP64__) || EIGEN_OS_WIN64
729 // 64-bit pointer operand constraint for inlined asm
730 asm(" prefetch.L1 [ %1 ];" : "=l"(addr) : "l"(addr));
731#else
732 // 32-bit pointer operand constraint for inlined asm
733 asm(" prefetch.L1 [ %1 ];" : "=r"(addr) : "r"(addr));
734#endif
735#elif (!EIGEN_COMP_MSVC) && (EIGEN_COMP_GNUC || EIGEN_COMP_CLANG || EIGEN_COMP_ICC)
736 __builtin_prefetch(addr);
737#endif
738}
739
741template<typename Packet> EIGEN_DEVICE_FUNC inline Packet preverse(const Packet& a)
742{ return a; }
743
745template<typename Packet> EIGEN_DEVICE_FUNC inline Packet pcplxflip(const Packet& a)
746{
747 return Packet(numext::imag(a),numext::real(a));
748}
749
750/**************************
751* Special math functions
752***************************/
753
755template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
756Packet psin(const Packet& a) { EIGEN_USING_STD(sin); return sin(a); }
757
759template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
760Packet pcos(const Packet& a) { EIGEN_USING_STD(cos); return cos(a); }
761
763template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
764Packet ptan(const Packet& a) { EIGEN_USING_STD(tan); return tan(a); }
765
767template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
768Packet pasin(const Packet& a) { EIGEN_USING_STD(asin); return asin(a); }
769
771template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
772Packet pacos(const Packet& a) { EIGEN_USING_STD(acos); return acos(a); }
773
775template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
776Packet patan(const Packet& a) { EIGEN_USING_STD(atan); return atan(a); }
777
779template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
780Packet psinh(const Packet& a) { EIGEN_USING_STD(sinh); return sinh(a); }
781
783template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
784Packet pcosh(const Packet& a) { EIGEN_USING_STD(cosh); return cosh(a); }
785
787template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
788Packet ptanh(const Packet& a) { EIGEN_USING_STD(tanh); return tanh(a); }
789
791template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
792Packet pexp(const Packet& a) { EIGEN_USING_STD(exp); return exp(a); }
793
795template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
796Packet pexpm1(const Packet& a) { return numext::expm1(a); }
797
799template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
800Packet plog(const Packet& a) { EIGEN_USING_STD(log); return log(a); }
801
803template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
804Packet plog1p(const Packet& a) { return numext::log1p(a); }
805
807template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
808Packet plog10(const Packet& a) { EIGEN_USING_STD(log10); return log10(a); }
809
811template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
812Packet plog2(const Packet& a) {
813 typedef typename internal::unpacket_traits<Packet>::type Scalar;
814 return pmul(pset1<Packet>(Scalar(EIGEN_LOG2E)), plog(a));
815}
816
818template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
819Packet psqrt(const Packet& a) { return numext::sqrt(a); }
820
822template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
823Packet prsqrt(const Packet& a) {
824 typedef typename internal::unpacket_traits<Packet>::type Scalar;
825 return pdiv(pset1<Packet>(Scalar(1)), psqrt(a));
826}
827
829template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
830Packet pround(const Packet& a) { using numext::round; return round(a); }
831
833template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
834Packet pfloor(const Packet& a) { using numext::floor; return floor(a); }
835
838template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
839Packet print(const Packet& a) { using numext::rint; return rint(a); }
840
842template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
843Packet pceil(const Packet& a) { using numext::ceil; return ceil(a); }
844
846template<typename Packet>
847EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type
848pfirst(const Packet& a)
849{ return a; }
850
855template<typename Packet>
856EIGEN_DEVICE_FUNC inline typename conditional<(unpacket_traits<Packet>::size%8)==0,typename unpacket_traits<Packet>::half,Packet>::type
857predux_half_dowto4(const Packet& a)
858{ return a; }
859
860// Slow generic implementation of Packet reduction.
861template <typename Packet, typename Op>
862EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type
863predux_helper(const Packet& a, Op op) {
864 typedef typename unpacket_traits<Packet>::type Scalar;
865 const size_t n = unpacket_traits<Packet>::size;
866 EIGEN_ALIGN_TO_BOUNDARY(sizeof(Packet)) Scalar elements[n];
867 pstoreu<Scalar>(elements, a);
868 for(size_t k = n / 2; k > 0; k /= 2) {
869 for(size_t i = 0; i < k; ++i) {
870 elements[i] = op(elements[i], elements[i + k]);
871 }
872 }
873 return elements[0];
874}
875
877template<typename Packet>
878EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type
879predux(const Packet& a)
880{
881 return a;
882}
883
885template <typename Packet>
886EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_mul(
887 const Packet& a) {
888 typedef typename unpacket_traits<Packet>::type Scalar;
889 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmul<Scalar>)));
890}
891
893template <typename Packet>
894EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_min(
895 const Packet &a) {
896 typedef typename unpacket_traits<Packet>::type Scalar;
897 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<PropagateFast, Scalar>)));
898}
899
900template <int NaNPropagation, typename Packet>
901EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_min(
902 const Packet& a) {
903 typedef typename unpacket_traits<Packet>::type Scalar;
904 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<NaNPropagation, Scalar>)));
905}
906
908template <typename Packet>
909EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_max(
910 const Packet &a) {
911 typedef typename unpacket_traits<Packet>::type Scalar;
912 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<PropagateFast, Scalar>)));
913}
914
915template <int NaNPropagation, typename Packet>
916EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_max(
917 const Packet& a) {
918 typedef typename unpacket_traits<Packet>::type Scalar;
919 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<NaNPropagation, Scalar>)));
920}
921
922#undef EIGEN_BINARY_OP_NAN_PROPAGATION
923
927// not needed yet
928// template<typename Packet> EIGEN_DEVICE_FUNC inline bool predux_all(const Packet& a)
929// { return bool(a); }
930
934template<typename Packet> EIGEN_DEVICE_FUNC inline bool predux_any(const Packet& a)
935{
936 // Dirty but generic implementation where "true" is assumed to be non 0 and all the sames.
937 // It is expected that "true" is either:
938 // - Scalar(1)
939 // - bits full of ones (NaN for floats),
940 // - or first bit equals to 1 (1 for ints, smallest denormal for floats).
941 // For all these cases, taking the sum is just fine, and this boils down to a no-op for scalars.
942 typedef typename unpacket_traits<Packet>::type Scalar;
943 return numext::not_equal_strict(predux(a), Scalar(0));
944}
945
946/***************************************************************************
947* The following functions might not have to be overwritten for vectorized types
948***************************************************************************/
949
951// NOTE: this function must really be templated on the packet type (think about different packet types for the same scalar type)
952template<typename Packet>
953inline void pstore1(typename unpacket_traits<Packet>::type* to, const typename unpacket_traits<Packet>::type& a)
954{
955 pstore(to, pset1<Packet>(a));
956}
957
959template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
960pmadd(const Packet& a,
961 const Packet& b,
962 const Packet& c)
963{ return padd(pmul(a, b),c); }
964
967template<typename Packet, int Alignment>
968EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits<Packet>::type* from)
969{
970 if(Alignment >= unpacket_traits<Packet>::alignment)
971 return pload<Packet>(from);
972 else
973 return ploadu<Packet>(from);
974}
975
978template<typename Scalar, typename Packet, int Alignment>
979EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from)
980{
981 if(Alignment >= unpacket_traits<Packet>::alignment)
982 pstore(to, from);
983 else
984 pstoreu(to, from);
985}
986
992template<typename Packet, int LoadMode>
993EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_ro(const typename unpacket_traits<Packet>::type* from)
994{
995 return ploadt<Packet, LoadMode>(from);
996}
997
998/***************************************************************************
999* Fast complex products (GCC generates a function call which is very slow)
1000***************************************************************************/
1001
1002// Eigen+CUDA does not support complexes.
1003#if !defined(EIGEN_GPUCC)
1004
1005template<> inline std::complex<float> pmul(const std::complex<float>& a, const std::complex<float>& b)
1006{ return std::complex<float>(a.real()*b.real() - a.imag()*b.imag(), a.imag()*b.real() + a.real()*b.imag()); }
1007
1008template<> inline std::complex<double> pmul(const std::complex<double>& a, const std::complex<double>& b)
1009{ return std::complex<double>(a.real()*b.real() - a.imag()*b.imag(), a.imag()*b.real() + a.real()*b.imag()); }
1010
1011#endif
1012
1013
1014/***************************************************************************
1015 * PacketBlock, that is a collection of N packets where the number of words
1016 * in the packet is a multiple of N.
1017***************************************************************************/
1018template <typename Packet,int N=unpacket_traits<Packet>::size> struct PacketBlock {
1019 Packet packet[N];
1020};
1021
1022template<typename Packet> EIGEN_DEVICE_FUNC inline void
1023ptranspose(PacketBlock<Packet,1>& /*kernel*/) {
1024 // Nothing to do in the scalar case, i.e. a 1x1 matrix.
1025}
1026
1027/***************************************************************************
1028 * Selector, i.e. vector of N boolean values used to select (i.e. blend)
1029 * words from 2 packets.
1030***************************************************************************/
1031template <size_t N> struct Selector {
1032 bool select[N];
1033};
1034
1035template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
1036pblend(const Selector<unpacket_traits<Packet>::size>& ifPacket, const Packet& thenPacket, const Packet& elsePacket) {
1037 return ifPacket.select[0] ? thenPacket : elsePacket;
1038}
1039
1040} // end namespace internal
1041
1042} // end namespace Eigen
1043
1044#endif // EIGEN_GENERIC_PACKET_MATH_H
Eigen::PropagateNaN
@ PropagateNaN
Definition Constants.h:343
Eigen::PropagateNumbers
@ PropagateNumbers
Definition Constants.h:345
Eigen
Namespace containing all symbols from the Eigen library.
Definition B01_Experimental.dox:1
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:74
Eigen::arg
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_arg_op< typename Derived::Scalar >, const Derived > arg(const Eigen::ArrayBase< Derived > &x)