Eigen  3.3.9
 
Loading...
Searching...
No Matches
GenericPacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_GENERIC_PACKET_MATH_H
12#define EIGEN_GENERIC_PACKET_MATH_H
13
14namespace Eigen {
15
16namespace internal {
17
25
26#ifndef EIGEN_DEBUG_ALIGNED_LOAD
27#define EIGEN_DEBUG_ALIGNED_LOAD
28#endif
29
30#ifndef EIGEN_DEBUG_UNALIGNED_LOAD
31#define EIGEN_DEBUG_UNALIGNED_LOAD
32#endif
33
34#ifndef EIGEN_DEBUG_ALIGNED_STORE
35#define EIGEN_DEBUG_ALIGNED_STORE
36#endif
37
38#ifndef EIGEN_DEBUG_UNALIGNED_STORE
39#define EIGEN_DEBUG_UNALIGNED_STORE
40#endif
41
42struct default_packet_traits
43{
44 enum {
45 HasHalfPacket = 0,
46
47 HasAdd = 1,
48 HasSub = 1,
49 HasMul = 1,
50 HasNegate = 1,
51 HasAbs = 1,
52 HasArg = 0,
53 HasAbs2 = 1,
54 HasMin = 1,
55 HasMax = 1,
56 HasConj = 1,
57 HasSetLinear = 1,
58 HasBlend = 0,
59
60 HasDiv = 0,
61 HasSqrt = 0,
62 HasRsqrt = 0,
63 HasExp = 0,
64 HasLog = 0,
65 HasLog1p = 0,
66 HasLog10 = 0,
67 HasPow = 0,
68
69 HasSin = 0,
70 HasCos = 0,
71 HasTan = 0,
72 HasASin = 0,
73 HasACos = 0,
74 HasATan = 0,
75 HasSinh = 0,
76 HasCosh = 0,
77 HasTanh = 0,
78 HasLGamma = 0,
79 HasDiGamma = 0,
80 HasZeta = 0,
81 HasPolygamma = 0,
82 HasErf = 0,
83 HasErfc = 0,
84 HasIGamma = 0,
85 HasIGammac = 0,
86 HasBetaInc = 0,
87
88 HasRound = 0,
89 HasFloor = 0,
90 HasCeil = 0,
91
92 HasSign = 0
93 };
94};
95
96template<typename T> struct packet_traits : default_packet_traits
97{
98 typedef T type;
99 typedef T half;
100 enum {
101 Vectorizable = 0,
102 size = 1,
103 AlignedOnScalar = 0,
104 HasHalfPacket = 0
105 };
106 enum {
107 HasAdd = 0,
108 HasSub = 0,
109 HasMul = 0,
110 HasNegate = 0,
111 HasAbs = 0,
112 HasAbs2 = 0,
113 HasMin = 0,
114 HasMax = 0,
115 HasConj = 0,
116 HasSetLinear = 0
117 };
118};
119
120template<typename T> struct packet_traits<const T> : packet_traits<T> { };
121
122template <typename Src, typename Tgt> struct type_casting_traits {
123 enum {
124 VectorizedCast = 0,
125 SrcCoeffRatio = 1,
126 TgtCoeffRatio = 1
127 };
128};
129
130
132template <typename SrcPacket, typename TgtPacket>
133EIGEN_DEVICE_FUNC inline TgtPacket
134pcast(const SrcPacket& a) {
135 return static_cast<TgtPacket>(a);
136}
137template <typename SrcPacket, typename TgtPacket>
138EIGEN_DEVICE_FUNC inline TgtPacket
139pcast(const SrcPacket& a, const SrcPacket& /*b*/) {
140 return static_cast<TgtPacket>(a);
141}
142
143template <typename SrcPacket, typename TgtPacket>
144EIGEN_DEVICE_FUNC inline TgtPacket
145pcast(const SrcPacket& a, const SrcPacket& /*b*/, const SrcPacket& /*c*/, const SrcPacket& /*d*/) {
146 return static_cast<TgtPacket>(a);
147}
148
150template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
151padd(const Packet& a,
152 const Packet& b) { return a+b; }
153
155template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
156psub(const Packet& a,
157 const Packet& b) { return a-b; }
158
160template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
161pnegate(const Packet& a) { return -a; }
162
164
165template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
166pconj(const Packet& a) { return numext::conj(a); }
167
169template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
170pmul(const Packet& a,
171 const Packet& b) { return a*b; }
172
174template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
175pdiv(const Packet& a,
176 const Packet& b) { return a/b; }
177
179template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
180pmin(const Packet& a,
181 const Packet& b) { return numext::mini(a, b); }
182
184template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
185pmax(const Packet& a,
186 const Packet& b) { return numext::maxi(a, b); }
187
189template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
190pabs(const Packet& a) { using std::abs; return abs(a); }
191
193template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
194parg(const Packet& a) { using numext::arg; return arg(a); }
195
197template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
198pand(const Packet& a, const Packet& b) { return a & b; }
199
201template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
202por(const Packet& a, const Packet& b) { return a | b; }
203
205template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
206pxor(const Packet& a, const Packet& b) { return a ^ b; }
207
209template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
210pandnot(const Packet& a, const Packet& b) { return a & (!b); }
211
213template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
214pload(const typename unpacket_traits<Packet>::type* from) { return *from; }
215
217template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
218ploadu(const typename unpacket_traits<Packet>::type* from) { return *from; }
219
221template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
222pset1(const typename unpacket_traits<Packet>::type& a) { return a; }
223
225template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
226pload1(const typename unpacket_traits<Packet>::type *a) { return pset1<Packet>(*a); }
227
233template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
234ploaddup(const typename unpacket_traits<Packet>::type* from) { return *from; }
235
242template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
243ploadquad(const typename unpacket_traits<Packet>::type* from)
244{ return pload1<Packet>(from); }
245
255template<typename Packet> EIGEN_DEVICE_FUNC
256inline void pbroadcast4(const typename unpacket_traits<Packet>::type *a,
257 Packet& a0, Packet& a1, Packet& a2, Packet& a3)
258{
259 a0 = pload1<Packet>(a+0);
260 a1 = pload1<Packet>(a+1);
261 a2 = pload1<Packet>(a+2);
262 a3 = pload1<Packet>(a+3);
263}
264
272template<typename Packet> EIGEN_DEVICE_FUNC
273inline void pbroadcast2(const typename unpacket_traits<Packet>::type *a,
274 Packet& a0, Packet& a1)
275{
276 a0 = pload1<Packet>(a+0);
277 a1 = pload1<Packet>(a+1);
278}
279
281template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
282plset(const typename unpacket_traits<Packet>::type& a) { return a; }
283
285template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pstore(Scalar* to, const Packet& from)
286{ (*to) = from; }
287
289template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pstoreu(Scalar* to, const Packet& from)
290{ (*to) = from; }
291
292 template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline Packet pgather(const Scalar* from, Index /*stride*/)
293 { return ploadu<Packet>(from); }
294
295 template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index /*stride*/)
296 { pstore(to, from); }
297
299template<typename Scalar> EIGEN_DEVICE_FUNC inline void prefetch(const Scalar* addr)
300{
301#ifdef __CUDA_ARCH__
302#if defined(__LP64__) || EIGEN_OS_WIN64
303 // 64-bit pointer operand constraint for inlined asm
304 asm(" prefetch.L1 [ %1 ];" : "=l"(addr) : "l"(addr));
305#else
306 // 32-bit pointer operand constraint for inlined asm
307 asm(" prefetch.L1 [ %1 ];" : "=r"(addr) : "r"(addr));
308#endif
309#elif (!EIGEN_COMP_MSVC) && (EIGEN_COMP_GNUC || EIGEN_COMP_CLANG || EIGEN_COMP_ICC)
310 __builtin_prefetch(addr);
311#endif
312}
313
315template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type pfirst(const Packet& a)
316{ return a; }
317
319template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
320preduxp(const Packet* vecs) { return vecs[0]; }
321
323template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux(const Packet& a)
324{ return a; }
325
330template<typename Packet> EIGEN_DEVICE_FUNC inline
331typename conditional<(unpacket_traits<Packet>::size%8)==0,typename unpacket_traits<Packet>::half,Packet>::type
332predux_downto4(const Packet& a)
333{ return a; }
334
336template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_mul(const Packet& a)
337{ return a; }
338
340template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_min(const Packet& a)
341{ return a; }
342
344template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_max(const Packet& a)
345{ return a; }
346
348template<typename Packet> EIGEN_DEVICE_FUNC inline Packet preverse(const Packet& a)
349{ return a; }
350
352template<typename Packet> EIGEN_DEVICE_FUNC inline Packet pcplxflip(const Packet& a)
353{
354 return Packet(a.imag(),a.real());
355}
356
357/**************************
358* Special math functions
359***************************/
360
362template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
363Packet psin(const Packet& a) { using std::sin; return sin(a); }
364
366template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
367Packet pcos(const Packet& a) { using std::cos; return cos(a); }
368
370template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
371Packet ptan(const Packet& a) { using std::tan; return tan(a); }
372
374template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
375Packet pasin(const Packet& a) { using std::asin; return asin(a); }
376
378template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
379Packet pacos(const Packet& a) { using std::acos; return acos(a); }
380
382template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
383Packet patan(const Packet& a) { using std::atan; return atan(a); }
384
386template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
387Packet psinh(const Packet& a) { using std::sinh; return sinh(a); }
388
390template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
391Packet pcosh(const Packet& a) { using std::cosh; return cosh(a); }
392
394template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
395Packet ptanh(const Packet& a) { using std::tanh; return tanh(a); }
396
398template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
399Packet pexp(const Packet& a) { using std::exp; return exp(a); }
400
402template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
403Packet plog(const Packet& a) { using std::log; return log(a); }
404
406template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
407Packet plog1p(const Packet& a) { return numext::log1p(a); }
408
410template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
411Packet plog10(const Packet& a) { using std::log10; return log10(a); }
412
414template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
415Packet psqrt(const Packet& a) { using std::sqrt; return sqrt(a); }
416
418template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
419Packet prsqrt(const Packet& a) {
420 return pdiv(pset1<Packet>(1), psqrt(a));
421}
422
424template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
425Packet pround(const Packet& a) { using numext::round; return round(a); }
426
428template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
429Packet pfloor(const Packet& a) { using numext::floor; return floor(a); }
430
432template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
433Packet pceil(const Packet& a) { using numext::ceil; return ceil(a); }
434
435/***************************************************************************
436* The following functions might not have to be overwritten for vectorized types
437***************************************************************************/
438
440// NOTE: this function must really be templated on the packet type (think about different packet types for the same scalar type)
441template<typename Packet>
442inline void pstore1(typename unpacket_traits<Packet>::type* to, const typename unpacket_traits<Packet>::type& a)
443{
444 pstore(to, pset1<Packet>(a));
445}
446
448template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
449pmadd(const Packet& a,
450 const Packet& b,
451 const Packet& c)
452{ return padd(pmul(a, b),c); }
453
456template<typename Packet, int Alignment>
457EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits<Packet>::type* from)
458{
459 if(Alignment >= unpacket_traits<Packet>::alignment)
460 return pload<Packet>(from);
461 else
462 return ploadu<Packet>(from);
463}
464
467template<typename Scalar, typename Packet, int Alignment>
468EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from)
469{
470 if(Alignment >= unpacket_traits<Packet>::alignment)
471 pstore(to, from);
472 else
473 pstoreu(to, from);
474}
475
481template<typename Packet, int LoadMode>
482EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_ro(const typename unpacket_traits<Packet>::type* from)
483{
484 return ploadt<Packet, LoadMode>(from);
485}
486
488template<int Offset,typename PacketType>
489struct palign_impl
490{
491 // by default data are aligned, so there is nothing to be done :)
492 static inline void run(PacketType&, const PacketType&) {}
493};
494
510template<int Offset,typename PacketType>
511inline void palign(PacketType& first, const PacketType& second)
512{
513 palign_impl<Offset,PacketType>::run(first,second);
514}
515
516/***************************************************************************
517* Fast complex products (GCC generates a function call which is very slow)
518***************************************************************************/
519
520// Eigen+CUDA does not support complexes.
521#ifndef EIGEN_CUDACC
522
523template<> inline std::complex<float> pmul(const std::complex<float>& a, const std::complex<float>& b)
524{ return std::complex<float>(a.real()*b.real() - a.imag()*b.imag(), a.imag()*b.real() + a.real()*b.imag()); }
525
526template<> inline std::complex<double> pmul(const std::complex<double>& a, const std::complex<double>& b)
527{ return std::complex<double>(a.real()*b.real() - a.imag()*b.imag(), a.imag()*b.real() + a.real()*b.imag()); }
528
529#endif
530
531
532/***************************************************************************
533 * PacketBlock, that is a collection of N packets where the number of words
534 * in the packet is a multiple of N.
535***************************************************************************/
536template <typename Packet,int N=unpacket_traits<Packet>::size> struct PacketBlock {
537 Packet packet[N];
538};
539
540template<typename Packet> EIGEN_DEVICE_FUNC inline void
541ptranspose(PacketBlock<Packet,1>& /*kernel*/) {
542 // Nothing to do in the scalar case, i.e. a 1x1 matrix.
543}
544
545/***************************************************************************
546 * Selector, i.e. vector of N boolean values used to select (i.e. blend)
547 * words from 2 packets.
548***************************************************************************/
549template <size_t N> struct Selector {
550 bool select[N];
551};
552
553template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
554pblend(const Selector<unpacket_traits<Packet>::size>& ifPacket, const Packet& thenPacket, const Packet& elsePacket) {
555 return ifPacket.select[0] ? thenPacket : elsePacket;
556}
557
559template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
560pinsertfirst(const Packet& a, typename unpacket_traits<Packet>::type b)
561{
562 // Default implementation based on pblend.
563 // It must be specialized for higher performance.
564 Selector<unpacket_traits<Packet>::size> mask;
565 mask.select[0] = true;
566 // This for loop should be optimized away by the compiler.
567 for(Index i=1; i<unpacket_traits<Packet>::size; ++i)
568 mask.select[i] = false;
569 return pblend(mask, pset1<Packet>(b), a);
570}
571
573template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
574pinsertlast(const Packet& a, typename unpacket_traits<Packet>::type b)
575{
576 // Default implementation based on pblend.
577 // It must be specialized for higher performance.
578 Selector<unpacket_traits<Packet>::size> mask;
579 // This for loop should be optimized away by the compiler.
580 for(Index i=0; i<unpacket_traits<Packet>::size-1; ++i)
581 mask.select[i] = false;
582 mask.select[unpacket_traits<Packet>::size-1] = true;
583 return pblend(mask, pset1<Packet>(b), a);
584}
585
586} // end namespace internal
587
588} // end namespace Eigen
589
590#endif // EIGEN_GENERIC_PACKET_MATH_H
Eigen
Namespace containing all symbols from the Eigen library.
Definition A05_PortingFrom2To3.dox:1
Eigen::cosh
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_cosh_op< typename Derived::Scalar >, const Derived > cosh(const Eigen::ArrayBase< Derived > &x)
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:65
Eigen::atan
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_atan_op< typename Derived::Scalar >, const Derived > atan(const Eigen::ArrayBase< Derived > &x)
Eigen::round
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_round_op< typename Derived::Scalar >, const Derived > round(const Eigen::ArrayBase< Derived > &x)
Eigen::asin
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_asin_op< typename Derived::Scalar >, const Derived > asin(const Eigen::ArrayBase< Derived > &x)
Eigen::acos
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_acos_op< typename Derived::Scalar >, const Derived > acos(const Eigen::ArrayBase< Derived > &x)
Eigen::sinh
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_sinh_op< typename Derived::Scalar >, const Derived > sinh(const Eigen::ArrayBase< Derived > &x)
Eigen::arg
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_arg_op< typename Derived::Scalar >, const Derived > arg(const Eigen::ArrayBase< Derived > &x)