Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GenericPacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_GENERIC_PACKET_MATH_H
13#define EIGEN_GENERIC_PACKET_MATH_H
14
15// IWYU pragma: private
16#include "./InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
29
30#ifndef EIGEN_DEBUG_ALIGNED_LOAD
31#define EIGEN_DEBUG_ALIGNED_LOAD
32#endif
33
34#ifndef EIGEN_DEBUG_UNALIGNED_LOAD
35#define EIGEN_DEBUG_UNALIGNED_LOAD
36#endif
37
38#ifndef EIGEN_DEBUG_ALIGNED_STORE
39#define EIGEN_DEBUG_ALIGNED_STORE
40#endif
41
42#ifndef EIGEN_DEBUG_UNALIGNED_STORE
43#define EIGEN_DEBUG_UNALIGNED_STORE
44#endif
45
46struct default_packet_traits {
47 enum {
48 // Ops that are implemented for most types.
49 HasAdd = 1,
50 HasSub = 1,
51 HasShift = 1,
52 HasMul = 1,
53 HasNegate = 1,
54 HasAbs = 1,
55 HasAbs2 = 1,
56 HasMin = 1,
57 HasMax = 1,
58 HasConj = 1,
59 HasSetLinear = 1,
60 HasSign = 1,
61 HasAbsDiff = 1,
62 // By default, the nearest integer functions (rint, round, floor, ceil, trunc) are enabled for all scalar and packet
63 // types
64 HasRound = 1,
65
66 HasArg = 0,
67 // This flag is used to indicate whether packet comparison is supported.
68 // pcmp_eq and pcmp_lt should be defined for it to be true.
69 HasCmp = 0,
70
71 HasDiv = 0,
72 HasFastIntDiv = 0,
73 HasReciprocal = 0,
74 HasSqrt = 0,
75 HasRsqrt = 0,
76 HasCbrt = 0,
77 HasExp = 0,
78 HasExpm1 = 0,
79 HasLog = 0,
80 HasLog1p = 0,
81 HasLog10 = 0,
82 HasPow = 0,
83 HasSin = 0,
84 HasCos = 0,
85 HasTan = 0,
86 HasASin = 0,
87 HasACos = 0,
88 HasATan = 0,
89 HasATanh = 0,
90 HasSinh = 0,
91 HasCosh = 0,
92 HasASinh = 0,
93 HasACosh = 0,
94 HasTanh = 0,
95 HasLGamma = 0,
96 HasDiGamma = 0,
97 HasZeta = 0,
98 HasPolygamma = 0,
99 HasErf = 0,
100 HasErfc = 0,
101 HasNdtri = 0,
102 HasBessel = 0,
103 HasIGamma = 0,
104 HasIGammaDerA = 0,
105 HasGammaSampleDerAlpha = 0,
106 HasIGammac = 0,
107 HasBetaInc = 0
108 };
109};
110
111template <typename T>
112struct packet_traits : default_packet_traits {
113 using type = T;
114 using half = T;
115 enum {
116 Vectorizable = 0,
117 size = 1,
118 AlignedOnScalar = 0,
119 };
120 enum {
121 HasAdd = 0,
122 HasSub = 0,
123 HasAbsDiff = 0,
124 HasMul = 0,
125 HasNegate = 0,
126 HasAbs = 0,
127 HasAbs2 = 0,
128 HasMin = 0,
129 HasMax = 0,
130 HasConj = 0,
131 HasSetLinear = 0
132 };
133};
134
135template <typename T>
136struct packet_traits<const T> : packet_traits<T> {};
137
138struct default_unpacket_traits {
139 enum { vectorizable = false, masked_load_available = false, masked_store_available = false };
140};
141
142template <typename T>
143struct unpacket_traits : default_unpacket_traits {
144 using type = T;
145 using half = T;
146 using integer_packet = typename numext::get_integer_by_size<sizeof(T)>::signed_type;
147 enum {
148 size = 1,
149 alignment = alignof(T),
150 };
151};
152
153template <typename T>
154struct unpacket_traits<const T> : unpacket_traits<T> {};
155
159template <typename Packet>
160struct is_scalar : std::is_same<Packet, typename unpacket_traits<Packet>::type> {};
161
162// automatically and succinctly define combinations of pcast<SrcPacket,TgtPacket> when
163// 1) the packets are the same type, or
164// 2) the packets differ only in sign.
165// In both of these cases, preinterpret (bit_cast) is equivalent to pcast (static_cast)
166template <typename SrcPacket, typename TgtPacket,
167 bool Scalar = is_scalar<SrcPacket>::value && is_scalar<TgtPacket>::value>
168struct is_degenerate_helper : std::is_same<SrcPacket, TgtPacket> {};
169template <>
170struct is_degenerate_helper<int8_t, uint8_t, true> : std::true_type {};
171template <>
172struct is_degenerate_helper<int16_t, uint16_t, true> : std::true_type {};
173template <>
174struct is_degenerate_helper<int32_t, uint32_t, true> : std::true_type {};
175template <>
176struct is_degenerate_helper<int64_t, uint64_t, true> : std::true_type {};
177
178template <typename SrcPacket, typename TgtPacket>
179struct is_degenerate_helper<SrcPacket, TgtPacket, false> {
180 using SrcScalar = typename unpacket_traits<SrcPacket>::type;
181 static constexpr int SrcSize = unpacket_traits<SrcPacket>::size;
182 using TgtScalar = typename unpacket_traits<TgtPacket>::type;
183 static constexpr int TgtSize = unpacket_traits<TgtPacket>::size;
184 static constexpr bool value = is_degenerate_helper<SrcScalar, TgtScalar, true>::value && (SrcSize == TgtSize);
185};
186
187// is_degenerate<T1,T2>::value == is_degenerate<T2,T1>::value
188template <typename SrcPacket, typename TgtPacket>
189struct is_degenerate {
190 static constexpr bool value =
191 is_degenerate_helper<SrcPacket, TgtPacket>::value || is_degenerate_helper<TgtPacket, SrcPacket>::value;
192};
193
194template <typename Packet>
195struct is_half {
196 using Scalar = typename unpacket_traits<Packet>::type;
197 static constexpr int Size = unpacket_traits<Packet>::size;
198 using DefaultPacket = typename packet_traits<Scalar>::type;
199 static constexpr int DefaultSize = unpacket_traits<DefaultPacket>::size;
200 static constexpr bool value = Size != 1 && Size < DefaultSize;
201};
202
203template <typename Src, typename Tgt>
204struct type_casting_traits {
205 enum {
206 VectorizedCast =
207 is_degenerate<Src, Tgt>::value && packet_traits<Src>::Vectorizable && packet_traits<Tgt>::Vectorizable,
208 SrcCoeffRatio = 1,
209 TgtCoeffRatio = 1
210 };
211};
212
213// provides a succinct template to define vectorized casting traits with respect to the largest accessible packet types
214template <typename Src, typename Tgt>
215struct vectorized_type_casting_traits {
216 enum : int {
217 DefaultSrcPacketSize = packet_traits<Src>::size,
218 DefaultTgtPacketSize = packet_traits<Tgt>::size,
219 VectorizedCast = 1,
220 SrcCoeffRatio = plain_enum_max(DefaultTgtPacketSize / DefaultSrcPacketSize, 1),
221 TgtCoeffRatio = plain_enum_max(DefaultSrcPacketSize / DefaultTgtPacketSize, 1)
222 };
223};
224
227template <typename T, int unique_id = 0>
228struct eigen_packet_wrapper {
229 EIGEN_ALWAYS_INLINE operator T&() { return m_val; }
230 EIGEN_ALWAYS_INLINE operator const T&() const { return m_val; }
231 EIGEN_ALWAYS_INLINE eigen_packet_wrapper() = default;
232 EIGEN_ALWAYS_INLINE eigen_packet_wrapper(const T& v) : m_val(v) {}
233 EIGEN_ALWAYS_INLINE eigen_packet_wrapper& operator=(const T& v) {
234 m_val = v;
235 return *this;
236 }
237
238 T m_val;
239};
240
241template <typename Target, typename Packet, bool IsSame = std::is_same<Target, Packet>::value>
242struct preinterpret_generic;
243
244template <typename Target, typename Packet>
245struct preinterpret_generic<Target, Packet, false> {
246 // the packets are not the same, attempt scalar bit_cast
247 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Target run(const Packet& a) {
248 return numext::bit_cast<Target, Packet>(a);
249 }
250};
251
252template <typename Packet>
253struct preinterpret_generic<Packet, Packet, true> {
254 // the packets are the same type: do nothing
255 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& a) { return a; }
256};
257
258template <typename ComplexPacket>
259struct preinterpret_generic<typename unpacket_traits<ComplexPacket>::as_real, ComplexPacket, false> {
260 using RealPacket = typename unpacket_traits<ComplexPacket>::as_real;
261 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RealPacket run(const ComplexPacket& a) { return a.v; }
262};
263
265template <typename Target, typename Packet>
266EIGEN_DEVICE_FUNC inline Target preinterpret(const Packet& a) {
267 return preinterpret_generic<Target, Packet>::run(a);
268}
269
270template <typename SrcPacket, typename TgtPacket, bool Degenerate = is_degenerate<SrcPacket, TgtPacket>::value,
271 bool TgtIsHalf = is_half<TgtPacket>::value>
272struct pcast_generic;
273
274template <typename SrcPacket, typename TgtPacket>
275struct pcast_generic<SrcPacket, TgtPacket, false, false> {
276 // the packets are not degenerate: attempt scalar static_cast
277 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(const SrcPacket& a) {
278 return cast_impl<SrcPacket, TgtPacket>::run(a);
279 }
280};
281
282template <typename Packet>
283struct pcast_generic<Packet, Packet, true, false> {
284 // the packets are the same: do nothing
285 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& a) { return a; }
286};
287
288template <typename SrcPacket, typename TgtPacket, bool TgtIsHalf>
289struct pcast_generic<SrcPacket, TgtPacket, true, TgtIsHalf> {
290 // the packets are degenerate: preinterpret is equivalent to pcast
291 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(const SrcPacket& a) { return preinterpret<TgtPacket>(a); }
292};
293
295template <typename SrcPacket, typename TgtPacket>
296EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a) {
297 return pcast_generic<SrcPacket, TgtPacket>::run(a);
298}
299template <typename SrcPacket, typename TgtPacket>
300EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a, const SrcPacket& b) {
301 return pcast_generic<SrcPacket, TgtPacket>::run(a, b);
302}
303template <typename SrcPacket, typename TgtPacket>
304EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a, const SrcPacket& b, const SrcPacket& c,
305 const SrcPacket& d) {
306 return pcast_generic<SrcPacket, TgtPacket>::run(a, b, c, d);
307}
308template <typename SrcPacket, typename TgtPacket>
309EIGEN_DEVICE_FUNC inline TgtPacket pcast(const SrcPacket& a, const SrcPacket& b, const SrcPacket& c, const SrcPacket& d,
310 const SrcPacket& e, const SrcPacket& f, const SrcPacket& g,
311 const SrcPacket& h) {
312 return pcast_generic<SrcPacket, TgtPacket>::run(a, b, c, d, e, f, g, h);
313}
314
315template <typename SrcPacket, typename TgtPacket>
316struct pcast_generic<SrcPacket, TgtPacket, false, true> {
317 // TgtPacket is a half packet of some other type
318 // perform cast and truncate result
319 using DefaultTgtPacket = typename is_half<TgtPacket>::DefaultPacket;
320 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(const SrcPacket& a) {
321 return preinterpret<TgtPacket>(pcast<SrcPacket, DefaultTgtPacket>(a));
322 }
323};
324
325// Make x opaque to the compiler without changing its value, where supported.
326template <typename Packet>
327EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pbarrier(Packet& x) {
328 EIGEN_UNUSED_VARIABLE(x);
329 EIGEN_OPTIMIZATION_BARRIER(x)
330}
331// The wrapper is a class, which EIGEN_OPTIMIZATION_BARRIER passes through memory; its vector can stay in a register.
332template <typename T, int unique_id>
333EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pbarrier(eigen_packet_wrapper<T, unique_id>& x) {
334 pbarrier(x.m_val);
335}
336
338template <typename Packet>
339EIGEN_DEVICE_FUNC inline Packet padd(const Packet& a, const Packet& b) {
340 return a + b;
341}
342// Bool arithmetic is specialized to avoid compiler warnings. Bitwise operations are intentional to keep scalar
343// evaluator loops branch-free.
344template <>
345EIGEN_DEVICE_FUNC inline bool padd(const bool& a, const bool& b) {
346 return a | b;
347}
348
353template <typename Packet>
354EIGEN_DEVICE_FUNC inline std::enable_if_t<unpacket_traits<Packet>::masked_fpops_available, Packet> padd(
355 const Packet& a, const Packet& b, typename unpacket_traits<Packet>::mask_t umask);
356
358template <typename Packet>
359EIGEN_DEVICE_FUNC inline Packet psub(const Packet& a, const Packet& b) {
360 return a - b;
361}
362
364template <typename Packet>
365EIGEN_DEVICE_FUNC inline Packet pnegate(const Packet& a) {
366 EIGEN_STATIC_ASSERT((!std::is_same<typename unpacket_traits<Packet>::type, bool>::value),
367 NEGATE IS NOT DEFINED FOR BOOLEAN TYPES)
368 return numext::negate(a);
369}
370
372template <typename Packet>
373EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pconj(const Packet& a) {
374 using Scalar = typename unpacket_traits<Packet>::type;
375 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex)
376 return numext::conj(a);
377 else
378 return a;
379}
380
382template <typename Packet>
383EIGEN_DEVICE_FUNC inline Packet pmul(const Packet& a, const Packet& b) {
384 return internal::mul(a, b);
385}
386template <>
387EIGEN_DEVICE_FUNC inline bool pmul(const bool& a, const bool& b) {
388 return a & b;
389}
390
392template <typename Packet>
393EIGEN_DEVICE_FUNC inline Packet pdiv(const Packet& a, const Packet& b) {
394 return a / b;
395}
396template <>
397EIGEN_DEVICE_FUNC inline bool pdiv(const bool& a, const bool& b) {
398 return a & b;
399}
400
401// In the generic packet case, memset to all one bits.
402template <typename Packet, typename EnableIf = void>
403struct ptrue_impl {
404 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& /*a*/) {
405 Packet b;
406 memset(static_cast<void*>(&b), 0xff, sizeof(Packet));
407 return b;
408 }
409};
410
411// Use a value of one for scalars.
412template <typename Scalar>
413struct ptrue_impl<Scalar, std::enable_if_t<is_scalar<Scalar>::value>> {
414 static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar&) { return Scalar(1); }
415};
416
417// For booleans, we can only directly set a valid `bool` value to avoid UB.
418template <>
419struct ptrue_impl<bool, void> {
420 static EIGEN_DEVICE_FUNC inline bool run(const bool&) { return true; }
421};
422
423// The all-ones value of a floating-point packet is a NaN bit pattern, which fast-math builds
424// may fold to poison; see EIGEN_FAST_MATH_CONSTANT_BARRIER in Macros.h. The same applies to
425// peven_mask, pinf and pnan below.
427template <typename Packet>
428EIGEN_DEVICE_FUNC inline Packet ptrue(const Packet& a) {
429 if (is_scalar<Packet>::value || std::is_same<Packet, bool>::value) {
430 // Scalar and boolean "masks" hold the value one, which is a legal value class; delegating
431 // to ptrue_impl (and its specializations) is safe here.
432 return ptrue_impl<Packet>::run(a);
433 }
434 Packet b;
435 memset(static_cast<void*>(&b), 0xff, sizeof(Packet));
436 // Every backend stores integer packets in integer vector types, which the poison folding does not affect.
437 EIGEN_IF_CONSTEXPR (!NumTraits<typename unpacket_traits<Packet>::type>::IsInteger) {
438 EIGEN_FAST_MATH_CONSTANT_BARRIER(b);
439 }
440 return b;
441}
442
443// In the general packet case, memset to zero.
444template <typename Packet, typename EnableIf = void>
445struct pzero_impl {
446 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& /*a*/) {
447 Packet b;
448 memset(static_cast<void*>(&b), 0x00, sizeof(Packet));
449 return b;
450 }
451};
452
453// For scalars, explicitly set to Scalar(0), since the underlying representation
454// for zero may not consist of all-zero bits.
455template <typename T>
456struct pzero_impl<T, std::enable_if_t<is_scalar<T>::value>> {
457 static EIGEN_DEVICE_FUNC inline T run(const T& /*a*/) { return T(0); }
458};
459
461template <typename Packet>
462EIGEN_DEVICE_FUNC inline Packet pzero(const Packet& a) {
463 return pzero_impl<Packet>::run(a);
464}
465
466template <typename T>
467struct bit_and {
468 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T operator()(const T& a, const T& b) const { return a & b; }
469};
470
471template <typename T>
472struct bit_or {
473 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T operator()(const T& a, const T& b) const { return a | b; }
474};
475
476template <typename T>
477struct bit_xor {
478 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T operator()(const T& a, const T& b) const { return a ^ b; }
479};
480
481template <typename T>
482struct bit_not {
483 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T operator()(const T& a) const { return ~a; }
484};
485
486template <>
487struct bit_and<bool> {
488 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE bool operator()(const bool& a, const bool& b) const { return a & b; }
489};
490
491template <>
492struct bit_or<bool> {
493 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE bool operator()(const bool& a, const bool& b) const { return a | b; }
494};
495
496template <>
497struct bit_xor<bool> {
498 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE bool operator()(const bool& a, const bool& b) const { return a != b; }
499};
500
501template <>
502struct bit_not<bool> {
503 EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE bool operator()(const bool& a) const { return !a; }
504};
505
506// Use operators &, |, ^, ~.
507template <typename T>
508struct operator_bitwise_helper {
509 EIGEN_DEVICE_FUNC static inline T bitwise_and(const T& a, const T& b) { return bit_and<T>()(a, b); }
510 EIGEN_DEVICE_FUNC static inline T bitwise_or(const T& a, const T& b) { return bit_or<T>()(a, b); }
511 EIGEN_DEVICE_FUNC static inline T bitwise_xor(const T& a, const T& b) { return bit_xor<T>()(a, b); }
512 EIGEN_DEVICE_FUNC static inline T bitwise_not(const T& a) { return bit_not<T>()(a); }
513};
514
515// Apply binary operations byte-by-byte
516template <typename T>
517struct bytewise_bitwise_helper {
518 EIGEN_DEVICE_FUNC static inline T bitwise_and(const T& a, const T& b) {
519 return binary(a, b, bit_and<unsigned char>());
520 }
521 EIGEN_DEVICE_FUNC static inline T bitwise_or(const T& a, const T& b) { return binary(a, b, bit_or<unsigned char>()); }
522 EIGEN_DEVICE_FUNC static inline T bitwise_xor(const T& a, const T& b) {
523 return binary(a, b, bit_xor<unsigned char>());
524 }
525 EIGEN_DEVICE_FUNC static inline T bitwise_not(const T& a) { return unary(a, bit_not<unsigned char>()); }
526
527 private:
528 template <typename Op>
529 EIGEN_DEVICE_FUNC static inline T unary(const T& a, Op op) {
530 const unsigned char* a_ptr = reinterpret_cast<const unsigned char*>(&a);
531 T c;
532 unsigned char* c_ptr = reinterpret_cast<unsigned char*>(&c);
533 for (size_t i = 0; i < sizeof(T); ++i) {
534 *c_ptr++ = op(*a_ptr++);
535 }
536 return c;
537 }
538
539 template <typename Op>
540 EIGEN_DEVICE_FUNC static inline T binary(const T& a, const T& b, Op op) {
541 const unsigned char* a_ptr = reinterpret_cast<const unsigned char*>(&a);
542 const unsigned char* b_ptr = reinterpret_cast<const unsigned char*>(&b);
543 T c;
544 unsigned char* c_ptr = reinterpret_cast<unsigned char*>(&c);
545 for (size_t i = 0; i < sizeof(T); ++i) {
546 *c_ptr++ = op(*a_ptr++, *b_ptr++);
547 }
548 return c;
549 }
550};
551
552// In the general case, use byte-by-byte manipulation.
553template <typename T, typename EnableIf = void>
554struct bitwise_helper : public bytewise_bitwise_helper<T> {};
555
556// For integers or non-trivial scalars, use binary operators.
557template <typename T>
558struct bitwise_helper<
559 T, std::enable_if_t<is_scalar<T>::value && (NumTraits<T>::IsInteger || NumTraits<T>::RequireInitialization)>>
560 : public operator_bitwise_helper<T> {};
561
563template <typename Packet>
564EIGEN_DEVICE_FUNC inline Packet pand(const Packet& a, const Packet& b) {
565 return bitwise_helper<Packet>::bitwise_and(a, b);
566}
567
569template <typename Packet>
570EIGEN_DEVICE_FUNC inline Packet por(const Packet& a, const Packet& b) {
571 return bitwise_helper<Packet>::bitwise_or(a, b);
572}
573
575template <typename Packet>
576EIGEN_DEVICE_FUNC inline Packet pxor(const Packet& a, const Packet& b) {
577 return bitwise_helper<Packet>::bitwise_xor(a, b);
578}
579
580// Packets use the backend's pxor and ptrue: compilers do not reliably vectorize the bytewise fallback (GCC 10 at -O2,
581// GCC and Clang at -O1). For boolean packets ptrue is true, so this is the logical not.
582template <typename Packet, bool IsScalar = is_scalar<Packet>::value>
583struct pnot_impl {
584 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return pxor(a, ptrue(a)); }
585};
586
587template <typename Packet>
588struct pnot_impl<Packet, true> {
589 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return bitwise_helper<Packet>::bitwise_not(a); }
590};
591
593template <typename Packet>
594EIGEN_DEVICE_FUNC inline Packet pnot(const Packet& a) {
595 return pnot_impl<Packet>::run(a);
596}
597
599template <typename Packet>
600EIGEN_DEVICE_FUNC inline Packet pandnot(const Packet& a, const Packet& b) {
601 return pand(a, pnot(b));
602}
603
605template <typename Packet>
606EIGEN_DEVICE_FUNC inline Packet pcmp_lt(const Packet& a, const Packet& b) {
607 return a < b ? ptrue(a) : pzero(a);
608}
609
611template <typename Packet>
612EIGEN_DEVICE_FUNC inline Packet pcmp_eq(const Packet& a, const Packet& b) {
613 return a == b ? ptrue(a) : pzero(a);
614}
615
617template <typename Packet>
618EIGEN_DEVICE_FUNC inline Packet pcmp_le(const Packet& a, const Packet& b) {
619 return por(pcmp_eq(a, b), pcmp_lt(a, b));
620}
621
623template <typename Packet>
624EIGEN_DEVICE_FUNC inline Packet pcmp_lt_or_nan(const Packet& a, const Packet& b) {
625 return a >= b ? pzero(a) : ptrue(a);
626}
627
628// In the general case, use bitwise select.
629template <typename Packet, bool is_scalar = is_scalar<Packet>::value>
630struct pselect_impl {
631 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& mask, const Packet& a, const Packet& b) {
632 return por(pand(a, mask), pandnot(b, mask));
633 }
634};
635
636// For scalars, use ternary select.
637template <typename Packet>
638struct pselect_impl<Packet, true> {
639 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& mask, const Packet& a, const Packet& b) {
640 return numext::select(mask, a, b);
641 }
642};
643
645template <typename Packet>
646EIGEN_DEVICE_FUNC inline Packet pselect(const Packet& mask, const Packet& a, const Packet& b) {
647 return pselect_impl<Packet>::run(mask, a, b);
648}
649
650template <>
651EIGEN_DEVICE_FUNC inline bool pselect<bool>(const bool& cond, const bool& a, const bool& b) {
652 return cond ? a : b;
653}
654
656template <typename Packet>
657struct pminmax_propagates_nan : bool_constant<false> {};
658
663template <int NaNPropagation, bool IsInteger, bool NativePropagatesNaN = false>
664struct pminmax_impl {
665 template <typename Packet, typename Op>
666 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a, const Packet& b, Op op) {
667 return op(a, b);
668 }
669};
670
673template <>
674struct pminmax_impl<PropagateNaN, false, false> {
675 template <typename Packet, typename Op>
676 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a, const Packet& b, Op op) {
677 // pselect is an ordinary call, so op(a, b) is evaluated even where an operand is NaN;
678 // only its result is discarded there. op therefore need not propagate NaN, but must
679 // still be well-defined on NaN input. Operands stay in the caller's order so that op
680 // selects the same one on a signed-zero tie as plain pmin/pmax does for this Packet.
681 Packet not_nan_mask_a = pcmp_eq(a, a);
682 Packet not_nan_mask_b = pcmp_eq(b, b);
683 return pselect(not_nan_mask_a, pselect(not_nan_mask_b, op(a, b), b), a);
684 }
685};
686
690template <bool NativePropagatesNaN>
691struct pminmax_impl<PropagateNumbers, false, NativePropagatesNaN> {
692 template <typename Packet, typename Op>
693 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a, const Packet& b, Op op) {
694 Packet not_nan_mask_a = pcmp_eq(a, a);
695 Packet not_nan_mask_b = pcmp_eq(b, b);
696 return pselect(not_nan_mask_a, pselect(not_nan_mask_b, op(a, b), a), b);
697 }
698};
699
700#define EIGEN_BINARY_OP_NAN_PROPAGATION(Type, Func) [](const Type& aa, const Type& bb) { return Func(aa, bb); }
701
704template <typename Packet>
705EIGEN_DEVICE_FUNC inline Packet pmin(const Packet& a, const Packet& b) {
706 return numext::mini(a, b);
707}
708
711template <int NaNPropagation, typename Packet>
712EIGEN_DEVICE_FUNC inline Packet pmin(const Packet& a, const Packet& b) {
713 constexpr bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger;
714 constexpr bool NativePropagatesNaN = pminmax_propagates_nan<Packet>::value;
715 return pminmax_impl<NaNPropagation, IsInteger, NativePropagatesNaN>::run(
716 a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet, (pmin<Packet>)));
717}
718
721template <typename Packet>
722EIGEN_DEVICE_FUNC inline Packet pmax(const Packet& a, const Packet& b) {
723 return numext::maxi(a, b);
724}
725
728template <int NaNPropagation, typename Packet>
729EIGEN_DEVICE_FUNC inline Packet pmax(const Packet& a, const Packet& b) {
730 constexpr bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger;
731 constexpr bool NativePropagatesNaN = pminmax_propagates_nan<Packet>::value;
732 return pminmax_impl<NaNPropagation, IsInteger, NativePropagatesNaN>::run(
733 a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet, (pmax<Packet>)));
734}
735
737template <typename Packet, std::enable_if_t<!(NumTraits<typename unpacket_traits<Packet>::type>::IsInteger &&
738 !NumTraits<typename unpacket_traits<Packet>::type>::IsSigned),
739 int> = 0>
740EIGEN_DEVICE_FUNC inline Packet pabs(const Packet& a) {
741 return numext::abs(a);
742}
743
744template <typename Packet, std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsInteger &&
745 !NumTraits<typename unpacket_traits<Packet>::type>::IsSigned,
746 int> = 0>
747EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pabs(const Packet& a) {
748 return a;
749}
750
752template <typename Packet>
753EIGEN_DEVICE_FUNC inline Packet paddsub(const Packet& a, const Packet& b) {
754 return pselect(peven_mask(a), padd(a, b), psub(a, b));
755}
756
758template <typename Packet>
759EIGEN_DEVICE_FUNC inline Packet parg(const Packet& a) {
760 return numext::arg(a);
761}
762
764template <int N, typename T>
765EIGEN_DEVICE_FUNC inline T parithmetic_shift_right(const T& a) {
766 return numext::arithmetic_shift_right(a, N);
767}
768
770template <int N, typename T>
771EIGEN_DEVICE_FUNC inline T plogical_shift_right(const T& a) {
772 return numext::logical_shift_right(a, N);
773}
774
776template <int N, typename T>
777EIGEN_DEVICE_FUNC inline T plogical_shift_left(const T& a) {
778 return numext::logical_shift_left(a, N);
779}
780
784template <typename Packet>
785EIGEN_DEVICE_FUNC inline Packet pfrexp(const Packet& a, Packet& exponent) {
786 int exp;
787 EIGEN_USING_STD(frexp);
788 Packet result = static_cast<Packet>(frexp(a, &exp));
789 exponent = static_cast<Packet>(exp);
790 return result;
791}
792
796template <typename Packet>
797EIGEN_DEVICE_FUNC inline Packet pldexp(const Packet& a, const Packet& exponent) {
798 return static_cast<Packet>(numext::ldexp(a, static_cast<int>(exponent)));
799}
800
802template <typename Packet>
803EIGEN_DEVICE_FUNC inline std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet>
804pabsdiff(const Packet& a, const Packet& b) {
805 return pselect(pcmp_lt(a, b), psub(b, a), psub(a, b));
806}
807template <typename Packet>
808EIGEN_DEVICE_FUNC inline std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet>
809pabsdiff(const Packet& a, const Packet& b) {
810 return pabs(psub(a, b));
811}
812
814template <typename Packet>
815EIGEN_DEVICE_FUNC inline Packet pload(const typename unpacket_traits<Packet>::type* from) {
816 return *from;
817}
818
823template <typename Packet>
824EIGEN_DEVICE_FUNC inline Packet pload_partial(const typename unpacket_traits<Packet>::type* from, const Index n,
825 const Index offset = 0) {
826 const Index packet_size = unpacket_traits<Packet>::size;
827 eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
828 using Scalar = typename unpacket_traits<Packet>::type;
829 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size] = {Scalar(0)};
830 for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) {
831 elements[i] = from[i - offset];
832 }
833 return pload<Packet>(elements);
834}
835
837template <typename Packet>
838EIGEN_DEVICE_FUNC inline Packet ploadu(const typename unpacket_traits<Packet>::type* from) {
839 return *from;
840}
841
844template <typename Packet>
845EIGEN_DEVICE_FUNC inline Packet ploadu_partial(const typename unpacket_traits<Packet>::type* from, const Index n,
846 const Index offset = 0) {
847 const Index packet_size = unpacket_traits<Packet>::size;
848 eigen_assert(n + offset <= packet_size && "number of elements plus offset will read past end of packet");
849 using Scalar = typename unpacket_traits<Packet>::type;
850 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size] = {Scalar(0)};
851 for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) {
852 elements[i] = from[i - offset];
853 }
854 return pload<Packet>(elements);
855}
856
861template <typename Packet>
862EIGEN_DEVICE_FUNC inline std::enable_if_t<unpacket_traits<Packet>::masked_load_available, Packet> ploadu(
863 const typename unpacket_traits<Packet>::type* from, typename unpacket_traits<Packet>::mask_t umask);
864
866template <typename Packet>
867EIGEN_DEVICE_FUNC inline Packet pset1(const typename unpacket_traits<Packet>::type& a) {
868 return a;
869}
870
872template <typename Packet, typename BitsType>
873EIGEN_DEVICE_FUNC inline Packet pset1frombits(BitsType a) {
874 using Scalar = typename unpacket_traits<Packet>::type;
875 return pset1<Packet>(numext::bit_cast<Scalar>(a));
876}
877
878template <typename Packet>
879struct packet_bit_pattern_traits {
880 using Scalar = typename unpacket_traits<Packet>::type;
881 using Bits = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type;
882 enum { HasIntegerBits = !std::is_void<Bits>::value };
883};
884
885// Widening an opaque IEEE binary32 bit pattern produces the target's native extended-scalar representation without
886// exposing a floating-point special-value literal to fast-math optimizers.
887template <typename Scalar>
888EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pscalar_from_float_bits(numext::uint32_t bits) {
889#if EIGEN_COMP_GNUC_STRICT && defined(__FINITE_MATH_ONLY__) && __FINITE_MATH_ONLY__ && \
890 !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(__SYCL_DEVICE_ONLY__)
891 // GCC also needs the integer pattern hidden before bit_cast and widening. Use a memory operand so this works on
892 // targets where EIGEN_OPTIMIZATION_BARRIER is intentionally unavailable.
893 __asm__("" : "+m"(bits));
894#endif
895 EIGEN_FAST_MATH_CONSTANT_BARRIER(bits);
896 return static_cast<Scalar>(numext::bit_cast<float>(bits));
897}
898
899template <typename Packet, bool HasIntegerBits = packet_bit_pattern_traits<Packet>::HasIntegerBits,
900 bool IsScalar = is_scalar<Packet>::value>
901struct psignmask_impl;
902
903template <typename Packet, bool IsScalar>
904struct psignmask_impl<Packet, true, IsScalar> {
905 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run() {
906 using Scalar = typename packet_bit_pattern_traits<Packet>::Scalar;
907 using Bits = typename packet_bit_pattern_traits<Packet>::Bits;
908 constexpr Bits kSignBit = static_cast<Bits>(Bits(1) << (CHAR_BIT * sizeof(Scalar) - 1));
909 return pset1frombits<Packet, Bits>(kSignBit);
910 }
911};
912
913template <typename Scalar>
914struct psignmask_impl<Scalar, false, true> {
915 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run() { return pscalar_from_float_bits<Scalar>(0x80000000u); }
916};
917
928template <typename Packet>
929EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet psignmask() {
930 return psignmask_impl<Packet>::run();
931}
932
943template <typename Packet, bool HasIntegerBits = packet_bit_pattern_traits<Packet>::HasIntegerBits,
944 bool IsScalar = is_scalar<Packet>::value>
945struct pinf_impl;
946
947template <typename Packet, bool IsScalar>
948struct pinf_impl<Packet, true, IsScalar> {
949 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run() {
950 using Scalar = typename packet_bit_pattern_traits<Packet>::Scalar;
951 using Bits = typename packet_bit_pattern_traits<Packet>::Bits;
952 constexpr int kMantissaBits = std::numeric_limits<Scalar>::digits - 1;
953 constexpr int kExponentBits = static_cast<int>(CHAR_BIT * sizeof(Scalar)) - 1 - kMantissaBits;
954 constexpr Bits kInf = static_cast<Bits>(((Bits(1) << kExponentBits) - 1) << kMantissaBits);
955 return pset1frombits<Packet, Bits>(kInf);
956 }
957};
958
959template <typename Scalar>
960struct pinf_impl<Scalar, false, true> {
961 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run() { return pscalar_from_float_bits<Scalar>(0x7f800000u); }
962};
963
964template <typename Packet>
965EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pinf() {
966 return pinf_impl<Packet>::run();
967}
968
970template <typename Packet, bool HasIntegerBits = packet_bit_pattern_traits<Packet>::HasIntegerBits,
971 bool IsScalar = is_scalar<Packet>::value>
972struct pnan_impl;
973
974template <typename Packet, bool IsScalar>
975struct pnan_impl<Packet, true, IsScalar> {
976 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run() {
977 using Scalar = typename packet_bit_pattern_traits<Packet>::Scalar;
978 using Bits = typename packet_bit_pattern_traits<Packet>::Bits;
979 constexpr int kMantissaBits = std::numeric_limits<Scalar>::digits - 1;
980 constexpr int kExponentBits = static_cast<int>(CHAR_BIT * sizeof(Scalar)) - 1 - kMantissaBits;
981 constexpr Bits kInf = static_cast<Bits>(((Bits(1) << kExponentBits) - 1) << kMantissaBits);
982 constexpr Bits kNaN = static_cast<Bits>(kInf | (Bits(1) << (kMantissaBits - 1)));
983 return pset1frombits<Packet, Bits>(kNaN);
984 }
985};
986
987template <typename Scalar>
988struct pnan_impl<Scalar, false, true> {
989 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run() { return pscalar_from_float_bits<Scalar>(0x7fc00000u); }
990};
991
992template <typename Packet>
993EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnan() {
994 return pnan_impl<Packet>::run();
995}
996
997template <typename Scalar, std::enable_if_t<std::is_trivially_copyable<Scalar>::value, int> = 0>
998EIGEN_DEVICE_FUNC inline Scalar pload1_scalar(const Scalar* a) {
999 Scalar scalar;
1000 EIGEN_USING_STD(memcpy)
1001 memcpy(&scalar, a, sizeof(Scalar));
1002 return scalar;
1003}
1004
1005template <typename Scalar, std::enable_if_t<!std::is_trivially_copyable<Scalar>::value, int> = 0>
1006EIGEN_DEVICE_FUNC inline Scalar pload1_scalar(const Scalar* a) {
1007 return Scalar(*a);
1008}
1009
1011template <typename Packet>
1012EIGEN_DEVICE_FUNC inline Packet pload1(const typename unpacket_traits<Packet>::type* a) {
1013 using Scalar = typename unpacket_traits<Packet>::type;
1014 return pset1<Packet>(pload1_scalar<Scalar>(a));
1015}
1016
1022template <typename Packet>
1023EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet ploaddup(const typename unpacket_traits<Packet>::type* from) {
1024 static_assert(unpacket_traits<Packet>::size <= 2, "Wider packets require a coefficient-duplicating ploaddup");
1025 return pload1<Packet>(from);
1026}
1027
1034template <typename Packet>
1035EIGEN_DEVICE_FUNC inline Packet ploadquad(const typename unpacket_traits<Packet>::type* from) {
1036 return pload1<Packet>(from);
1037}
1038
1048template <typename Packet>
1049EIGEN_DEVICE_FUNC inline void pbroadcast4(const typename unpacket_traits<Packet>::type* a, Packet& a0, Packet& a1,
1050 Packet& a2, Packet& a3) {
1051 a0 = pload1<Packet>(a + 0);
1052 a1 = pload1<Packet>(a + 1);
1053 a2 = pload1<Packet>(a + 2);
1054 a3 = pload1<Packet>(a + 3);
1055}
1056
1064template <typename Packet>
1065EIGEN_DEVICE_FUNC inline void pbroadcast2(const typename unpacket_traits<Packet>::type* a, Packet& a0, Packet& a1) {
1066 a0 = pload1<Packet>(a + 0);
1067 a1 = pload1<Packet>(a + 1);
1068}
1069
1071template <typename Packet>
1072EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet plset(const typename unpacket_traits<Packet>::type& a) {
1073 return a;
1074}
1075
1078template <typename Packet>
1079EIGEN_DEVICE_FUNC inline Packet peven_mask(const Packet& /*a*/) {
1080 using Scalar = typename unpacket_traits<Packet>::type;
1081 if (is_scalar<Packet>::value) {
1082 // The scalar "mask" is numeric: true is represented by the value one.
1083 return pset1<Packet>(Scalar(1));
1084 }
1085 const size_t n = unpacket_traits<Packet>::size;
1086 Packet b;
1087 char* bytes = reinterpret_cast<char*>(&b);
1088 for (size_t i = 0; i < n; ++i) {
1089 memset(bytes + i * sizeof(Scalar), ((i & 1) == 0 ? 0xff : 0), sizeof(Scalar));
1090 }
1091 EIGEN_FAST_MATH_CONSTANT_BARRIER(b);
1092 return b;
1093}
1094
1096template <typename Scalar, typename Packet>
1097EIGEN_DEVICE_FUNC inline void pstore(Scalar* to, const Packet& from) {
1098 (*to) = from;
1099}
1100
1104template <typename Scalar, typename Packet>
1105EIGEN_DEVICE_FUNC inline void pstore_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) {
1106 const Index packet_size = unpacket_traits<Packet>::size;
1107 eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
1108 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size];
1109 pstore<Scalar>(elements, from);
1110 for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) {
1111 to[i] = elements[i + offset];
1112 }
1113}
1114
1116template <typename Scalar, typename Packet>
1117EIGEN_DEVICE_FUNC inline void pstoreu(Scalar* to, const Packet& from) {
1118 (*to) = from;
1119}
1120
1122template <typename Scalar, typename Packet>
1123EIGEN_DEVICE_FUNC inline void pstoreu_partial(Scalar* to, const Packet& from, const Index n, const Index offset = 0) {
1124 const Index packet_size = unpacket_traits<Packet>::size;
1125 eigen_assert(n + offset <= packet_size && "number of elements plus offset will write past end of packet");
1126 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size];
1127 pstore<Scalar>(elements, from);
1128 for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) {
1129 to[i] = elements[i + offset];
1130 }
1131}
1132
1137template <typename Scalar, typename Packet>
1138EIGEN_DEVICE_FUNC inline std::enable_if_t<unpacket_traits<Packet>::masked_store_available, void> pstoreu(
1139 Scalar* to, const Packet& from, typename unpacket_traits<Packet>::mask_t umask);
1140
1141template <typename Scalar, typename Packet>
1142EIGEN_DEVICE_FUNC inline Packet pgather(const Scalar* from, Index /*stride*/) {
1143 return ploadu<Packet>(from);
1144}
1145
1146template <typename Scalar, typename Packet>
1147EIGEN_DEVICE_FUNC inline Packet pgather_partial(const Scalar* from, Index stride, const Index n) {
1148 const Index packet_size = unpacket_traits<Packet>::size;
1149 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size] = {Scalar(0)};
1150 for (Index i = 0; i < numext::mini(n, packet_size); i++) {
1151 elements[i] = from[i * stride];
1152 }
1153 return pload<Packet>(elements);
1154}
1155
1156template <typename Scalar, typename Packet>
1157EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index /*stride*/) {
1158 pstore(to, from);
1159}
1160
1161template <typename Scalar, typename Packet>
1162EIGEN_DEVICE_FUNC inline void pscatter_partial(Scalar* to, const Packet& from, Index stride, const Index n) {
1163 const Index packet_size = unpacket_traits<Packet>::size;
1164 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size];
1165 pstore<Scalar>(elements, from);
1166 for (Index i = 0; i < numext::mini(n, packet_size); i++) {
1167 to[i * stride] = elements[i];
1168 }
1169}
1170
1172template <typename Scalar>
1173EIGEN_DEVICE_FUNC inline void prefetch(const Scalar* addr) {
1174#if defined(EIGEN_HIP_DEVICE_COMPILE)
1175 // do nothing
1176#elif defined(EIGEN_CUDA_ARCH)
1177#if defined(__LP64__) || EIGEN_OS_WIN64
1178 // 64-bit pointer operand constraint for inlined asm
1179 asm(" prefetch.L1 [ %1 ];" : "=l"(addr) : "l"(addr));
1180#else
1181 // 32-bit pointer operand constraint for inlined asm
1182 asm(" prefetch.L1 [ %1 ];" : "=r"(addr) : "r"(addr));
1183#endif
1184#elif (!EIGEN_COMP_MSVC) && (EIGEN_COMP_GNUC || EIGEN_COMP_CLANG || EIGEN_COMP_ICC)
1185 __builtin_prefetch(addr);
1186#endif
1187}
1188
1190template <typename Packet>
1191EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet preverse(const Packet& a) {
1192 return a;
1193}
1194
1196template <typename Packet>
1197EIGEN_DEVICE_FUNC inline Packet pcplxflip(const Packet& a) {
1198 return Packet(numext::imag(a), numext::real(a));
1199}
1200
1202// TODO(rmlarsen): Define and use in all complex backends.
1203template <typename Packet>
1204EIGEN_DEVICE_FUNC inline Packet pdupreal(const Packet& a) {
1205 return Packet(numext::real(a), numext::real(a));
1206}
1207
1209// TODO(rmlarsen): Define and use in all complex backends.
1210template <typename Packet>
1211EIGEN_DEVICE_FUNC inline Packet pdupimag(const Packet& a) {
1212 return Packet(numext::imag(a), numext::imag(a));
1213}
1214
1215/**************************
1216 * Special math functions
1217 ***************************/
1218
1219// Declared ahead of pisnan_impl so the complex case can use a backend's pisnan for the real packet.
1220template <typename Packet>
1221EIGEN_DEVICE_FUNC inline Packet pisnan(const Packet& a);
1222
1223// Implemented without ptrue: an all-ones float packet is a NaN bit pattern, which under
1224// fast-math flags clang turns into a poison constant that deletes any expression it flows
1225// into.
1226template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex,
1227 bool IsScalar = is_scalar<Packet>::value,
1228 bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1229struct pisnan_impl {
1230 // Equivalent to !(a == a).
1231 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return pcmp_lt_or_nan(a, a); }
1232};
1233
1234// Integer scalars have no NaN; the answer is the all-false mask. The generic path is unusable
1235// here: pcmp_lt_or_nan has no meaningful integer semantics (and its generic form does not even
1236// compile for integer SIMD packets).
1237template <typename Packet, bool IsScalar>
1238struct pisnan_impl<Packet, false, IsScalar, true> {
1239 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return pzero(a); }
1240};
1241
1242template <typename Packet, bool IsInteger>
1243struct pisnan_impl<Packet, true, false, IsInteger> {
1244 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) {
1245 using RealPacket = typename unpacket_traits<Packet>::as_real;
1246 // A NaN in either the real or the imaginary lane marks the whole complex element.
1247 Packet nan_lanes = Packet(pisnan<RealPacket>(a.v));
1248 return por(nan_lanes, pcplxflip(nan_lanes));
1249 }
1250};
1251
1252// Scalar complex arguments have no wrapped real packet; combine the per-component results in the
1253// value domain, where the scalar mask convention is Scalar(1)/Scalar(0).
1254template <typename Scalar, bool IsInteger>
1255struct pisnan_impl<Scalar, true, true, IsInteger> {
1256 static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& a) {
1257 using RealScalar = typename NumTraits<Scalar>::Real;
1258 const RealScalar nan_mask =
1259 por(pisnan_impl<RealScalar>::run(numext::real(a)), pisnan_impl<RealScalar>::run(numext::imag(a)));
1260 return Scalar(nan_mask);
1261 }
1262};
1263
1265template <typename Packet>
1266EIGEN_DEVICE_FUNC inline Packet pisnan(const Packet& a) {
1267 return pisnan_impl<Packet>::run(a);
1268}
1269
1270template <typename Packet, bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1271struct pisinf_impl {
1272 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return pcmp_eq(pabs(a), pinf<Packet>()); }
1273};
1274
1275// Integer scalars have no infinity; the answer is the all-false mask. The generic path is wrong
1276// for them: pinf() synthesizes its bit pattern from numeric_limits digits, which for int32 yields
1277// 2^30, so |a| == 2^30 would read as "inf".
1278template <typename Packet>
1279struct pisinf_impl<Packet, true> {
1280 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return pzero(a); }
1281};
1282
1284template <typename Packet>
1285EIGEN_DEVICE_FUNC inline Packet pisinf(const Packet& a) {
1286 return pisinf_impl<Packet>::run(a);
1287}
1288
1289template <typename Packet, bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1290struct pisfinite_impl {
1291 // |a| < inf is a single comparison that is false for both NaN and infinities.
1292 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return pcmp_lt(pabs(a), pinf<Packet>()); }
1293};
1294
1295// Integer scalars are always finite; the answer is the all-true mask (safe for integer packets,
1296// where all-ones is not a NaN bit pattern).
1297template <typename Packet>
1298struct pisfinite_impl<Packet, true> {
1299 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return ptrue(a); }
1300};
1301
1303template <typename Packet>
1304EIGEN_DEVICE_FUNC inline Packet pisfinite(const Packet& a) {
1305 return pisfinite_impl<Packet>::run(a);
1306}
1307
1309template <typename Packet>
1310EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psin(const Packet& a) {
1311 return numext::sin(a);
1312}
1313
1315template <typename Packet>
1316EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcos(const Packet& a) {
1317 return numext::cos(a);
1318}
1319
1321template <typename Packet>
1322EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptan(const Packet& a) {
1323 return numext::tan(a);
1324}
1325
1327template <typename Packet>
1328EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasin(const Packet& a) {
1329 return numext::asin(a);
1330}
1331
1333template <typename Packet>
1334EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacos(const Packet& a) {
1335 return numext::acos(a);
1336}
1337
1339template <typename Packet>
1340EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psinh(const Packet& a) {
1341 return numext::sinh(a);
1342}
1343
1345template <typename Packet>
1346EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcosh(const Packet& a) {
1347 return numext::cosh(a);
1348}
1349
1351template <typename Packet>
1352EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patan(const Packet& a) {
1353 return numext::atan(a);
1354}
1355
1357template <typename Packet>
1358EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptanh(const Packet& a) {
1359 return numext::tanh(a);
1360}
1361
1363template <typename Packet>
1364EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patanh(const Packet& a) {
1365 return numext::atanh(a);
1366}
1367
1369template <typename Packet>
1370EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasinh(const Packet& a) {
1371 return numext::asinh(a);
1372}
1373
1375template <typename Packet>
1376EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacosh(const Packet& a) {
1377 return numext::acosh(a);
1378}
1379
1381template <typename Packet>
1382EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp(const Packet& a) {
1383 return numext::exp(a);
1384}
1385
1387template <typename Packet>
1388EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp2(const Packet& a) {
1389 return numext::exp2(a);
1390}
1391
1393template <typename Packet>
1394EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexpm1(const Packet& a) {
1395 return numext::expm1(a);
1396}
1397
1399template <typename Packet>
1400EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog(const Packet& a) {
1401 return numext::log(a);
1402}
1403
1405template <typename Packet>
1406EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog1p(const Packet& a) {
1407 return numext::log1p(a);
1408}
1409
1411template <typename Packet>
1412EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog10(const Packet& a) {
1413 return numext::log10(a);
1414}
1415
1416template <typename Packet, bool IsComplexPacket = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex &&
1417 !is_scalar<Packet>::value>
1418struct plog2_impl {
1419 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) {
1420 using Scalar = typename internal::unpacket_traits<Packet>::type;
1421 using RealScalar = typename NumTraits<Scalar>::Real;
1422 return pmul(pset1<Packet>(Scalar(RealScalar(EIGEN_LOG2E))), plog(a));
1423 }
1424};
1425
1426// Defined with plog_complex in GenericPacketMathComplex.h; declared here so that plog2 on a complex packet cannot
1427// silently take the primary template before that header.
1428template <typename Packet>
1429struct plog2_impl<Packet, true>;
1430
1432template <typename Packet>
1433EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog2(const Packet& a) {
1434 return plog2_impl<Packet>::run(a);
1435}
1436
1438template <typename Packet>
1439EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psqrt(const Packet& a) {
1440 return numext::sqrt(a);
1441}
1442
1444template <typename Packet>
1445EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcbrt(const Packet& a) {
1446 return numext::cbrt(a);
1447}
1448
1449template <typename Packet, bool IsScalar = is_scalar<Packet>::value,
1450 bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1451struct nearest_integer_packetop_impl {
1452 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_floor(const Packet& x) { return numext::floor(x); }
1453 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_ceil(const Packet& x) { return numext::ceil(x); }
1454 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_rint(const Packet& x) { return numext::rint(x); }
1455 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_round(const Packet& x) { return numext::round(x); }
1456 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_trunc(const Packet& x) { return numext::trunc(x); }
1457};
1458
1460template <typename Packet>
1461EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pround(const Packet& a) {
1462 return nearest_integer_packetop_impl<Packet>::run_round(a);
1463}
1464
1466template <typename Packet>
1467EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pfloor(const Packet& a) {
1468 return nearest_integer_packetop_impl<Packet>::run_floor(a);
1469}
1470
1473template <typename Packet>
1474EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet print(const Packet& a) {
1475 return nearest_integer_packetop_impl<Packet>::run_rint(a);
1476}
1477
1479template <typename Packet>
1480EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pceil(const Packet& a) {
1481 return nearest_integer_packetop_impl<Packet>::run_ceil(a);
1482}
1483
1485template <typename Packet>
1486EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet ptrunc(const Packet& a) {
1487 return nearest_integer_packetop_impl<Packet>::run_trunc(a);
1488}
1489
1490template <typename Packet, typename EnableIf = void>
1491struct psign_impl {
1492 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return numext::sign(a); }
1493};
1494
1496template <typename Packet>
1497EIGEN_DEVICE_FUNC inline Packet psign(const Packet& a) {
1498 return psign_impl<Packet>::run(a);
1499}
1500
1501template <>
1502EIGEN_DEVICE_FUNC inline bool psign(const bool& a) {
1503 return a;
1504}
1505
1507template <typename Packet>
1508EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type pfirst(const Packet& a) {
1509 return a;
1510}
1511
1516template <typename Packet>
1517EIGEN_DEVICE_FUNC inline std::conditional_t<(unpacket_traits<Packet>::size % 8) == 0,
1518 typename unpacket_traits<Packet>::half, Packet>
1519predux_half(const Packet& a) {
1520 return a;
1521}
1522
1523// Slow generic implementation of Packet reduction.
1524template <typename Packet, typename Op>
1525EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_helper(const Packet& a, Op op) {
1526 using Scalar = typename unpacket_traits<Packet>::type;
1527 const size_t n = unpacket_traits<Packet>::size;
1528 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[n];
1529 pstoreu<Scalar>(elements, a);
1530 for (size_t k = n / 2; k > 0; k /= 2) {
1531 for (size_t i = 0; i < k; ++i) {
1532 elements[i] = op(elements[i], elements[i + k]);
1533 }
1534 }
1535 return elements[0];
1536}
1537
1538template <typename Packet, std::enable_if_t<unpacket_traits<Packet>::size == 1, int> = 0>
1539EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename unpacket_traits<Packet>::type predux_one_element(const Packet& a) {
1540 return pfirst(a);
1541}
1542
1544template <typename Packet>
1545EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename unpacket_traits<Packet>::type predux(const Packet& a) {
1546 return predux_one_element(a);
1547}
1548
1550template <typename Packet>
1551EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename unpacket_traits<Packet>::type predux_mul(const Packet& a) {
1552 using Scalar = typename unpacket_traits<Packet>::type;
1553 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmul<Scalar>)));
1554}
1555
1557template <typename Packet>
1558EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_min(const Packet& a) {
1559 using Scalar = typename unpacket_traits<Packet>::type;
1560 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<Scalar>)));
1561}
1562
1564template <typename Packet>
1565EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_max(const Packet& a) {
1566 using Scalar = typename unpacket_traits<Packet>::type;
1567 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<Scalar>)));
1568}
1569
1570template <int NaNPropagation, typename Packet>
1571struct predux_min_max_helper_impl {
1572 using Scalar = typename unpacket_traits<Packet>::type;
1573 static constexpr bool UsePredux_ = NaNPropagation == PropagateFast || NumTraits<Scalar>::IsInteger;
1574 template <bool UsePredux = UsePredux_, std::enable_if_t<!UsePredux, bool> = true>
1575 static EIGEN_DEVICE_FUNC inline Scalar run_min(const Packet& a) {
1576 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<NaNPropagation, Scalar>)));
1577 }
1578 template <bool UsePredux = UsePredux_, std::enable_if_t<!UsePredux, bool> = true>
1579 static EIGEN_DEVICE_FUNC inline Scalar run_max(const Packet& a) {
1580 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<NaNPropagation, Scalar>)));
1581 }
1582 template <bool UsePredux = UsePredux_, std::enable_if_t<UsePredux, bool> = true>
1583 static EIGEN_DEVICE_FUNC inline Scalar run_min(const Packet& a) {
1584 return predux_min(a);
1585 }
1586 template <bool UsePredux = UsePredux_, std::enable_if_t<UsePredux, bool> = true>
1587 static EIGEN_DEVICE_FUNC inline Scalar run_max(const Packet& a) {
1588 return predux_max(a);
1589 }
1590};
1591
1592template <int NaNPropagation, typename Packet>
1593EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_min(const Packet& a) {
1594 return predux_min_max_helper_impl<NaNPropagation, Packet>::run_min(a);
1595}
1596
1597template <int NaNPropagation, typename Packet>
1598EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_max(const Packet& a) {
1599 return predux_min_max_helper_impl<NaNPropagation, Packet>::run_max(a);
1600}
1601
1602#undef EIGEN_BINARY_OP_NAN_PROPAGATION
1603
1604template <typename Packet, bool IsBoolean = std::is_same<typename unpacket_traits<Packet>::type, bool>::value>
1605struct predux_count_impl {
1606 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index run(const Packet& a) {
1607 using Scalar = typename unpacket_traits<Packet>::type;
1608 const Packet true_values = pandnot(pset1<Packet>(Scalar(1)), pcmp_eq(a, pzero(a)));
1609 return static_cast<Index>(numext::real(predux(true_values)));
1610 }
1611};
1612
1613template <typename Packet>
1614struct predux_count_impl<Packet, true> {
1615 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index run(const Packet& a) {
1616 using Scalar = typename unpacket_traits<Packet>::type;
1617 constexpr int PacketSize = unpacket_traits<Packet>::size;
1618 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar values[PacketSize];
1619 pstoreu<Scalar>(values, a);
1620 Index result = 0;
1621 for (int i = 0; i < PacketSize; ++i) result += values[i] ? 1 : 0;
1622 return result;
1623 }
1624};
1625
1627template <typename Packet>
1628EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index predux_count(const Packet& a) {
1629 return predux_count_impl<Packet>::run(a);
1630}
1631
1635template <typename Packet>
1636EIGEN_DEVICE_FUNC inline bool predux_any(const Packet& a) {
1637 // Dirty but generic implementation where "true" is assumed to be non 0 and all the sames.
1638 // It is expected that "true" is either:
1639 // - Scalar(1)
1640 // - bits full of ones (NaN for floats),
1641 // - or first bit equals to 1 (1 for ints, smallest denormal for floats).
1642 // This arithmetic fallback boils down to a no-op for scalars. Vector backends whose masks use floating-point bit
1643 // patterns must specialize this with an integer-bit reduction because fast-math or FTZ can discard those values.
1644 using Scalar = typename unpacket_traits<Packet>::type;
1645 return numext::not_equal_strict(predux(a), Scalar(0));
1646}
1647
1648template <typename Packet, bool IsBoolean = std::is_same<typename unpacket_traits<Packet>::type, bool>::value>
1649struct predux_all_impl {
1650 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE bool run(const Packet& a) { return !predux_any(pcmp_eq(a, pzero(a))); }
1651};
1652
1653template <typename Packet>
1654struct predux_all_impl<Packet, true> {
1655 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE bool run(const Packet& a) { return predux_mul(a); }
1656};
1657
1659template <typename Packet>
1660EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_all(const Packet& a) {
1661 return predux_all_impl<Packet>::run(a);
1662}
1663
1664/***************************************************************************
1665 * The following functions might not have to be overwritten for vectorized types
1666 ***************************************************************************/
1667
1668template <typename Packet>
1669EIGEN_DEVICE_FUNC inline Packet pmadd(const Packet& a, const Packet& b, const Packet& c);
1670
1671// pmadd<Packet> does not round a*b to the packet's scalar type before adding c.
1672// Neither an architecture-wide FMA flag nor a custom pmadd overload proves this property.
1673template <typename Packet>
1674struct is_fused_madd : false_type {};
1675
1676template <typename Packet, typename EnableIf = void>
1677struct pmadd_impl {
1678 // Separate pmsub/pnmadd paths require pmul and psub; derived paths require pmadd and pnegate.
1679 // pnmsub requires pmadd and pnegate. Opting into fusion also requires pnegate for derived subtract variants.
1680 static constexpr bool Fused = is_fused_madd<Packet>::value;
1681 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pmadd(const Packet& a, const Packet& b, const Packet& c) {
1682 return padd(pmul(a, b), c);
1683 }
1684 template <bool Fuse = Fused, std::enable_if_t<!Fuse, int> = 0>
1685 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pmsub(const Packet& a, const Packet& b, const Packet& c) {
1686 return psub(pmul(a, b), c);
1687 }
1688 template <bool Fuse = Fused, std::enable_if_t<Fuse, int> = 0>
1689 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pmsub(const Packet& a, const Packet& b, const Packet& c) {
1690 return internal::pmadd<Packet>(a, b, pnegate(c));
1691 }
1692 template <bool Fuse = Fused, std::enable_if_t<!Fuse, int> = 0>
1693 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnmadd(const Packet& a, const Packet& b, const Packet& c) {
1694 return psub(c, pmul(a, b));
1695 }
1696 template <bool Fuse = Fused, std::enable_if_t<Fuse, int> = 0>
1697 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnmadd(const Packet& a, const Packet& b, const Packet& c) {
1698 return internal::pmadd<Packet>(pnegate(a), b, c);
1699 }
1700 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnmsub(const Packet& a, const Packet& b, const Packet& c) {
1701 return pnegate(internal::pmadd<Packet>(a, b, c));
1702 }
1703};
1704
1705#if defined(EIGEN_VECTORIZE_FMA)
1706// Scalar path for pmadd with FMA to ensure consistency with vectorized path.
1707template <typename Scalar>
1708struct pmadd_impl<Scalar, std::enable_if_t<std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value>> {
1709 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmadd(const Scalar& a, const Scalar& b, const Scalar& c) {
1710 return numext::fma(a, b, c);
1711 }
1712 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmsub(const Scalar& a, const Scalar& b, const Scalar& c) {
1713 return numext::fma(a, b, Scalar(-c));
1714 }
1715 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmadd(const Scalar& a, const Scalar& b, const Scalar& c) {
1716 return numext::fma(Scalar(-a), b, c);
1717 }
1718 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmsub(const Scalar& a, const Scalar& b, const Scalar& c) {
1719 return -Scalar(numext::fma(a, b, c));
1720 }
1721};
1722
1723template <typename Scalar>
1724struct pmadd_impl<Scalar,
1725 std::enable_if_t<is_scalar<Scalar>::value && NumTraits<Scalar>::IsSigned &&
1726 !std::is_same<Scalar, float>::value && !std::is_same<Scalar, double>::value>> {
1727#else
1728template <typename Scalar>
1729struct pmadd_impl<Scalar, std::enable_if_t<is_scalar<Scalar>::value && NumTraits<Scalar>::IsSigned>> {
1730#endif
1731 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmadd(const Scalar& a, const Scalar& b, const Scalar& c) {
1732 return numext::madd<Scalar>(a, b, c);
1733 }
1734 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmsub(const Scalar& a, const Scalar& b, const Scalar& c) {
1735 return numext::madd<Scalar>(a, b, Scalar(-c));
1736 }
1737 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmadd(const Scalar& a, const Scalar& b, const Scalar& c) {
1738 return numext::madd<Scalar>(Scalar(-a), b, c);
1739 }
1740 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmsub(const Scalar& a, const Scalar& b, const Scalar& c) {
1741 return -Scalar(numext::madd<Scalar>(a, b, c));
1742 }
1743};
1744
1745// Multiply-add instructions.
1747template <typename Packet>
1748EIGEN_DEVICE_FUNC inline Packet pmadd(const Packet& a, const Packet& b, const Packet& c) {
1749 return pmadd_impl<Packet>::pmadd(a, b, c);
1750}
1751
1753template <typename Packet>
1754EIGEN_DEVICE_FUNC inline Packet pmsub(const Packet& a, const Packet& b, const Packet& c) {
1755 return pmadd_impl<Packet>::pmsub(a, b, c);
1756}
1757
1759template <typename Packet>
1760EIGEN_DEVICE_FUNC inline Packet pnmadd(const Packet& a, const Packet& b, const Packet& c) {
1761 return pmadd_impl<Packet>::pnmadd(a, b, c);
1762}
1763
1765template <typename Packet>
1766EIGEN_DEVICE_FUNC inline Packet pnmsub(const Packet& a, const Packet& b, const Packet& c) {
1767 return pmadd_impl<Packet>::pnmsub(a, b, c);
1768}
1769
1772// NOTE: this function must really be templated on the packet type (think about different packet types for the same
1773// scalar type)
1774template <typename Packet>
1775inline void pstore1(typename unpacket_traits<Packet>::type* to, const typename unpacket_traits<Packet>::type& a) {
1776 pstore(to, pset1<Packet>(a));
1777}
1778
1781template <typename Packet, int Alignment>
1782EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits<Packet>::type* from) {
1783 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1784 return pload<Packet>(from);
1785 } else {
1786 return ploadu<Packet>(from);
1787 }
1788}
1789
1792template <typename Packet, int Alignment>
1793EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(const typename unpacket_traits<Packet>::type* from,
1794 const Index n, const Index offset = 0) {
1795 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1796 return pload_partial<Packet>(from, n, offset);
1797 } else {
1798 return ploadu_partial<Packet>(from, n, offset);
1799 }
1800}
1801
1804template <typename Scalar, typename Packet, int Alignment>
1805EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from) {
1806 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1807 pstore(to, from);
1808 } else {
1809 pstoreu(to, from);
1810 }
1811}
1812
1815template <typename Scalar, typename Packet, int Alignment>
1816EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret_partial(Scalar* to, const Packet& from, const Index n,
1817 const Index offset = 0) {
1818 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1819 pstore_partial(to, from, n, offset);
1820 } else {
1821 pstoreu_partial(to, from, n, offset);
1822 }
1823}
1824
1830template <typename Packet, int LoadMode>
1831EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_ro(const typename unpacket_traits<Packet>::type* from) {
1832 return ploadt<Packet, LoadMode>(from);
1833}
1834
1835/***************************************************************************
1836 * Fast complex products (GCC generates a function call which is very slow)
1837 ***************************************************************************/
1838
1839// Eigen+CUDA does not support complexes.
1840#if !defined(EIGEN_GPUCC)
1841
1842template <>
1843inline std::complex<float> pmul(const std::complex<float>& a, const std::complex<float>& b) {
1844 return std::complex<float>(a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag());
1845}
1846
1847template <>
1848inline std::complex<double> pmul(const std::complex<double>& a, const std::complex<double>& b) {
1849 return std::complex<double>(a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag());
1850}
1851
1852#endif
1853
1854/***************************************************************************
1855 * PacketBlock, that is a collection of N packets where the number of words
1856 * in the packet is a multiple of N.
1857 ***************************************************************************/
1858template <typename Packet, int N = unpacket_traits<Packet>::size>
1859struct PacketBlock {
1860 Packet packet[N];
1861};
1862
1863template <typename Packet, int size = 1>
1864EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet, size>& /*kernel*/) {
1865 // Nothing to do in the scalar case, i.e. a 1x1 matrix.
1866}
1867
1869template <typename Packet>
1870EIGEN_DEVICE_FUNC inline Packet preciprocal(const Packet& a) {
1871 using Scalar = typename unpacket_traits<Packet>::type;
1872 return pdiv(pset1<Packet>(Scalar(1)), a);
1873}
1874
1876template <typename Packet>
1877EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet prsqrt(const Packet& a) {
1878 return preciprocal<Packet>(psqrt(a));
1879}
1880
1881template <typename Packet, bool IsScalar = is_scalar<Packet>::value,
1882 bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger,
1883 bool IsUnsigned = IsInteger && !NumTraits<typename unpacket_traits<Packet>::type>::IsSigned>
1884struct psignbit_impl;
1885template <typename Packet, bool IsInteger, bool IsUnsigned>
1886struct psignbit_impl<Packet, true, IsInteger, IsUnsigned> {
1887 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Packet run(const Packet& a) { return numext::signbit(a); }
1888};
1889template <typename Packet>
1890struct psignbit_impl<Packet, false, false, false> {
1891 // generic implementation if not specialized in PacketMath.h
1892 // slower than arithmetic shift
1893 using Scalar = typename unpacket_traits<Packet>::type;
1894 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static Packet run(const Packet& a) {
1895 const Packet cst_pos_one = pset1<Packet>(Scalar(1));
1896 const Packet cst_neg_one = pset1<Packet>(Scalar(-1));
1897 return pcmp_eq(por(pand(a, cst_neg_one), cst_pos_one), cst_neg_one);
1898 }
1899};
1900template <typename Packet>
1901struct psignbit_impl<Packet, false, true, false> {
1902 // generic implementation for signed integer packets
1903 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Packet run(const Packet& a) { return pcmp_lt(a, pzero(a)); }
1904};
1905template <typename Packet>
1906struct psignbit_impl<Packet, false, true, true> {
1907 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Packet run(const Packet& a) { return pzero(a); }
1908};
1910template <typename Packet>
1911EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr Packet psignbit(const Packet& a) {
1912 return psignbit_impl<Packet>::run(a);
1913}
1914
1916template <typename Packet, std::enable_if_t<is_scalar<Packet>::value, int> = 0>
1917EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet patan2(const Packet& y, const Packet& x) {
1918 return numext::atan2(y, x);
1919}
1920
1922template <typename Packet, std::enable_if_t<!is_scalar<Packet>::value, int> = 0>
1923EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet patan2(const Packet& y, const Packet& x) {
1924 using Scalar = typename internal::unpacket_traits<Packet>::type;
1925
1926 // See https://en.cppreference.com/w/cpp/numeric/math/atan2
1927 // for how corner cases are supposed to be handled according to the
1928 // IEEE floating-point standard (IEC 60559).
1929 const Packet kSignMask = psignmask<Packet>();
1930 const Packet kZero = pzero(x);
1931 const Packet kOne = pset1<Packet>(Scalar(1));
1932 const Packet kPi = pset1<Packet>(Scalar(EIGEN_PI));
1933
1934 const Packet x_has_signbit = psignbit(x);
1935 const Packet y_signmask = pand(y, kSignMask);
1936 const Packet x_signmask = pand(x, kSignMask);
1937 const Packet result_signmask = pxor(y_signmask, x_signmask);
1938 const Packet shift = por(pand(x_has_signbit, kPi), y_signmask);
1939
1940 const Packet x_and_y_are_same = pcmp_eq(pabs(x), pabs(y));
1941 const Packet x_and_y_are_zero = pcmp_eq(por(x, y), kZero);
1942
1943 Packet arg = pdiv(y, x);
1944 arg = pselect(x_and_y_are_same, por(kOne, result_signmask), arg);
1945 arg = pselect(x_and_y_are_zero, result_signmask, arg);
1946
1947 Packet result = patan(arg);
1948 result = padd(result, shift);
1949 return result;
1950}
1951
1953template <typename Packet, std::enable_if_t<is_scalar<Packet>::value, int> = 0>
1954EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pcarg(const Packet& a) {
1955 return Packet(numext::arg(a));
1956}
1957
1959template <typename Packet, std::enable_if_t<!is_scalar<Packet>::value, int> = 0>
1960EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pcarg(const Packet& a) {
1961 EIGEN_STATIC_ASSERT(NumTraits<typename unpacket_traits<Packet>::type>::IsComplex,
1962 THIS METHOD IS FOR COMPLEX TYPES ONLY)
1963 using RealPacket = typename unpacket_traits<Packet>::as_real;
1964 // a // r i r i ...
1965 RealPacket aflip = pcplxflip(a).v; // i r i r ...
1966 RealPacket result = patan2(aflip, a.v); // atan2 crap atan2 crap ...
1967 return (Packet)pand(result, peven_mask(result)); // atan2 0 atan2 0 ...
1968}
1969
1972template <typename Packet>
1973EIGEN_DEVICE_FUNC inline Packet ploaduSegment(const typename unpacket_traits<Packet>::type* from, Index begin,
1974 Index count) {
1975 using Scalar = typename unpacket_traits<Packet>::type;
1976 constexpr Index PacketSize = unpacket_traits<Packet>::size;
1977 eigen_assert((begin >= 0 && count >= 0 && begin + count <= PacketSize) && "invalid range");
1978 Scalar aux[PacketSize] = {};
1979 for (Index k = begin; k < begin + count; k++) {
1980 aux[k] = from[k];
1981 }
1982 return ploadu<Packet>(aux);
1983}
1984
1987template <typename Packet>
1988EIGEN_DEVICE_FUNC inline Packet ploadSegment(const typename unpacket_traits<Packet>::type* from, Index begin,
1989 Index count) {
1990 return ploaduSegment<Packet>(from, begin, count);
1991}
1992
1996template <typename Scalar, typename Packet>
1997EIGEN_DEVICE_FUNC inline void pstoreuSegment(Scalar* to, const Packet& from, Index begin, Index count) {
1998 constexpr Index PacketSize = unpacket_traits<Packet>::size;
1999 eigen_assert((begin >= 0 && count >= 0 && begin + count <= PacketSize) && "invalid range");
2000 Scalar aux[PacketSize];
2001 pstoreu<Scalar, Packet>(aux, from);
2002 for (Index k = begin; k < begin + count; k++) {
2003 to[k] = aux[k];
2004 }
2005}
2006
2010template <typename Scalar, typename Packet>
2011EIGEN_DEVICE_FUNC inline void pstoreSegment(Scalar* to, const Packet& from, Index begin, Index count) {
2012 return pstoreuSegment(to, from, begin, count);
2013}
2014
2017template <typename Packet, int Alignment>
2018EIGEN_DEVICE_FUNC inline Packet ploadtSegment(const typename unpacket_traits<Packet>::type* from, Index begin,
2019 Index count) {
2020 constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
2021 EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
2022 return ploadSegment<Packet>(from, begin, count);
2023 } else {
2024 return ploaduSegment<Packet>(from, begin, count);
2025 }
2026}
2027
2030template <typename Scalar, typename Packet, int Alignment>
2031EIGEN_DEVICE_FUNC inline void pstoretSegment(Scalar* to, const Packet& from, Index begin, Index count) {
2032 constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
2033 EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
2034 pstoreSegment<Scalar, Packet>(to, from, begin, count);
2035 } else {
2036 pstoreuSegment<Scalar, Packet>(to, from, begin, count);
2037 }
2038}
2039
2040#ifndef EIGEN_NO_IO
2041
2042template <typename Packet>
2043class StreamablePacket {
2044 public:
2045 using Scalar = typename unpacket_traits<Packet>::type;
2046 StreamablePacket(const Packet& packet) { pstoreu(v_, packet); }
2047
2048 friend std::ostream& operator<<(std::ostream& os, const StreamablePacket& packet) {
2049 os << "{" << packet.v_[0];
2050 for (int i = 1; i < unpacket_traits<Packet>::size; ++i) {
2051 os << "," << packet.v_[i];
2052 }
2053 os << "}";
2054 return os;
2055 }
2056
2057 private:
2058 Scalar v_[unpacket_traits<Packet>::size];
2059};
2060
2064template <typename Packet>
2065StreamablePacket<Packet> postream(const Packet& packet) {
2066 return StreamablePacket<Packet>(packet);
2067}
2068
2069#endif // EIGEN_NO_IO
2070
2071} // end namespace internal
2072
2073} // end namespace Eigen
2074
2075#endif // EIGEN_GENERIC_PACKET_MATH_H
@ PropagateNaN
Definition Constants.h:343
@ PropagateNumbers
Definition Constants.h:345
@ PropagateFast
Definition Constants.h:341