12#ifndef EIGEN_GENERIC_PACKET_MATH_H
13#define EIGEN_GENERIC_PACKET_MATH_H
16#include "./InternalHeaderCheck.h"
30#ifndef EIGEN_DEBUG_ALIGNED_LOAD
31#define EIGEN_DEBUG_ALIGNED_LOAD
34#ifndef EIGEN_DEBUG_UNALIGNED_LOAD
35#define EIGEN_DEBUG_UNALIGNED_LOAD
38#ifndef EIGEN_DEBUG_ALIGNED_STORE
39#define EIGEN_DEBUG_ALIGNED_STORE
42#ifndef EIGEN_DEBUG_UNALIGNED_STORE
43#define EIGEN_DEBUG_UNALIGNED_STORE
46struct default_packet_traits {
105 HasGammaSampleDerAlpha = 0,
112struct packet_traits : default_packet_traits {
136struct packet_traits<const T> : packet_traits<T> {};
138struct default_unpacket_traits {
139 enum { vectorizable =
false, masked_load_available =
false, masked_store_available =
false };
143struct unpacket_traits : default_unpacket_traits {
146 using integer_packet =
typename numext::get_integer_by_size<
sizeof(T)>::signed_type;
149 alignment =
alignof(T),
154struct unpacket_traits<const T> : unpacket_traits<T> {};
159template <
typename Packet>
160struct is_scalar : std::is_same<Packet, typename unpacket_traits<Packet>::type> {};
166template <
typename SrcPacket,
typename TgtPacket,
167 bool Scalar = is_scalar<SrcPacket>::value && is_scalar<TgtPacket>::value>
168struct is_degenerate_helper : std::is_same<SrcPacket, TgtPacket> {};
170struct is_degenerate_helper<int8_t, uint8_t, true> : std::true_type {};
172struct is_degenerate_helper<int16_t, uint16_t, true> : std::true_type {};
174struct is_degenerate_helper<int32_t, uint32_t, true> : std::true_type {};
176struct is_degenerate_helper<int64_t, uint64_t, true> : std::true_type {};
178template <
typename SrcPacket,
typename TgtPacket>
179struct is_degenerate_helper<SrcPacket, TgtPacket, false> {
180 using SrcScalar =
typename unpacket_traits<SrcPacket>::type;
181 static constexpr int SrcSize = unpacket_traits<SrcPacket>::size;
182 using TgtScalar =
typename unpacket_traits<TgtPacket>::type;
183 static constexpr int TgtSize = unpacket_traits<TgtPacket>::size;
184 static constexpr bool value = is_degenerate_helper<SrcScalar, TgtScalar, true>::value && (SrcSize == TgtSize);
188template <
typename SrcPacket,
typename TgtPacket>
189struct is_degenerate {
190 static constexpr bool value =
191 is_degenerate_helper<SrcPacket, TgtPacket>::value || is_degenerate_helper<TgtPacket, SrcPacket>::value;
194template <
typename Packet>
196 using Scalar =
typename unpacket_traits<Packet>::type;
197 static constexpr int Size = unpacket_traits<Packet>::size;
198 using DefaultPacket =
typename packet_traits<Scalar>::type;
199 static constexpr int DefaultSize = unpacket_traits<DefaultPacket>::size;
200 static constexpr bool value = Size != 1 && Size < DefaultSize;
203template <
typename Src,
typename Tgt>
204struct type_casting_traits {
207 is_degenerate<Src, Tgt>::value && packet_traits<Src>::Vectorizable && packet_traits<Tgt>::Vectorizable,
214template <
typename Src,
typename Tgt>
215struct vectorized_type_casting_traits {
217 DefaultSrcPacketSize = packet_traits<Src>::size,
218 DefaultTgtPacketSize = packet_traits<Tgt>::size,
220 SrcCoeffRatio = plain_enum_max(DefaultTgtPacketSize / DefaultSrcPacketSize, 1),
221 TgtCoeffRatio = plain_enum_max(DefaultSrcPacketSize / DefaultTgtPacketSize, 1)
227template <
typename T,
int unique_
id = 0>
228struct eigen_packet_wrapper {
229 EIGEN_ALWAYS_INLINE
operator T&() {
return m_val; }
230 EIGEN_ALWAYS_INLINE
operator const T&()
const {
return m_val; }
231 EIGEN_ALWAYS_INLINE eigen_packet_wrapper() =
default;
232 EIGEN_ALWAYS_INLINE eigen_packet_wrapper(
const T& v) : m_val(v) {}
233 EIGEN_ALWAYS_INLINE eigen_packet_wrapper& operator=(
const T& v) {
241template <typename Target, typename Packet, bool IsSame = std::is_same<Target, Packet>::value>
242struct preinterpret_generic;
244template <
typename Target,
typename Packet>
245struct preinterpret_generic<Target, Packet, false> {
247 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Target run(
const Packet& a) {
248 return numext::bit_cast<Target, Packet>(a);
252template <
typename Packet>
253struct preinterpret_generic<Packet, Packet, true> {
255 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(
const Packet& a) {
return a; }
258template <
typename ComplexPacket>
259struct preinterpret_generic<typename unpacket_traits<ComplexPacket>::as_real, ComplexPacket, false> {
260 using RealPacket =
typename unpacket_traits<ComplexPacket>::as_real;
261 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE RealPacket run(
const ComplexPacket& a) {
return a.v; }
265template <
typename Target,
typename Packet>
266EIGEN_DEVICE_FUNC
inline Target preinterpret(
const Packet& a) {
267 return preinterpret_generic<Target, Packet>::run(a);
270template <typename SrcPacket, typename TgtPacket, bool Degenerate = is_degenerate<SrcPacket, TgtPacket>::value,
271 bool TgtIsHalf = is_half<TgtPacket>::value>
274template <
typename SrcPacket,
typename TgtPacket>
275struct pcast_generic<SrcPacket, TgtPacket, false, false> {
277 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(
const SrcPacket& a) {
278 return cast_impl<SrcPacket, TgtPacket>::run(a);
282template <
typename Packet>
283struct pcast_generic<Packet, Packet, true, false> {
285 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(
const Packet& a) {
return a; }
288template <
typename SrcPacket,
typename TgtPacket,
bool TgtIsHalf>
289struct pcast_generic<SrcPacket, TgtPacket, true, TgtIsHalf> {
291 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(
const SrcPacket& a) {
return preinterpret<TgtPacket>(a); }
295template <
typename SrcPacket,
typename TgtPacket>
296EIGEN_DEVICE_FUNC
inline TgtPacket pcast(
const SrcPacket& a) {
297 return pcast_generic<SrcPacket, TgtPacket>::run(a);
299template <
typename SrcPacket,
typename TgtPacket>
300EIGEN_DEVICE_FUNC
inline TgtPacket pcast(
const SrcPacket& a,
const SrcPacket& b) {
301 return pcast_generic<SrcPacket, TgtPacket>::run(a, b);
303template <
typename SrcPacket,
typename TgtPacket>
304EIGEN_DEVICE_FUNC
inline TgtPacket pcast(
const SrcPacket& a,
const SrcPacket& b,
const SrcPacket& c,
305 const SrcPacket& d) {
306 return pcast_generic<SrcPacket, TgtPacket>::run(a, b, c, d);
308template <
typename SrcPacket,
typename TgtPacket>
309EIGEN_DEVICE_FUNC
inline TgtPacket pcast(
const SrcPacket& a,
const SrcPacket& b,
const SrcPacket& c,
const SrcPacket& d,
310 const SrcPacket& e,
const SrcPacket& f,
const SrcPacket& g,
311 const SrcPacket& h) {
312 return pcast_generic<SrcPacket, TgtPacket>::run(a, b, c, d, e, f, g, h);
315template <
typename SrcPacket,
typename TgtPacket>
316struct pcast_generic<SrcPacket, TgtPacket, false, true> {
319 using DefaultTgtPacket =
typename is_half<TgtPacket>::DefaultPacket;
320 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TgtPacket run(
const SrcPacket& a) {
321 return preinterpret<TgtPacket>(pcast<SrcPacket, DefaultTgtPacket>(a));
326template <
typename Packet>
327EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pbarrier(Packet& x) {
328 EIGEN_UNUSED_VARIABLE(x);
329 EIGEN_OPTIMIZATION_BARRIER(x)
332template <
typename T,
int unique_
id>
333EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pbarrier(eigen_packet_wrapper<T, unique_id>& x) {
338template <
typename Packet>
339EIGEN_DEVICE_FUNC
inline Packet padd(
const Packet& a,
const Packet& b) {
345EIGEN_DEVICE_FUNC
inline bool padd(
const bool& a,
const bool& b) {
353template <
typename Packet>
354EIGEN_DEVICE_FUNC
inline std::enable_if_t<unpacket_traits<Packet>::masked_fpops_available, Packet> padd(
355 const Packet& a,
const Packet& b,
typename unpacket_traits<Packet>::mask_t umask);
358template <
typename Packet>
359EIGEN_DEVICE_FUNC
inline Packet psub(
const Packet& a,
const Packet& b) {
364template <
typename Packet>
365EIGEN_DEVICE_FUNC
inline Packet pnegate(
const Packet& a) {
366 EIGEN_STATIC_ASSERT((!std::is_same<
typename unpacket_traits<Packet>::type,
bool>::value),
367 NEGATE IS NOT DEFINED FOR BOOLEAN TYPES)
368 return numext::negate(a);
372template <
typename Packet>
373EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pconj(
const Packet& a) {
374 using Scalar =
typename unpacket_traits<Packet>::type;
375 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex)
376 return numext::conj(a);
382template <
typename Packet>
383EIGEN_DEVICE_FUNC
inline Packet pmul(
const Packet& a,
const Packet& b) {
384 return internal::mul(a, b);
387EIGEN_DEVICE_FUNC
inline bool pmul(
const bool& a,
const bool& b) {
392template <
typename Packet>
393EIGEN_DEVICE_FUNC
inline Packet pdiv(
const Packet& a,
const Packet& b) {
397EIGEN_DEVICE_FUNC
inline bool pdiv(
const bool& a,
const bool& b) {
402template <
typename Packet,
typename EnableIf =
void>
404 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& ) {
406 memset(
static_cast<void*
>(&b), 0xff,
sizeof(Packet));
412template <
typename Scalar>
413struct ptrue_impl<Scalar, std::enable_if_t<is_scalar<Scalar>::value>> {
414 static EIGEN_DEVICE_FUNC
inline Scalar run(
const Scalar&) {
return Scalar(1); }
419struct ptrue_impl<bool, void> {
420 static EIGEN_DEVICE_FUNC
inline bool run(
const bool&) {
return true; }
427template <
typename Packet>
428EIGEN_DEVICE_FUNC
inline Packet ptrue(
const Packet& a) {
429 if (is_scalar<Packet>::value || std::is_same<Packet, bool>::value) {
432 return ptrue_impl<Packet>::run(a);
435 memset(
static_cast<void*
>(&b), 0xff,
sizeof(Packet));
437 EIGEN_IF_CONSTEXPR (!NumTraits<
typename unpacket_traits<Packet>::type>::IsInteger) {
438 EIGEN_FAST_MATH_CONSTANT_BARRIER(b);
444template <
typename Packet,
typename EnableIf =
void>
446 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& ) {
448 memset(
static_cast<void*
>(&b), 0x00,
sizeof(Packet));
456struct pzero_impl<T, std::enable_if_t<is_scalar<T>::value>> {
457 static EIGEN_DEVICE_FUNC
inline T run(
const T& ) {
return T(0); }
461template <
typename Packet>
462EIGEN_DEVICE_FUNC
inline Packet pzero(
const Packet& a) {
463 return pzero_impl<Packet>::run(a);
468 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE T operator()(
const T& a,
const T& b)
const {
return a & b; }
473 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE T operator()(
const T& a,
const T& b)
const {
return a | b; }
478 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE T operator()(
const T& a,
const T& b)
const {
return a ^ b; }
483 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE T operator()(
const T& a)
const {
return ~a; }
487struct bit_and<bool> {
488 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
bool operator()(
const bool& a,
const bool& b)
const {
return a & b; }
493 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
bool operator()(
const bool& a,
const bool& b)
const {
return a | b; }
497struct bit_xor<bool> {
498 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
bool operator()(
const bool& a,
const bool& b)
const {
return a != b; }
502struct bit_not<bool> {
503 EIGEN_DEVICE_FUNC
constexpr EIGEN_ALWAYS_INLINE
bool operator()(
const bool& a)
const {
return !a; }
508struct operator_bitwise_helper {
509 EIGEN_DEVICE_FUNC
static inline T bitwise_and(
const T& a,
const T& b) {
return bit_and<T>()(a, b); }
510 EIGEN_DEVICE_FUNC
static inline T bitwise_or(
const T& a,
const T& b) {
return bit_or<T>()(a, b); }
511 EIGEN_DEVICE_FUNC
static inline T bitwise_xor(
const T& a,
const T& b) {
return bit_xor<T>()(a, b); }
512 EIGEN_DEVICE_FUNC
static inline T bitwise_not(
const T& a) {
return bit_not<T>()(a); }
517struct bytewise_bitwise_helper {
518 EIGEN_DEVICE_FUNC
static inline T bitwise_and(
const T& a,
const T& b) {
519 return binary(a, b, bit_and<unsigned char>());
521 EIGEN_DEVICE_FUNC
static inline T bitwise_or(
const T& a,
const T& b) {
return binary(a, b, bit_or<unsigned char>()); }
522 EIGEN_DEVICE_FUNC
static inline T bitwise_xor(
const T& a,
const T& b) {
523 return binary(a, b, bit_xor<unsigned char>());
525 EIGEN_DEVICE_FUNC
static inline T bitwise_not(
const T& a) {
return unary(a, bit_not<unsigned char>()); }
528 template <
typename Op>
529 EIGEN_DEVICE_FUNC
static inline T unary(
const T& a, Op op) {
530 const unsigned char* a_ptr =
reinterpret_cast<const unsigned char*
>(&a);
532 unsigned char* c_ptr =
reinterpret_cast<unsigned char*
>(&c);
533 for (
size_t i = 0; i <
sizeof(T); ++i) {
534 *c_ptr++ = op(*a_ptr++);
539 template <
typename Op>
540 EIGEN_DEVICE_FUNC
static inline T binary(
const T& a,
const T& b, Op op) {
541 const unsigned char* a_ptr =
reinterpret_cast<const unsigned char*
>(&a);
542 const unsigned char* b_ptr =
reinterpret_cast<const unsigned char*
>(&b);
544 unsigned char* c_ptr =
reinterpret_cast<unsigned char*
>(&c);
545 for (
size_t i = 0; i <
sizeof(T); ++i) {
546 *c_ptr++ = op(*a_ptr++, *b_ptr++);
553template <
typename T,
typename EnableIf =
void>
554struct bitwise_helper :
public bytewise_bitwise_helper<T> {};
558struct bitwise_helper<
559 T, std::enable_if_t<is_scalar<T>::value && (NumTraits<T>::IsInteger || NumTraits<T>::RequireInitialization)>>
560 :
public operator_bitwise_helper<T> {};
563template <
typename Packet>
564EIGEN_DEVICE_FUNC
inline Packet pand(
const Packet& a,
const Packet& b) {
565 return bitwise_helper<Packet>::bitwise_and(a, b);
569template <
typename Packet>
570EIGEN_DEVICE_FUNC
inline Packet por(
const Packet& a,
const Packet& b) {
571 return bitwise_helper<Packet>::bitwise_or(a, b);
575template <
typename Packet>
576EIGEN_DEVICE_FUNC
inline Packet pxor(
const Packet& a,
const Packet& b) {
577 return bitwise_helper<Packet>::bitwise_xor(a, b);
582template <typename Packet, bool IsScalar = is_scalar<Packet>::value>
584 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return pxor(a, ptrue(a)); }
587template <
typename Packet>
588struct pnot_impl<Packet, true> {
589 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return bitwise_helper<Packet>::bitwise_not(a); }
593template <
typename Packet>
594EIGEN_DEVICE_FUNC
inline Packet pnot(
const Packet& a) {
595 return pnot_impl<Packet>::run(a);
599template <
typename Packet>
600EIGEN_DEVICE_FUNC
inline Packet pandnot(
const Packet& a,
const Packet& b) {
601 return pand(a, pnot(b));
605template <
typename Packet>
606EIGEN_DEVICE_FUNC
inline Packet pcmp_lt(
const Packet& a,
const Packet& b) {
607 return a < b ? ptrue(a) : pzero(a);
611template <
typename Packet>
612EIGEN_DEVICE_FUNC
inline Packet pcmp_eq(
const Packet& a,
const Packet& b) {
613 return a == b ? ptrue(a) : pzero(a);
617template <
typename Packet>
618EIGEN_DEVICE_FUNC
inline Packet pcmp_le(
const Packet& a,
const Packet& b) {
619 return por(pcmp_eq(a, b), pcmp_lt(a, b));
623template <
typename Packet>
624EIGEN_DEVICE_FUNC
inline Packet pcmp_lt_or_nan(
const Packet& a,
const Packet& b) {
625 return a >= b ? pzero(a) : ptrue(a);
629template <typename Packet, bool is_scalar = is_scalar<Packet>::value>
631 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& mask,
const Packet& a,
const Packet& b) {
632 return por(pand(a, mask), pandnot(b, mask));
637template <
typename Packet>
638struct pselect_impl<Packet, true> {
639 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& mask,
const Packet& a,
const Packet& b) {
640 return numext::select(mask, a, b);
645template <
typename Packet>
646EIGEN_DEVICE_FUNC
inline Packet pselect(
const Packet& mask,
const Packet& a,
const Packet& b) {
647 return pselect_impl<Packet>::run(mask, a, b);
651EIGEN_DEVICE_FUNC
inline bool pselect<bool>(
const bool& cond,
const bool& a,
const bool& b) {
656template <
typename Packet>
657struct pminmax_propagates_nan : bool_constant<false> {};
663template <
int NaNPropagation,
bool IsInteger,
bool NativePropagatesNaN = false>
665 template <
typename Packet,
typename Op>
666 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a,
const Packet& b, Op op) {
675 template <
typename Packet,
typename Op>
676 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a,
const Packet& b, Op op) {
681 Packet not_nan_mask_a = pcmp_eq(a, a);
682 Packet not_nan_mask_b = pcmp_eq(b, b);
683 return pselect(not_nan_mask_a, pselect(not_nan_mask_b, op(a, b), b), a);
690template <
bool NativePropagatesNaN>
692 template <
typename Packet,
typename Op>
693 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a,
const Packet& b, Op op) {
694 Packet not_nan_mask_a = pcmp_eq(a, a);
695 Packet not_nan_mask_b = pcmp_eq(b, b);
696 return pselect(not_nan_mask_a, pselect(not_nan_mask_b, op(a, b), a), b);
700#define EIGEN_BINARY_OP_NAN_PROPAGATION(Type, Func) [](const Type& aa, const Type& bb) { return Func(aa, bb); }
704template <
typename Packet>
705EIGEN_DEVICE_FUNC
inline Packet pmin(
const Packet& a,
const Packet& b) {
706 return numext::mini(a, b);
711template <
int NaNPropagation,
typename Packet>
712EIGEN_DEVICE_FUNC
inline Packet pmin(
const Packet& a,
const Packet& b) {
713 constexpr bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger;
714 constexpr bool NativePropagatesNaN = pminmax_propagates_nan<Packet>::value;
715 return pminmax_impl<NaNPropagation, IsInteger, NativePropagatesNaN>::run(
716 a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet, (pmin<Packet>)));
721template <
typename Packet>
722EIGEN_DEVICE_FUNC
inline Packet pmax(
const Packet& a,
const Packet& b) {
723 return numext::maxi(a, b);
728template <
int NaNPropagation,
typename Packet>
729EIGEN_DEVICE_FUNC
inline Packet pmax(
const Packet& a,
const Packet& b) {
730 constexpr bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger;
731 constexpr bool NativePropagatesNaN = pminmax_propagates_nan<Packet>::value;
732 return pminmax_impl<NaNPropagation, IsInteger, NativePropagatesNaN>::run(
733 a, b, EIGEN_BINARY_OP_NAN_PROPAGATION(Packet, (pmax<Packet>)));
737template <
typename Packet, std::enable_if_t<!(NumTraits<
typename unpacket_traits<Packet>::type>::IsInteger &&
738 !NumTraits<
typename unpacket_traits<Packet>::type>::IsSigned),
740EIGEN_DEVICE_FUNC
inline Packet pabs(
const Packet& a) {
741 return numext::abs(a);
744template <typename Packet, std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsInteger &&
745 !NumTraits<typename unpacket_traits<Packet>::type>::IsSigned,
747EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pabs(
const Packet& a) {
752template <
typename Packet>
753EIGEN_DEVICE_FUNC
inline Packet paddsub(
const Packet& a,
const Packet& b) {
754 return pselect(peven_mask(a), padd(a, b), psub(a, b));
758template <
typename Packet>
759EIGEN_DEVICE_FUNC
inline Packet parg(
const Packet& a) {
760 return numext::arg(a);
764template <
int N,
typename T>
765EIGEN_DEVICE_FUNC
inline T parithmetic_shift_right(
const T& a) {
766 return numext::arithmetic_shift_right(a, N);
770template <
int N,
typename T>
771EIGEN_DEVICE_FUNC
inline T plogical_shift_right(
const T& a) {
772 return numext::logical_shift_right(a, N);
776template <
int N,
typename T>
777EIGEN_DEVICE_FUNC
inline T plogical_shift_left(
const T& a) {
778 return numext::logical_shift_left(a, N);
784template <
typename Packet>
785EIGEN_DEVICE_FUNC
inline Packet pfrexp(
const Packet& a, Packet& exponent) {
787 EIGEN_USING_STD(frexp);
788 Packet result =
static_cast<Packet
>(frexp(a, &exp));
789 exponent =
static_cast<Packet
>(exp);
796template <
typename Packet>
797EIGEN_DEVICE_FUNC
inline Packet pldexp(
const Packet& a,
const Packet& exponent) {
798 return static_cast<Packet
>(numext::ldexp(a,
static_cast<int>(exponent)));
802template <
typename Packet>
803EIGEN_DEVICE_FUNC
inline std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet>
804pabsdiff(
const Packet& a,
const Packet& b) {
805 return pselect(pcmp_lt(a, b), psub(b, a), psub(a, b));
807template <
typename Packet>
808EIGEN_DEVICE_FUNC
inline std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsInteger, Packet>
809pabsdiff(
const Packet& a,
const Packet& b) {
810 return pabs(psub(a, b));
814template <
typename Packet>
815EIGEN_DEVICE_FUNC
inline Packet pload(
const typename unpacket_traits<Packet>::type* from) {
823template <
typename Packet>
824EIGEN_DEVICE_FUNC
inline Packet pload_partial(
const typename unpacket_traits<Packet>::type* from,
const Index n,
825 const Index offset = 0) {
826 const Index packet_size = unpacket_traits<Packet>::size;
827 eigen_assert(n + offset <= packet_size &&
"number of elements plus offset will read past end of packet");
828 using Scalar =
typename unpacket_traits<Packet>::type;
829 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size] = {Scalar(0)};
830 for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) {
831 elements[i] = from[i - offset];
833 return pload<Packet>(elements);
837template <
typename Packet>
838EIGEN_DEVICE_FUNC
inline Packet ploadu(
const typename unpacket_traits<Packet>::type* from) {
844template <
typename Packet>
845EIGEN_DEVICE_FUNC
inline Packet ploadu_partial(
const typename unpacket_traits<Packet>::type* from,
const Index n,
846 const Index offset = 0) {
847 const Index packet_size = unpacket_traits<Packet>::size;
848 eigen_assert(n + offset <= packet_size &&
"number of elements plus offset will read past end of packet");
849 using Scalar =
typename unpacket_traits<Packet>::type;
850 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size] = {Scalar(0)};
851 for (Index i = offset; i < numext::mini(n + offset, packet_size); i++) {
852 elements[i] = from[i - offset];
854 return pload<Packet>(elements);
861template <
typename Packet>
862EIGEN_DEVICE_FUNC
inline std::enable_if_t<unpacket_traits<Packet>::masked_load_available, Packet> ploadu(
863 const typename unpacket_traits<Packet>::type* from,
typename unpacket_traits<Packet>::mask_t umask);
866template <
typename Packet>
867EIGEN_DEVICE_FUNC
inline Packet pset1(
const typename unpacket_traits<Packet>::type& a) {
872template <
typename Packet,
typename BitsType>
873EIGEN_DEVICE_FUNC
inline Packet pset1frombits(BitsType a) {
874 using Scalar =
typename unpacket_traits<Packet>::type;
875 return pset1<Packet>(numext::bit_cast<Scalar>(a));
878template <
typename Packet>
879struct packet_bit_pattern_traits {
880 using Scalar =
typename unpacket_traits<Packet>::type;
881 using Bits =
typename numext::get_integer_by_size<
sizeof(Scalar)>::unsigned_type;
882 enum { HasIntegerBits = !std::is_void<Bits>::value };
887template <
typename Scalar>
888EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pscalar_from_float_bits(numext::uint32_t bits) {
889#if EIGEN_COMP_GNUC_STRICT && defined(__FINITE_MATH_ONLY__) && __FINITE_MATH_ONLY__ && \
890 !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(__SYCL_DEVICE_ONLY__)
893 __asm__(
"" :
"+m"(bits));
895 EIGEN_FAST_MATH_CONSTANT_BARRIER(bits);
896 return static_cast<Scalar
>(numext::bit_cast<float>(bits));
899template <typename Packet, bool HasIntegerBits = packet_bit_pattern_traits<Packet>::HasIntegerBits,
900 bool IsScalar = is_scalar<Packet>::value>
901struct psignmask_impl;
903template <
typename Packet,
bool IsScalar>
904struct psignmask_impl<Packet, true, IsScalar> {
905 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run() {
906 using Scalar =
typename packet_bit_pattern_traits<Packet>::Scalar;
907 using Bits =
typename packet_bit_pattern_traits<Packet>::Bits;
908 constexpr Bits kSignBit =
static_cast<Bits
>(Bits(1) << (CHAR_BIT *
sizeof(Scalar) - 1));
909 return pset1frombits<Packet, Bits>(kSignBit);
913template <
typename Scalar>
914struct psignmask_impl<Scalar, false, true> {
915 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run() {
return pscalar_from_float_bits<Scalar>(0x80000000u); }
928template <
typename Packet>
929EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet psignmask() {
930 return psignmask_impl<Packet>::run();
943template <typename Packet, bool HasIntegerBits = packet_bit_pattern_traits<Packet>::HasIntegerBits,
944 bool IsScalar = is_scalar<Packet>::value>
947template <
typename Packet,
bool IsScalar>
948struct pinf_impl<Packet, true, IsScalar> {
949 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run() {
950 using Scalar =
typename packet_bit_pattern_traits<Packet>::Scalar;
951 using Bits =
typename packet_bit_pattern_traits<Packet>::Bits;
952 constexpr int kMantissaBits = std::numeric_limits<Scalar>::digits - 1;
953 constexpr int kExponentBits =
static_cast<int>(CHAR_BIT *
sizeof(Scalar)) - 1 - kMantissaBits;
954 constexpr Bits kInf =
static_cast<Bits
>(((Bits(1) << kExponentBits) - 1) << kMantissaBits);
955 return pset1frombits<Packet, Bits>(kInf);
959template <
typename Scalar>
960struct pinf_impl<Scalar, false, true> {
961 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run() {
return pscalar_from_float_bits<Scalar>(0x7f800000u); }
964template <
typename Packet>
965EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pinf() {
966 return pinf_impl<Packet>::run();
970template <typename Packet, bool HasIntegerBits = packet_bit_pattern_traits<Packet>::HasIntegerBits,
971 bool IsScalar = is_scalar<Packet>::value>
974template <
typename Packet,
bool IsScalar>
975struct pnan_impl<Packet, true, IsScalar> {
976 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run() {
977 using Scalar =
typename packet_bit_pattern_traits<Packet>::Scalar;
978 using Bits =
typename packet_bit_pattern_traits<Packet>::Bits;
979 constexpr int kMantissaBits = std::numeric_limits<Scalar>::digits - 1;
980 constexpr int kExponentBits =
static_cast<int>(CHAR_BIT *
sizeof(Scalar)) - 1 - kMantissaBits;
981 constexpr Bits kInf =
static_cast<Bits
>(((Bits(1) << kExponentBits) - 1) << kMantissaBits);
982 constexpr Bits kNaN =
static_cast<Bits
>(kInf | (Bits(1) << (kMantissaBits - 1)));
983 return pset1frombits<Packet, Bits>(kNaN);
987template <
typename Scalar>
988struct pnan_impl<Scalar, false, true> {
989 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run() {
return pscalar_from_float_bits<Scalar>(0x7fc00000u); }
992template <
typename Packet>
993EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnan() {
994 return pnan_impl<Packet>::run();
997template <typename Scalar, std::enable_if_t<std::is_trivially_copyable<Scalar>::value,
int> = 0>
998EIGEN_DEVICE_FUNC
inline Scalar pload1_scalar(
const Scalar* a) {
1000 EIGEN_USING_STD(memcpy)
1001 memcpy(&scalar, a,
sizeof(Scalar));
1005template <typename Scalar, std::enable_if_t<!std::is_trivially_copyable<Scalar>::value,
int> = 0>
1006EIGEN_DEVICE_FUNC
inline Scalar pload1_scalar(
const Scalar* a) {
1011template <
typename Packet>
1012EIGEN_DEVICE_FUNC
inline Packet pload1(
const typename unpacket_traits<Packet>::type* a) {
1013 using Scalar =
typename unpacket_traits<Packet>::type;
1014 return pset1<Packet>(pload1_scalar<Scalar>(a));
1022template <
typename Packet>
1023EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet ploaddup(
const typename unpacket_traits<Packet>::type* from) {
1024 static_assert(unpacket_traits<Packet>::size <= 2,
"Wider packets require a coefficient-duplicating ploaddup");
1025 return pload1<Packet>(from);
1034template <
typename Packet>
1035EIGEN_DEVICE_FUNC
inline Packet ploadquad(
const typename unpacket_traits<Packet>::type* from) {
1036 return pload1<Packet>(from);
1048template <
typename Packet>
1049EIGEN_DEVICE_FUNC
inline void pbroadcast4(
const typename unpacket_traits<Packet>::type* a, Packet& a0, Packet& a1,
1050 Packet& a2, Packet& a3) {
1051 a0 = pload1<Packet>(a + 0);
1052 a1 = pload1<Packet>(a + 1);
1053 a2 = pload1<Packet>(a + 2);
1054 a3 = pload1<Packet>(a + 3);
1064template <
typename Packet>
1065EIGEN_DEVICE_FUNC
inline void pbroadcast2(
const typename unpacket_traits<Packet>::type* a, Packet& a0, Packet& a1) {
1066 a0 = pload1<Packet>(a + 0);
1067 a1 = pload1<Packet>(a + 1);
1071template <
typename Packet>
1072EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet plset(
const typename unpacket_traits<Packet>::type& a) {
1078template <
typename Packet>
1079EIGEN_DEVICE_FUNC
inline Packet peven_mask(
const Packet& ) {
1080 using Scalar =
typename unpacket_traits<Packet>::type;
1081 if (is_scalar<Packet>::value) {
1083 return pset1<Packet>(Scalar(1));
1085 const size_t n = unpacket_traits<Packet>::size;
1087 char* bytes =
reinterpret_cast<char*
>(&b);
1088 for (
size_t i = 0; i < n; ++i) {
1089 memset(bytes + i *
sizeof(Scalar), ((i & 1) == 0 ? 0xff : 0),
sizeof(Scalar));
1091 EIGEN_FAST_MATH_CONSTANT_BARRIER(b);
1096template <
typename Scalar,
typename Packet>
1097EIGEN_DEVICE_FUNC
inline void pstore(Scalar* to,
const Packet& from) {
1104template <
typename Scalar,
typename Packet>
1105EIGEN_DEVICE_FUNC
inline void pstore_partial(Scalar* to,
const Packet& from,
const Index n,
const Index offset = 0) {
1106 const Index packet_size = unpacket_traits<Packet>::size;
1107 eigen_assert(n + offset <= packet_size &&
"number of elements plus offset will write past end of packet");
1108 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size];
1109 pstore<Scalar>(elements, from);
1110 for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) {
1111 to[i] = elements[i + offset];
1116template <
typename Scalar,
typename Packet>
1117EIGEN_DEVICE_FUNC
inline void pstoreu(Scalar* to,
const Packet& from) {
1122template <
typename Scalar,
typename Packet>
1123EIGEN_DEVICE_FUNC
inline void pstoreu_partial(Scalar* to,
const Packet& from,
const Index n,
const Index offset = 0) {
1124 const Index packet_size = unpacket_traits<Packet>::size;
1125 eigen_assert(n + offset <= packet_size &&
"number of elements plus offset will write past end of packet");
1126 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size];
1127 pstore<Scalar>(elements, from);
1128 for (Index i = 0; i < numext::mini(n, packet_size - offset); i++) {
1129 to[i] = elements[i + offset];
1137template <
typename Scalar,
typename Packet>
1138EIGEN_DEVICE_FUNC
inline std::enable_if_t<unpacket_traits<Packet>::masked_store_available,
void> pstoreu(
1139 Scalar* to,
const Packet& from,
typename unpacket_traits<Packet>::mask_t umask);
1141template <
typename Scalar,
typename Packet>
1142EIGEN_DEVICE_FUNC
inline Packet pgather(
const Scalar* from, Index ) {
1143 return ploadu<Packet>(from);
1146template <
typename Scalar,
typename Packet>
1147EIGEN_DEVICE_FUNC
inline Packet pgather_partial(
const Scalar* from, Index stride,
const Index n) {
1148 const Index packet_size = unpacket_traits<Packet>::size;
1149 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size] = {Scalar(0)};
1150 for (Index i = 0; i < numext::mini(n, packet_size); i++) {
1151 elements[i] = from[i * stride];
1153 return pload<Packet>(elements);
1156template <
typename Scalar,
typename Packet>
1157EIGEN_DEVICE_FUNC
inline void pscatter(Scalar* to,
const Packet& from, Index ) {
1161template <
typename Scalar,
typename Packet>
1162EIGEN_DEVICE_FUNC
inline void pscatter_partial(Scalar* to,
const Packet& from, Index stride,
const Index n) {
1163 const Index packet_size = unpacket_traits<Packet>::size;
1164 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[packet_size];
1165 pstore<Scalar>(elements, from);
1166 for (Index i = 0; i < numext::mini(n, packet_size); i++) {
1167 to[i * stride] = elements[i];
1172template <
typename Scalar>
1173EIGEN_DEVICE_FUNC
inline void prefetch(
const Scalar* addr) {
1174#if defined(EIGEN_HIP_DEVICE_COMPILE)
1176#elif defined(EIGEN_CUDA_ARCH)
1177#if defined(__LP64__) || EIGEN_OS_WIN64
1179 asm(
" prefetch.L1 [ %1 ];" :
"=l"(addr) :
"l"(addr));
1182 asm(
" prefetch.L1 [ %1 ];" :
"=r"(addr) :
"r"(addr));
1184#elif (!EIGEN_COMP_MSVC) && (EIGEN_COMP_GNUC || EIGEN_COMP_CLANG || EIGEN_COMP_ICC)
1185 __builtin_prefetch(addr);
1190template <
typename Packet>
1191EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet preverse(
const Packet& a) {
1196template <
typename Packet>
1197EIGEN_DEVICE_FUNC
inline Packet pcplxflip(
const Packet& a) {
1198 return Packet(numext::imag(a), numext::real(a));
1203template <
typename Packet>
1204EIGEN_DEVICE_FUNC
inline Packet pdupreal(
const Packet& a) {
1205 return Packet(numext::real(a), numext::real(a));
1210template <
typename Packet>
1211EIGEN_DEVICE_FUNC
inline Packet pdupimag(
const Packet& a) {
1212 return Packet(numext::imag(a), numext::imag(a));
1220template <
typename Packet>
1221EIGEN_DEVICE_FUNC
inline Packet pisnan(
const Packet& a);
1226template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex,
1227 bool IsScalar = is_scalar<Packet>::value,
1228 bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1231 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return pcmp_lt_or_nan(a, a); }
1237template <
typename Packet,
bool IsScalar>
1238struct pisnan_impl<Packet, false, IsScalar, true> {
1239 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return pzero(a); }
1242template <
typename Packet,
bool IsInteger>
1243struct pisnan_impl<Packet, true, false, IsInteger> {
1244 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
1245 using RealPacket =
typename unpacket_traits<Packet>::as_real;
1247 Packet nan_lanes = Packet(pisnan<RealPacket>(a.v));
1248 return por(nan_lanes, pcplxflip(nan_lanes));
1254template <
typename Scalar,
bool IsInteger>
1255struct pisnan_impl<Scalar, true, true, IsInteger> {
1256 static EIGEN_DEVICE_FUNC
inline Scalar run(
const Scalar& a) {
1257 using RealScalar =
typename NumTraits<Scalar>::Real;
1258 const RealScalar nan_mask =
1259 por(pisnan_impl<RealScalar>::run(numext::real(a)), pisnan_impl<RealScalar>::run(numext::imag(a)));
1260 return Scalar(nan_mask);
1265template <
typename Packet>
1266EIGEN_DEVICE_FUNC
inline Packet pisnan(
const Packet& a) {
1267 return pisnan_impl<Packet>::run(a);
1270template <typename Packet, bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1272 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return pcmp_eq(pabs(a), pinf<Packet>()); }
1278template <
typename Packet>
1279struct pisinf_impl<Packet, true> {
1280 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return pzero(a); }
1284template <
typename Packet>
1285EIGEN_DEVICE_FUNC
inline Packet pisinf(
const Packet& a) {
1286 return pisinf_impl<Packet>::run(a);
1289template <typename Packet, bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1290struct pisfinite_impl {
1292 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return pcmp_lt(pabs(a), pinf<Packet>()); }
1297template <
typename Packet>
1298struct pisfinite_impl<Packet, true> {
1299 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return ptrue(a); }
1303template <
typename Packet>
1304EIGEN_DEVICE_FUNC
inline Packet pisfinite(
const Packet& a) {
1305 return pisfinite_impl<Packet>::run(a);
1309template <
typename Packet>
1310EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psin(
const Packet& a) {
1311 return numext::sin(a);
1315template <
typename Packet>
1316EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcos(
const Packet& a) {
1317 return numext::cos(a);
1321template <
typename Packet>
1322EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptan(
const Packet& a) {
1323 return numext::tan(a);
1327template <
typename Packet>
1328EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasin(
const Packet& a) {
1329 return numext::asin(a);
1333template <
typename Packet>
1334EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacos(
const Packet& a) {
1335 return numext::acos(a);
1339template <
typename Packet>
1340EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psinh(
const Packet& a) {
1341 return numext::sinh(a);
1345template <
typename Packet>
1346EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcosh(
const Packet& a) {
1347 return numext::cosh(a);
1351template <
typename Packet>
1352EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patan(
const Packet& a) {
1353 return numext::atan(a);
1357template <
typename Packet>
1358EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptanh(
const Packet& a) {
1359 return numext::tanh(a);
1363template <
typename Packet>
1364EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patanh(
const Packet& a) {
1365 return numext::atanh(a);
1369template <
typename Packet>
1370EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasinh(
const Packet& a) {
1371 return numext::asinh(a);
1375template <
typename Packet>
1376EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacosh(
const Packet& a) {
1377 return numext::acosh(a);
1381template <
typename Packet>
1382EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp(
const Packet& a) {
1383 return numext::exp(a);
1387template <
typename Packet>
1388EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp2(
const Packet& a) {
1389 return numext::exp2(a);
1393template <
typename Packet>
1394EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexpm1(
const Packet& a) {
1395 return numext::expm1(a);
1399template <
typename Packet>
1400EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog(
const Packet& a) {
1401 return numext::log(a);
1405template <
typename Packet>
1406EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog1p(
const Packet& a) {
1407 return numext::log1p(a);
1411template <
typename Packet>
1412EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog10(
const Packet& a) {
1413 return numext::log10(a);
1416template <typename Packet, bool IsComplexPacket = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex &&
1417 !is_scalar<Packet>::value>
1419 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
1420 using Scalar =
typename internal::unpacket_traits<Packet>::type;
1421 using RealScalar =
typename NumTraits<Scalar>::Real;
1422 return pmul(pset1<Packet>(Scalar(RealScalar(EIGEN_LOG2E))), plog(a));
1428template <
typename Packet>
1429struct plog2_impl<Packet, true>;
1432template <
typename Packet>
1433EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog2(
const Packet& a) {
1434 return plog2_impl<Packet>::run(a);
1438template <
typename Packet>
1439EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psqrt(
const Packet& a) {
1440 return numext::sqrt(a);
1444template <
typename Packet>
1445EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcbrt(
const Packet& a) {
1446 return numext::cbrt(a);
1449template <typename Packet, bool IsScalar = is_scalar<Packet>::value,
1450 bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>
1451struct nearest_integer_packetop_impl {
1452 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_floor(
const Packet& x) {
return numext::floor(x); }
1453 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_ceil(
const Packet& x) {
return numext::ceil(x); }
1454 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_rint(
const Packet& x) {
return numext::rint(x); }
1455 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_round(
const Packet& x) {
return numext::round(x); }
1456 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_trunc(
const Packet& x) {
return numext::trunc(x); }
1460template <
typename Packet>
1461EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pround(
const Packet& a) {
1462 return nearest_integer_packetop_impl<Packet>::run_round(a);
1466template <
typename Packet>
1467EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pfloor(
const Packet& a) {
1468 return nearest_integer_packetop_impl<Packet>::run_floor(a);
1473template <
typename Packet>
1474EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet print(
const Packet& a) {
1475 return nearest_integer_packetop_impl<Packet>::run_rint(a);
1479template <
typename Packet>
1480EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pceil(
const Packet& a) {
1481 return nearest_integer_packetop_impl<Packet>::run_ceil(a);
1485template <
typename Packet>
1486EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet ptrunc(
const Packet& a) {
1487 return nearest_integer_packetop_impl<Packet>::run_trunc(a);
1490template <
typename Packet,
typename EnableIf =
void>
1492 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return numext::sign(a); }
1496template <
typename Packet>
1497EIGEN_DEVICE_FUNC
inline Packet psign(
const Packet& a) {
1498 return psign_impl<Packet>::run(a);
1502EIGEN_DEVICE_FUNC
inline bool psign(
const bool& a) {
1507template <
typename Packet>
1508EIGEN_DEVICE_FUNC
inline typename unpacket_traits<Packet>::type pfirst(
const Packet& a) {
1516template <
typename Packet>
1517EIGEN_DEVICE_FUNC
inline std::conditional_t<(unpacket_traits<Packet>::size % 8) == 0,
1518 typename unpacket_traits<Packet>::half, Packet>
1519predux_half(
const Packet& a) {
1524template <
typename Packet,
typename Op>
1525EIGEN_DEVICE_FUNC
inline typename unpacket_traits<Packet>::type predux_helper(
const Packet& a, Op op) {
1526 using Scalar =
typename unpacket_traits<Packet>::type;
1527 const size_t n = unpacket_traits<Packet>::size;
1528 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar elements[n];
1529 pstoreu<Scalar>(elements, a);
1530 for (
size_t k = n / 2; k > 0; k /= 2) {
1531 for (
size_t i = 0; i < k; ++i) {
1532 elements[i] = op(elements[i], elements[i + k]);
1538template <typename Packet, std::enable_if_t<unpacket_traits<Packet>::size == 1,
int> = 0>
1539EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename unpacket_traits<Packet>::type predux_one_element(
const Packet& a) {
1544template <
typename Packet>
1545EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename unpacket_traits<Packet>::type predux(
const Packet& a) {
1546 return predux_one_element(a);
1550template <
typename Packet>
1551EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename unpacket_traits<Packet>::type predux_mul(
const Packet& a) {
1552 using Scalar =
typename unpacket_traits<Packet>::type;
1553 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmul<Scalar>)));
1557template <
typename Packet>
1558EIGEN_DEVICE_FUNC
inline typename unpacket_traits<Packet>::type predux_min(
const Packet& a) {
1559 using Scalar =
typename unpacket_traits<Packet>::type;
1560 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<Scalar>)));
1564template <
typename Packet>
1565EIGEN_DEVICE_FUNC
inline typename unpacket_traits<Packet>::type predux_max(
const Packet& a) {
1566 using Scalar =
typename unpacket_traits<Packet>::type;
1567 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<Scalar>)));
1570template <
int NaNPropagation,
typename Packet>
1571struct predux_min_max_helper_impl {
1572 using Scalar =
typename unpacket_traits<Packet>::type;
1573 static constexpr bool UsePredux_ = NaNPropagation ==
PropagateFast || NumTraits<Scalar>::IsInteger;
1574 template <
bool UsePredux = UsePredux_, std::enable_if_t<!UsePredux,
bool> = true>
1575 static EIGEN_DEVICE_FUNC
inline Scalar run_min(
const Packet& a) {
1576 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmin<NaNPropagation, Scalar>)));
1578 template <
bool UsePredux = UsePredux_, std::enable_if_t<!UsePredux,
bool> = true>
1579 static EIGEN_DEVICE_FUNC
inline Scalar run_max(
const Packet& a) {
1580 return predux_helper(a, EIGEN_BINARY_OP_NAN_PROPAGATION(Scalar, (pmax<NaNPropagation, Scalar>)));
1582 template <
bool UsePredux = UsePredux_, std::enable_if_t<UsePredux,
bool> = true>
1583 static EIGEN_DEVICE_FUNC
inline Scalar run_min(
const Packet& a) {
1584 return predux_min(a);
1586 template <
bool UsePredux = UsePredux_, std::enable_if_t<UsePredux,
bool> = true>
1587 static EIGEN_DEVICE_FUNC
inline Scalar run_max(
const Packet& a) {
1588 return predux_max(a);
1592template <
int NaNPropagation,
typename Packet>
1593EIGEN_DEVICE_FUNC
inline typename unpacket_traits<Packet>::type predux_min(
const Packet& a) {
1594 return predux_min_max_helper_impl<NaNPropagation, Packet>::run_min(a);
1597template <
int NaNPropagation,
typename Packet>
1598EIGEN_DEVICE_FUNC
inline typename unpacket_traits<Packet>::type predux_max(
const Packet& a) {
1599 return predux_min_max_helper_impl<NaNPropagation, Packet>::run_max(a);
1602#undef EIGEN_BINARY_OP_NAN_PROPAGATION
1604template <typename Packet, bool IsBoolean = std::is_same<typename unpacket_traits<Packet>::type,
bool>::value>
1605struct predux_count_impl {
1606 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Index run(
const Packet& a) {
1607 using Scalar =
typename unpacket_traits<Packet>::type;
1608 const Packet true_values = pandnot(pset1<Packet>(Scalar(1)), pcmp_eq(a, pzero(a)));
1609 return static_cast<Index
>(numext::real(predux(true_values)));
1613template <
typename Packet>
1614struct predux_count_impl<Packet, true> {
1615 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Index run(
const Packet& a) {
1616 using Scalar =
typename unpacket_traits<Packet>::type;
1617 constexpr int PacketSize = unpacket_traits<Packet>::size;
1618 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar values[PacketSize];
1619 pstoreu<Scalar>(values, a);
1621 for (
int i = 0; i < PacketSize; ++i) result += values[i] ? 1 : 0;
1627template <
typename Packet>
1628EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index predux_count(
const Packet& a) {
1629 return predux_count_impl<Packet>::run(a);
1635template <
typename Packet>
1636EIGEN_DEVICE_FUNC
inline bool predux_any(
const Packet& a) {
1644 using Scalar =
typename unpacket_traits<Packet>::type;
1645 return numext::not_equal_strict(predux(a), Scalar(0));
1648template <typename Packet, bool IsBoolean = std::is_same<typename unpacket_traits<Packet>::type,
bool>::value>
1649struct predux_all_impl {
1650 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool run(
const Packet& a) {
return !predux_any(pcmp_eq(a, pzero(a))); }
1653template <
typename Packet>
1654struct predux_all_impl<Packet, true> {
1655 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE
bool run(
const Packet& a) {
return predux_mul(a); }
1659template <
typename Packet>
1660EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool predux_all(
const Packet& a) {
1661 return predux_all_impl<Packet>::run(a);
1668template <
typename Packet>
1669EIGEN_DEVICE_FUNC
inline Packet pmadd(
const Packet& a,
const Packet& b,
const Packet& c);
1673template <
typename Packet>
1674struct is_fused_madd : false_type {};
1676template <
typename Packet,
typename EnableIf =
void>
1680 static constexpr bool Fused = is_fused_madd<Packet>::value;
1681 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pmadd(
const Packet& a,
const Packet& b,
const Packet& c) {
1682 return padd(pmul(a, b), c);
1684 template <
bool Fuse = Fused, std::enable_if_t<!Fuse,
int> = 0>
1685 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pmsub(
const Packet& a,
const Packet& b,
const Packet& c) {
1686 return psub(pmul(a, b), c);
1688 template <
bool Fuse = Fused, std::enable_if_t<Fuse,
int> = 0>
1689 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pmsub(
const Packet& a,
const Packet& b,
const Packet& c) {
1690 return internal::pmadd<Packet>(a, b, pnegate(c));
1692 template <
bool Fuse = Fused, std::enable_if_t<!Fuse,
int> = 0>
1693 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnmadd(
const Packet& a,
const Packet& b,
const Packet& c) {
1694 return psub(c, pmul(a, b));
1696 template <
bool Fuse = Fused, std::enable_if_t<Fuse,
int> = 0>
1697 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnmadd(
const Packet& a,
const Packet& b,
const Packet& c) {
1698 return internal::pmadd<Packet>(pnegate(a), b, c);
1700 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pnmsub(
const Packet& a,
const Packet& b,
const Packet& c) {
1701 return pnegate(internal::pmadd<Packet>(a, b, c));
1705#if defined(EIGEN_VECTORIZE_FMA)
1707template <
typename Scalar>
1708struct pmadd_impl<Scalar, std::enable_if_t<std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value>> {
1709 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmadd(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1710 return numext::fma(a, b, c);
1712 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmsub(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1713 return numext::fma(a, b, Scalar(-c));
1715 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmadd(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1716 return numext::fma(Scalar(-a), b, c);
1718 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmsub(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1719 return -Scalar(numext::fma(a, b, c));
1723template <
typename Scalar>
1724struct pmadd_impl<Scalar,
1725 std::enable_if_t<is_scalar<Scalar>::value && NumTraits<Scalar>::IsSigned &&
1726 !std::is_same<Scalar, float>::value && !std::is_same<Scalar, double>::value>> {
1728template <
typename Scalar>
1729struct pmadd_impl<Scalar, std::enable_if_t<is_scalar<Scalar>::value && NumTraits<Scalar>::IsSigned>> {
1731 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmadd(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1732 return numext::madd<Scalar>(a, b, c);
1734 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pmsub(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1735 return numext::madd<Scalar>(a, b, Scalar(-c));
1737 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmadd(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1738 return numext::madd<Scalar>(Scalar(-a), b, c);
1740 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar pnmsub(
const Scalar& a,
const Scalar& b,
const Scalar& c) {
1741 return -Scalar(numext::madd<Scalar>(a, b, c));
1747template <
typename Packet>
1748EIGEN_DEVICE_FUNC
inline Packet pmadd(
const Packet& a,
const Packet& b,
const Packet& c) {
1749 return pmadd_impl<Packet>::pmadd(a, b, c);
1753template <
typename Packet>
1754EIGEN_DEVICE_FUNC
inline Packet pmsub(
const Packet& a,
const Packet& b,
const Packet& c) {
1755 return pmadd_impl<Packet>::pmsub(a, b, c);
1759template <
typename Packet>
1760EIGEN_DEVICE_FUNC
inline Packet pnmadd(
const Packet& a,
const Packet& b,
const Packet& c) {
1761 return pmadd_impl<Packet>::pnmadd(a, b, c);
1765template <
typename Packet>
1766EIGEN_DEVICE_FUNC
inline Packet pnmsub(
const Packet& a,
const Packet& b,
const Packet& c) {
1767 return pmadd_impl<Packet>::pnmsub(a, b, c);
1774template <
typename Packet>
1775inline void pstore1(
typename unpacket_traits<Packet>::type* to,
const typename unpacket_traits<Packet>::type& a) {
1776 pstore(to, pset1<Packet>(a));
1781template <
typename Packet,
int Alignment>
1782EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(
const typename unpacket_traits<Packet>::type* from) {
1783 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1784 return pload<Packet>(from);
1786 return ploadu<Packet>(from);
1792template <
typename Packet,
int Alignment>
1793EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_partial(
const typename unpacket_traits<Packet>::type* from,
1794 const Index n,
const Index offset = 0) {
1795 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1796 return pload_partial<Packet>(from, n, offset);
1798 return ploadu_partial<Packet>(from, n, offset);
1804template <
typename Scalar,
typename Packet,
int Alignment>
1805EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pstoret(Scalar* to,
const Packet& from) {
1806 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1815template <
typename Scalar,
typename Packet,
int Alignment>
1816EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void pstoret_partial(Scalar* to,
const Packet& from,
const Index n,
1817 const Index offset = 0) {
1818 EIGEN_IF_CONSTEXPR (Alignment >= unpacket_traits<Packet>::alignment) {
1819 pstore_partial(to, from, n, offset);
1821 pstoreu_partial(to, from, n, offset);
1830template <
typename Packet,
int LoadMode>
1831EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_ro(
const typename unpacket_traits<Packet>::type* from) {
1832 return ploadt<Packet, LoadMode>(from);
1840#if !defined(EIGEN_GPUCC)
1843inline std::complex<float> pmul(
const std::complex<float>& a,
const std::complex<float>& b) {
1844 return std::complex<float>(a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag());
1848inline std::complex<double> pmul(
const std::complex<double>& a,
const std::complex<double>& b) {
1849 return std::complex<double>(a.real() * b.real() - a.imag() * b.imag(), a.imag() * b.real() + a.real() * b.imag());
1858template <typename Packet, int N = unpacket_traits<Packet>::size>
1863template <
typename Packet,
int size = 1>
1864EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet, size>& ) {
1869template <
typename Packet>
1870EIGEN_DEVICE_FUNC
inline Packet preciprocal(
const Packet& a) {
1871 using Scalar =
typename unpacket_traits<Packet>::type;
1872 return pdiv(pset1<Packet>(Scalar(1)), a);
1876template <
typename Packet>
1877EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet prsqrt(
const Packet& a) {
1878 return preciprocal<Packet>(psqrt(a));
1881template <typename Packet, bool IsScalar = is_scalar<Packet>::value,
1882 bool IsInteger = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger,
1883 bool IsUnsigned = IsInteger && !NumTraits<typename unpacket_traits<Packet>::type>::IsSigned>
1884struct psignbit_impl;
1885template <
typename Packet,
bool IsInteger,
bool IsUn
signed>
1886struct psignbit_impl<Packet, true, IsInteger, IsUnsigned> {
1887 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
static constexpr Packet run(
const Packet& a) {
return numext::signbit(a); }
1889template <
typename Packet>
1890struct psignbit_impl<Packet, false, false, false> {
1893 using Scalar =
typename unpacket_traits<Packet>::type;
1894 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
static Packet run(
const Packet& a) {
1895 const Packet cst_pos_one = pset1<Packet>(Scalar(1));
1896 const Packet cst_neg_one = pset1<Packet>(Scalar(-1));
1897 return pcmp_eq(por(pand(a, cst_neg_one), cst_pos_one), cst_neg_one);
1900template <
typename Packet>
1901struct psignbit_impl<Packet, false, true, false> {
1903 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
static constexpr Packet run(
const Packet& a) {
return pcmp_lt(a, pzero(a)); }
1905template <
typename Packet>
1906struct psignbit_impl<Packet, false, true, true> {
1907 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
static constexpr Packet run(
const Packet& a) {
return pzero(a); }
1910template <
typename Packet>
1911EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
constexpr Packet psignbit(
const Packet& a) {
1912 return psignbit_impl<Packet>::run(a);
1916template <typename Packet, std::enable_if_t<is_scalar<Packet>::value,
int> = 0>
1917EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet patan2(
const Packet& y,
const Packet& x) {
1918 return numext::atan2(y, x);
1922template <typename Packet, std::enable_if_t<!is_scalar<Packet>::value,
int> = 0>
1923EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet patan2(
const Packet& y,
const Packet& x) {
1924 using Scalar =
typename internal::unpacket_traits<Packet>::type;
1929 const Packet kSignMask = psignmask<Packet>();
1930 const Packet kZero = pzero(x);
1931 const Packet kOne = pset1<Packet>(Scalar(1));
1932 const Packet kPi = pset1<Packet>(Scalar(EIGEN_PI));
1934 const Packet x_has_signbit = psignbit(x);
1935 const Packet y_signmask = pand(y, kSignMask);
1936 const Packet x_signmask = pand(x, kSignMask);
1937 const Packet result_signmask = pxor(y_signmask, x_signmask);
1938 const Packet shift = por(pand(x_has_signbit, kPi), y_signmask);
1940 const Packet x_and_y_are_same = pcmp_eq(pabs(x), pabs(y));
1941 const Packet x_and_y_are_zero = pcmp_eq(por(x, y), kZero);
1943 Packet arg = pdiv(y, x);
1944 arg = pselect(x_and_y_are_same, por(kOne, result_signmask), arg);
1945 arg = pselect(x_and_y_are_zero, result_signmask, arg);
1947 Packet result = patan(arg);
1948 result = padd(result, shift);
1953template <typename Packet, std::enable_if_t<is_scalar<Packet>::value,
int> = 0>
1954EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pcarg(
const Packet& a) {
1955 return Packet(numext::arg(a));
1959template <typename Packet, std::enable_if_t<!is_scalar<Packet>::value,
int> = 0>
1960EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet pcarg(
const Packet& a) {
1961 EIGEN_STATIC_ASSERT(NumTraits<
typename unpacket_traits<Packet>::type>::IsComplex,
1962 THIS METHOD IS FOR COMPLEX TYPES ONLY)
1963 using RealPacket =
typename unpacket_traits<Packet>::as_real;
1965 RealPacket aflip = pcplxflip(a).v;
1966 RealPacket result = patan2(aflip, a.v);
1967 return (Packet)pand(result, peven_mask(result));
1972template <
typename Packet>
1973EIGEN_DEVICE_FUNC
inline Packet ploaduSegment(
const typename unpacket_traits<Packet>::type* from, Index begin,
1975 using Scalar =
typename unpacket_traits<Packet>::type;
1976 constexpr Index PacketSize = unpacket_traits<Packet>::size;
1977 eigen_assert((begin >= 0 && count >= 0 && begin + count <= PacketSize) &&
"invalid range");
1978 Scalar aux[PacketSize] = {};
1979 for (Index k = begin; k < begin + count; k++) {
1982 return ploadu<Packet>(aux);
1987template <
typename Packet>
1988EIGEN_DEVICE_FUNC
inline Packet ploadSegment(
const typename unpacket_traits<Packet>::type* from, Index begin,
1990 return ploaduSegment<Packet>(from, begin, count);
1996template <
typename Scalar,
typename Packet>
1997EIGEN_DEVICE_FUNC
inline void pstoreuSegment(Scalar* to,
const Packet& from, Index begin, Index count) {
1998 constexpr Index PacketSize = unpacket_traits<Packet>::size;
1999 eigen_assert((begin >= 0 && count >= 0 && begin + count <= PacketSize) &&
"invalid range");
2000 Scalar aux[PacketSize];
2001 pstoreu<Scalar, Packet>(aux, from);
2002 for (Index k = begin; k < begin + count; k++) {
2010template <
typename Scalar,
typename Packet>
2011EIGEN_DEVICE_FUNC
inline void pstoreSegment(Scalar* to,
const Packet& from, Index begin, Index count) {
2012 return pstoreuSegment(to, from, begin, count);
2017template <
typename Packet,
int Alignment>
2018EIGEN_DEVICE_FUNC
inline Packet ploadtSegment(
const typename unpacket_traits<Packet>::type* from, Index begin,
2020 constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
2021 EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
2022 return ploadSegment<Packet>(from, begin, count);
2024 return ploaduSegment<Packet>(from, begin, count);
2030template <
typename Scalar,
typename Packet,
int Alignment>
2031EIGEN_DEVICE_FUNC
inline void pstoretSegment(Scalar* to,
const Packet& from, Index begin, Index count) {
2032 constexpr int RequiredAlignment = unpacket_traits<Packet>::alignment;
2033 EIGEN_IF_CONSTEXPR (Alignment >= RequiredAlignment) {
2034 pstoreSegment<Scalar, Packet>(to, from, begin, count);
2036 pstoreuSegment<Scalar, Packet>(to, from, begin, count);
2042template <
typename Packet>
2043class StreamablePacket {
2045 using Scalar =
typename unpacket_traits<Packet>::type;
2046 StreamablePacket(
const Packet& packet) { pstoreu(v_, packet); }
2048 friend std::ostream& operator<<(std::ostream& os,
const StreamablePacket& packet) {
2049 os <<
"{" << packet.v_[0];
2050 for (
int i = 1; i < unpacket_traits<Packet>::size; ++i) {
2051 os <<
"," << packet.v_[i];
2058 Scalar v_[unpacket_traits<Packet>::size];
2064template <
typename Packet>
2065StreamablePacket<Packet> postream(
const Packet& packet) {
2066 return StreamablePacket<Packet>(packet);
@ PropagateNaN
Definition Constants.h:343
@ PropagateNumbers
Definition Constants.h:345
@ PropagateFast
Definition Constants.h:341