11#ifndef EIGEN_PACKET_MATH_GPU_H
12#define EIGEN_PACKET_MATH_GPU_H
15#include "../../InternalHeaderCheck.h"
28#if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU)
31struct is_arithmetic<float4> : std::true_type {};
33struct is_arithmetic<double2> : std::true_type {};
36struct packet_traits<float> : default_packet_traits {
39 static constexpr int Vectorizable = 1;
40 static constexpr int AlignedOnScalar = 1;
41 static constexpr int size = 4;
43 static constexpr int HasDiv = 1;
44 static constexpr int HasSin = 0;
45 static constexpr int HasCos = 0;
46 static constexpr int HasLog = 1;
47 static constexpr int HasExp = 1;
48 static constexpr int HasSqrt = 1;
49 static constexpr int HasRsqrt = 1;
50 static constexpr int HasLGamma = 1;
51 static constexpr int HasDiGamma = 1;
52 static constexpr int HasZeta = 1;
53 static constexpr int HasPolygamma = 1;
54 static constexpr int HasErf = 1;
55 static constexpr int HasErfc = 1;
56 static constexpr int HasNdtri = 1;
57 static constexpr int HasBessel = 1;
58 static constexpr int HasIGamma = 1;
59 static constexpr int HasIGammaDerA = 1;
60 static constexpr int HasGammaSampleDerAlpha = 1;
61 static constexpr int HasIGammac = 1;
62 static constexpr int HasBetaInc = 1;
64 static constexpr int HasCmp = 1;
68struct packet_traits<double> : default_packet_traits {
71 static constexpr int Vectorizable = 1;
72 static constexpr int AlignedOnScalar = 1;
73 static constexpr int size = 2;
75 static constexpr int HasDiv = 1;
76 static constexpr int HasLog = 1;
77 static constexpr int HasExp = 1;
78 static constexpr int HasSqrt = 1;
79 static constexpr int HasRsqrt = 1;
80 static constexpr int HasLGamma = 1;
81 static constexpr int HasDiGamma = 1;
82 static constexpr int HasZeta = 1;
83 static constexpr int HasPolygamma = 1;
84 static constexpr int HasErf = 1;
85 static constexpr int HasErfc = 1;
86 static constexpr int HasNdtri = 1;
87 static constexpr int HasBessel = 1;
88 static constexpr int HasIGamma = 1;
89 static constexpr int HasIGammaDerA = 1;
90 static constexpr int HasGammaSampleDerAlpha = 1;
91 static constexpr int HasIGammac = 1;
92 static constexpr int HasBetaInc = 1;
94 static constexpr int HasCmp = 1;
98struct unpacket_traits<float4> {
100 static constexpr int size = 4;
101 static constexpr int alignment =
Aligned16;
102 static constexpr bool vectorizable =
true;
103 static constexpr bool masked_load_available =
false;
104 static constexpr bool masked_store_available =
false;
108struct unpacket_traits<double2> {
110 static constexpr int size = 2;
111 static constexpr int alignment =
Aligned16;
112 static constexpr bool vectorizable =
true;
113 static constexpr bool masked_load_available =
false;
114 static constexpr bool masked_store_available =
false;
115 using half = double2;
119EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pset1<float4>(
const float& from) {
120 return make_float4(from, from, from, from);
123EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pset1<double2>(
const double& from) {
124 return make_double2(from, from);
131using lane_bits_t =
typename numext::get_integer_by_size<
sizeof(T)>::unsigned_type;
133#define EIGEN_MAKE_BITWISE_BINOP(name, op) \
134 template <typename T> \
135 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T bitwise_##name(const T& a, const T& b) { \
136 using Bits = lane_bits_t<T>; \
137 return numext::bit_cast<T>(static_cast<Bits>(numext::bit_cast<Bits>(a) op numext::bit_cast<Bits>(b))); \
140EIGEN_MAKE_BITWISE_BINOP(and, &)
141EIGEN_MAKE_BITWISE_BINOP(or, |)
142EIGEN_MAKE_BITWISE_BINOP(xor, ^)
143EIGEN_MAKE_BITWISE_BINOP(andnot, &~)
145#undef EIGEN_MAKE_BITWISE_BINOP
150EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T mask_from(
bool condition) {
151 using Bits = lane_bits_t<T>;
152 return numext::bit_cast<T>(condition ? ~Bits(0) : Bits(0));
156EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T eq_mask(
const T& a,
const T& b) {
157 return mask_from<T>(a == b);
160EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T lt_mask(
const T& a,
const T& b) {
161 return mask_from<T>(a < b);
164EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T le_mask(
const T& a,
const T& b) {
165 return mask_from<T>(a <= b);
169EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T lt_or_nan_mask(
const T& a,
const T& b) {
170 return mask_from<T>(!(a >= b));
174EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pand<float4>(
const float4& a,
const float4& b) {
175 return make_float4(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y), bitwise_and(a.z, b.z), bitwise_and(a.w, b.w));
178EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pand<double2>(
const double2& a,
const double2& b) {
179 return make_double2(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y));
183EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 por<float4>(
const float4& a,
const float4& b) {
184 return make_float4(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y), bitwise_or(a.z, b.z), bitwise_or(a.w, b.w));
187EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 por<double2>(
const double2& a,
const double2& b) {
188 return make_double2(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y));
192EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pxor<float4>(
const float4& a,
const float4& b) {
193 return make_float4(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y), bitwise_xor(a.z, b.z), bitwise_xor(a.w, b.w));
196EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pxor<double2>(
const double2& a,
const double2& b) {
197 return make_double2(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y));
201EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pandnot<float4>(
const float4& a,
const float4& b) {
202 return make_float4(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y), bitwise_andnot(a.z, b.z),
203 bitwise_andnot(a.w, b.w));
206EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pandnot<double2>(
const double2& a,
const double2& b) {
207 return make_double2(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y));
211EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_eq<float4>(
const float4& a,
const float4& b) {
212 return make_float4(eq_mask(a.x, b.x), eq_mask(a.y, b.y), eq_mask(a.z, b.z), eq_mask(a.w, b.w));
215EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_lt<float4>(
const float4& a,
const float4& b) {
216 return make_float4(lt_mask(a.x, b.x), lt_mask(a.y, b.y), lt_mask(a.z, b.z), lt_mask(a.w, b.w));
219EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_le<float4>(
const float4& a,
const float4& b) {
220 return make_float4(le_mask(a.x, b.x), le_mask(a.y, b.y), le_mask(a.z, b.z), le_mask(a.w, b.w));
223EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_lt_or_nan<float4>(
const float4& a,
const float4& b) {
224 return make_float4(lt_or_nan_mask(a.x, b.x), lt_or_nan_mask(a.y, b.y), lt_or_nan_mask(a.z, b.z),
225 lt_or_nan_mask(a.w, b.w));
228EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_eq<double2>(
const double2& a,
const double2& b) {
229 return make_double2(eq_mask(a.x, b.x), eq_mask(a.y, b.y));
232EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_lt<double2>(
const double2& a,
const double2& b) {
233 return make_double2(lt_mask(a.x, b.x), lt_mask(a.y, b.y));
236EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_le<double2>(
const double2& a,
const double2& b) {
237 return make_double2(le_mask(a.x, b.x), le_mask(a.y, b.y));
240EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_lt_or_nan<double2>(
const double2& a,
const double2& b) {
241 return make_double2(lt_or_nan_mask(a.x, b.x), lt_or_nan_mask(a.y, b.y));
245EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool predux_any(
const float4& a) {
246 using Bits = lane_bits_t<float>;
247 return (numext::bit_cast<Bits>(a.x) | numext::bit_cast<Bits>(a.y) | numext::bit_cast<Bits>(a.z) |
248 numext::bit_cast<Bits>(a.w)) != 0;
252EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool predux_any(
const double2& a) {
253 using Bits = lane_bits_t<double>;
254 return (numext::bit_cast<Bits>(a.x) | numext::bit_cast<Bits>(a.y)) != 0;
260EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 psign<float4>(
const float4& a) {
261 return make_float4(numext::sign(a.x), numext::sign(a.y), numext::sign(a.z), numext::sign(a.w));
264EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 psign<double2>(
const double2& a) {
265 return make_double2(numext::sign(a.x), numext::sign(a.y));
269EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plset<float4>(
const float& a) {
270 return make_float4(a, a + 1, a + 2, a + 3);
273EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 plset<double2>(
const double& a) {
274 return make_double2(a, a + 1);
278EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 padd<float4>(
const float4& a,
const float4& b) {
279 return make_float4(a.x + b.x, a.y + b.y, a.z + b.z, a.w + b.w);
282EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 padd<double2>(
const double2& a,
const double2& b) {
283 return make_double2(a.x + b.x, a.y + b.y);
287EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 psub<float4>(
const float4& a,
const float4& b) {
288 return make_float4(a.x - b.x, a.y - b.y, a.z - b.z, a.w - b.w);
291EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 psub<double2>(
const double2& a,
const double2& b) {
292 return make_double2(a.x - b.x, a.y - b.y);
296EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pnegate(
const float4& a) {
297 return make_float4(-a.x, -a.y, -a.z, -a.w);
300EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pnegate(
const double2& a) {
301 return make_double2(-a.x, -a.y);
305EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmul<float4>(
const float4& a,
const float4& b) {
306 return make_float4(a.x * b.x, a.y * b.y, a.z * b.z, a.w * b.w);
309EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmul<double2>(
const double2& a,
const double2& b) {
310 return make_double2(a.x * b.x, a.y * b.y);
314EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pdiv<float4>(
const float4& a,
const float4& b) {
315 return make_float4(a.x / b.x, a.y / b.y, a.z / b.z, a.w / b.w);
318EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pdiv<double2>(
const double2& a,
const double2& b) {
319 return make_double2(a.x / b.x, a.y / b.y);
323EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmin<float4>(
const float4& a,
const float4& b) {
324 return make_float4(fminf(a.x, b.x), fminf(a.y, b.y), fminf(a.z, b.z), fminf(a.w, b.w));
327EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmin<double2>(
const double2& a,
const double2& b) {
328 return make_double2(fmin(a.x, b.x), fmin(a.y, b.y));
332EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmax<float4>(
const float4& a,
const float4& b) {
333 return make_float4(fmaxf(a.x, b.x), fmaxf(a.y, b.y), fmaxf(a.z, b.z), fmaxf(a.w, b.w));
336EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmax<double2>(
const double2& a,
const double2& b) {
337 return make_double2(fmax(a.x, b.x), fmax(a.y, b.y));
341EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pload<float4>(
const float* from) {
342 return *
reinterpret_cast<const float4*
>(from);
346EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pload<double2>(
const double* from) {
347 return *
reinterpret_cast<const double2*
>(from);
351EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 ploadu<float4>(
const float* from) {
352 return make_float4(from[0], from[1], from[2], from[3]);
355EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 ploadu<double2>(
const double* from) {
356 return make_double2(from[0], from[1]);
360EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 ploaddup<float4>(
const float* from) {
361 return make_float4(from[0], from[0], from[1], from[1]);
364EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 ploaddup<double2>(
const double* from) {
365 return make_double2(from[0], from[0]);
369EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const float4& from) {
370 *
reinterpret_cast<float4*
>(to) = from;
374EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const double2& from) {
375 *
reinterpret_cast<double2*
>(to) = from;
379EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const float4& from) {
387EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const double2& from) {
393EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro<float4, Aligned>(
const float* from) {
394#if defined(EIGEN_GPU_COMPILE_PHASE)
395 return __ldg(
reinterpret_cast<const float4*
>(from));
397 return make_float4(from[0], from[1], from[2], from[3]);
401EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro<double2, Aligned>(
const double* from) {
402#if defined(EIGEN_GPU_COMPILE_PHASE)
403 return __ldg(
reinterpret_cast<const double2*
>(from));
405 return make_double2(from[0], from[1]);
410EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro<float4, Unaligned>(
const float* from) {
411#if defined(EIGEN_GPU_COMPILE_PHASE)
412 return make_float4(__ldg(from + 0), __ldg(from + 1), __ldg(from + 2), __ldg(from + 3));
414 return make_float4(from[0], from[1], from[2], from[3]);
418EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro<double2, Unaligned>(
const double* from) {
419#if defined(EIGEN_GPU_COMPILE_PHASE)
420 return make_double2(__ldg(from + 0), __ldg(from + 1));
422 return make_double2(from[0], from[1]);
427EIGEN_DEVICE_FUNC
inline float4 pgather<float, float4>(
const float* from, Index stride) {
428 return make_float4(from[0 * stride], from[1 * stride], from[2 * stride], from[3 * stride]);
432EIGEN_DEVICE_FUNC
inline double2 pgather<double, double2>(
const double* from, Index stride) {
433 return make_double2(from[0 * stride], from[1 * stride]);
437EIGEN_DEVICE_FUNC
inline void pscatter<float, float4>(
float* to,
const float4& from, Index stride) {
438 to[stride * 0] = from.x;
439 to[stride * 1] = from.y;
440 to[stride * 2] = from.z;
441 to[stride * 3] = from.w;
444EIGEN_DEVICE_FUNC
inline void pscatter<double, double2>(
double* to,
const double2& from, Index stride) {
445 to[stride * 0] = from.x;
446 to[stride * 1] = from.y;
450EIGEN_DEVICE_FUNC
inline float4 preverse(
const float4& a) {
451 return make_float4(a.w, a.z, a.y, a.x);
454EIGEN_DEVICE_FUNC
inline double2 preverse(
const double2& a) {
455 return make_double2(a.y, a.x);
459EIGEN_DEVICE_FUNC
inline float pfirst<float4>(
const float4& a) {
463EIGEN_DEVICE_FUNC
inline double pfirst<double2>(
const double2& a) {
468EIGEN_DEVICE_FUNC
inline float predux<float4>(
const float4& a) {
469 return a.x + a.y + a.z + a.w;
472EIGEN_DEVICE_FUNC
inline double predux<double2>(
const double2& a) {
477EIGEN_DEVICE_FUNC
inline float predux_max<float4>(
const float4& a) {
478 return fmaxf(fmaxf(a.x, a.y), fmaxf(a.z, a.w));
481EIGEN_DEVICE_FUNC
inline double predux_max<double2>(
const double2& a) {
482 return fmax(a.x, a.y);
486EIGEN_DEVICE_FUNC
inline float predux_min<float4>(
const float4& a) {
487 return fminf(fminf(a.x, a.y), fminf(a.z, a.w));
490EIGEN_DEVICE_FUNC
inline double predux_min<double2>(
const double2& a) {
491 return fmin(a.x, a.y);
495EIGEN_DEVICE_FUNC
inline float predux_mul<float4>(
const float4& a) {
496 return a.x * a.y * a.z * a.w;
499EIGEN_DEVICE_FUNC
inline double predux_mul<double2>(
const double2& a) {
504EIGEN_DEVICE_FUNC
inline float4 pabs<float4>(
const float4& a) {
505 return make_float4(fabsf(a.x), fabsf(a.y), fabsf(a.z), fabsf(a.w));
508EIGEN_DEVICE_FUNC
inline double2 pabs<double2>(
const double2& a) {
509 return make_double2(fabs(a.x), fabs(a.y));
513EIGEN_DEVICE_FUNC
inline float4 pfloor<float4>(
const float4& a) {
514 return make_float4(floorf(a.x), floorf(a.y), floorf(a.z), floorf(a.w));
517EIGEN_DEVICE_FUNC
inline double2 pfloor<double2>(
const double2& a) {
518 return make_double2(floor(a.x), floor(a.y));
522EIGEN_DEVICE_FUNC
inline float4 pceil<float4>(
const float4& a) {
523 return make_float4(ceilf(a.x), ceilf(a.y), ceilf(a.z), ceilf(a.w));
526EIGEN_DEVICE_FUNC
inline double2 pceil<double2>(
const double2& a) {
527 return make_double2(ceil(a.x), ceil(a.y));
531EIGEN_DEVICE_FUNC
inline float4 print<float4>(
const float4& a) {
532 return make_float4(rintf(a.x), rintf(a.y), rintf(a.z), rintf(a.w));
535EIGEN_DEVICE_FUNC
inline double2 print<double2>(
const double2& a) {
536 return make_double2(rint(a.x), rint(a.y));
540EIGEN_DEVICE_FUNC
inline float4 ptrunc<float4>(
const float4& a) {
541 return make_float4(truncf(a.x), truncf(a.y), truncf(a.z), truncf(a.w));
544EIGEN_DEVICE_FUNC
inline double2 ptrunc<double2>(
const double2& a) {
545 return make_double2(trunc(a.x), trunc(a.y));
549EIGEN_DEVICE_FUNC
inline float4 pround<float4>(
const float4& a) {
550 return make_float4(roundf(a.x), roundf(a.y), roundf(a.z), roundf(a.w));
553EIGEN_DEVICE_FUNC
inline double2 pround<double2>(
const double2& a) {
554 return make_double2(round(a.x), round(a.y));
557EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<float4, 4>& kernel) {
558 float tmp = kernel.packet[0].y;
559 kernel.packet[0].y = kernel.packet[1].x;
560 kernel.packet[1].x = tmp;
562 tmp = kernel.packet[0].z;
563 kernel.packet[0].z = kernel.packet[2].x;
564 kernel.packet[2].x = tmp;
566 tmp = kernel.packet[0].w;
567 kernel.packet[0].w = kernel.packet[3].x;
568 kernel.packet[3].x = tmp;
570 tmp = kernel.packet[1].z;
571 kernel.packet[1].z = kernel.packet[2].y;
572 kernel.packet[2].y = tmp;
574 tmp = kernel.packet[1].w;
575 kernel.packet[1].w = kernel.packet[3].y;
576 kernel.packet[3].y = tmp;
578 tmp = kernel.packet[2].w;
579 kernel.packet[2].w = kernel.packet[3].z;
580 kernel.packet[3].z = tmp;
583EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<double2, 2>& kernel) {
584 double tmp = kernel.packet[0].y;
585 kernel.packet[0].y = kernel.packet[1].x;
586 kernel.packet[1].x = tmp;
593#if defined(EIGEN_GPU_COMPILE_PHASE)
597using Packet4h2 = ulonglong2;
599struct unpacket_traits<Packet4h2> {
600 using type = Eigen::half;
601 static constexpr int size = 8;
602 static constexpr int alignment =
Aligned16;
603 static constexpr bool vectorizable =
true;
604 static constexpr bool masked_load_available =
false;
605 static constexpr bool masked_store_available =
false;
606 using half = Packet4h2;
609struct is_arithmetic<Packet4h2> : std::true_type {};
612struct unpacket_traits<half2> {
613 using type = Eigen::half;
614 static constexpr int size = 2;
616 static constexpr int alignment =
Aligned8;
617 static constexpr bool vectorizable =
true;
618 static constexpr bool masked_load_available =
false;
619 static constexpr bool masked_store_available =
false;
623struct is_arithmetic<half2> : std::true_type {};
626struct packet_traits<Eigen::half> : default_packet_traits {
627 using type = Packet4h2;
628 using half = Packet4h2;
629 static constexpr int Vectorizable = 1;
630 static constexpr int AlignedOnScalar = 1;
631 static constexpr int size = 8;
632 static constexpr int HasAdd = 1;
633 static constexpr int HasSub = 1;
634 static constexpr int HasMul = 1;
635 static constexpr int HasDiv = 1;
636 static constexpr int HasSqrt = 1;
637 static constexpr int HasRsqrt = 1;
638 static constexpr int HasExp = 1;
639 static constexpr int HasExpm1 = 1;
640 static constexpr int HasLog = 1;
641 static constexpr int HasLog1p = 1;
644 static constexpr int HasRound = 0;
645 static constexpr int HasSign = 0;
651#if (defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || defined(EIGEN_HIP_DEVICE_COMPILE)
652#define EIGEN_GPU_HAS_FP16_ARITHMETIC 1
656EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pset1<half2>(
const Eigen::half& from) {
657 return __half2half2(from);
661EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pload<half2>(
const Eigen::half* from) {
662 return *
reinterpret_cast<const half2*
>(from);
666EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploadu<half2>(
const Eigen::half* from) {
667 return __halves2half2(from[0], from[1]);
671EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploaddup<half2>(
const Eigen::half* from) {
672 return __halves2half2(from[0], from[0]);
676EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstore<Eigen::half>(Eigen::half* to,
const half2& from) {
677 *
reinterpret_cast<half2*
>(to) = from;
681EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstoreu<Eigen::half>(Eigen::half* to,
const half2& from) {
682 to[0] = __low2half(from);
683 to[1] = __high2half(from);
687EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Aligned>(
const Eigen::half* from) {
689 return __ldg(
reinterpret_cast<const half2*
>(from));
693EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Unaligned>(
const Eigen::half* from) {
694 return __halves2half2(__ldg(from + 0), __ldg(from + 1));
698EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pgather<Eigen::half, half2>(
const Eigen::half* from, Index stride) {
699 return __halves2half2(from[0 * stride], from[1 * stride]);
703EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<Eigen::half, half2>(Eigen::half* to,
const half2& from,
705 to[stride * 0] = __low2half(from);
706 to[stride * 1] = __high2half(from);
710EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 preverse(
const half2& a) {
711 return __halves2half2(__high2half(a), __low2half(a));
715EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half pfirst<half2>(
const half2& a) {
716 return __low2half(a);
720EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE numext::uint16_t low_bits(
const half2& a) {
721 const Eigen::half low = __low2half(a);
724EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE numext::uint16_t high_bits(
const half2& a) {
725 const Eigen::half high = __high2half(a);
728EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 half2_from_bits(numext::uint16_t low, numext::uint16_t high) {
729 return __halves2half2(half_impl::raw_uint16_to_half(low), half_impl::raw_uint16_to_half(high));
732EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half half_mask_from(
bool condition) {
733 return half_impl::raw_uint16_to_half(condition ? 0xffffu : 0x0000u);
737EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pabs<half2>(
const half2& a) {
738 return half2_from_bits(low_bits(a) & 0x7FFF, high_bits(a) & 0x7FFF);
742EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ptrue<half2>(
const half2& ) {
743 return pset1<half2>(half_mask_from(
true));
747EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pzero<half2>(
const half2& ) {
748 return pset1<half2>(half_mask_from(
false));
751EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<half2, 2>& kernel) {
752 const Eigen::half a1 = __low2half(kernel.packet[0]);
753 const Eigen::half a2 = __high2half(kernel.packet[0]);
754 const Eigen::half b1 = __low2half(kernel.packet[1]);
755 const Eigen::half b2 = __high2half(kernel.packet[1]);
756 kernel.packet[0] = __halves2half2(a1, b1);
757 kernel.packet[1] = __halves2half2(a2, b2);
761EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plset<half2>(
const Eigen::half& a) {
762#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
763 return __halves2half2(a, __hadd(a, __float2half(1.0f)));
765 return __halves2half2(a, __float2half(__half2float(a) + 1.0f));
770EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pselect<half2>(
const half2& mask,
const half2& a,
const half2& b) {
771 const Eigen::half mask_low = __low2half(mask);
772 const Eigen::half mask_high = __high2half(mask);
773 const Eigen::half low = mask_low == Eigen::half(0) ? __low2half(b) : __low2half(a);
774 const Eigen::half high = mask_high == Eigen::half(0) ? __high2half(b) : __high2half(a);
775 return __halves2half2(low, high);
780EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_eq<half2>(
const half2& a,
const half2& b) {
781 return __halves2half2(half_mask_from(__low2float(a) == __low2float(b)),
782 half_mask_from(__high2float(a) == __high2float(b)));
785EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_lt<half2>(
const half2& a,
const half2& b) {
786 return __halves2half2(half_mask_from(__low2float(a) < __low2float(b)),
787 half_mask_from(__high2float(a) < __high2float(b)));
790EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_le<half2>(
const half2& a,
const half2& b) {
791 return __halves2half2(half_mask_from(__low2float(a) <= __low2float(b)),
792 half_mask_from(__high2float(a) <= __high2float(b)));
796EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pand<half2>(
const half2& a,
const half2& b) {
797 return half2_from_bits(low_bits(a) & low_bits(b), high_bits(a) & high_bits(b));
800EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 por<half2>(
const half2& a,
const half2& b) {
801 return half2_from_bits(low_bits(a) | low_bits(b), high_bits(a) | high_bits(b));
804EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pxor<half2>(
const half2& a,
const half2& b) {
805 return half2_from_bits(low_bits(a) ^ low_bits(b), high_bits(a) ^ high_bits(b));
808EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pandnot<half2>(
const half2& a,
const half2& b) {
809 return half2_from_bits(low_bits(a) & ~low_bits(b), high_bits(a) & ~high_bits(b));
813EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 padd<half2>(
const half2& a,
const half2& b) {
814#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
815 return __hadd2(a, b);
817 return __floats2half2_rn(__low2float(a) + __low2float(b), __high2float(a) + __high2float(b));
822EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psub<half2>(
const half2& a,
const half2& b) {
823#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
824 return __hsub2(a, b);
826 return __floats2half2_rn(__low2float(a) - __low2float(b), __high2float(a) - __high2float(b));
831EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pnegate(
const half2& a) {
832#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
835 return __floats2half2_rn(-__low2float(a), -__high2float(a));
840EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmul<half2>(
const half2& a,
const half2& b) {
841#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
842 return __hmul2(a, b);
844 return __floats2half2_rn(__low2float(a) * __low2float(b), __high2float(a) * __high2float(b));
849struct is_fused_madd<half2> : true_type {};
851EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmadd(
const half2& a,
const half2& b,
const half2& c) {
852#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
853 return __hfma2(a, b, c);
855 return __floats2half2_rn(fmaf(__low2float(a), __low2float(b), __low2float(c)),
856 fmaf(__high2float(a), __high2float(b), __high2float(c)));
861EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pdiv<half2>(
const half2& a,
const half2& b) {
862#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
863 return __h2div(a, b);
865 return __floats2half2_rn(__low2float(a) / __low2float(b), __high2float(a) / __high2float(b));
873EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmin<half2>(
const half2& a,
const half2& b) {
874 const __half low = __low2float(a) < __low2float(b) ? __low2half(a) : __low2half(b);
875 const __half high = __high2float(a) < __high2float(b) ? __high2half(a) : __high2half(b);
876 return __halves2half2(low, high);
880EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmax<half2>(
const half2& a,
const half2& b) {
881 const __half low = __low2float(a) > __low2float(b) ? __low2half(a) : __low2half(b);
882 const __half high = __high2float(a) > __high2float(b) ? __high2half(a) : __high2half(b);
883 return __halves2half2(low, high);
887EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux<half2>(
const half2& a) {
888#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
889 return __hadd(__low2half(a), __high2half(a));
891 return __float2half(__low2float(a) + __high2float(a));
896EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool predux_any(
const half2& a) {
897 return (low_bits(a) | high_bits(a)) != 0;
901EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max<half2>(
const half2& a) {
902#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
903 const __half first = __low2half(a);
904 const __half second = __high2half(a);
905 return __hgt(first, second) ? first : second;
907 return __low2float(a) > __high2float(a) ? __low2half(a) : __high2half(a);
912EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_min<half2>(
const half2& a) {
913#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
914 const __half first = __low2half(a);
915 const __half second = __high2half(a);
916 return __hlt(first, second) ? first : second;
918 return __low2float(a) < __high2float(a) ? __low2half(a) : __high2half(a);
923EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_mul<half2>(
const half2& a) {
924#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
925 return __hmul(__low2half(a), __high2half(a));
927 return __float2half(__low2float(a) * __high2float(a));
932EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog<half2>(
const half2& a) {
933#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
936 return __floats2half2_rn(logf(__low2float(a)), logf(__high2float(a)));
941EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexp<half2>(
const half2& a) {
942#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
945 return __floats2half2_rn(expf(__low2float(a)), expf(__high2float(a)));
950EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psqrt<half2>(
const half2& a) {
951#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
954 return __floats2half2_rn(sqrtf(__low2float(a)), sqrtf(__high2float(a)));
959EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 prsqrt<half2>(
const half2& a) {
960#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
963 return __floats2half2_rn(rsqrtf(__low2float(a)), rsqrtf(__high2float(a)));
969EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog1p<half2>(
const half2& a) {
970 return __floats2half2_rn(log1pf(__low2float(a)), log1pf(__high2float(a)));
974EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexpm1<half2>(
const half2& a) {
975 return __floats2half2_rn(expm1f(__low2float(a)), expm1f(__high2float(a)));
983EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 lane_half2(
const Packet4h2& p,
int i) {
984 return reinterpret_cast<const half2*
>(&p)[i];
987EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 make_packet4h2(
const half2& l0,
const half2& l1,
const half2& l2,
990 half2* lanes =
reinterpret_cast<half2*
>(&r);
999#define EIGEN_GPU_PACKET4H2_UNARY(NAME) \
1001 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 NAME<Packet4h2>(const Packet4h2& a) { \
1002 return make_packet4h2(NAME(lane_half2(a, 0)), NAME(lane_half2(a, 1)), NAME(lane_half2(a, 2)), \
1003 NAME(lane_half2(a, 3))); \
1006#define EIGEN_GPU_PACKET4H2_BINARY(NAME) \
1008 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 NAME<Packet4h2>(const Packet4h2& a, const Packet4h2& b) { \
1009 return make_packet4h2(NAME(lane_half2(a, 0), lane_half2(b, 0)), NAME(lane_half2(a, 1), lane_half2(b, 1)), \
1010 NAME(lane_half2(a, 2), lane_half2(b, 2)), NAME(lane_half2(a, 3), lane_half2(b, 3))); \
1014EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pset1<Packet4h2>(
const Eigen::half& from) {
1015 const half2 lane = pset1<half2>(from);
1016 return make_packet4h2(lane, lane, lane, lane);
1020EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pload<Packet4h2>(
const Eigen::half* from) {
1021 return *
reinterpret_cast<const Packet4h2*
>(from);
1025EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ploadu<Packet4h2>(
const Eigen::half* from) {
1026 return make_packet4h2(ploadu<half2>(from + 0), ploadu<half2>(from + 2), ploadu<half2>(from + 4),
1027 ploadu<half2>(from + 6));
1031EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ploaddup<Packet4h2>(
const Eigen::half* from) {
1032 return make_packet4h2(ploaddup<half2>(from + 0), ploaddup<half2>(from + 1), ploaddup<half2>(from + 2),
1033 ploaddup<half2>(from + 3));
1037EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstore<Eigen::half>(Eigen::half* to,
const Packet4h2& from) {
1038 *
reinterpret_cast<Packet4h2*
>(to) = from;
1042EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pstoreu<Eigen::half>(Eigen::half* to,
const Packet4h2& from) {
1043 pstoreu<Eigen::half>(to + 0, lane_half2(from, 0));
1044 pstoreu<Eigen::half>(to + 2, lane_half2(from, 1));
1045 pstoreu<Eigen::half>(to + 4, lane_half2(from, 2));
1046 pstoreu<Eigen::half>(to + 6, lane_half2(from, 3));
1050EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4h2 ploadt_ro<Packet4h2, Aligned>(
const Eigen::half* from) {
1051 return __ldg(
reinterpret_cast<const Packet4h2*
>(from));
1055EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4h2 ploadt_ro<Packet4h2, Unaligned>(
const Eigen::half* from) {
1056 return make_packet4h2(ploadt_ro<half2, Unaligned>(from + 0), ploadt_ro<half2, Unaligned>(from + 2),
1057 ploadt_ro<half2, Unaligned>(from + 4), ploadt_ro<half2, Unaligned>(from + 6));
1061EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pgather<Eigen::half, Packet4h2>(
const Eigen::half* from, Index stride) {
1062 return make_packet4h2(
1063 __halves2half2(from[0 * stride], from[1 * stride]), __halves2half2(from[2 * stride], from[3 * stride]),
1064 __halves2half2(from[4 * stride], from[5 * stride]), __halves2half2(from[6 * stride], from[7 * stride]));
1068EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<Eigen::half, Packet4h2>(Eigen::half* to,
const Packet4h2& from,
1070 pscatter<Eigen::half, half2>(to + stride * 0, lane_half2(from, 0), stride);
1071 pscatter<Eigen::half, half2>(to + stride * 2, lane_half2(from, 1), stride);
1072 pscatter<Eigen::half, half2>(to + stride * 4, lane_half2(from, 2), stride);
1073 pscatter<Eigen::half, half2>(to + stride * 6, lane_half2(from, 3), stride);
1077EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 preverse(
const Packet4h2& a) {
1078 return make_packet4h2(preverse(lane_half2(a, 3)), preverse(lane_half2(a, 2)), preverse(lane_half2(a, 1)),
1079 preverse(lane_half2(a, 0)));
1083EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4h2>(
const Packet4h2& a) {
1084 return pfirst<half2>(lane_half2(a, 0));
1088EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ptrue<Packet4h2>(
const Packet4h2& ) {
1089 return pset1<Packet4h2>(half_impl::raw_uint16_to_half(0xffffu));
1093EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pzero<Packet4h2>(
const Packet4h2& ) {
1094 return pset1<Packet4h2>(half_impl::raw_uint16_to_half(0x0000u));
1098EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4h2, 8>& kernel) {
1099 Eigen::half elements[8][8];
1101 for (
int row = 0; row < 8; ++row) {
1103 for (
int lane = 0; lane < 4; ++lane) {
1104 const half2 pair = lane_half2(kernel.packet[row], lane);
1105 elements[row][2 * lane] = __low2half(pair);
1106 elements[row][2 * lane + 1] = __high2half(pair);
1110 for (
int row = 0; row < 8; ++row) {
1111 kernel.packet[row] = make_packet4h2(
1112 __halves2half2(elements[0][row], elements[1][row]), __halves2half2(elements[2][row], elements[3][row]),
1113 __halves2half2(elements[4][row], elements[5][row]), __halves2half2(elements[6][row], elements[7][row]));
1118EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 plset<Packet4h2>(
const Eigen::half& a) {
1120 const half2 base = pset1<half2>(a);
1121 return make_packet4h2(plset<half2>(a), padd<half2>(base, __floats2half2_rn(2.0f, 3.0f)),
1122 padd<half2>(base, __floats2half2_rn(4.0f, 5.0f)),
1123 padd<half2>(base, __floats2half2_rn(6.0f, 7.0f)));
1127EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pselect<Packet4h2>(
const Packet4h2& mask,
const Packet4h2& a,
1128 const Packet4h2& b) {
1129 return make_packet4h2(pselect<half2>(lane_half2(mask, 0), lane_half2(a, 0), lane_half2(b, 0)),
1130 pselect<half2>(lane_half2(mask, 1), lane_half2(a, 1), lane_half2(b, 1)),
1131 pselect<half2>(lane_half2(mask, 2), lane_half2(a, 2), lane_half2(b, 2)),
1132 pselect<half2>(lane_half2(mask, 3), lane_half2(a, 3), lane_half2(b, 3)));
1136struct is_fused_madd<Packet4h2> : is_fused_madd<half2> {};
1138EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pmadd(
const Packet4h2& a,
const Packet4h2& b,
const Packet4h2& c) {
1139 return make_packet4h2(pmadd<half2>(lane_half2(a, 0), lane_half2(b, 0), lane_half2(c, 0)),
1140 pmadd<half2>(lane_half2(a, 1), lane_half2(b, 1), lane_half2(c, 1)),
1141 pmadd<half2>(lane_half2(a, 2), lane_half2(b, 2), lane_half2(c, 2)),
1142 pmadd<half2>(lane_half2(a, 3), lane_half2(b, 3), lane_half2(c, 3)));
1145EIGEN_GPU_PACKET4H2_UNARY(pabs)
1146EIGEN_GPU_PACKET4H2_UNARY(pnegate)
1147EIGEN_GPU_PACKET4H2_UNARY(plog)
1148EIGEN_GPU_PACKET4H2_UNARY(pexp)
1149EIGEN_GPU_PACKET4H2_UNARY(psqrt)
1150EIGEN_GPU_PACKET4H2_UNARY(prsqrt)
1151EIGEN_GPU_PACKET4H2_UNARY(plog1p)
1152EIGEN_GPU_PACKET4H2_UNARY(pexpm1)
1154EIGEN_GPU_PACKET4H2_BINARY(padd)
1155EIGEN_GPU_PACKET4H2_BINARY(psub)
1156EIGEN_GPU_PACKET4H2_BINARY(pmul)
1157EIGEN_GPU_PACKET4H2_BINARY(pdiv)
1158EIGEN_GPU_PACKET4H2_BINARY(pmin)
1159EIGEN_GPU_PACKET4H2_BINARY(pmax)
1160EIGEN_GPU_PACKET4H2_BINARY(pand)
1161EIGEN_GPU_PACKET4H2_BINARY(por)
1162EIGEN_GPU_PACKET4H2_BINARY(pxor)
1163EIGEN_GPU_PACKET4H2_BINARY(pandnot)
1164EIGEN_GPU_PACKET4H2_BINARY(pcmp_eq)
1165EIGEN_GPU_PACKET4H2_BINARY(pcmp_lt)
1166EIGEN_GPU_PACKET4H2_BINARY(pcmp_le)
1168#undef EIGEN_GPU_PACKET4H2_UNARY
1169#undef EIGEN_GPU_PACKET4H2_BINARY
1172EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux<Packet4h2>(
const Packet4h2& a) {
1173 return predux(lane_half2(a, 0)) + predux(lane_half2(a, 1)) + predux(lane_half2(a, 2)) + predux(lane_half2(a, 3));
1177EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool predux_any(
const Packet4h2& a) {
1178 return predux_any(lane_half2(a, 0)) | predux_any(lane_half2(a, 1)) | predux_any(lane_half2(a, 2)) |
1179 predux_any(lane_half2(a, 3));
1183EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4h2>(
const Packet4h2& a) {
1185 pmax<half2>(pmax<half2>(lane_half2(a, 0), lane_half2(a, 1)), pmax<half2>(lane_half2(a, 2), lane_half2(a, 3)));
1186 return predux_max<half2>(m);
1190EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_min<Packet4h2>(
const Packet4h2& a) {
1192 pmin<half2>(pmin<half2>(lane_half2(a, 0), lane_half2(a, 1)), pmin<half2>(lane_half2(a, 2), lane_half2(a, 3)));
1193 return predux_min<half2>(m);
1198EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet4h2>(
const Packet4h2& a) {
1199 const half2 product =
1200 pmul<half2>(pmul<half2>(lane_half2(a, 0), lane_half2(a, 1)), pmul<half2>(lane_half2(a, 2), lane_half2(a, 3)));
1201 return predux_mul<half2>(product);
1204#undef EIGEN_GPU_HAS_FP16_ARITHMETIC
@ Aligned8
Definition Constants.h:237
@ Aligned16
Definition Constants.h:238