Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_PACKET_MATH_GPU_H
12#define EIGEN_PACKET_MATH_GPU_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21// Read-only data cached load (__ldg) is available on all supported GPU architectures (sm_50+ for CUDA, GFX906+ for
22// HIP). Native FP16 arithmetic needs sm_53+ (EIGEN_GPU_HAS_FP16_ARITHMETIC); below that the half2 packet ops
23// compute in float.
24
25// Make sure this is only available when targeting a GPU: we don't want to
26// introduce conflicts between these packet_traits definitions and the ones
27// we'll use on the host side (SSE, AVX, ...)
28#if defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU)
29
30template <>
31struct is_arithmetic<float4> : std::true_type {};
32template <>
33struct is_arithmetic<double2> : std::true_type {};
34
35template <>
36struct packet_traits<float> : default_packet_traits {
37 using type = float4;
38 using half = float4;
39 static constexpr int Vectorizable = 1;
40 static constexpr int AlignedOnScalar = 1;
41 static constexpr int size = 4;
42
43 static constexpr int HasDiv = 1;
44 static constexpr int HasSin = 0;
45 static constexpr int HasCos = 0;
46 static constexpr int HasLog = 1;
47 static constexpr int HasExp = 1;
48 static constexpr int HasSqrt = 1;
49 static constexpr int HasRsqrt = 1;
50 static constexpr int HasLGamma = 1;
51 static constexpr int HasDiGamma = 1;
52 static constexpr int HasZeta = 1;
53 static constexpr int HasPolygamma = 1;
54 static constexpr int HasErf = 1;
55 static constexpr int HasErfc = 1;
56 static constexpr int HasNdtri = 1;
57 static constexpr int HasBessel = 1;
58 static constexpr int HasIGamma = 1;
59 static constexpr int HasIGammaDerA = 1;
60 static constexpr int HasGammaSampleDerAlpha = 1;
61 static constexpr int HasIGammac = 1;
62 static constexpr int HasBetaInc = 1;
63
64 static constexpr int HasCmp = 1;
65};
66
67template <>
68struct packet_traits<double> : default_packet_traits {
69 using type = double2;
70 using half = double2;
71 static constexpr int Vectorizable = 1;
72 static constexpr int AlignedOnScalar = 1;
73 static constexpr int size = 2;
74
75 static constexpr int HasDiv = 1;
76 static constexpr int HasLog = 1;
77 static constexpr int HasExp = 1;
78 static constexpr int HasSqrt = 1;
79 static constexpr int HasRsqrt = 1;
80 static constexpr int HasLGamma = 1;
81 static constexpr int HasDiGamma = 1;
82 static constexpr int HasZeta = 1;
83 static constexpr int HasPolygamma = 1;
84 static constexpr int HasErf = 1;
85 static constexpr int HasErfc = 1;
86 static constexpr int HasNdtri = 1;
87 static constexpr int HasBessel = 1;
88 static constexpr int HasIGamma = 1;
89 static constexpr int HasIGammaDerA = 1;
90 static constexpr int HasGammaSampleDerAlpha = 1;
91 static constexpr int HasIGammac = 1;
92 static constexpr int HasBetaInc = 1;
93
94 static constexpr int HasCmp = 1;
95};
96
97template <>
98struct unpacket_traits<float4> {
99 using type = float;
100 static constexpr int size = 4;
101 static constexpr int alignment = Aligned16;
102 static constexpr bool vectorizable = true;
103 static constexpr bool masked_load_available = false;
104 static constexpr bool masked_store_available = false;
105 using half = float4;
106};
107template <>
108struct unpacket_traits<double2> {
109 using type = double;
110 static constexpr int size = 2;
111 static constexpr int alignment = Aligned16;
112 static constexpr bool vectorizable = true;
113 static constexpr bool masked_load_available = false;
114 static constexpr bool masked_store_available = false;
115 using half = double2;
116};
117
118template <>
119EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pset1<float4>(const float& from) {
120 return make_float4(from, from, from, from);
121}
122template <>
123EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pset1<double2>(const double& from) {
124 return make_double2(from, from);
125}
126
127// Bit-level helpers on the scalar lanes. numext::bit_cast rather than the __int_as_float family, which are device
128// intrinsics: a .cu translation unit that defines EIGEN_USE_GPU instantiates these packet types in the host pass
129// too, and an operation that exists in only one of the two passes makes packet_traits differ between them.
130template <typename T>
131using lane_bits_t = typename numext::get_integer_by_size<sizeof(T)>::unsigned_type;
132
133#define EIGEN_MAKE_BITWISE_BINOP(name, op) \
134 template <typename T> \
135 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T bitwise_##name(const T& a, const T& b) { \
136 using Bits = lane_bits_t<T>; \
137 return numext::bit_cast<T>(static_cast<Bits>(numext::bit_cast<Bits>(a) op numext::bit_cast<Bits>(b))); \
138 }
139
140EIGEN_MAKE_BITWISE_BINOP(and, &)
141EIGEN_MAKE_BITWISE_BINOP(or, |)
142EIGEN_MAKE_BITWISE_BINOP(xor, ^)
143EIGEN_MAKE_BITWISE_BINOP(andnot, &~)
144
145#undef EIGEN_MAKE_BITWISE_BINOP
146
147// A comparison returns an all-ones lane where it holds and an all-zero lane elsewhere, so that the result can be
148// consumed bitwise by pselect, pand and pandnot.
149template <typename T>
150EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T mask_from(bool condition) {
151 using Bits = lane_bits_t<T>;
152 return numext::bit_cast<T>(condition ? ~Bits(0) : Bits(0));
153}
154
155template <typename T>
156EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T eq_mask(const T& a, const T& b) {
157 return mask_from<T>(a == b);
158}
159template <typename T>
160EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T lt_mask(const T& a, const T& b) {
161 return mask_from<T>(a < b);
162}
163template <typename T>
164EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T le_mask(const T& a, const T& b) {
165 return mask_from<T>(a <= b);
166}
167// !(a >= b), so a NaN operand makes the lane true.
168template <typename T>
169EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T lt_or_nan_mask(const T& a, const T& b) {
170 return mask_from<T>(!(a >= b));
171}
172
173template <>
174EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pand<float4>(const float4& a, const float4& b) {
175 return make_float4(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y), bitwise_and(a.z, b.z), bitwise_and(a.w, b.w));
176}
177template <>
178EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pand<double2>(const double2& a, const double2& b) {
179 return make_double2(bitwise_and(a.x, b.x), bitwise_and(a.y, b.y));
180}
181
182template <>
183EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 por<float4>(const float4& a, const float4& b) {
184 return make_float4(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y), bitwise_or(a.z, b.z), bitwise_or(a.w, b.w));
185}
186template <>
187EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 por<double2>(const double2& a, const double2& b) {
188 return make_double2(bitwise_or(a.x, b.x), bitwise_or(a.y, b.y));
189}
190
191template <>
192EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pxor<float4>(const float4& a, const float4& b) {
193 return make_float4(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y), bitwise_xor(a.z, b.z), bitwise_xor(a.w, b.w));
194}
195template <>
196EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pxor<double2>(const double2& a, const double2& b) {
197 return make_double2(bitwise_xor(a.x, b.x), bitwise_xor(a.y, b.y));
198}
199
200template <>
201EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pandnot<float4>(const float4& a, const float4& b) {
202 return make_float4(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y), bitwise_andnot(a.z, b.z),
203 bitwise_andnot(a.w, b.w));
204}
205template <>
206EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pandnot<double2>(const double2& a, const double2& b) {
207 return make_double2(bitwise_andnot(a.x, b.x), bitwise_andnot(a.y, b.y));
208}
209
210template <>
211EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_eq<float4>(const float4& a, const float4& b) {
212 return make_float4(eq_mask(a.x, b.x), eq_mask(a.y, b.y), eq_mask(a.z, b.z), eq_mask(a.w, b.w));
213}
214template <>
215EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_lt<float4>(const float4& a, const float4& b) {
216 return make_float4(lt_mask(a.x, b.x), lt_mask(a.y, b.y), lt_mask(a.z, b.z), lt_mask(a.w, b.w));
217}
218template <>
219EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_le<float4>(const float4& a, const float4& b) {
220 return make_float4(le_mask(a.x, b.x), le_mask(a.y, b.y), le_mask(a.z, b.z), le_mask(a.w, b.w));
221}
222template <>
223EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pcmp_lt_or_nan<float4>(const float4& a, const float4& b) {
224 return make_float4(lt_or_nan_mask(a.x, b.x), lt_or_nan_mask(a.y, b.y), lt_or_nan_mask(a.z, b.z),
225 lt_or_nan_mask(a.w, b.w));
226}
227template <>
228EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_eq<double2>(const double2& a, const double2& b) {
229 return make_double2(eq_mask(a.x, b.x), eq_mask(a.y, b.y));
230}
231template <>
232EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_lt<double2>(const double2& a, const double2& b) {
233 return make_double2(lt_mask(a.x, b.x), lt_mask(a.y, b.y));
234}
235template <>
236EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_le<double2>(const double2& a, const double2& b) {
237 return make_double2(le_mask(a.x, b.x), le_mask(a.y, b.y));
238}
239template <>
240EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pcmp_lt_or_nan<double2>(const double2& a, const double2& b) {
241 return make_double2(lt_or_nan_mask(a.x, b.x), lt_or_nan_mask(a.y, b.y));
242}
243
244template <>
245EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const float4& a) {
246 using Bits = lane_bits_t<float>;
247 return (numext::bit_cast<Bits>(a.x) | numext::bit_cast<Bits>(a.y) | numext::bit_cast<Bits>(a.z) |
248 numext::bit_cast<Bits>(a.w)) != 0;
249}
250
251template <>
252EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const double2& a) {
253 using Bits = lane_bits_t<double>;
254 return (numext::bit_cast<Bits>(a.x) | numext::bit_cast<Bits>(a.y)) != 0;
255}
256
257// numext::sign: 0 for either zero, NaN for NaN, +/-1 otherwise. default_packet_traits advertises HasSign, so
258// without these the flag was a promise the backend did not keep (the generic form does not compile for float4).
259template <>
260EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 psign<float4>(const float4& a) {
261 return make_float4(numext::sign(a.x), numext::sign(a.y), numext::sign(a.z), numext::sign(a.w));
262}
263template <>
264EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 psign<double2>(const double2& a) {
265 return make_double2(numext::sign(a.x), numext::sign(a.y));
266}
267
268template <>
269EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 plset<float4>(const float& a) {
270 return make_float4(a, a + 1, a + 2, a + 3);
271}
272template <>
273EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 plset<double2>(const double& a) {
274 return make_double2(a, a + 1);
275}
276
277template <>
278EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 padd<float4>(const float4& a, const float4& b) {
279 return make_float4(a.x + b.x, a.y + b.y, a.z + b.z, a.w + b.w);
280}
281template <>
282EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 padd<double2>(const double2& a, const double2& b) {
283 return make_double2(a.x + b.x, a.y + b.y);
284}
285
286template <>
287EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 psub<float4>(const float4& a, const float4& b) {
288 return make_float4(a.x - b.x, a.y - b.y, a.z - b.z, a.w - b.w);
289}
290template <>
291EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 psub<double2>(const double2& a, const double2& b) {
292 return make_double2(a.x - b.x, a.y - b.y);
293}
294
295template <>
296EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pnegate(const float4& a) {
297 return make_float4(-a.x, -a.y, -a.z, -a.w);
298}
299template <>
300EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pnegate(const double2& a) {
301 return make_double2(-a.x, -a.y);
302}
303
304template <>
305EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmul<float4>(const float4& a, const float4& b) {
306 return make_float4(a.x * b.x, a.y * b.y, a.z * b.z, a.w * b.w);
307}
308template <>
309EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmul<double2>(const double2& a, const double2& b) {
310 return make_double2(a.x * b.x, a.y * b.y);
311}
312
313template <>
314EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pdiv<float4>(const float4& a, const float4& b) {
315 return make_float4(a.x / b.x, a.y / b.y, a.z / b.z, a.w / b.w);
316}
317template <>
318EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pdiv<double2>(const double2& a, const double2& b) {
319 return make_double2(a.x / b.x, a.y / b.y);
320}
321
322template <>
323EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmin<float4>(const float4& a, const float4& b) {
324 return make_float4(fminf(a.x, b.x), fminf(a.y, b.y), fminf(a.z, b.z), fminf(a.w, b.w));
325}
326template <>
327EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmin<double2>(const double2& a, const double2& b) {
328 return make_double2(fmin(a.x, b.x), fmin(a.y, b.y));
329}
330
331template <>
332EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pmax<float4>(const float4& a, const float4& b) {
333 return make_float4(fmaxf(a.x, b.x), fmaxf(a.y, b.y), fmaxf(a.z, b.z), fmaxf(a.w, b.w));
334}
335template <>
336EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pmax<double2>(const double2& a, const double2& b) {
337 return make_double2(fmax(a.x, b.x), fmax(a.y, b.y));
338}
339
340template <>
341EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 pload<float4>(const float* from) {
342 return *reinterpret_cast<const float4*>(from);
343}
344
345template <>
346EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 pload<double2>(const double* from) {
347 return *reinterpret_cast<const double2*>(from);
348}
349
350template <>
351EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 ploadu<float4>(const float* from) {
352 return make_float4(from[0], from[1], from[2], from[3]);
353}
354template <>
355EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 ploadu<double2>(const double* from) {
356 return make_double2(from[0], from[1]);
357}
358
359template <>
360EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float4 ploaddup<float4>(const float* from) {
361 return make_float4(from[0], from[0], from[1], from[1]);
362}
363template <>
364EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double2 ploaddup<double2>(const double* from) {
365 return make_double2(from[0], from[0]);
366}
367
368template <>
369EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore<float>(float* to, const float4& from) {
370 *reinterpret_cast<float4*>(to) = from;
371}
372
373template <>
374EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore<double>(double* to, const double2& from) {
375 *reinterpret_cast<double2*>(to) = from;
376}
377
378template <>
379EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const float4& from) {
380 to[0] = from.x;
381 to[1] = from.y;
382 to[2] = from.z;
383 to[3] = from.w;
384}
385
386template <>
387EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const double2& from) {
388 to[0] = from.x;
389 to[1] = from.y;
390}
391
392template <>
393EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro<float4, Aligned>(const float* from) {
394#if defined(EIGEN_GPU_COMPILE_PHASE)
395 return __ldg(reinterpret_cast<const float4*>(from));
396#else
397 return make_float4(from[0], from[1], from[2], from[3]);
398#endif
399}
400template <>
401EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro<double2, Aligned>(const double* from) {
402#if defined(EIGEN_GPU_COMPILE_PHASE)
403 return __ldg(reinterpret_cast<const double2*>(from));
404#else
405 return make_double2(from[0], from[1]);
406#endif
407}
408
409template <>
410EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float4 ploadt_ro<float4, Unaligned>(const float* from) {
411#if defined(EIGEN_GPU_COMPILE_PHASE)
412 return make_float4(__ldg(from + 0), __ldg(from + 1), __ldg(from + 2), __ldg(from + 3));
413#else
414 return make_float4(from[0], from[1], from[2], from[3]);
415#endif
416}
417template <>
418EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double2 ploadt_ro<double2, Unaligned>(const double* from) {
419#if defined(EIGEN_GPU_COMPILE_PHASE)
420 return make_double2(__ldg(from + 0), __ldg(from + 1));
421#else
422 return make_double2(from[0], from[1]);
423#endif
424}
425
426template <>
427EIGEN_DEVICE_FUNC inline float4 pgather<float, float4>(const float* from, Index stride) {
428 return make_float4(from[0 * stride], from[1 * stride], from[2 * stride], from[3 * stride]);
429}
430
431template <>
432EIGEN_DEVICE_FUNC inline double2 pgather<double, double2>(const double* from, Index stride) {
433 return make_double2(from[0 * stride], from[1 * stride]);
434}
435
436template <>
437EIGEN_DEVICE_FUNC inline void pscatter<float, float4>(float* to, const float4& from, Index stride) {
438 to[stride * 0] = from.x;
439 to[stride * 1] = from.y;
440 to[stride * 2] = from.z;
441 to[stride * 3] = from.w;
442}
443template <>
444EIGEN_DEVICE_FUNC inline void pscatter<double, double2>(double* to, const double2& from, Index stride) {
445 to[stride * 0] = from.x;
446 to[stride * 1] = from.y;
447}
448
449template <>
450EIGEN_DEVICE_FUNC inline float4 preverse(const float4& a) {
451 return make_float4(a.w, a.z, a.y, a.x);
452}
453template <>
454EIGEN_DEVICE_FUNC inline double2 preverse(const double2& a) {
455 return make_double2(a.y, a.x);
456}
457
458template <>
459EIGEN_DEVICE_FUNC inline float pfirst<float4>(const float4& a) {
460 return a.x;
461}
462template <>
463EIGEN_DEVICE_FUNC inline double pfirst<double2>(const double2& a) {
464 return a.x;
465}
466
467template <>
468EIGEN_DEVICE_FUNC inline float predux<float4>(const float4& a) {
469 return a.x + a.y + a.z + a.w;
470}
471template <>
472EIGEN_DEVICE_FUNC inline double predux<double2>(const double2& a) {
473 return a.x + a.y;
474}
475
476template <>
477EIGEN_DEVICE_FUNC inline float predux_max<float4>(const float4& a) {
478 return fmaxf(fmaxf(a.x, a.y), fmaxf(a.z, a.w));
479}
480template <>
481EIGEN_DEVICE_FUNC inline double predux_max<double2>(const double2& a) {
482 return fmax(a.x, a.y);
483}
484
485template <>
486EIGEN_DEVICE_FUNC inline float predux_min<float4>(const float4& a) {
487 return fminf(fminf(a.x, a.y), fminf(a.z, a.w));
488}
489template <>
490EIGEN_DEVICE_FUNC inline double predux_min<double2>(const double2& a) {
491 return fmin(a.x, a.y);
492}
493
494template <>
495EIGEN_DEVICE_FUNC inline float predux_mul<float4>(const float4& a) {
496 return a.x * a.y * a.z * a.w;
497}
498template <>
499EIGEN_DEVICE_FUNC inline double predux_mul<double2>(const double2& a) {
500 return a.x * a.y;
501}
502
503template <>
504EIGEN_DEVICE_FUNC inline float4 pabs<float4>(const float4& a) {
505 return make_float4(fabsf(a.x), fabsf(a.y), fabsf(a.z), fabsf(a.w));
506}
507template <>
508EIGEN_DEVICE_FUNC inline double2 pabs<double2>(const double2& a) {
509 return make_double2(fabs(a.x), fabs(a.y));
510}
511
512template <>
513EIGEN_DEVICE_FUNC inline float4 pfloor<float4>(const float4& a) {
514 return make_float4(floorf(a.x), floorf(a.y), floorf(a.z), floorf(a.w));
515}
516template <>
517EIGEN_DEVICE_FUNC inline double2 pfloor<double2>(const double2& a) {
518 return make_double2(floor(a.x), floor(a.y));
519}
520
521template <>
522EIGEN_DEVICE_FUNC inline float4 pceil<float4>(const float4& a) {
523 return make_float4(ceilf(a.x), ceilf(a.y), ceilf(a.z), ceilf(a.w));
524}
525template <>
526EIGEN_DEVICE_FUNC inline double2 pceil<double2>(const double2& a) {
527 return make_double2(ceil(a.x), ceil(a.y));
528}
529
530template <>
531EIGEN_DEVICE_FUNC inline float4 print<float4>(const float4& a) {
532 return make_float4(rintf(a.x), rintf(a.y), rintf(a.z), rintf(a.w));
533}
534template <>
535EIGEN_DEVICE_FUNC inline double2 print<double2>(const double2& a) {
536 return make_double2(rint(a.x), rint(a.y));
537}
538
539template <>
540EIGEN_DEVICE_FUNC inline float4 ptrunc<float4>(const float4& a) {
541 return make_float4(truncf(a.x), truncf(a.y), truncf(a.z), truncf(a.w));
542}
543template <>
544EIGEN_DEVICE_FUNC inline double2 ptrunc<double2>(const double2& a) {
545 return make_double2(trunc(a.x), trunc(a.y));
546}
547
548template <>
549EIGEN_DEVICE_FUNC inline float4 pround<float4>(const float4& a) {
550 return make_float4(roundf(a.x), roundf(a.y), roundf(a.z), roundf(a.w));
551}
552template <>
553EIGEN_DEVICE_FUNC inline double2 pround<double2>(const double2& a) {
554 return make_double2(round(a.x), round(a.y));
555}
556
557EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<float4, 4>& kernel) {
558 float tmp = kernel.packet[0].y;
559 kernel.packet[0].y = kernel.packet[1].x;
560 kernel.packet[1].x = tmp;
561
562 tmp = kernel.packet[0].z;
563 kernel.packet[0].z = kernel.packet[2].x;
564 kernel.packet[2].x = tmp;
565
566 tmp = kernel.packet[0].w;
567 kernel.packet[0].w = kernel.packet[3].x;
568 kernel.packet[3].x = tmp;
569
570 tmp = kernel.packet[1].z;
571 kernel.packet[1].z = kernel.packet[2].y;
572 kernel.packet[2].y = tmp;
573
574 tmp = kernel.packet[1].w;
575 kernel.packet[1].w = kernel.packet[3].y;
576 kernel.packet[3].y = tmp;
577
578 tmp = kernel.packet[2].w;
579 kernel.packet[2].w = kernel.packet[3].z;
580 kernel.packet[3].z = tmp;
581}
582
583EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<double2, 2>& kernel) {
584 double tmp = kernel.packet[0].y;
585 kernel.packet[0].y = kernel.packet[1].x;
586 kernel.packet[1].x = tmp;
587}
588
589#endif // defined(EIGEN_GPUCC) && defined(EIGEN_USE_GPU)
590
591// Half-packet functions are only available in GPU device compilation — they use
592// intrinsics (__half2, etc.) that have no host-side benefit.
593#if defined(EIGEN_GPU_COMPILE_PHASE)
594
595// Two packets over Eigen::half: the native two-lane half2, and Packet4h2, eight halves in four half2 lanes.
596// Packet4h2 stays an alias of ulonglong2 because the Tensor GPU kernels and TensorFlow name it that way.
597using Packet4h2 = ulonglong2;
598template <>
599struct unpacket_traits<Packet4h2> {
600 using type = Eigen::half;
601 static constexpr int size = 8;
602 static constexpr int alignment = Aligned16;
603 static constexpr bool vectorizable = true;
604 static constexpr bool masked_load_available = false;
605 static constexpr bool masked_store_available = false;
606 using half = Packet4h2;
607};
608template <>
609struct is_arithmetic<Packet4h2> : std::true_type {};
610
611template <>
612struct unpacket_traits<half2> {
613 using type = Eigen::half;
614 static constexpr int size = 2;
615 // half2 needs 4-byte alignment; Aligned8 is the smallest value the enum offers that satisfies it.
616 static constexpr int alignment = Aligned8;
617 static constexpr bool vectorizable = true;
618 static constexpr bool masked_load_available = false;
619 static constexpr bool masked_store_available = false;
620 using half = half2;
621};
622template <>
623struct is_arithmetic<half2> : std::true_type {};
624
625template <>
626struct packet_traits<Eigen::half> : default_packet_traits {
627 using type = Packet4h2;
628 using half = Packet4h2;
629 static constexpr int Vectorizable = 1;
630 static constexpr int AlignedOnScalar = 1;
631 static constexpr int size = 8;
632 static constexpr int HasAdd = 1;
633 static constexpr int HasSub = 1;
634 static constexpr int HasMul = 1;
635 static constexpr int HasDiv = 1;
636 static constexpr int HasSqrt = 1;
637 static constexpr int HasRsqrt = 1;
638 static constexpr int HasExp = 1;
639 static constexpr int HasExpm1 = 1;
640 static constexpr int HasLog = 1;
641 static constexpr int HasLog1p = 1;
642 // default_packet_traits turns these on, but there is no pfloor/pceil/print/ptrunc/pround and no psign for the
643 // half packets; advertising them makes the generic fallback the evaluator picks fail to compile.
644 static constexpr int HasRound = 0;
645 static constexpr int HasSign = 0;
646};
647
648// ---------------------------------------------------------------------------------------------------------------
649// half2, the native two-lane packet.
650
651#if (defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || defined(EIGEN_HIP_DEVICE_COMPILE)
652#define EIGEN_GPU_HAS_FP16_ARITHMETIC 1
653#endif
654
655template <>
656EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pset1<half2>(const Eigen::half& from) {
657 return __half2half2(from);
658}
659
660template <>
661EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pload<half2>(const Eigen::half* from) {
662 return *reinterpret_cast<const half2*>(from);
663}
664
665template <>
666EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploadu<half2>(const Eigen::half* from) {
667 return __halves2half2(from[0], from[1]);
668}
669
670template <>
671EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ploaddup<half2>(const Eigen::half* from) {
672 return __halves2half2(from[0], from[0]);
673}
674
675template <>
676EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const half2& from) {
677 *reinterpret_cast<half2*>(to) = from;
678}
679
680template <>
681EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const half2& from) {
682 to[0] = __low2half(from);
683 to[1] = __high2half(from);
684}
685
686template <>
687EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Aligned>(const Eigen::half* from) {
688 // Input is guaranteed to be properly aligned.
689 return __ldg(reinterpret_cast<const half2*>(from));
690}
691
692template <>
693EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Unaligned>(const Eigen::half* from) {
694 return __halves2half2(__ldg(from + 0), __ldg(from + 1));
695}
696
697template <>
698EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pgather<Eigen::half, half2>(const Eigen::half* from, Index stride) {
699 return __halves2half2(from[0 * stride], from[1 * stride]);
700}
701
702template <>
703EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<Eigen::half, half2>(Eigen::half* to, const half2& from,
704 Index stride) {
705 to[stride * 0] = __low2half(from);
706 to[stride * 1] = __high2half(from);
707}
708
709template <>
710EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 preverse(const half2& a) {
711 return __halves2half2(__high2half(a), __low2half(a));
712}
713
714template <>
715EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half pfirst<half2>(const half2& a) {
716 return __low2half(a);
717}
718
719// __low2half returns the CUDA type; Eigen::half is what carries the accessible raw field.
720EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE numext::uint16_t low_bits(const half2& a) {
721 const Eigen::half low = __low2half(a);
722 return low.x;
723}
724EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE numext::uint16_t high_bits(const half2& a) {
725 const Eigen::half high = __high2half(a);
726 return high.x;
727}
728EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 half2_from_bits(numext::uint16_t low, numext::uint16_t high) {
729 return __halves2half2(half_impl::raw_uint16_to_half(low), half_impl::raw_uint16_to_half(high));
730}
731// An all-ones or all-zero lane: the mask form pselect and the bitwise operations consume.
732EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half half_mask_from(bool condition) {
733 return half_impl::raw_uint16_to_half(condition ? 0xffffu : 0x0000u);
734}
735
736template <>
737EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pabs<half2>(const half2& a) {
738 return half2_from_bits(low_bits(a) & 0x7FFF, high_bits(a) & 0x7FFF);
739}
740
741template <>
742EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 ptrue<half2>(const half2& /*a*/) {
743 return pset1<half2>(half_mask_from(true));
744}
745
746template <>
747EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pzero<half2>(const half2& /*a*/) {
748 return pset1<half2>(half_mask_from(false));
749}
750
751EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<half2, 2>& kernel) {
752 const Eigen::half a1 = __low2half(kernel.packet[0]);
753 const Eigen::half a2 = __high2half(kernel.packet[0]);
754 const Eigen::half b1 = __low2half(kernel.packet[1]);
755 const Eigen::half b2 = __high2half(kernel.packet[1]);
756 kernel.packet[0] = __halves2half2(a1, b1);
757 kernel.packet[1] = __halves2half2(a2, b2);
758}
759
760template <>
761EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plset<half2>(const Eigen::half& a) {
762#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
763 return __halves2half2(a, __hadd(a, __float2half(1.0f)));
764#else
765 return __halves2half2(a, __float2half(__half2float(a) + 1.0f));
766#endif
767}
768
769template <>
770EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pselect<half2>(const half2& mask, const half2& a, const half2& b) {
771 const Eigen::half mask_low = __low2half(mask);
772 const Eigen::half mask_high = __high2half(mask);
773 const Eigen::half low = mask_low == Eigen::half(0) ? __low2half(b) : __low2half(a);
774 const Eigen::half high = mask_high == Eigen::half(0) ? __high2half(b) : __high2half(a);
775 return __halves2half2(low, high);
776}
777
778// Conversion to float is exact for both half operands.
779template <>
780EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_eq<half2>(const half2& a, const half2& b) {
781 return __halves2half2(half_mask_from(__low2float(a) == __low2float(b)),
782 half_mask_from(__high2float(a) == __high2float(b)));
783}
784template <>
785EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_lt<half2>(const half2& a, const half2& b) {
786 return __halves2half2(half_mask_from(__low2float(a) < __low2float(b)),
787 half_mask_from(__high2float(a) < __high2float(b)));
788}
789template <>
790EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pcmp_le<half2>(const half2& a, const half2& b) {
791 return __halves2half2(half_mask_from(__low2float(a) <= __low2float(b)),
792 half_mask_from(__high2float(a) <= __high2float(b)));
793}
794
795template <>
796EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pand<half2>(const half2& a, const half2& b) {
797 return half2_from_bits(low_bits(a) & low_bits(b), high_bits(a) & high_bits(b));
798}
799template <>
800EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 por<half2>(const half2& a, const half2& b) {
801 return half2_from_bits(low_bits(a) | low_bits(b), high_bits(a) | high_bits(b));
802}
803template <>
804EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pxor<half2>(const half2& a, const half2& b) {
805 return half2_from_bits(low_bits(a) ^ low_bits(b), high_bits(a) ^ high_bits(b));
806}
807template <>
808EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pandnot<half2>(const half2& a, const half2& b) {
809 return half2_from_bits(low_bits(a) & ~low_bits(b), high_bits(a) & ~high_bits(b));
810}
811
812template <>
813EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 padd<half2>(const half2& a, const half2& b) {
814#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
815 return __hadd2(a, b);
816#else
817 return __floats2half2_rn(__low2float(a) + __low2float(b), __high2float(a) + __high2float(b));
818#endif
819}
820
821template <>
822EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psub<half2>(const half2& a, const half2& b) {
823#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
824 return __hsub2(a, b);
825#else
826 return __floats2half2_rn(__low2float(a) - __low2float(b), __high2float(a) - __high2float(b));
827#endif
828}
829
830template <>
831EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pnegate(const half2& a) {
832#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
833 return __hneg2(a);
834#else
835 return __floats2half2_rn(-__low2float(a), -__high2float(a));
836#endif
837}
838
839template <>
840EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmul<half2>(const half2& a, const half2& b) {
841#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
842 return __hmul2(a, b);
843#else
844 return __floats2half2_rn(__low2float(a) * __low2float(b), __high2float(a) * __high2float(b));
845#endif
846}
847
848template <>
849struct is_fused_madd<half2> : true_type {};
850template <>
851EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmadd(const half2& a, const half2& b, const half2& c) {
852#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
853 return __hfma2(a, b, c);
854#else
855 return __floats2half2_rn(fmaf(__low2float(a), __low2float(b), __low2float(c)),
856 fmaf(__high2float(a), __high2float(b), __high2float(c)));
857#endif
858}
859
860template <>
861EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pdiv<half2>(const half2& a, const half2& b) {
862#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
863 return __h2div(a, b);
864#else
865 return __floats2half2_rn(__low2float(a) / __low2float(b), __high2float(a) / __high2float(b));
866#endif
867}
868
869// Compared through float, and every comparison against a NaN is false, so the result is b. This differs from
870// the fminf/fmaxf of the float packets, which return the non-NaN operand: here a NaN in b propagates and a
871// NaN in a does not.
872template <>
873EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmin<half2>(const half2& a, const half2& b) {
874 const __half low = __low2float(a) < __low2float(b) ? __low2half(a) : __low2half(b);
875 const __half high = __high2float(a) < __high2float(b) ? __high2half(a) : __high2half(b);
876 return __halves2half2(low, high);
877}
878
879template <>
880EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pmax<half2>(const half2& a, const half2& b) {
881 const __half low = __low2float(a) > __low2float(b) ? __low2half(a) : __low2half(b);
882 const __half high = __high2float(a) > __high2float(b) ? __high2half(a) : __high2half(b);
883 return __halves2half2(low, high);
884}
885
886template <>
887EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux<half2>(const half2& a) {
888#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
889 return __hadd(__low2half(a), __high2half(a));
890#else
891 return __float2half(__low2float(a) + __high2float(a));
892#endif
893}
894
895template <>
896EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const half2& a) {
897 return (low_bits(a) | high_bits(a)) != 0;
898}
899
900template <>
901EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max<half2>(const half2& a) {
902#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
903 const __half first = __low2half(a);
904 const __half second = __high2half(a);
905 return __hgt(first, second) ? first : second;
906#else
907 return __low2float(a) > __high2float(a) ? __low2half(a) : __high2half(a);
908#endif
909}
910
911template <>
912EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_min<half2>(const half2& a) {
913#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
914 const __half first = __low2half(a);
915 const __half second = __high2half(a);
916 return __hlt(first, second) ? first : second;
917#else
918 return __low2float(a) < __high2float(a) ? __low2half(a) : __high2half(a);
919#endif
920}
921
922template <>
923EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_mul<half2>(const half2& a) {
924#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
925 return __hmul(__low2half(a), __high2half(a));
926#else
927 return __float2half(__low2float(a) * __high2float(a));
928#endif
929}
930
931template <>
932EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog<half2>(const half2& a) {
933#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
934 return h2log(a);
935#else
936 return __floats2half2_rn(logf(__low2float(a)), logf(__high2float(a)));
937#endif
938}
939
940template <>
941EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexp<half2>(const half2& a) {
942#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
943 return h2exp(a);
944#else
945 return __floats2half2_rn(expf(__low2float(a)), expf(__high2float(a)));
946#endif
947}
948
949template <>
950EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 psqrt<half2>(const half2& a) {
951#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
952 return h2sqrt(a);
953#else
954 return __floats2half2_rn(sqrtf(__low2float(a)), sqrtf(__high2float(a)));
955#endif
956}
957
958template <>
959EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 prsqrt<half2>(const half2& a) {
960#if defined(EIGEN_GPU_HAS_FP16_ARITHMETIC)
961 return h2rsqrt(a);
962#else
963 return __floats2half2_rn(rsqrtf(__low2float(a)), rsqrtf(__high2float(a)));
964#endif
965}
966
967// No native h2log1p/h2expm1; both go through float.
968template <>
969EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 plog1p<half2>(const half2& a) {
970 return __floats2half2_rn(log1pf(__low2float(a)), log1pf(__high2float(a)));
971}
972
973template <>
974EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 pexpm1<half2>(const half2& a) {
975 return __floats2half2_rn(expm1f(__low2float(a)), expm1f(__high2float(a)));
976}
977
978// ---------------------------------------------------------------------------------------------------------------
979// Packet4h2 is four half2 lanes in the two words of a ulonglong2, the alias the Tensor GPU kernels expect. Do not
980// replace the cast: reaching the lanes conformingly (shift and reassemble, or __half2_raw) stops the compiler
981// keeping them in registers, costing 168 SASS instructions against 152 for eight mixed half operations and 152
982// against 64 for an 8x8 ptranspose (sm_89, nvcc 13.3).
983EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE half2 lane_half2(const Packet4h2& p, int i) {
984 return reinterpret_cast<const half2*>(&p)[i];
985}
986
987EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 make_packet4h2(const half2& l0, const half2& l1, const half2& l2,
988 const half2& l3) {
989 Packet4h2 r;
990 half2* lanes = reinterpret_cast<half2*>(&r);
991 lanes[0] = l0;
992 lanes[1] = l1;
993 lanes[2] = l2;
994 lanes[3] = l3;
995 return r;
996}
997
998// Every lane-wise operation is its half2 form applied to the four lanes; the macros keep that unroll in one place.
999#define EIGEN_GPU_PACKET4H2_UNARY(NAME) \
1000 template <> \
1001 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 NAME<Packet4h2>(const Packet4h2& a) { \
1002 return make_packet4h2(NAME(lane_half2(a, 0)), NAME(lane_half2(a, 1)), NAME(lane_half2(a, 2)), \
1003 NAME(lane_half2(a, 3))); \
1004 }
1005
1006#define EIGEN_GPU_PACKET4H2_BINARY(NAME) \
1007 template <> \
1008 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 NAME<Packet4h2>(const Packet4h2& a, const Packet4h2& b) { \
1009 return make_packet4h2(NAME(lane_half2(a, 0), lane_half2(b, 0)), NAME(lane_half2(a, 1), lane_half2(b, 1)), \
1010 NAME(lane_half2(a, 2), lane_half2(b, 2)), NAME(lane_half2(a, 3), lane_half2(b, 3))); \
1011 }
1012
1013template <>
1014EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pset1<Packet4h2>(const Eigen::half& from) {
1015 const half2 lane = pset1<half2>(from);
1016 return make_packet4h2(lane, lane, lane, lane);
1017}
1018
1019template <>
1020EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pload<Packet4h2>(const Eigen::half* from) {
1021 return *reinterpret_cast<const Packet4h2*>(from);
1022}
1023
1024template <>
1025EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ploadu<Packet4h2>(const Eigen::half* from) {
1026 return make_packet4h2(ploadu<half2>(from + 0), ploadu<half2>(from + 2), ploadu<half2>(from + 4),
1027 ploadu<half2>(from + 6));
1028}
1029
1030template <>
1031EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ploaddup<Packet4h2>(const Eigen::half* from) {
1032 return make_packet4h2(ploaddup<half2>(from + 0), ploaddup<half2>(from + 1), ploaddup<half2>(from + 2),
1033 ploaddup<half2>(from + 3));
1034}
1035
1036template <>
1037EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4h2& from) {
1038 *reinterpret_cast<Packet4h2*>(to) = from;
1039}
1040
1041template <>
1042EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet4h2& from) {
1043 pstoreu<Eigen::half>(to + 0, lane_half2(from, 0));
1044 pstoreu<Eigen::half>(to + 2, lane_half2(from, 1));
1045 pstoreu<Eigen::half>(to + 4, lane_half2(from, 2));
1046 pstoreu<Eigen::half>(to + 6, lane_half2(from, 3));
1047}
1048
1049template <>
1050EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4h2 ploadt_ro<Packet4h2, Aligned>(const Eigen::half* from) {
1051 return __ldg(reinterpret_cast<const Packet4h2*>(from));
1052}
1053
1054template <>
1055EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4h2 ploadt_ro<Packet4h2, Unaligned>(const Eigen::half* from) {
1056 return make_packet4h2(ploadt_ro<half2, Unaligned>(from + 0), ploadt_ro<half2, Unaligned>(from + 2),
1057 ploadt_ro<half2, Unaligned>(from + 4), ploadt_ro<half2, Unaligned>(from + 6));
1058}
1059
1060template <>
1061EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pgather<Eigen::half, Packet4h2>(const Eigen::half* from, Index stride) {
1062 return make_packet4h2(
1063 __halves2half2(from[0 * stride], from[1 * stride]), __halves2half2(from[2 * stride], from[3 * stride]),
1064 __halves2half2(from[4 * stride], from[5 * stride]), __halves2half2(from[6 * stride], from[7 * stride]));
1065}
1066
1067template <>
1068EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet4h2>(Eigen::half* to, const Packet4h2& from,
1069 Index stride) {
1070 pscatter<Eigen::half, half2>(to + stride * 0, lane_half2(from, 0), stride);
1071 pscatter<Eigen::half, half2>(to + stride * 2, lane_half2(from, 1), stride);
1072 pscatter<Eigen::half, half2>(to + stride * 4, lane_half2(from, 2), stride);
1073 pscatter<Eigen::half, half2>(to + stride * 6, lane_half2(from, 3), stride);
1074}
1075
1076template <>
1077EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 preverse(const Packet4h2& a) {
1078 return make_packet4h2(preverse(lane_half2(a, 3)), preverse(lane_half2(a, 2)), preverse(lane_half2(a, 1)),
1079 preverse(lane_half2(a, 0)));
1080}
1081
1082template <>
1083EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4h2>(const Packet4h2& a) {
1084 return pfirst<half2>(lane_half2(a, 0));
1085}
1086
1087template <>
1088EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 ptrue<Packet4h2>(const Packet4h2& /*a*/) {
1089 return pset1<Packet4h2>(half_impl::raw_uint16_to_half(0xffffu));
1090}
1091
1092template <>
1093EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pzero<Packet4h2>(const Packet4h2& /*a*/) {
1094 return pset1<Packet4h2>(half_impl::raw_uint16_to_half(0x0000u));
1095}
1096
1097// An 8x8 transpose of halves: packet r of the result is column r of the input.
1098EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4h2, 8>& kernel) {
1099 Eigen::half elements[8][8];
1100 EIGEN_UNROLL_LOOP
1101 for (int row = 0; row < 8; ++row) {
1102 EIGEN_UNROLL_LOOP
1103 for (int lane = 0; lane < 4; ++lane) {
1104 const half2 pair = lane_half2(kernel.packet[row], lane);
1105 elements[row][2 * lane] = __low2half(pair);
1106 elements[row][2 * lane + 1] = __high2half(pair);
1107 }
1108 }
1109 EIGEN_UNROLL_LOOP
1110 for (int row = 0; row < 8; ++row) {
1111 kernel.packet[row] = make_packet4h2(
1112 __halves2half2(elements[0][row], elements[1][row]), __halves2half2(elements[2][row], elements[3][row]),
1113 __halves2half2(elements[4][row], elements[5][row]), __halves2half2(elements[6][row], elements[7][row]));
1114 }
1115}
1116
1117template <>
1118EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 plset<Packet4h2>(const Eigen::half& a) {
1119 // Add each offset once: (a + 2*k) + 1 can round twice. Preserve a's sign in lane zero.
1120 const half2 base = pset1<half2>(a);
1121 return make_packet4h2(plset<half2>(a), padd<half2>(base, __floats2half2_rn(2.0f, 3.0f)),
1122 padd<half2>(base, __floats2half2_rn(4.0f, 5.0f)),
1123 padd<half2>(base, __floats2half2_rn(6.0f, 7.0f)));
1124}
1125
1126template <>
1127EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pselect<Packet4h2>(const Packet4h2& mask, const Packet4h2& a,
1128 const Packet4h2& b) {
1129 return make_packet4h2(pselect<half2>(lane_half2(mask, 0), lane_half2(a, 0), lane_half2(b, 0)),
1130 pselect<half2>(lane_half2(mask, 1), lane_half2(a, 1), lane_half2(b, 1)),
1131 pselect<half2>(lane_half2(mask, 2), lane_half2(a, 2), lane_half2(b, 2)),
1132 pselect<half2>(lane_half2(mask, 3), lane_half2(a, 3), lane_half2(b, 3)));
1133}
1134
1135template <>
1136struct is_fused_madd<Packet4h2> : is_fused_madd<half2> {};
1137template <>
1138EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4h2 pmadd(const Packet4h2& a, const Packet4h2& b, const Packet4h2& c) {
1139 return make_packet4h2(pmadd<half2>(lane_half2(a, 0), lane_half2(b, 0), lane_half2(c, 0)),
1140 pmadd<half2>(lane_half2(a, 1), lane_half2(b, 1), lane_half2(c, 1)),
1141 pmadd<half2>(lane_half2(a, 2), lane_half2(b, 2), lane_half2(c, 2)),
1142 pmadd<half2>(lane_half2(a, 3), lane_half2(b, 3), lane_half2(c, 3)));
1143}
1144
1145EIGEN_GPU_PACKET4H2_UNARY(pabs)
1146EIGEN_GPU_PACKET4H2_UNARY(pnegate)
1147EIGEN_GPU_PACKET4H2_UNARY(plog)
1148EIGEN_GPU_PACKET4H2_UNARY(pexp)
1149EIGEN_GPU_PACKET4H2_UNARY(psqrt)
1150EIGEN_GPU_PACKET4H2_UNARY(prsqrt)
1151EIGEN_GPU_PACKET4H2_UNARY(plog1p)
1152EIGEN_GPU_PACKET4H2_UNARY(pexpm1)
1153
1154EIGEN_GPU_PACKET4H2_BINARY(padd)
1155EIGEN_GPU_PACKET4H2_BINARY(psub)
1156EIGEN_GPU_PACKET4H2_BINARY(pmul)
1157EIGEN_GPU_PACKET4H2_BINARY(pdiv)
1158EIGEN_GPU_PACKET4H2_BINARY(pmin)
1159EIGEN_GPU_PACKET4H2_BINARY(pmax)
1160EIGEN_GPU_PACKET4H2_BINARY(pand)
1161EIGEN_GPU_PACKET4H2_BINARY(por)
1162EIGEN_GPU_PACKET4H2_BINARY(pxor)
1163EIGEN_GPU_PACKET4H2_BINARY(pandnot)
1164EIGEN_GPU_PACKET4H2_BINARY(pcmp_eq)
1165EIGEN_GPU_PACKET4H2_BINARY(pcmp_lt)
1166EIGEN_GPU_PACKET4H2_BINARY(pcmp_le)
1167
1168#undef EIGEN_GPU_PACKET4H2_UNARY
1169#undef EIGEN_GPU_PACKET4H2_BINARY
1170
1171template <>
1172EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux<Packet4h2>(const Packet4h2& a) {
1173 return predux(lane_half2(a, 0)) + predux(lane_half2(a, 1)) + predux(lane_half2(a, 2)) + predux(lane_half2(a, 3));
1174}
1175
1176template <>
1177EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool predux_any(const Packet4h2& a) {
1178 return predux_any(lane_half2(a, 0)) | predux_any(lane_half2(a, 1)) | predux_any(lane_half2(a, 2)) |
1179 predux_any(lane_half2(a, 3));
1180}
1181
1182template <>
1183EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4h2>(const Packet4h2& a) {
1184 const half2 m =
1185 pmax<half2>(pmax<half2>(lane_half2(a, 0), lane_half2(a, 1)), pmax<half2>(lane_half2(a, 2), lane_half2(a, 3)));
1186 return predux_max<half2>(m);
1187}
1188
1189template <>
1190EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_min<Packet4h2>(const Packet4h2& a) {
1191 const half2 m =
1192 pmin<half2>(pmin<half2>(lane_half2(a, 0), lane_half2(a, 1)), pmin<half2>(lane_half2(a, 2), lane_half2(a, 3)));
1193 return predux_min<half2>(m);
1194}
1195
1196// Likely to overflow or underflow: eight halves multiplied together.
1197template <>
1198EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet4h2>(const Packet4h2& a) {
1199 const half2 product =
1200 pmul<half2>(pmul<half2>(lane_half2(a, 0), lane_half2(a, 1)), pmul<half2>(lane_half2(a, 2), lane_half2(a, 3)));
1201 return predux_mul<half2>(product);
1202}
1203
1204#undef EIGEN_GPU_HAS_FP16_ARITHMETIC
1205
1206#endif // defined(EIGEN_GPU_COMPILE_PHASE)
1207
1208} // end namespace internal
1209
1210} // end namespace Eigen
1211
1212#endif // EIGEN_PACKET_MATH_GPU_H
@ Aligned8
Definition Constants.h:237
@ Aligned16
Definition Constants.h:238