Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
Half.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// This Source Code Form is subject to the terms of the Mozilla
5// Public License v. 2.0. If a copy of the MPL was not distributed
6// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
7//
8// The conversion routines are Copyright (c) Fabian Giesen, 2016.
9// The original license follows:
10//
11// Copyright (c) Fabian Giesen, 2016
12// All rights reserved.
13// Redistribution and use in source and binary forms, with or without
14// modification, are permitted.
15// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
18// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
19// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
21// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26// SPDX-License-Identifier: MPL-2.0
27
28// Standard 16-bit float type, mostly useful for GPUs. Defines a new
29// type Eigen::half (inheriting either from CUDA's or HIP's __half struct) with
30// operator overloads such that it behaves basically as an arithmetic
31// type. It will be quite slow on CPUs (so it is recommended to stay
32// in fp32 for CPUs, except for simple parameter conversions, I/O
33// to disk and the likes), but fast on GPUs.
34
35#ifndef EIGEN_HALF_H
36#define EIGEN_HALF_H
37
38// IWYU pragma: private
39#include "../../InternalHeaderCheck.h"
40
41// When compiling with GPU support, the "__half_raw" base class as well as
42// some other routines are defined in the GPU compiler header files
43// (cuda_fp16.h, hip_fp16.h), and they are not tagged constexpr
44// As a consequence, we get compile failures when compiling Eigen with
45// GPU support. Hence the need to disable EIGEN_CONSTEXPR when building
46// Eigen with GPU support.
47// Any functions that require `numext::bit_cast` may also not be constexpr,
48// including any native types when setting via raw bit values.
49#if defined(EIGEN_HAS_GPU_FP16) || defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) || defined(EIGEN_HAS_BUILTIN_FLOAT16)
50#define _EIGEN_MAYBE_CONSTEXPR
51#else
52#define _EIGEN_MAYBE_CONSTEXPR constexpr
53#endif
54
55#define F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, METHOD) \
56 template <> \
57 EIGEN_UNUSED EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC PACKET_F16 METHOD<PACKET_F16>(const PACKET_F16& _x) { \
58 return float2half(METHOD<PACKET_F>(half2float(_x))); \
59 }
60
61#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_F16(PACKET_F, PACKET_F16) \
62 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pcos) \
63 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, psin) \
64 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, psinh) \
65 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pcosh) \
66 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pasinh) \
67 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pacosh) \
68 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pexp) \
69 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pexp2) \
70 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pexpm1) \
71 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, plog) \
72 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, plog1p) \
73 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, plog2) \
74 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, plog10) \
75 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, preciprocal) \
76 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, prsqrt) \
77 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pcbrt) \
78 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, psqrt) \
79 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, ptanh)
80
81// F16 wrappers for contrib/SpecialFunctions.
82#define EIGEN_INSTANTIATE_SPECIAL_FUNCS_F16(PACKET_F, PACKET_F16) \
83 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, perf) \
84 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pndtri)
85
86#define EIGEN_INSTANTIATE_BESSEL_FUNCS_F16(PACKET_F, PACKET_F16) \
87 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_i0) \
88 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_i0e) \
89 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_i1) \
90 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_i1e) \
91 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_j0) \
92 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_j1) \
93 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_k0) \
94 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_k0e) \
95 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_k1) \
96 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_k1e) \
97 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_y0) \
98 F16_PACKET_FUNCTION(PACKET_F, PACKET_F16, pbessel_y1)
99
100namespace Eigen {
101
102struct half;
103
104namespace half_impl {
105
106// We want to use the __half_raw struct from the HIP header file only during the device compile phase.
107// This is required because of a quirk in the way TensorFlow GPU builds are done.
108// When compiling TensorFlow source code with GPU support, files that
109// * contain GPU kernels (i.e. *.cu.cc files) are compiled via hipcc
110// * do not contain GPU kernels ( i.e. *.cc files) are compiled via gcc (typically)
111//
112// Tensorflow uses the Eigen::half type as its FP16 type, and there are functions that
113// * are defined in a file that gets compiled via hipcc AND
114// * have Eigen::half as a pass-by-value argument AND
115// * are called in a file that gets compiled via gcc
116//
117// In the scenario described above the caller and callee will see different versions
118// of the Eigen::half base class __half_raw, and they will be compiled by different compilers
119//
120// There appears to be an ABI mismatch between gcc and clang (which is called by hipcc) that results in
121// the callee getting corrupted values for the Eigen::half argument.
122//
123// Making the host side compile phase of hipcc use the same Eigen::half impl, as the gcc compile, resolves
124// this error, and hence the following convoluted #if condition
125#if !defined(EIGEN_GPUCC) || !defined(EIGEN_GPU_COMPILE_PHASE)
126
127// Make our own __half_raw definition that is similar to CUDA's.
128struct __half_raw {
129 struct construct_from_rep_tag {};
130#if (defined(EIGEN_GPUCC) && !defined(EIGEN_GPU_COMPILE_PHASE))
131 // Eigen::half can be used as the datatype for shared memory declarations (in Eigen and TF)
132 // The element type for shared memory cannot have non-trivial constructors
133 // and hence the following special casing (which skips the zero-initialization).
134 // Note that this check gets done even in the host compilation phase, and
135 // hence the need for this
136 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR __half_raw() {}
137#else
138 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR __half_raw() : x(0) {}
139#endif
140
141 // The type of x is part of the ABI of half: under AAPCS64, a class whose only member is __fp16 is passed in a SIMD
142 // register, and one whose member is uint16_t in a general-purpose register. So x is __fp16 only with FP16 scalar
143 // arithmetic, as in Eigen 5.0. Targets that have only the __fp16 interchange type (EIGEN_HAS_ARM64_FP16) store
144 // uint16_t and reach __fp16 through native_half() and the __fp16 rep constructor below.
145#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC)
146 explicit EIGEN_DEVICE_FUNC __half_raw(numext::uint16_t raw) : x(numext::bit_cast<__fp16>(raw)) {}
147 EIGEN_DEVICE_FUNC constexpr __half_raw(construct_from_rep_tag, __fp16 rep) : x{rep} {}
148 __fp16 x;
149#elif defined(EIGEN_HAS_BUILTIN_FLOAT16)
150 explicit EIGEN_DEVICE_FUNC __half_raw(numext::uint16_t raw) : x(numext::bit_cast<_Float16>(raw)) {}
151 EIGEN_DEVICE_FUNC constexpr __half_raw(construct_from_rep_tag, _Float16 rep) : x{rep} {}
152 _Float16 x;
153#else
154 explicit EIGEN_DEVICE_FUNC constexpr __half_raw(numext::uint16_t raw) : x(raw) {}
155 EIGEN_DEVICE_FUNC constexpr __half_raw(construct_from_rep_tag, numext::uint16_t rep) : x{rep} {}
156#if EIGEN_HAS_ARM64_FP16
157 EIGEN_DEVICE_FUNC __half_raw(construct_from_rep_tag, __fp16 rep) : x(numext::bit_cast<numext::uint16_t>(rep)) {}
158#endif
159 numext::uint16_t x;
160#endif
161};
162
163#elif defined(EIGEN_HIPCC)
164// HIP GPU compile phase: nothing to do here.
165// HIP fp16 header file has a definition for __half_raw
166#elif defined(EIGEN_CUDACC)
167
168// CUDA GPU compile phase.
169
170#elif defined(EIGEN_SYCL_DEVICE_ONLY)
171using __half_raw = cl::sycl::half;
172#endif
173
174EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR __half_raw raw_uint16_to_half(numext::uint16_t x);
175EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC numext::uint16_t raw_half_as_uint16(const __half_raw& h);
176template <bool AssumeArgumentIsNormalOrInfinityOrZero = false>
177EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne(float ff);
178template <>
179EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne<false>(float ff);
180template <>
181EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne<true>(float ff);
182EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half_raw h);
183
184// Narrowing a wider floating-point value or an integer of more than 24 bits through float rounds twice. Rounding to
185// odd in the first step keeps the second rounding, to a format at least two bits narrower than float, correct.
186// Paths: 1 for wider floating-point types; 2 for integers of at most 32 bits, rounded to odd on the multiples of 2^8
187// where |x| >= 2^24, which float holds exactly and which keep a second rounding to p <= 15 bits correct (its midpoints
188// there are multiples of 2^(24 - p) >= 2^9), with no double arithmetic in device code; 3 for 64-bit integers, rounded
189// to odd on the multiples of 2^11 where |x| >= 2^53, which double holds exactly; 0 casts directly.
190template <typename T>
191struct narrow_to_float_path
192 : std::integral_constant<int, std::is_floating_point<T>::value ? (sizeof(T) > sizeof(float) ? 1 : 0)
193 : !std::is_integral<T>::value || std::numeric_limits<T>::digits <= 24 ? 0
194 : std::numeric_limits<T>::digits <= 32 ? 2
195 : std::numeric_limits<T>::digits <= 64 ? 3
196 : 0> {};
197
198template <typename T, int Path = narrow_to_float_path<T>::value>
199struct narrow_to_float_impl {
200 EIGEN_DEVICE_FUNC static float run(const T& val) { return static_cast<float>(val); }
201};
202template <typename T>
203struct narrow_to_float_impl<T, 1> {
204 EIGEN_DEVICE_FUNC static float run(const T& val) {
205 const float f = static_cast<float>(val);
206 if (static_cast<T>(f) == val || (numext::isnan)(val)) return f;
207 // Inexact: truncate toward zero, then set the last bit.
208 const bool away = numext::abs(static_cast<T>(f)) > numext::abs(val);
209 return numext::bit_cast<float>((numext::bit_cast<numext::uint32_t>(f) - (away ? 1u : 0u)) | 1u);
210 }
211};
212// In two's complement, clearing the low bits rounds toward -inf for either sign; setting the next bit then picks the
213// odd neighbor.
214template <typename T>
215struct narrow_to_float_impl<T, 2> {
216 EIGEN_DEVICE_FUNC static float run(const T& val) {
217 using U = std::make_unsigned_t<T>;
218 const U u = static_cast<U>(val);
219 const U low = 0xff;
220 const U odd = (u & ~low) | ((u & low) != 0 ? U(0x100) : U(0));
221 const U magnitude = std::is_signed<T>::value && (u >> (sizeof(U) * CHAR_BIT - 1)) ? U(0) - u : u;
222 return static_cast<float>(magnitude >= (U(1) << 24) ? numext::bit_cast<T>(odd) : val);
223 }
224};
225template <typename T>
226struct narrow_to_float_impl<T, 3> {
227 EIGEN_DEVICE_FUNC static float run(const T& val) {
228 using U = std::make_unsigned_t<T>;
229 const U u = static_cast<U>(val);
230 const U low = 0x7ff;
231 const U odd = (u & ~low) | ((u & low) != 0 ? U(0x800) : U(0));
232 const U magnitude = std::is_signed<T>::value && (u >> (sizeof(U) * CHAR_BIT - 1)) ? U(0) - u : u;
233 return narrow_to_float_impl<double>::run(
234 static_cast<double>(magnitude >= (U(1) << 53) ? numext::bit_cast<T>(odd) : val));
235 }
236};
237template <typename T>
238EIGEN_DEVICE_FUNC inline float narrow_to_float(const T& val) {
239 return narrow_to_float_impl<T>::run(val);
240}
241
242struct half_base : public __half_raw {
243 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half_base() {}
244 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half_base(const __half_raw& h) : __half_raw(h) {}
245
246#if defined(EIGEN_GPUCC)
247#if defined(EIGEN_HIPCC)
248 // Delegate to raw_uint16_to_half, which reinterprets the raw bits for every storage type of
249 // __half_raw::x. In the host compile phase on platforms with a native fp16 type (e.g. __fp16 on
250 // arm64), a direct "x = __half_as_ushort(h)" would perform a numeric integer-to-float conversion.
251 // numext::bit_cast is used to extract the bits because hip_fp16.h defines __half_as_ushort for
252 // the device compile phase only (host references fail to link against ROCm 6.3).
253 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half_base(const __half& h)
254 : __half_raw(raw_uint16_to_half(numext::bit_cast<numext::uint16_t>(h))) {}
255#elif defined(EIGEN_CUDACC)
256 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half_base(const __half& h) : __half_raw(*(__half_raw*)&h) {}
257#endif
258#endif
259};
260
261} // namespace half_impl
262
263// Class definition.
264struct half : public half_impl::half_base {
265 // Writing this out as separate #if-else blocks to make the code easier to follow
266 // The same applies to most #if-else blocks in this file
267#if !defined(EIGEN_GPUCC) || !defined(EIGEN_GPU_COMPILE_PHASE)
268 // Use the same base class for the following two scenarios
269 // * when compiling without GPU support enabled
270 // * during host compile phase when compiling with GPU support enabled
271 using __half_raw = half_impl::__half_raw;
272#elif defined(EIGEN_HIPCC)
273 // Nothing to do here
274 // HIP fp16 header file has a definition for __half_raw
275#elif defined(EIGEN_CUDACC)
276 // Nothing to do here.
277#endif
278
279 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half() {}
280
281 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half(const __half_raw& h) : half_impl::half_base(h) {}
282
283#if defined(EIGEN_GPUCC)
284#if defined(EIGEN_HIPCC)
285 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half(const __half& h) : half_impl::half_base(h) {}
286#elif defined(EIGEN_CUDACC)
287 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half(const __half& h) : half_impl::half_base(h) {}
288#endif
289#endif
290
291// In the device compile phase __half_raw is the vendor type, which has no construct_from_rep_tag,
292// so these constructors are restricted to the host compile phase and non-GPU builds.
293#if EIGEN_HAS_ARM64_FP16 && !defined(EIGEN_GPU_COMPILE_PHASE)
294 explicit EIGEN_DEVICE_FUNC half(__fp16 b) : half(__half_raw(__half_raw::construct_from_rep_tag(), b)) {}
295#elif defined(EIGEN_HAS_BUILTIN_FLOAT16) && !defined(EIGEN_GPU_COMPILE_PHASE)
296 explicit EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half(_Float16 b)
297 : half(__half_raw(__half_raw::construct_from_rep_tag(), b)) {}
298#endif
299
300 explicit EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR half(bool b)
301 : half_impl::half_base(half_impl::raw_uint16_to_half(b ? 0x3c00 : 0)) {}
302 template <class T, std::enable_if_t<!std::is_integral<T>::value, int> = 0>
303 explicit EIGEN_DEVICE_FUNC half(T val)
304 : half_impl::half_base(half_impl::float_to_half_rtne<false>(half_impl::narrow_to_float(val))) {}
305 // Float rounds only integers beyond 2^24, which half overflows either way.
306 template <class T, std::enable_if_t<std::is_integral<T>::value, int> = 0>
307 explicit EIGEN_DEVICE_FUNC half(T val)
308 : half_impl::half_base(half_impl::float_to_half_rtne<true>(static_cast<float>(val))) {}
309 explicit EIGEN_DEVICE_FUNC half(float f) : half_impl::half_base(half_impl::float_to_half_rtne<false>(f)) {}
310
311 // Following the convention of numpy, converting between complex and
312 // float will lead to loss of imag value.
313 template <typename RealScalar>
314 explicit EIGEN_DEVICE_FUNC half(std::complex<RealScalar> c)
315 : half_impl::half_base(half_impl::float_to_half_rtne<false>(half_impl::narrow_to_float(c.real()))) {}
316
317 EIGEN_DEVICE_FUNC operator float() const { // NOLINT: Allow implicit conversion to float, because it is lossless.
318 return half_impl::half_to_float(*this);
319 }
320
321#if defined(EIGEN_HAS_GPU_FP16) && !defined(EIGEN_GPU_COMPILE_PHASE)
322 EIGEN_DEVICE_FUNC operator __half() const {
323 ::__half_raw hr;
324 // raw_half_as_uint16 reinterprets the raw bits for every storage type of __half_raw::x.
325 // A direct "hr.x = x" would perform a numeric float-to-integer conversion when x has a
326 // native fp16 type (e.g. __fp16 on arm64), since the vendor ::__half_raw::x is an integer.
327 hr.x = half_impl::raw_half_as_uint16(*this);
328 return __half(hr);
329 }
330#endif
331};
332
333// TODO(majnemer): Get rid of this once we can rely on C++17 inline variables do
334// solve the ODR issue.
335namespace half_impl {
336template <typename = void>
337struct numeric_limits_half_impl {
338 static constexpr const bool is_specialized = true;
339 static constexpr const bool is_signed = true;
340 static constexpr const bool is_integer = false;
341 static constexpr const bool is_exact = false;
342 static constexpr const bool has_infinity = true;
343 static constexpr const bool has_quiet_NaN = true;
344 static constexpr const bool has_signaling_NaN = true;
345 EIGEN_DIAGNOSTICS(push)
346 EIGEN_DISABLE_DEPRECATED_WARNING
347 static constexpr const std::float_denorm_style has_denorm = std::denorm_present;
348 static constexpr const bool has_denorm_loss = false;
349 EIGEN_DIAGNOSTICS(pop)
350 static constexpr const std::float_round_style round_style = std::round_to_nearest;
351 static constexpr const bool is_iec559 = true;
352 // The C++ standard defines this as "true if the set of values representable
353 // by the type is finite." Half has finite precision.
354 static constexpr const bool is_bounded = true;
355 static constexpr const bool is_modulo = false;
356 static constexpr const int digits = 11;
357 static constexpr const int digits10 =
358 3; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html
359 static constexpr const int max_digits10 =
360 5; // according to http://half.sourceforge.net/structstd_1_1numeric__limits_3_01half__float_1_1half_01_4.html
361 static constexpr const int radix = std::numeric_limits<float>::radix;
362 static constexpr const int min_exponent = -13;
363 static constexpr const int min_exponent10 = -4;
364 static constexpr const int max_exponent = 16;
365 static constexpr const int max_exponent10 = 4;
366 static constexpr const bool traps = std::numeric_limits<float>::traps;
367 // IEEE754: "The implementer shall choose how tininess is detected, but shall
368 // detect tininess in the same way for all operations in radix two"
369 static constexpr const bool tinyness_before = std::numeric_limits<float>::tinyness_before;
370
371 static _EIGEN_MAYBE_CONSTEXPR Eigen::half(min)() { return Eigen::half_impl::raw_uint16_to_half(0x0400); }
372 static _EIGEN_MAYBE_CONSTEXPR Eigen::half lowest() { return Eigen::half_impl::raw_uint16_to_half(0xfbff); }
373 static _EIGEN_MAYBE_CONSTEXPR Eigen::half(max)() { return Eigen::half_impl::raw_uint16_to_half(0x7bff); }
374 static _EIGEN_MAYBE_CONSTEXPR Eigen::half epsilon() { return Eigen::half_impl::raw_uint16_to_half(0x1400); }
375 static _EIGEN_MAYBE_CONSTEXPR Eigen::half round_error() { return Eigen::half_impl::raw_uint16_to_half(0x3800); }
376 static _EIGEN_MAYBE_CONSTEXPR Eigen::half infinity() { return Eigen::half_impl::raw_uint16_to_half(0x7c00); }
377 static _EIGEN_MAYBE_CONSTEXPR Eigen::half quiet_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7e00); }
378 static _EIGEN_MAYBE_CONSTEXPR Eigen::half signaling_NaN() { return Eigen::half_impl::raw_uint16_to_half(0x7d00); }
379 static _EIGEN_MAYBE_CONSTEXPR Eigen::half denorm_min() { return Eigen::half_impl::raw_uint16_to_half(0x0001); }
380};
381
382// Redundant out-of-class definitions are required pre-C++17 but deprecated since.
383#if EIGEN_COMP_CXXVER < 17
384template <typename T>
385constexpr const bool numeric_limits_half_impl<T>::is_specialized;
386template <typename T>
387constexpr const bool numeric_limits_half_impl<T>::is_signed;
388template <typename T>
389constexpr const bool numeric_limits_half_impl<T>::is_integer;
390template <typename T>
391constexpr const bool numeric_limits_half_impl<T>::is_exact;
392template <typename T>
393constexpr const bool numeric_limits_half_impl<T>::has_infinity;
394template <typename T>
395constexpr const bool numeric_limits_half_impl<T>::has_quiet_NaN;
396template <typename T>
397constexpr const bool numeric_limits_half_impl<T>::has_signaling_NaN;
398EIGEN_DIAGNOSTICS(push)
399EIGEN_DISABLE_DEPRECATED_WARNING
400template <typename T>
401constexpr const std::float_denorm_style numeric_limits_half_impl<T>::has_denorm;
402template <typename T>
403constexpr const bool numeric_limits_half_impl<T>::has_denorm_loss;
404EIGEN_DIAGNOSTICS(pop)
405template <typename T>
406constexpr const std::float_round_style numeric_limits_half_impl<T>::round_style;
407template <typename T>
408constexpr const bool numeric_limits_half_impl<T>::is_iec559;
409template <typename T>
410constexpr const bool numeric_limits_half_impl<T>::is_bounded;
411template <typename T>
412constexpr const bool numeric_limits_half_impl<T>::is_modulo;
413template <typename T>
414constexpr const int numeric_limits_half_impl<T>::digits;
415template <typename T>
416constexpr const int numeric_limits_half_impl<T>::digits10;
417template <typename T>
418constexpr const int numeric_limits_half_impl<T>::max_digits10;
419template <typename T>
420constexpr const int numeric_limits_half_impl<T>::radix;
421template <typename T>
422constexpr const int numeric_limits_half_impl<T>::min_exponent;
423template <typename T>
424constexpr const int numeric_limits_half_impl<T>::min_exponent10;
425template <typename T>
426constexpr const int numeric_limits_half_impl<T>::max_exponent;
427template <typename T>
428constexpr const int numeric_limits_half_impl<T>::max_exponent10;
429template <typename T>
430constexpr const bool numeric_limits_half_impl<T>::traps;
431template <typename T>
432constexpr const bool numeric_limits_half_impl<T>::tinyness_before;
433#endif
434} // end namespace half_impl
435} // end namespace Eigen
436
437namespace std {
438// If std::numeric_limits<T> is specialized, should also specialize
439// std::numeric_limits<const T>, std::numeric_limits<volatile T>, and
440// std::numeric_limits<const volatile T>
441// https://stackoverflow.com/a/16519653/
442template <>
443class numeric_limits<Eigen::half> : public Eigen::half_impl::numeric_limits_half_impl<> {};
444template <>
445class numeric_limits<const Eigen::half> : public numeric_limits<Eigen::half> {};
446template <>
447class numeric_limits<volatile Eigen::half> : public numeric_limits<Eigen::half> {};
448template <>
449class numeric_limits<const volatile Eigen::half> : public numeric_limits<Eigen::half> {};
450} // end namespace std
451
452namespace Eigen {
453
454namespace half_impl {
455
456#if (defined(EIGEN_CUDA_ARCH) && EIGEN_CUDA_ARCH >= 530) || defined(EIGEN_HIP_DEVICE_COMPILE)
457// Note: We deliberately do *not* define this to 1 even if we have Arm's native
458// fp16 type since GPU half types are rather different from native CPU half types.
459#define EIGEN_HAS_NATIVE_GPU_FP16
460#endif
461
462// Intrinsics for native fp16 support. Note that on current hardware,
463// these are no faster than fp32 arithmetic (you need to use the half2
464// versions to get the ALU speed increased), but you do save the
465// conversion steps back and forth.
466
467#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
468EIGEN_STRONG_INLINE __device__ half operator+(const half& a, const half& b) { return __hadd(::__half(a), ::__half(b)); }
469EIGEN_STRONG_INLINE __device__ half operator*(const half& a, const half& b) { return __hmul(a, b); }
470EIGEN_STRONG_INLINE __device__ half operator-(const half& a, const half& b) { return __hsub(a, b); }
471EIGEN_STRONG_INLINE __device__ half operator/(const half& a, const half& b) { return __hdiv(a, b); }
472EIGEN_STRONG_INLINE __device__ half operator-(const half& a) { return __hneg(a); }
473EIGEN_STRONG_INLINE __device__ half& operator+=(half& a, const half& b) {
474 a = a + b;
475 return a;
476}
477EIGEN_STRONG_INLINE __device__ half& operator*=(half& a, const half& b) {
478 a = a * b;
479 return a;
480}
481EIGEN_STRONG_INLINE __device__ half& operator-=(half& a, const half& b) {
482 a = a - b;
483 return a;
484}
485EIGEN_STRONG_INLINE __device__ half& operator/=(half& a, const half& b) {
486 a = a / b;
487 return a;
488}
489EIGEN_STRONG_INLINE __device__ bool operator==(const half& a, const half& b) { return __heq(a, b); }
490// __hne is the ordered comparison, false for a NaN operand; != must be true there.
491EIGEN_STRONG_INLINE __device__ bool operator!=(const half& a, const half& b) { return !__heq(a, b); }
492EIGEN_STRONG_INLINE __device__ bool operator<(const half& a, const half& b) { return __hlt(a, b); }
493EIGEN_STRONG_INLINE __device__ bool operator<=(const half& a, const half& b) { return __hle(a, b); }
494EIGEN_STRONG_INLINE __device__ bool operator>(const half& a, const half& b) { return __hgt(a, b); }
495EIGEN_STRONG_INLINE __device__ bool operator>=(const half& a, const half& b) { return __hge(a, b); }
496
497#endif // EIGEN_HAS_NATIVE_GPU_FP16
498
499#if (EIGEN_HAS_ARM64_FP16 || defined(EIGEN_HAS_BUILTIN_FLOAT16)) && !defined(EIGEN_GPU_COMPILE_PHASE)
500#if EIGEN_HAS_ARM64_FP16
501using native_half_t = __fp16;
502#else
503using native_half_t = _Float16;
504#endif
505
506// The value of h as the compiler's half type, whatever the type of __half_raw::x.
507EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC native_half_t native_half(const __half_raw& h) {
508 return numext::bit_cast<native_half_t>(h.x);
509}
510
511// nvcc's EDG front end does not promote __fp16 arithmetic to float, and it ranks the mandatory
512// __fp16 -> float promotion as an exact match, so constructing a half from a native fp16 expression
513// is an ambiguous tie between half(__fp16) and half(float). Every nvcc arm64 translation unit that
514// includes Eigen then fails to compile. Building __half_raw names one constructor, and is what
515// half(__fp16) and half(_Float16) do internally, so the stored value is unchanged.
516EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half half_from_rep(native_half_t rep) {
517 return half(__half_raw(__half_raw::construct_from_rep_tag(), rep));
518}
519#endif
520
521#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC) && !defined(EIGEN_GPU_COMPILE_PHASE)
522EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator+(const half& a, const half& b) {
523 return half_from_rep(vaddh_f16(a.x, b.x));
524}
525EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator*(const half& a, const half& b) {
526 return half_from_rep(vmulh_f16(a.x, b.x));
527}
528EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator-(const half& a, const half& b) {
529 return half_from_rep(vsubh_f16(a.x, b.x));
530}
531EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator/(const half& a, const half& b) {
532 return half_from_rep(vdivh_f16(a.x, b.x));
533}
534EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator-(const half& a) { return half_from_rep(vnegh_f16(a.x)); }
535EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator+=(half& a, const half& b) {
536 a = half_from_rep(vaddh_f16(a.x, b.x));
537 return a;
538}
539EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator*=(half& a, const half& b) {
540 a = half_from_rep(vmulh_f16(a.x, b.x));
541 return a;
542}
543EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator-=(half& a, const half& b) {
544 a = half_from_rep(vsubh_f16(a.x, b.x));
545 return a;
546}
547EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator/=(half& a, const half& b) {
548 a = half_from_rep(vdivh_f16(a.x, b.x));
549 return a;
550}
551EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator==(const half& a, const half& b) { return vceqh_f16(a.x, b.x); }
552EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const half& a, const half& b) { return !vceqh_f16(a.x, b.x); }
553EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const half& a, const half& b) { return vclth_f16(a.x, b.x); }
554EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const half& a, const half& b) { return vcleh_f16(a.x, b.x); }
555EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const half& a, const half& b) { return vcgth_f16(a.x, b.x); }
556EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const half& a, const half& b) { return vcgeh_f16(a.x, b.x); }
557
558#elif (EIGEN_HAS_ARM64_FP16 || defined(EIGEN_HAS_BUILTIN_FLOAT16)) && !defined(EIGEN_GPU_COMPILE_PHASE)
559
560// NOTE: While GCC appears to only apply `-Wdouble-promotion` to single-to-double
561// conversions, Clang applies the warning to half-to-single implicit conversion also. For
562// the operations below, implicit conversion is intentional so that---depending on the
563// target features---the compiler can choose between using native half-precision
564// instructions or converting to single-precision.
565#if EIGEN_COMP_CLANG
566#pragma clang diagnostic push
567#pragma clang diagnostic ignored "-Wdouble-promotion"
568#endif
569
570EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator+(const half& a, const half& b) {
571 return half_from_rep(static_cast<native_half_t>(native_half(a) + native_half(b)));
572}
573EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator*(const half& a, const half& b) {
574 return half_from_rep(static_cast<native_half_t>(native_half(a) * native_half(b)));
575}
576EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator-(const half& a, const half& b) {
577 return half_from_rep(static_cast<native_half_t>(native_half(a) - native_half(b)));
578}
579EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator/(const half& a, const half& b) {
580 return half_from_rep(static_cast<native_half_t>(native_half(a) / native_half(b)));
581}
582EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator-(const half& a) {
583 return half_from_rep(static_cast<native_half_t>(-native_half(a)));
584}
585EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator+=(half& a, const half& b) {
586 a = a + b;
587 return a;
588}
589EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator*=(half& a, const half& b) {
590 a = a * b;
591 return a;
592}
593EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator-=(half& a, const half& b) {
594 a = a - b;
595 return a;
596}
597EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator/=(half& a, const half& b) {
598 a = a / b;
599 return a;
600}
601EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator==(const half& a, const half& b) {
602 return native_half(a) == native_half(b);
603}
604EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const half& a, const half& b) {
605 return native_half(a) != native_half(b);
606}
607EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const half& a, const half& b) {
608 return native_half(a) < native_half(b);
609}
610EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const half& a, const half& b) {
611 return native_half(a) <= native_half(b);
612}
613EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const half& a, const half& b) {
614 return native_half(a) > native_half(b);
615}
616EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const half& a, const half& b) {
617 return native_half(a) >= native_half(b);
618}
619
620#if EIGEN_COMP_CLANG
621#pragma clang diagnostic pop
622#endif
623
624// We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler,
625// invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation
626// of the functions, while the latter can only deal with one of them.
627#elif !defined(EIGEN_HAS_NATIVE_GPU_FP16) || (EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) // Emulate support for half floats
628
629#if EIGEN_COMP_CLANG && defined(EIGEN_GPUCC)
630// We need to provide emulated *host-side* FP16 operators for clang.
631#pragma push_macro("EIGEN_DEVICE_FUNC")
632#undef EIGEN_DEVICE_FUNC
633#if defined(EIGEN_GPUCC) && defined(EIGEN_HAS_NATIVE_GPU_FP16)
634#define EIGEN_DEVICE_FUNC __host__
635#else // both host and device need emulated ops.
636#define EIGEN_DEVICE_FUNC __host__ __device__
637#endif
638#endif
639
640// Definitions for CPUs and older HIP+CUDA, mostly working through conversion
641// to/from fp32.
642EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator+(const half& a, const half& b) { return half(float(a) + float(b)); }
643EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator*(const half& a, const half& b) { return half(float(a) * float(b)); }
644EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator-(const half& a, const half& b) { return half(float(a) - float(b)); }
645EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator/(const half& a, const half& b) { return half(float(a) / float(b)); }
646EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator-(const half& a) {
647 return raw_uint16_to_half(static_cast<numext::uint16_t>(raw_half_as_uint16(a) ^ 0x8000));
648}
649EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator+=(half& a, const half& b) {
650 a = half(float(a) + float(b));
651 return a;
652}
653EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator*=(half& a, const half& b) {
654 a = half(float(a) * float(b));
655 return a;
656}
657EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator-=(half& a, const half& b) {
658 a = half(float(a) - float(b));
659 return a;
660}
661EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator/=(half& a, const half& b) {
662 a = half(float(a) / float(b));
663 return a;
664}
665
666// Non-negative floating point numbers have a monotonic mapping to non-negative integers.
667// This property allows floating point numbers to be reinterpreted as integers for comparisons, which is useful if there
668// is no native floating point comparison operator. Floating point signedness is handled by the sign-magnitude
669// representation, whereas integers typically use two's complement. Converting the bit pattern from sign-magnitude to
670// two's complement allows the transformed bit patterns be compared as signed integers. All edge cases (+/-0 and +/-
671// infinity) are handled automatically, except NaN.
672//
673// fp16 uses 1 sign bit, 5 exponent bits, and 10 mantissa bits. The bit pattern conveys NaN when all the exponent
674// bits (5) are set, and at least one mantissa bit is set. The sign bit is irrelevant for determining NaN. To check for
675// NaN, clear the sign bit and check if the integral representation is greater than 01111100000000. To test
676// for non-NaN, clear the sign bit and check if the integral representation is less than or equal to 01111100000000.
677
678// convert sign-magnitude representation to two's complement
679EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC int16_t mapToSigned(uint16_t a) {
680#if EIGEN_COMP_NVHPC
681 // NVHPC through at least 26.5 can miscompile repeated inlined comparisons of the transformed integer representations.
682 EIGEN_OPTIMIZATION_BARRIER(a)
683#endif
684 constexpr uint16_t kAbsMask = (1 << 15) - 1;
685 // If the sign bit is set, clear the sign bit and return the (integer) negation. Otherwise, return the input.
686 return (a >> 15) ? -(a & kAbsMask) : a;
687}
688EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool isOrdered(const half& a, const half& b) {
689 constexpr uint16_t kInf = ((1 << 5) - 1) << 10;
690 constexpr uint16_t kAbsMask = (1 << 15) - 1;
691 return numext::maxi(a.x & kAbsMask, b.x & kAbsMask) <= kInf;
692}
693EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator==(const half& a, const half& b) {
694 bool result = mapToSigned(a.x) == mapToSigned(b.x);
695 result &= isOrdered(a, b);
696 return result;
697}
698EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const half& a, const half& b) { return !(a == b); }
699EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const half& a, const half& b) {
700 bool result = mapToSigned(a.x) < mapToSigned(b.x);
701 result &= isOrdered(a, b);
702 return result;
703}
704EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const half& a, const half& b) {
705 bool result = mapToSigned(a.x) <= mapToSigned(b.x);
706 result &= isOrdered(a, b);
707 return result;
708}
709EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const half& a, const half& b) {
710 bool result = mapToSigned(a.x) > mapToSigned(b.x);
711 result &= isOrdered(a, b);
712 return result;
713}
714EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const half& a, const half& b) {
715 bool result = mapToSigned(a.x) >= mapToSigned(b.x);
716 result &= isOrdered(a, b);
717 return result;
718}
719
720#if EIGEN_COMP_CLANG && defined(EIGEN_GPUCC)
721#pragma pop_macro("EIGEN_DEVICE_FUNC")
722#endif
723
724#endif // Emulate support for half floats
725
726// Division by an index. Do it in full float precision to avoid accuracy
727// issues in converting the denominator to half.
728EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator/(const half& a, Index b) {
729 return half(static_cast<float>(a) / static_cast<float>(b));
730}
731
732EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator++(half& a) {
733 a += half(1);
734 return a;
735}
736
737EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator--(half& a) {
738 a -= half(1);
739 return a;
740}
741
742EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator++(half& a, int) {
743 half original_value = a;
744 ++a;
745 return original_value;
746}
747
748EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator--(half& a, int) {
749 half original_value = a;
750 --a;
751 return original_value;
752}
753
754// Conversion routines, including fallbacks for the host or older CUDA.
755// Note that newer Intel CPUs (Haswell or newer) have vectorized versions of
756// these in hardware. If we need more performance on older/other CPUs, they are
757// also possible to vectorize directly.
758
759EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR __half_raw raw_uint16_to_half(numext::uint16_t x) {
760 // In the device compile phase of a GPU build we cannot simply do a "return __half_raw(x)",
761 // because there __half_raw is the vendor type (a union in the hip_fp16 header file) that has
762 // no uint16 constructor, and that will trigger a compile error. There "h.x = x" assigns the
763 // raw bits, since the vendor member x is an integer.
764 // In the host compile phase (and in non-GPU builds) Eigen's own __half_raw is in effect and its
765 // member x may be a native fp16 type (__fp16 on arm64, _Float16 with AVX512FP16 or riscv-zfh),
766 // so "h.x = x" would perform a numeric integer-to-float conversion that corrupts the raw bits;
767 // the explicit uint16 constructor reinterprets the bits for every storage type instead.
768 // Fortunately, since we need to disable EIGEN_CONSTEXPR for GPU anyway, we can get out
769 // of this catch22 by having separate bodies for the GPU device phase / everything else.
770#if defined(EIGEN_GPUCC) && defined(EIGEN_GPU_COMPILE_PHASE)
771 __half_raw h;
772 h.x = x;
773 return h;
774#else
775 return __half_raw(x);
776#endif
777}
778
779EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC numext::uint16_t raw_half_as_uint16(const __half_raw& h) {
780 // HIP/CUDA/Default have a member 'x' of type uint16_t.
781 // On ARM64 with FP16 scalar arithmetic, the member 'x' is of type __fp16, so we need to bit-cast.
782 // For SYCL, cl::sycl::half is _Float16, so cast directly.
783#if EIGEN_HAS_ARM64_FP16
784 return numext::bit_cast<numext::uint16_t>(h.x);
785#elif defined(EIGEN_HAS_BUILTIN_FLOAT16)
786 return numext::bit_cast<numext::uint16_t>(h.x);
787#elif defined(EIGEN_SYCL_DEVICE_ONLY)
788 return numext::bit_cast<numext::uint16_t>(h);
789#else
790 return h.x;
791#endif
792}
793
794template <>
795EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne<false>(float ff) {
796#if defined(EIGEN_GPU_COMPILE_PHASE)
797 __half tmp_ff = __float2half(ff);
798 return *(__half_raw*)&tmp_ff;
799
800#elif EIGEN_HAS_ARM64_FP16
801 return __half_raw(__half_raw::construct_from_rep_tag(), static_cast<__fp16>(ff));
802
803#elif defined(EIGEN_HAS_BUILTIN_FLOAT16)
804 __half_raw h;
805 h.x = static_cast<_Float16>(ff);
806 return h;
807
808#elif defined(EIGEN_HAS_FP16_C)
809 __half_raw h;
810 // Spell out the vector conversion to avoid Clang warning about the C99 compound literal used by _cvtss_sh.
811 h.x = static_cast<numext::uint16_t>(_mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(ff), 0), 0));
812 return h;
813
814#else
815 uint32_t f_bits = Eigen::numext::bit_cast<uint32_t>(ff);
816 const uint32_t f32infty_bits = {255 << 23};
817 const uint32_t f16max_bits = {(127 + 16) << 23};
818 const uint32_t denorm_magic_bits = {((127 - 15) + (23 - 10) + 1) << 23};
819 const uint32_t sign_mask = 0x80000000u;
820 __half_raw o;
821 o.x = static_cast<uint16_t>(0x0u);
822
823 const uint32_t sign = f_bits & sign_mask;
824 f_bits ^= sign;
825
826 // NOTE all the integer compares in this function can be safely
827 // compiled into signed compares since all operands are below
828 // 0x80000000. Important if you want fast straight SSE2 code
829 // (since there's no unsigned PCMPGTD).
830
831 if (f_bits >= f16max_bits) { // result is Inf or NaN (all exponent bits set)
832 o.x = (f_bits > f32infty_bits) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf
833 } else { // (De)normalized number or zero
834 if (f_bits < (113 << 23)) { // resulting FP16 is subnormal or zero
835 // use a magic value to align our 10 mantissa bits at the bottom of
836 // the float. as long as FP addition is round-to-nearest-even this
837 // just works.
838 f_bits = Eigen::numext::bit_cast<uint32_t>(Eigen::numext::bit_cast<float>(f_bits) +
839 Eigen::numext::bit_cast<float>(denorm_magic_bits));
840
841 // and one integer subtract of the bias later, we have our final float!
842 o.x = static_cast<numext::uint16_t>(f_bits - denorm_magic_bits);
843 } else {
844 const uint32_t mant_odd = (f_bits >> 13) & 1; // resulting mantissa is odd
845
846 // update exponent, rounding bias part 1
847 // Equivalent to `f.u += ((unsigned int)(15 - 127) << 23) + 0xfff`, but
848 // without arithmetic overflow.
849 f_bits += 0xc8000fffU;
850 // rounding bias part 2
851 f_bits += mant_odd;
852 // take the bits!
853 o.x = static_cast<numext::uint16_t>(f_bits >> 13);
854 }
855 }
856
857 o.x |= static_cast<numext::uint16_t>(sign >> 16);
858 return o;
859#endif
860}
861
862// float_to_half_rtne template specialization that assumes that its function
863// argument (ff) is either a normal floating-point number that does not
864// underflow to a half subnormal, +/-infinity, or zero. Used to improve
865// compile-time constant folding and conversion from an integer type to half.
866template <>
867EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half_raw float_to_half_rtne<true>(float ff) {
868#if defined(EIGEN_GPU_COMPILE_PHASE)
869 __half tmp_ff = __float2half(ff);
870 return *(__half_raw*)&tmp_ff;
871
872#elif EIGEN_HAS_ARM64_FP16
873 return __half_raw(__half_raw::construct_from_rep_tag(), static_cast<__fp16>(ff));
874
875#elif defined(EIGEN_HAS_BUILTIN_FLOAT16)
876 __half_raw h;
877 h.x = static_cast<_Float16>(ff);
878 return h;
879
880#else
881 uint32_t f_bits = Eigen::numext::bit_cast<uint32_t>(ff);
882 const uint32_t f16max_bits = {(127 + 16) << 23};
883 const uint32_t sign = f_bits & 0x80000000u;
884 f_bits ^= sign;
885
886 numext::uint16_t raw = 0x0000u;
887 if (f_bits >= f16max_bits) {
888 raw = 0x7c00u;
889 } else if (f_bits != 0) {
890 const uint32_t mant_odd = (f_bits >> 13) & 1u;
891 f_bits += 0xc8000fffU + mant_odd;
892 raw = static_cast<numext::uint16_t>(f_bits >> 13);
893 }
894 return raw_uint16_to_half(raw | static_cast<numext::uint16_t>(sign >> 16));
895#endif
896}
897
898EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float half_to_float(__half_raw h) {
899#if defined(EIGEN_GPU_COMPILE_PHASE)
900 return __half2float(h);
901#elif EIGEN_HAS_ARM64_FP16 || defined(EIGEN_HAS_BUILTIN_FLOAT16)
902 return static_cast<float>(native_half(h));
903#elif defined(EIGEN_HAS_FP16_C)
904#if EIGEN_COMP_MSVC
905 // MSVC does not have scalar instructions.
906 return _mm_cvtss_f32(_mm_cvtph_ps(_mm_set1_epi16(h.x)));
907#else
908 return _cvtsh_ss(h.x);
909#endif
910#else
911 const float magic = Eigen::numext::bit_cast<float>(static_cast<uint32_t>(113 << 23));
912 const uint32_t shifted_exp = 0x7c00 << 13; // exponent mask after shift
913 uint32_t o_bits = (h.x & 0x7fff) << 13; // exponent/mantissa bits
914 const uint32_t exp = shifted_exp & o_bits; // just the exponent
915 o_bits += (127 - 15) << 23; // exponent adjust
916
917 // handle exponent special cases
918 if (exp == shifted_exp) { // Inf/NaN?
919 o_bits += (128 - 16) << 23; // extra exp adjust
920 } else if (exp == 0) { // Zero/Denormal?
921 o_bits += 1 << 23; // extra exp adjust
922 // renormalize
923 o_bits = Eigen::numext::bit_cast<uint32_t>(Eigen::numext::bit_cast<float>(o_bits) - magic);
924 }
925
926 o_bits |= (h.x & 0x8000u) << 16; // sign bit
927 return Eigen::numext::bit_cast<float>(o_bits);
928#endif
929}
930
931// --- standard functions ---
932
933EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isinf)(const half& a) { return (raw_half_as_uint16(a) & 0x7fff) == 0x7c00; }
934EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isnan)(const half& a) {
935#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
936 return __hisnan(a);
937#else
938 return (raw_half_as_uint16(a) & 0x7fff) > 0x7c00;
939#endif
940}
941EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isfinite)(const half& a) {
942 return (raw_half_as_uint16(a) & 0x7fff) < 0x7c00;
943}
944
945EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half abs(const half& a) {
946#if defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC)
947 return half_from_rep(vabsh_f16(a.x));
948#else
949 return raw_uint16_to_half(static_cast<numext::uint16_t>(raw_half_as_uint16(a) & 0x7FFF));
950#endif
951}
952EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half exp(const half& a) {
953#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
954 return half(hexp(::__half(a)));
955#else
956 return half(numext::exp(float(a)));
957#endif
958}
959EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half exp2(const half& a) {
960#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
961 return half(hexp2(::__half(a)));
962#else
963 return half(numext::exp2(float(a)));
964#endif
965}
966EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half expm1(const half& a) { return half(numext::expm1(float(a))); }
967// float covers the half range, so the single conversion back rounds correctly.
968EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half ldexp(const half& a, int exponent) {
969 return half(numext::ldexp(float(a), exponent));
970}
971EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log(const half& a) {
972#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
973 return half(hlog(::__half(a)));
974#else
975 return half(numext::log(float(a)));
976#endif
977}
978EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log1p(const half& a) { return half(numext::log1p(float(a))); }
979EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log10(const half& a) { return half(numext::log10(float(a))); }
980EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log2(const half& a) { return half(numext::log2(float(a))); }
981
982EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sqrt(const half& a) {
983#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
984 return half(hsqrt(::__half(a)));
985#else
986 return half(numext::sqrt(float(a)));
987#endif
988}
989EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half pow(const half& a, const half& b) {
990 return half(numext::pow(float(a), float(b)));
991}
992EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half atan2(const half& a, const half& b) {
993 return half(numext::atan2(float(a), float(b)));
994}
995EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sin(const half& a) { return half(numext::sin(float(a))); }
996EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half cos(const half& a) { return half(numext::cos(float(a))); }
997EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tan(const half& a) { return half(numext::tan(float(a))); }
998EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tanh(const half& a) {
999 // Select the scalar template, not the host fast-math overload, to preserve half rounding.
1000 return half(numext::tanh<float>(float(a)));
1001}
1002EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half asin(const half& a) { return half(numext::asin(float(a))); }
1003EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half acos(const half& a) { return half(numext::acos(float(a))); }
1004EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half atan(const half& a) { return half(numext::atan(float(a))); }
1005EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half atanh(const half& a) { return half(numext::atanh(float(a))); }
1006EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half floor(const half& a) {
1007#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
1008 return half(hfloor(::__half(a)));
1009#else
1010 return half(numext::floor(float(a)));
1011#endif
1012}
1013EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half ceil(const half& a) {
1014#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
1015 return half(hceil(::__half(a)));
1016#else
1017 return half(numext::ceil(float(a)));
1018#endif
1019}
1020EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half rint(const half& a) { return half(numext::rint(float(a))); }
1021EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half round(const half& a) { return half(numext::round(float(a))); }
1022EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half trunc(const half& a) { return half(numext::trunc(float(a))); }
1023EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half fmod(const half& a, const half& b) {
1024 return half(numext::fmod(float(a), float(b)));
1025}
1026
1027EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half(min)(const half& a, const half& b) { return b < a ? b : a; }
1028
1029EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half(max)(const half& a, const half& b) { return a < b ? b : a; }
1030
1031// a * b + c for half or bfloat16 a, b and c, rounded to odd in double: narrowing the result to half or bfloat16, which
1032// rounds to odd in float and then to nearest, then rounds a * b + c once. A float fma rounds twice instead: for half,
1033// fma(33/32, 65/64, 2^-24) rounds to the midpoint 2145/2048 in float, then to even, 1 + 3/64, not up to 1 + 49/1024.
1034// a * b is exact in double, and TwoSum (Knuth) gives the error e of s = a * b + c exactly. Device code does not use
1035// it: GPUs run double at a fraction of the float rate, and SYCL devices need not support it.
1036EIGEN_DEVICE_FUNC inline double fma_round_to_odd(float a, float b, float c) {
1037 const double p = static_cast<double>(a) * static_cast<double>(b);
1038 const double c_wide = static_cast<double>(c);
1039 const double s = p + c_wide;
1040 // p is exact, so s is nonfinite only for a nonfinite operand; TwoSum would then evaluate inf - inf (FE_INVALID).
1041 if (!(numext::isfinite)(s)) return s;
1042 const double p_part = s - c_wide;
1043 const double e = (p - p_part) + (c_wide - (s - p_part));
1044 if (e == 0) return s;
1045 // Inexact: truncate toward zero, then set the last bit.
1046 const bool away = (s > 0) != (e > 0);
1047 return numext::bit_cast<double>((numext::bit_cast<numext::uint64_t>(s) - (away ? 1u : 0u)) | 1u);
1048}
1049
1050EIGEN_DEVICE_FUNC inline half fma(const half& a, const half& b, const half& c) {
1051#if defined(EIGEN_HAS_NATIVE_GPU_FP16)
1052 // Rounded once in hardware.
1053 return __hfma(::__half(a), ::__half(b), ::__half(c));
1054#elif defined(EIGEN_GPU_COMPILE_PHASE) || defined(__SYCL_DEVICE_ONLY__)
1055 // Rounds twice; see fma_round_to_odd.
1056 return half(numext::fma(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c)));
1057#elif defined(EIGEN_HAS_ARM64_FP16_SCALAR_ARITHMETIC)
1058 return half_from_rep(vfmah_f16(c.x, a.x, b.x));
1059#elif defined(EIGEN_VECTORIZE_AVX512FP16)
1060 // Reduces to vfmadd213sh.
1061 return half(_mm_cvtsh_h(_mm_fmadd_ph(_mm_set_sh(a.x), _mm_set_sh(b.x), _mm_set_sh(c.x))));
1062#else
1063 return half(fma_round_to_odd(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c)));
1064#endif
1065}
1066
1067#ifndef EIGEN_NO_IO
1068EIGEN_ALWAYS_INLINE std::ostream& operator<<(std::ostream& os, const half& v) {
1069 os << static_cast<float>(v);
1070 return os;
1071}
1072#endif
1073
1074} // end namespace half_impl
1075
1076// import Eigen::half_impl::half into Eigen namespace
1077// using half_impl::half;
1078
1079namespace internal {
1080
1081template <>
1082struct is_arithmetic<half> : std::true_type {};
1083
1084template <>
1085struct random_impl<half> {
1086 enum : int { MantissaBits = 10 };
1087 using Impl = random_impl<float>;
1088 static EIGEN_DEVICE_FUNC inline half run(const half& x, const half& y) {
1089 float result = Impl::run(x, y, MantissaBits);
1090 return half(result);
1091 }
1092 static EIGEN_DEVICE_FUNC inline half run() {
1093 float result = Impl::run(MantissaBits);
1094 return half(result);
1095 }
1096};
1097
1098} // end namespace internal
1099
1100template <>
1101struct NumTraits<Eigen::half> : GenericNumTraits<Eigen::half> {
1102 enum { IsSigned = true, IsInteger = false, IsComplex = false, RequireInitialization = false };
1103
1104 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half epsilon() {
1105 // 0x1400 is 2^-10, the fp16 machine epsilon, matching std::numeric_limits<Eigen::half>::epsilon().
1106 return half_impl::raw_uint16_to_half(0x1400);
1107 }
1108 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half dummy_precision() {
1109 return half_impl::raw_uint16_to_half(0x211f); // Eigen::half(1e-2f);
1110 }
1111 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half highest() {
1112 return half_impl::raw_uint16_to_half(0x7bff);
1113 }
1114 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half lowest() {
1115 return half_impl::raw_uint16_to_half(0xfbff);
1116 }
1117 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half infinity() {
1118 return half_impl::raw_uint16_to_half(0x7c00);
1119 }
1120 EIGEN_DEVICE_FUNC _EIGEN_MAYBE_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::half quiet_NaN() {
1121 return half_impl::raw_uint16_to_half(0x7e00);
1122 }
1123};
1124
1125} // end namespace Eigen
1126
1127#undef _EIGEN_MAYBE_CONSTEXPR
1128
1129namespace Eigen {
1130namespace numext {
1131
1132#if defined(EIGEN_GPU_COMPILE_PHASE)
1133
1134template <>
1135EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isnan)(const Eigen::half& h) {
1136 return (half_impl::isnan)(h);
1137}
1138
1139template <>
1140EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isinf)(const Eigen::half& h) {
1141 return (half_impl::isinf)(h);
1142}
1143
1144template <>
1145EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isfinite)(const Eigen::half& h) {
1146 return (half_impl::isfinite)(h);
1147}
1148
1149#endif
1150
1151template <>
1152EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half bit_cast<Eigen::half, uint16_t>(const uint16_t& src) {
1153 return Eigen::half(Eigen::half_impl::raw_uint16_to_half(src));
1154}
1155
1156template <>
1157EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast<uint16_t, Eigen::half>(const Eigen::half& src) {
1158 return Eigen::half_impl::raw_half_as_uint16(src);
1159}
1160
1161EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half nextafter(const Eigen::half& from, const Eigen::half& to) {
1162 if (numext::isnan EIGEN_NOT_A_MACRO(from)) {
1163 return from;
1164 }
1165 if (numext::isnan EIGEN_NOT_A_MACRO(to)) {
1166 return to;
1167 }
1168 if (from == to) {
1169 return to;
1170 }
1171 uint16_t from_bits = numext::bit_cast<uint16_t>(from);
1172 bool from_sign = from_bits >> 15;
1173 if ((from_bits & 0x7fff) == 0) {
1174 // From ±0 toward a nonzero value: the neighbor is the smallest subnormal
1175 // carrying the sign of the direction (IEEE-754 nextUp/nextDown of zero).
1176 from_bits = (to > from) ? uint16_t(0x0001) : uint16_t(0x8001);
1177 } else if ((to > from) != from_sign) {
1178 // Toward the infinity with the same sign as from: increase the magnitude.
1179 ++from_bits;
1180 } else {
1181 --from_bits;
1182 }
1183 return numext::bit_cast<Eigen::half>(from_bits);
1184}
1185
1186// Specialize multiply-add to match packet operations and reduce conversions to/from float.
1187template <>
1188EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half madd<Eigen::half>(const Eigen::half& x, const Eigen::half& y,
1189 const Eigen::half& z) {
1190 return Eigen::half(static_cast<float>(x) * static_cast<float>(y) + static_cast<float>(z));
1191}
1192
1193} // namespace numext
1194} // namespace Eigen
1195
1196// Warp shuffle overloads for Eigen::half.
1197// CUDA uses __shfl_*_sync (with mask); HIP uses __shfl_* (no mask).
1198// Note that the following are __device__ - only functions.
1199#if defined(EIGEN_CUDACC) || defined(EIGEN_HIPCC)
1200
1201#if defined(EIGEN_CUDACC)
1202
1203__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_sync(unsigned mask, Eigen::half var, int srcLane,
1204 int width = warpSize) {
1205 const __half h = var;
1206 return static_cast<Eigen::half>(__shfl_sync(mask, h, srcLane, width));
1207}
1208
1209__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_up_sync(unsigned mask, Eigen::half var, unsigned int delta,
1210 int width = warpSize) {
1211 const __half h = var;
1212 return static_cast<Eigen::half>(__shfl_up_sync(mask, h, delta, width));
1213}
1214
1215__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_down_sync(unsigned mask, Eigen::half var, unsigned int delta,
1216 int width = warpSize) {
1217 const __half h = var;
1218 return static_cast<Eigen::half>(__shfl_down_sync(mask, h, delta, width));
1219}
1220
1221__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_xor_sync(unsigned mask, Eigen::half var, int laneMask,
1222 int width = warpSize) {
1223 const __half h = var;
1224 return static_cast<Eigen::half>(__shfl_xor_sync(mask, h, laneMask, width));
1225}
1226
1227#else // HIP
1228
1229__device__ EIGEN_STRONG_INLINE Eigen::half __shfl(Eigen::half var, int srcLane, int width = warpSize) {
1230 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1231 return Eigen::numext::bit_cast<Eigen::half>(static_cast<Eigen::numext::uint16_t>(__shfl(ivar, srcLane, width)));
1232}
1233
1234__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_up(Eigen::half var, unsigned int delta, int width = warpSize) {
1235 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1236 return Eigen::numext::bit_cast<Eigen::half>(static_cast<Eigen::numext::uint16_t>(__shfl_up(ivar, delta, width)));
1237}
1238
1239__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_down(Eigen::half var, unsigned int delta, int width = warpSize) {
1240 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1241 return Eigen::numext::bit_cast<Eigen::half>(static_cast<Eigen::numext::uint16_t>(__shfl_down(ivar, delta, width)));
1242}
1243
1244__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_xor(Eigen::half var, int laneMask, int width = warpSize) {
1245 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1246 return Eigen::numext::bit_cast<Eigen::half>(static_cast<Eigen::numext::uint16_t>(__shfl_xor(ivar, laneMask, width)));
1247}
1248
1249#endif // HIP vs CUDA
1250#endif // __shfl*
1251
1252// ldg() has an overload for __half_raw, but we also need one for Eigen::half.
1253#if defined(EIGEN_CUDACC) || defined(EIGEN_HIPCC)
1254EIGEN_STRONG_INLINE __device__ Eigen::half __ldg(const Eigen::half* ptr) {
1255 return Eigen::half_impl::raw_uint16_to_half(__ldg(reinterpret_cast<const Eigen::numext::uint16_t*>(ptr)));
1256}
1257#endif // __ldg
1258
1259#if EIGEN_HAS_STD_HASH
1260namespace std {
1261template <>
1262struct hash<Eigen::half> {
1263 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::half& a) const {
1264 // -0 == +0, so both must hash alike.
1265 const Eigen::numext::uint16_t bits = Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a);
1266 return static_cast<std::size_t>((bits & 0x7fff) == 0 ? 0 : bits);
1267 }
1268};
1269} // end namespace std
1270#endif
1271
1272namespace Eigen {
1273namespace internal {
1274
1275template <>
1276struct cast_impl<float, half> {
1277 EIGEN_DEVICE_FUNC static inline half run(const float& a) {
1278#if defined(EIGEN_GPU_COMPILE_PHASE)
1279 return __float2half(a);
1280#else
1281 return half(a);
1282#endif
1283 }
1284};
1285
1286template <>
1287struct cast_impl<int, half> {
1288 EIGEN_DEVICE_FUNC static inline half run(const int& a) {
1289#if defined(EIGEN_GPU_COMPILE_PHASE)
1290 return __float2half(static_cast<float>(a));
1291#else
1292 return half(static_cast<float>(a));
1293#endif
1294 }
1295};
1296
1297template <>
1298struct cast_impl<half, float> {
1299 EIGEN_DEVICE_FUNC static inline float run(const half& a) {
1300#if defined(EIGEN_GPU_COMPILE_PHASE)
1301 return __half2float(a);
1302#else
1303 return static_cast<float>(a);
1304#endif
1305 }
1306};
1307
1308} // namespace internal
1309} // namespace Eigen
1310
1311#endif // EIGEN_HALF_H