17#ifndef EIGEN_BFLOAT16_H
18#define EIGEN_BFLOAT16_H
21#include "../../InternalHeaderCheck.h"
23#if defined(EIGEN_HAS_HIP_BF16)
30#pragma push_macro("EIGEN_CONSTEXPR")
32#define EIGEN_CONSTEXPR
35#define BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, METHOD) \
37 EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED PACKET_BF16 METHOD<PACKET_BF16>( \
38 const PACKET_BF16& _x) { \
39 return F32ToBf16(METHOD<PACKET_F>(Bf16ToF32(_x))); \
46#define BF16_PACKET_FUNCTION_HALVES(PACKET_F, PACKET_BF16, METHOD) \
48 EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED PACKET_BF16 METHOD<PACKET_BF16>( \
49 const PACKET_BF16& _x) { \
50 return F32ToBf16(METHOD<PACKET_F>(Bf16ToF32Low(_x)), METHOD<PACKET_F>(Bf16ToF32High(_x))); \
56#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \
57 FUNCTION(PACKET_F, PACKET_BF16, pcos) \
58 FUNCTION(PACKET_F, PACKET_BF16, psin) \
59 FUNCTION(PACKET_F, PACKET_BF16, psinh) \
60 FUNCTION(PACKET_F, PACKET_BF16, pcosh) \
61 FUNCTION(PACKET_F, PACKET_BF16, pasinh) \
62 FUNCTION(PACKET_F, PACKET_BF16, pacosh) \
63 FUNCTION(PACKET_F, PACKET_BF16, pexp) \
64 FUNCTION(PACKET_F, PACKET_BF16, pexp2) \
65 FUNCTION(PACKET_F, PACKET_BF16, pexpm1) \
66 FUNCTION(PACKET_F, PACKET_BF16, plog) \
67 FUNCTION(PACKET_F, PACKET_BF16, plog1p) \
68 FUNCTION(PACKET_F, PACKET_BF16, plog2) \
69 FUNCTION(PACKET_F, PACKET_BF16, plog10) \
70 FUNCTION(PACKET_F, PACKET_BF16, preciprocal) \
71 FUNCTION(PACKET_F, PACKET_BF16, pcbrt) \
72 FUNCTION(PACKET_F, PACKET_BF16, ptanh)
74#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(PACKET_F, PACKET_BF16) \
75 EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16)
78#define EIGEN_INSTANTIATE_GENERIC_SQRT_FUNCS_BF16(PACKET_F, PACKET_BF16) \
79 BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, psqrt) \
80 BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, prsqrt)
83#define EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \
84 FUNCTION(PACKET_F, PACKET_BF16, perf) \
85 FUNCTION(PACKET_F, PACKET_BF16, pndtri)
87#define EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16(PACKET_F, PACKET_BF16) \
88 EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16)
90#define EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \
91 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0) \
92 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0e) \
93 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1) \
94 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1e) \
95 FUNCTION(PACKET_F, PACKET_BF16, pbessel_j0) \
96 FUNCTION(PACKET_F, PACKET_BF16, pbessel_j1) \
97 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0) \
98 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0e) \
99 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1) \
100 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1e) \
101 FUNCTION(PACKET_F, PACKET_BF16, pbessel_y0) \
102 FUNCTION(PACKET_F, PACKET_BF16, pbessel_y1)
104#define EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16(PACKET_F, PACKET_BF16) \
105 EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16)
108#if defined(EIGEN_HAS_HIP_BF16) && defined(EIGEN_GPU_COMPILE_PHASE)
109#define EIGEN_USE_HIP_BF16
118EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast<Eigen::bfloat16, uint16_t>(
const uint16_t& src);
121EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast<uint16_t, Eigen::bfloat16>(
const Eigen::bfloat16& src);
123namespace bfloat16_impl {
125#if defined(EIGEN_CUDACC)
132#define EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR
134#define EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR EIGEN_CONSTEXPR
137#if defined(EIGEN_USE_HIP_BF16)
139struct __bfloat16_raw :
public hip_bfloat16 {
140 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() {}
141 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(hip_bfloat16 hb) : hip_bfloat16(hb) {}
142 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(
unsigned short raw) : hip_bfloat16(raw) {}
148struct __bfloat16_raw {
149#if defined(EIGEN_CUDACC) || (defined(EIGEN_HAS_HIP_BF16) && !defined(EIGEN_GPU_COMPILE_PHASE))
150 EIGEN_DEVICE_FUNC EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR __bfloat16_raw() {}
152 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() : value(0) {}
154 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(
unsigned short raw) : value(raw) {}
155 unsigned short value;
160EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(
unsigned short value);
161template <
bool AssumeArgumentIsNormalOrInfinityOrZero>
162EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(
float ff);
166EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<false>(
float ff);
168EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<true>(
float ff);
169EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
float bfloat16_to_float(__bfloat16_raw h);
171struct bfloat16_base :
public __bfloat16_raw {
172 EIGEN_DEVICE_FUNC EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR bfloat16_base() {}
173 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16_base(
const __bfloat16_raw& h) : __bfloat16_raw(h) {}
179struct bfloat16 :
public bfloat16_impl::bfloat16_base {
180 using __bfloat16_raw = bfloat16_impl::__bfloat16_raw;
182 EIGEN_DEVICE_FUNC EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR bfloat16() {}
184 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(
const __bfloat16_raw& h) : bfloat16_impl::bfloat16_base(h) {}
186 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(
bool b)
187 : bfloat16_impl::bfloat16_base(bfloat16_impl::raw_uint16_to_bfloat16(b ? 0x3f80 : 0)) {}
190 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(T val)
191 : bfloat16_impl::bfloat16_base(
192 bfloat16_impl::float_to_bfloat16_rtne<std::is_integral<T>::value>(half_impl::narrow_to_float(val))) {}
194 explicit EIGEN_DEVICE_FUNC bfloat16(
float f)
195 : bfloat16_impl::bfloat16_base(bfloat16_impl::float_to_bfloat16_rtne<false>(f)) {}
199 template <
typename RealScalar>
200 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(
const std::complex<RealScalar>& val)
201 : bfloat16_impl::bfloat16_base(
202 bfloat16_impl::float_to_bfloat16_rtne<false>(half_impl::narrow_to_float(val.real()))) {}
204 EIGEN_DEVICE_FUNC
operator float()
const {
205 return bfloat16_impl::bfloat16_to_float(*
this);
211namespace bfloat16_impl {
212template <
typename =
void>
213struct numeric_limits_bfloat16_impl {
214 static EIGEN_CONSTEXPR
const bool is_specialized =
true;
215 static EIGEN_CONSTEXPR
const bool is_signed =
true;
216 static EIGEN_CONSTEXPR
const bool is_integer =
false;
217 static EIGEN_CONSTEXPR
const bool is_exact =
false;
218 static EIGEN_CONSTEXPR
const bool has_infinity =
true;
219 static EIGEN_CONSTEXPR
const bool has_quiet_NaN =
true;
220 static EIGEN_CONSTEXPR
const bool has_signaling_NaN =
true;
221 EIGEN_DIAGNOSTICS(push)
222 EIGEN_DISABLE_DEPRECATED_WARNING
223 static EIGEN_CONSTEXPR
const std::float_denorm_style has_denorm = std::denorm_present;
224 static EIGEN_CONSTEXPR
const bool has_denorm_loss =
false;
225 EIGEN_DIAGNOSTICS(pop)
226 static EIGEN_CONSTEXPR
const std::float_round_style round_style = std::numeric_limits<float>::round_style;
227 static EIGEN_CONSTEXPR
const bool is_iec559 =
true;
230 static EIGEN_CONSTEXPR
const bool is_bounded =
true;
231 static EIGEN_CONSTEXPR
const bool is_modulo =
false;
232 static EIGEN_CONSTEXPR
const int digits = 8;
233 static EIGEN_CONSTEXPR
const int digits10 = 2;
234 static EIGEN_CONSTEXPR
const int max_digits10 = 4;
235 static EIGEN_CONSTEXPR
const int radix = std::numeric_limits<float>::radix;
236 static EIGEN_CONSTEXPR
const int min_exponent = std::numeric_limits<float>::min_exponent;
237 static EIGEN_CONSTEXPR
const int min_exponent10 = std::numeric_limits<float>::min_exponent10;
238 static EIGEN_CONSTEXPR
const int max_exponent = std::numeric_limits<float>::max_exponent;
239 static EIGEN_CONSTEXPR
const int max_exponent10 = std::numeric_limits<float>::max_exponent10;
240 static EIGEN_CONSTEXPR
const bool traps = std::numeric_limits<float>::traps;
243 static EIGEN_CONSTEXPR
const bool tinyness_before = std::numeric_limits<float>::tinyness_before;
245 static EIGEN_CONSTEXPR Eigen::bfloat16(min)() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0080); }
246 static EIGEN_CONSTEXPR Eigen::bfloat16 lowest() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0xff7f); }
247 static EIGEN_CONSTEXPR Eigen::bfloat16(max)() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f7f); }
248 static EIGEN_CONSTEXPR Eigen::bfloat16 epsilon() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3c00); }
249 static EIGEN_CONSTEXPR Eigen::bfloat16 round_error() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3f00); }
250 static EIGEN_CONSTEXPR Eigen::bfloat16 infinity() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f80); }
251 static EIGEN_CONSTEXPR Eigen::bfloat16 quiet_NaN() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0); }
252 static EIGEN_CONSTEXPR Eigen::bfloat16 signaling_NaN() {
253 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fa0);
255 static EIGEN_CONSTEXPR Eigen::bfloat16 denorm_min() {
return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0001); }
259#if EIGEN_COMP_CXXVER < 17
261EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::is_specialized;
263EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::is_signed;
265EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::is_integer;
267EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::is_exact;
269EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::has_infinity;
271EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::has_quiet_NaN;
273EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::has_signaling_NaN;
274EIGEN_DIAGNOSTICS(push)
275EIGEN_DISABLE_DEPRECATED_WARNING
277EIGEN_CONSTEXPR
const std::float_denorm_style numeric_limits_bfloat16_impl<T>::has_denorm;
279EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::has_denorm_loss;
280EIGEN_DIAGNOSTICS(pop)
282EIGEN_CONSTEXPR
const std::float_round_style numeric_limits_bfloat16_impl<T>::round_style;
284EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::is_iec559;
286EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::is_bounded;
288EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::is_modulo;
290EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::digits;
292EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::digits10;
294EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::max_digits10;
296EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::radix;
298EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::min_exponent;
300EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::min_exponent10;
302EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::max_exponent;
304EIGEN_CONSTEXPR
const int numeric_limits_bfloat16_impl<T>::max_exponent10;
306EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::traps;
308EIGEN_CONSTEXPR
const bool numeric_limits_bfloat16_impl<T>::tinyness_before;
319class numeric_limits<Eigen::bfloat16> :
public Eigen::bfloat16_impl::numeric_limits_bfloat16_impl<> {};
321class numeric_limits<const Eigen::bfloat16> :
public numeric_limits<Eigen::bfloat16> {};
323class numeric_limits<volatile Eigen::bfloat16> :
public numeric_limits<Eigen::bfloat16> {};
325class numeric_limits<const volatile Eigen::bfloat16> :
public numeric_limits<Eigen::bfloat16> {};
330namespace bfloat16_impl {
335#if !defined(EIGEN_HAS_NATIVE_BF16) || (EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC)
337#if EIGEN_COMP_CLANG && defined(EIGEN_GPUCC)
339#pragma push_macro("EIGEN_DEVICE_FUNC")
340#undef EIGEN_DEVICE_FUNC
341#if (defined(EIGEN_HAS_GPU_BF16) && defined(EIGEN_HAS_NATIVE_BF16))
342#define EIGEN_DEVICE_FUNC __host__
344#define EIGEN_DEVICE_FUNC __host__ __device__
351EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(
const bfloat16& a,
const bfloat16& b) {
352 return bfloat16(
float(a) +
float(b));
354EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(
const bfloat16& a,
const int& b) {
355 return bfloat16(
float(a) +
static_cast<float>(b));
357EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(
const int& a,
const bfloat16& b) {
358 return bfloat16(
static_cast<float>(a) +
float(b));
360EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator*(
const bfloat16& a,
const bfloat16& b) {
361 return bfloat16(
float(a) *
float(b));
363EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator-(
const bfloat16& a,
const bfloat16& b) {
364 return bfloat16(
float(a) -
float(b));
366EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator/(
const bfloat16& a,
const bfloat16& b) {
367 return bfloat16(
float(a) /
float(b));
369EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator-(
const bfloat16& a) {
370 numext::uint16_t x = numext::bit_cast<uint16_t>(a) ^ 0x8000;
371 return numext::bit_cast<bfloat16>(x);
373EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator+=(bfloat16& a,
const bfloat16& b) {
374 a = bfloat16(
float(a) +
float(b));
377EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator*=(bfloat16& a,
const bfloat16& b) {
378 a = bfloat16(
float(a) *
float(b));
381EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator-=(bfloat16& a,
const bfloat16& b) {
382 a = bfloat16(
float(a) -
float(b));
385EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator/=(bfloat16& a,
const bfloat16& b) {
386 a = bfloat16(
float(a) /
float(b));
389EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a) {
393EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a) {
397EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a,
int) {
398 bfloat16 original_value = a;
400 return original_value;
402EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a,
int) {
403 bfloat16 original_value = a;
405 return original_value;
407#if defined(EIGEN_GPU_COMPILE_PHASE)
414EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator==(
const bfloat16& a,
const bfloat16& b) {
415 return numext::equal_strict(
float(a),
float(b));
417EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator!=(
const bfloat16& a,
const bfloat16& b) {
418 return numext::not_equal_strict(
float(a),
float(b));
420EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator<(
const bfloat16& a,
const bfloat16& b) {
421 return float(a) < float(b);
423EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator<=(
const bfloat16& a,
const bfloat16& b) {
424 return float(a) <= float(b);
426EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator>(
const bfloat16& a,
const bfloat16& b) {
427 return float(a) > float(b);
429EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator>=(
const bfloat16& a,
const bfloat16& b) {
430 return float(a) >= float(b);
433EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC int16_t bfloat16_map_to_signed(numext::uint16_t bits) {
434 constexpr numext::uint16_t kAbsMask = 0x7fff;
435 return (bits >> 15) ? -(bits & kAbsMask) : bits;
437EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool bfloat16_is_ordered(numext::uint16_t a, numext::uint16_t b) {
438 constexpr numext::uint16_t kAbsMask = 0x7fff;
439 constexpr numext::uint16_t kInf = 0x7f80;
440 return numext::maxi(a & kAbsMask, b & kAbsMask) <= kInf;
444EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator==(
const bfloat16& a,
const bfloat16& b) {
445 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
446 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
447 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) == bfloat16_map_to_signed(b_bits)) &
448 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
450EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator!=(
const bfloat16& a,
const bfloat16& b) {
return !(a == b); }
451EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator<(
const bfloat16& a,
const bfloat16& b) {
452 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
453 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
454 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) < bfloat16_map_to_signed(b_bits)) &
455 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
457EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator<=(
const bfloat16& a,
const bfloat16& b) {
458 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
459 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
460 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) <= bfloat16_map_to_signed(b_bits)) &
461 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
463EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator>(
const bfloat16& a,
const bfloat16& b) {
464 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
465 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
466 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) > bfloat16_map_to_signed(b_bits)) &
467 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
469EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator>=(
const bfloat16& a,
const bfloat16& b) {
470 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
471 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
472 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) >= bfloat16_map_to_signed(b_bits)) &
473 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
477#if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC)
478#pragma pop_macro("EIGEN_DEVICE_FUNC")
484EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator/(
const bfloat16& a, Index b) {
485 return bfloat16(
static_cast<float>(a) /
static_cast<float>(b));
488EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw truncate_to_bfloat16(
const float v) {
489#if defined(EIGEN_USE_HIP_BF16)
490 return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(v, __bfloat16_raw::truncate));
492 __bfloat16_raw output;
493 if (numext::isnan EIGEN_NOT_A_MACRO(v)) {
494 output.value = std::signbit(v) ? 0xFFC0 : 0x7FC0;
497 output.value =
static_cast<numext::uint16_t
>(numext::bit_cast<numext::uint32_t>(v) >> 16);
502EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(numext::uint16_t value) {
503#if defined(EIGEN_USE_HIP_BF16)
508 return __bfloat16_raw(value);
512EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR numext::uint16_t raw_bfloat16_as_uint16(
513 const __bfloat16_raw& bf) {
514#if defined(EIGEN_USE_HIP_BF16)
524EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<false>(
float ff) {
525#if defined(EIGEN_USE_HIP_BF16)
526 return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff));
528 __bfloat16_raw output;
530 if (numext::isnan EIGEN_NOT_A_MACRO(ff)) {
536 output.value = std::signbit(ff) ? 0xFFC0 : 0x7FC0;
687 output = float_to_bfloat16_rtne<true>(ff);
698EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<true>(
float ff) {
699#if defined(EIGEN_USE_HIP_BF16)
700 return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff));
702 numext::uint32_t input = numext::bit_cast<numext::uint32_t>(ff);
703 __bfloat16_raw output;
706 numext::uint32_t lsb = (input >> 16) & 1;
707 numext::uint32_t rounding_bias = 0x7fff + lsb;
708 input += rounding_bias;
709 output.value =
static_cast<numext::uint16_t
>(input >> 16);
714EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
float bfloat16_to_float(__bfloat16_raw h) {
715#if defined(EIGEN_USE_HIP_BF16)
716 return static_cast<float>(h);
718 return numext::bit_cast<float>(
static_cast<numext::uint32_t
>(h.value) << 16);
724EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isinf)(
const bfloat16& a) {
725 return (raw_bfloat16_as_uint16(a) & 0x7fff) == 0x7f80;
727EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isnan)(
const bfloat16& a) {
728 return (raw_bfloat16_as_uint16(a) & 0x7fff) > 0x7f80;
730EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isfinite)(
const bfloat16& a) {
731 return (raw_bfloat16_as_uint16(a) & 0x7fff) < 0x7f80;
734EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 abs(
const bfloat16& a) {
735 numext::uint16_t x = numext::bit_cast<numext::uint16_t>(a) & 0x7FFF;
736 return numext::bit_cast<bfloat16>(x);
738EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp(
const bfloat16& a) {
return bfloat16(numext::exp(
float(a))); }
739EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp2(
const bfloat16& a) {
return bfloat16(numext::exp2(
float(a))); }
740EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 expm1(
const bfloat16& a) {
return bfloat16(numext::expm1(
float(a))); }
742EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 ldexp(
const bfloat16& a,
int exponent) {
743 return bfloat16(numext::ldexp(
float(a), exponent));
745EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log(
const bfloat16& a) {
return bfloat16(numext::log(
float(a))); }
746EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log1p(
const bfloat16& a) {
return bfloat16(numext::log1p(
float(a))); }
747EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log10(
const bfloat16& a) {
return bfloat16(numext::log10(
float(a))); }
748EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log2(
const bfloat16& a) {
return bfloat16(numext::log2(
float(a))); }
749EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sqrt(
const bfloat16& a) {
return bfloat16(numext::sqrt(
float(a))); }
750EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cbrt(
const bfloat16& a) {
return bfloat16(numext::cbrt(
float(a))); }
751EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 pow(
const bfloat16& a,
const bfloat16& b) {
752 return bfloat16(numext::pow(
float(a),
float(b)));
754EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan2(
const bfloat16& a,
const bfloat16& b) {
755 return bfloat16(numext::atan2(
float(a),
float(b)));
757EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sin(
const bfloat16& a) {
return bfloat16(numext::sin(
float(a))); }
758EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cos(
const bfloat16& a) {
return bfloat16(numext::cos(
float(a))); }
759EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tan(
const bfloat16& a) {
return bfloat16(numext::tan(
float(a))); }
760EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asin(
const bfloat16& a) {
return bfloat16(numext::asin(
float(a))); }
761EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acos(
const bfloat16& a) {
return bfloat16(numext::acos(
float(a))); }
762EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan(
const bfloat16& a) {
return bfloat16(numext::atan(
float(a))); }
763EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sinh(
const bfloat16& a) {
return bfloat16(numext::sinh(
float(a))); }
764EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cosh(
const bfloat16& a) {
return bfloat16(numext::cosh(
float(a))); }
765EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tanh(
const bfloat16& a) {
767 return bfloat16(numext::tanh<float>(
float(a)));
769EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asinh(
const bfloat16& a) {
return bfloat16(numext::asinh(
float(a))); }
770EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acosh(
const bfloat16& a) {
return bfloat16(numext::acosh(
float(a))); }
771EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atanh(
const bfloat16& a) {
return bfloat16(numext::atanh(
float(a))); }
772EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exact_float_to_bfloat16(
float f) {
773 return raw_uint16_to_bfloat16(
static_cast<numext::uint16_t
>(numext::bit_cast<numext::uint32_t>(f) >> 16));
775EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 floor(
const bfloat16& a) {
776 return exact_float_to_bfloat16(numext::floor(
float(a)));
778EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 ceil(
const bfloat16& a) {
779 return exact_float_to_bfloat16(numext::ceil(
float(a)));
781EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 rint(
const bfloat16& a) {
782 return exact_float_to_bfloat16(numext::rint(
float(a)));
784EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 round(
const bfloat16& a) {
785 return exact_float_to_bfloat16(numext::round(
float(a)));
787EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 trunc(
const bfloat16& a) {
788 return exact_float_to_bfloat16(numext::trunc(
float(a)));
791EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmod(
const bfloat16& a,
const bfloat16& b) {
792 return exact_float_to_bfloat16(numext::fmod(
float(a),
float(b)));
795EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16(min)(
const bfloat16& a,
const bfloat16& b) {
796 const float f1 =
static_cast<float>(a);
797 const float f2 =
static_cast<float>(b);
798 return f2 < f1 ? b : a;
801EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16(max)(
const bfloat16& a,
const bfloat16& b) {
802 const float f1 =
static_cast<float>(a);
803 const float f2 =
static_cast<float>(b);
804 return f1 < f2 ? b : a;
807EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmin(
const bfloat16& a,
const bfloat16& b) {
808 const float f1 =
static_cast<float>(a);
809 const float f2 =
static_cast<float>(b);
810 EIGEN_USING_STD(fmin);
811 return exact_float_to_bfloat16(fmin(f1, f2));
814EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmax(
const bfloat16& a,
const bfloat16& b) {
815 const float f1 =
static_cast<float>(a);
816 const float f2 =
static_cast<float>(b);
817 EIGEN_USING_STD(fmax);
818 return exact_float_to_bfloat16(fmax(f1, f2));
821EIGEN_DEVICE_FUNC
inline bfloat16 fma(
const bfloat16& a,
const bfloat16& b,
const bfloat16& c) {
822#if defined(EIGEN_GPU_COMPILE_PHASE) || defined(__SYCL_DEVICE_ONLY__)
824 return bfloat16(numext::fma(
static_cast<float>(a),
static_cast<float>(b),
static_cast<float>(c)));
826 return bfloat16(half_impl::fma_round_to_odd(
static_cast<float>(a),
static_cast<float>(b),
static_cast<float>(c)));
831EIGEN_ALWAYS_INLINE std::ostream& operator<<(std::ostream& os,
const bfloat16& v) {
832 os << static_cast<float>(v);
842struct is_arithmetic<bfloat16> : std::true_type {};
845struct random_impl<bfloat16> {
846 enum :
int { MantissaBits = 7 };
847 using Impl = random_impl<float>;
848 static EIGEN_DEVICE_FUNC
inline bfloat16 run(
const bfloat16& x,
const bfloat16& y) {
849 float result = Impl::run(x, y, MantissaBits);
850 return bfloat16(result);
852 static EIGEN_DEVICE_FUNC
inline bfloat16 run() {
853 float result = Impl::run(MantissaBits);
854 return bfloat16_impl::exact_float_to_bfloat16(result);
861struct NumTraits<Eigen::bfloat16> : GenericNumTraits<Eigen::bfloat16> {
862 enum { IsSigned =
true, IsInteger =
false, IsComplex =
false, RequireInitialization =
false };
864 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
static EIGEN_STRONG_INLINE Eigen::bfloat16 epsilon() {
865 return bfloat16_impl::raw_uint16_to_bfloat16(0x3c00);
867 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
static EIGEN_STRONG_INLINE Eigen::bfloat16 dummy_precision() {
868 return bfloat16_impl::raw_uint16_to_bfloat16(0x3D4D);
870 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
static EIGEN_STRONG_INLINE Eigen::bfloat16 highest() {
871 return bfloat16_impl::raw_uint16_to_bfloat16(0x7F7F);
873 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
static EIGEN_STRONG_INLINE Eigen::bfloat16 lowest() {
874 return bfloat16_impl::raw_uint16_to_bfloat16(0xFF7F);
876 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
static EIGEN_STRONG_INLINE Eigen::bfloat16 infinity() {
877 return bfloat16_impl::raw_uint16_to_bfloat16(0x7f80);
879 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR
static EIGEN_STRONG_INLINE Eigen::bfloat16 quiet_NaN() {
880 return bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0);
886#if defined(EIGEN_HAS_HIP_BF16)
887#pragma pop_macro("EIGEN_CONSTEXPR")
894EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isnan)(
const Eigen::bfloat16& h) {
895 return (bfloat16_impl::isnan)(h);
899EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isinf)(
const Eigen::bfloat16& h) {
900 return (bfloat16_impl::isinf)(h);
904EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isfinite)(
const Eigen::bfloat16& h) {
905 return (bfloat16_impl::isfinite)(h);
909EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast<Eigen::bfloat16, uint16_t>(
const uint16_t& src) {
910 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(src);
914EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast<uint16_t, Eigen::bfloat16>(
const Eigen::bfloat16& src) {
915 return Eigen::bfloat16_impl::raw_bfloat16_as_uint16(src);
918EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 nextafter(
const bfloat16& from,
const bfloat16& to) {
919 if (numext::isnan EIGEN_NOT_A_MACRO(from)) {
922 if (numext::isnan EIGEN_NOT_A_MACRO(to)) {
928 uint16_t from_bits = numext::bit_cast<uint16_t>(from);
929 bool from_sign = from_bits >> 15;
930 if ((from_bits & 0x7fff) == 0) {
933 from_bits = (to > from) ? uint16_t(0x0001) : uint16_t(0x8001);
934 }
else if ((to > from) != from_sign) {
940 return numext::bit_cast<bfloat16>(from_bits);
945EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 madd<Eigen::bfloat16>(
const Eigen::bfloat16& x,
946 const Eigen::bfloat16& y,
947 const Eigen::bfloat16& z) {
948 return Eigen::bfloat16(
static_cast<float>(x) *
static_cast<float>(y) +
static_cast<float>(z));
954#if EIGEN_HAS_STD_HASH
957struct hash<Eigen::bfloat16> {
958 EIGEN_STRONG_INLINE std::size_t operator()(
const Eigen::bfloat16& a)
const {
960 const Eigen::numext::uint16_t bits = Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a);
961 return static_cast<std::size_t
>((bits & 0x7fff) == 0 ? 0 : bits);
972#if defined(EIGEN_HIPCC)
974#if defined(EIGEN_HAS_HIP_BF16)
976__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl(Eigen::bfloat16 var,
int srcLane,
int width = warpSize) {
977 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
978 return Eigen::numext::bit_cast<Eigen::bfloat16>(
static_cast<Eigen::numext::uint16_t
>(__shfl(ivar, srcLane, width)));
981__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_up(Eigen::bfloat16 var,
unsigned int delta,
982 int width = warpSize) {
983 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
984 return Eigen::numext::bit_cast<Eigen::bfloat16>(
static_cast<Eigen::numext::uint16_t
>(__shfl_up(ivar, delta, width)));
987__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_down(Eigen::bfloat16 var,
unsigned int delta,
988 int width = warpSize) {
989 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
990 return Eigen::numext::bit_cast<Eigen::bfloat16>(
991 static_cast<Eigen::numext::uint16_t
>(__shfl_down(ivar, delta, width)));
994__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_xor(Eigen::bfloat16 var,
int laneMask,
int width = warpSize) {
995 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
996 return Eigen::numext::bit_cast<Eigen::bfloat16>(
997 static_cast<Eigen::numext::uint16_t
>(__shfl_xor(ivar, laneMask, width)));
1002#elif defined(EIGEN_CUDACC)
1004__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_sync(
unsigned mask, Eigen::bfloat16 var,
int srcLane,
1005 int width = warpSize) {
1006 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1007 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1008 static_cast<Eigen::numext::uint16_t
>(__shfl_sync(mask, ivar, srcLane, width)));
1011__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_up_sync(
unsigned mask, Eigen::bfloat16 var,
unsigned int delta,
1012 int width = warpSize) {
1013 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1014 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1015 static_cast<Eigen::numext::uint16_t
>(__shfl_up_sync(mask, ivar, delta, width)));
1018__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_down_sync(
unsigned mask, Eigen::bfloat16 var,
unsigned int delta,
1019 int width = warpSize) {
1020 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1021 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1022 static_cast<Eigen::numext::uint16_t
>(__shfl_down_sync(mask, ivar, delta, width)));
1025__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_xor_sync(
unsigned mask, Eigen::bfloat16 var,
int laneMask,
1026 int width = warpSize) {
1027 const int ivar =
static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1028 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1029 static_cast<Eigen::numext::uint16_t
>(__shfl_xor_sync(mask, ivar, laneMask, width)));
1034#if defined(EIGEN_HIPCC)
1035EIGEN_STRONG_INLINE __device__ Eigen::bfloat16 __ldg(
const Eigen::bfloat16* ptr) {
1036 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(
1037 __ldg(Eigen::numext::bit_cast<const Eigen::numext::uint16_t*>(ptr)));
1041#undef EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR
Holds information about the various numeric (i.e. scalar) types allowed by Eigen.
Definition NumTraits.h:267