Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
BFloat16.h
1/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3Licensed under the Apache License, Version 2.0 (the "License");
4you may not use this file except in compliance with the License.
5You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9Unless required by applicable law or agreed to in writing, software
10distributed under the License is distributed on an "AS IS" BASIS,
11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12See the License for the specific language governing permissions and
13limitations under the License.
14==============================================================================*/
15// SPDX-License-Identifier: Apache-2.0
16
17#ifndef EIGEN_BFLOAT16_H
18#define EIGEN_BFLOAT16_H
19
20// IWYU pragma: private
21#include "../../InternalHeaderCheck.h"
22
23#if defined(EIGEN_HAS_HIP_BF16)
24// When compiling with GPU support, the "hip_bfloat16" base class as well as
25// some other routines are defined in the GPU compiler header files
26// (hip_bfloat16.h), and they are not tagged constexpr
27// As a consequence, we get compile failures when compiling Eigen with
28// GPU support. Hence the need to disable EIGEN_CONSTEXPR when building
29// Eigen with GPU support
30#pragma push_macro("EIGEN_CONSTEXPR")
31#undef EIGEN_CONSTEXPR
32#define EIGEN_CONSTEXPR
33#endif
34
35#define BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, METHOD) \
36 template <> \
37 EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED PACKET_BF16 METHOD<PACKET_BF16>( \
38 const PACKET_BF16& _x) { \
39 return F32ToBf16(METHOD<PACKET_F>(Bf16ToF32(_x))); \
40 }
41
42// BF16_PACKET_FUNCTION for a bfloat16 packet that spans two PACKET_F, through the backend's
43// Bf16ToF32Low, Bf16ToF32High and two-argument F32ToBf16.
44// METHOD is a template name, which cannot be parenthesized.
45// NOLINTBEGIN(bugprone-macro-parentheses)
46#define BF16_PACKET_FUNCTION_HALVES(PACKET_F, PACKET_BF16, METHOD) \
47 template <> \
48 EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_UNUSED PACKET_BF16 METHOD<PACKET_BF16>( \
49 const PACKET_BF16& _x) { \
50 return F32ToBf16(METHOD<PACKET_F>(Bf16ToF32Low(_x)), METHOD<PACKET_F>(Bf16ToF32High(_x))); \
51 }
52// NOLINTEND(bugprone-macro-parentheses)
53
54// The _WITH forms take the macro that defines each function, BF16_PACKET_FUNCTION or
55// BF16_PACKET_FUNCTION_HALVES, so that every backend instantiates the same list.
56#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \
57 FUNCTION(PACKET_F, PACKET_BF16, pcos) \
58 FUNCTION(PACKET_F, PACKET_BF16, psin) \
59 FUNCTION(PACKET_F, PACKET_BF16, psinh) \
60 FUNCTION(PACKET_F, PACKET_BF16, pcosh) \
61 FUNCTION(PACKET_F, PACKET_BF16, pasinh) \
62 FUNCTION(PACKET_F, PACKET_BF16, pacosh) \
63 FUNCTION(PACKET_F, PACKET_BF16, pexp) \
64 FUNCTION(PACKET_F, PACKET_BF16, pexp2) \
65 FUNCTION(PACKET_F, PACKET_BF16, pexpm1) \
66 FUNCTION(PACKET_F, PACKET_BF16, plog) \
67 FUNCTION(PACKET_F, PACKET_BF16, plog1p) \
68 FUNCTION(PACKET_F, PACKET_BF16, plog2) \
69 FUNCTION(PACKET_F, PACKET_BF16, plog10) \
70 FUNCTION(PACKET_F, PACKET_BF16, preciprocal) \
71 FUNCTION(PACKET_F, PACKET_BF16, pcbrt) \
72 FUNCTION(PACKET_F, PACKET_BF16, ptanh)
73
74#define EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(PACKET_F, PACKET_BF16) \
75 EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16)
76
77// psqrt and prsqrt through the float round trip, for backends without their own bfloat16 kernels.
78#define EIGEN_INSTANTIATE_GENERIC_SQRT_FUNCS_BF16(PACKET_F, PACKET_BF16) \
79 BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, psqrt) \
80 BF16_PACKET_FUNCTION(PACKET_F, PACKET_BF16, prsqrt)
81
82// BF16 wrappers for contrib/SpecialFunctions.
83#define EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \
84 FUNCTION(PACKET_F, PACKET_BF16, perf) \
85 FUNCTION(PACKET_F, PACKET_BF16, pndtri)
86
87#define EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16(PACKET_F, PACKET_BF16) \
88 EIGEN_INSTANTIATE_SPECIAL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16)
89
90#define EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16_WITH(FUNCTION, PACKET_F, PACKET_BF16) \
91 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0) \
92 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i0e) \
93 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1) \
94 FUNCTION(PACKET_F, PACKET_BF16, pbessel_i1e) \
95 FUNCTION(PACKET_F, PACKET_BF16, pbessel_j0) \
96 FUNCTION(PACKET_F, PACKET_BF16, pbessel_j1) \
97 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0) \
98 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k0e) \
99 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1) \
100 FUNCTION(PACKET_F, PACKET_BF16, pbessel_k1e) \
101 FUNCTION(PACKET_F, PACKET_BF16, pbessel_y0) \
102 FUNCTION(PACKET_F, PACKET_BF16, pbessel_y1)
103
104#define EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16(PACKET_F, PACKET_BF16) \
105 EIGEN_INSTANTIATE_BESSEL_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION, PACKET_F, PACKET_BF16)
106
107// Only use HIP GPU bf16 in kernels
108#if defined(EIGEN_HAS_HIP_BF16) && defined(EIGEN_GPU_COMPILE_PHASE)
109#define EIGEN_USE_HIP_BF16
110#endif
111
112namespace Eigen {
113
114struct bfloat16;
115
116namespace numext {
117template <>
118EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast<Eigen::bfloat16, uint16_t>(const uint16_t& src);
119
120template <>
121EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast<uint16_t, Eigen::bfloat16>(const Eigen::bfloat16& src);
122} // namespace numext
123namespace bfloat16_impl {
124
125#if defined(EIGEN_CUDACC)
126// Like Eigen::half, bfloat16 is used as the element type of __shared__ arrays, whose type must have an empty default
127// constructor: clang's CUDA mode rejects `__shared__ bfloat16 a[N]` when a constructor in the chain initializes value,
128// in both compile phases. Hence, in a CUDA translation unit, the default constructors of __bfloat16_raw,
129// bfloat16_base and bfloat16 leave value indeterminate, in the host phase too: `bfloat16()`, `bfloat16{}` and
130// `std::vector<bfloat16>(n)` do not yield zero there, as for Eigen::half. Other translation units keep the zero
131// initialization. Such constructors cannot be constexpr before C++20.
132#define EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR
133#else
134#define EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR EIGEN_CONSTEXPR
135#endif
136
137#if defined(EIGEN_USE_HIP_BF16)
138
139struct __bfloat16_raw : public hip_bfloat16 {
140 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() {}
141 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(hip_bfloat16 hb) : hip_bfloat16(hb) {}
142 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(unsigned short raw) : hip_bfloat16(raw) {}
143};
144
145#else
146
147// Make our own __bfloat16_raw definition.
148struct __bfloat16_raw {
149#if defined(EIGEN_CUDACC) || (defined(EIGEN_HAS_HIP_BF16) && !defined(EIGEN_GPU_COMPILE_PHASE))
150 EIGEN_DEVICE_FUNC EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR __bfloat16_raw() {}
151#else
152 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw() : value(0) {}
153#endif
154 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw(unsigned short raw) : value(raw) {}
155 unsigned short value;
156};
157
158#endif // defined(EIGEN_USE_HIP_BF16)
159
160EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(unsigned short value);
161template <bool AssumeArgumentIsNormalOrInfinityOrZero>
162EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne(float ff);
163// Forward declarations of template specializations, to avoid Visual C++ 2019 errors, saying:
164// > error C2908: explicit specialization; 'float_to_bfloat16_rtne' has already been instantiated
165template <>
166EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<false>(float ff);
167template <>
168EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<true>(float ff);
169EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float bfloat16_to_float(__bfloat16_raw h);
170
171struct bfloat16_base : public __bfloat16_raw {
172 EIGEN_DEVICE_FUNC EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR bfloat16_base() {}
173 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16_base(const __bfloat16_raw& h) : __bfloat16_raw(h) {}
174};
175
176} // namespace bfloat16_impl
177
178// Class definition.
179struct bfloat16 : public bfloat16_impl::bfloat16_base {
180 using __bfloat16_raw = bfloat16_impl::__bfloat16_raw;
181
182 EIGEN_DEVICE_FUNC EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR bfloat16() {}
183
184 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(const __bfloat16_raw& h) : bfloat16_impl::bfloat16_base(h) {}
185
186 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(bool b)
187 : bfloat16_impl::bfloat16_base(bfloat16_impl::raw_uint16_to_bfloat16(b ? 0x3f80 : 0)) {}
188
189 template <class T>
190 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(T val)
191 : bfloat16_impl::bfloat16_base(
192 bfloat16_impl::float_to_bfloat16_rtne<std::is_integral<T>::value>(half_impl::narrow_to_float(val))) {}
193
194 explicit EIGEN_DEVICE_FUNC bfloat16(float f)
195 : bfloat16_impl::bfloat16_base(bfloat16_impl::float_to_bfloat16_rtne<false>(f)) {}
196
197 // Following the convention of numpy, converting between complex and
198 // float will lead to loss of imag value.
199 template <typename RealScalar>
200 explicit EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR bfloat16(const std::complex<RealScalar>& val)
201 : bfloat16_impl::bfloat16_base(
202 bfloat16_impl::float_to_bfloat16_rtne<false>(half_impl::narrow_to_float(val.real()))) {}
203
204 EIGEN_DEVICE_FUNC operator float() const { // NOLINT: Allow implicit conversion to float, because it is lossless.
205 return bfloat16_impl::bfloat16_to_float(*this);
206 }
207};
208
209// TODO(majnemer): Get rid of this once we can rely on C++17 inline variables to
210// solve the ODR issue.
211namespace bfloat16_impl {
212template <typename = void>
213struct numeric_limits_bfloat16_impl {
214 static EIGEN_CONSTEXPR const bool is_specialized = true;
215 static EIGEN_CONSTEXPR const bool is_signed = true;
216 static EIGEN_CONSTEXPR const bool is_integer = false;
217 static EIGEN_CONSTEXPR const bool is_exact = false;
218 static EIGEN_CONSTEXPR const bool has_infinity = true;
219 static EIGEN_CONSTEXPR const bool has_quiet_NaN = true;
220 static EIGEN_CONSTEXPR const bool has_signaling_NaN = true;
221 EIGEN_DIAGNOSTICS(push)
222 EIGEN_DISABLE_DEPRECATED_WARNING
223 static EIGEN_CONSTEXPR const std::float_denorm_style has_denorm = std::denorm_present;
224 static EIGEN_CONSTEXPR const bool has_denorm_loss = false;
225 EIGEN_DIAGNOSTICS(pop)
226 static EIGEN_CONSTEXPR const std::float_round_style round_style = std::numeric_limits<float>::round_style;
227 static EIGEN_CONSTEXPR const bool is_iec559 = true;
228 // The C++ standard defines this as "true if the set of values representable
229 // by the type is finite." BFloat16 has finite precision.
230 static EIGEN_CONSTEXPR const bool is_bounded = true;
231 static EIGEN_CONSTEXPR const bool is_modulo = false;
232 static EIGEN_CONSTEXPR const int digits = 8;
233 static EIGEN_CONSTEXPR const int digits10 = 2;
234 static EIGEN_CONSTEXPR const int max_digits10 = 4;
235 static EIGEN_CONSTEXPR const int radix = std::numeric_limits<float>::radix;
236 static EIGEN_CONSTEXPR const int min_exponent = std::numeric_limits<float>::min_exponent;
237 static EIGEN_CONSTEXPR const int min_exponent10 = std::numeric_limits<float>::min_exponent10;
238 static EIGEN_CONSTEXPR const int max_exponent = std::numeric_limits<float>::max_exponent;
239 static EIGEN_CONSTEXPR const int max_exponent10 = std::numeric_limits<float>::max_exponent10;
240 static EIGEN_CONSTEXPR const bool traps = std::numeric_limits<float>::traps;
241 // IEEE754: "The implementer shall choose how tininess is detected, but shall
242 // detect tininess in the same way for all operations in radix two"
243 static EIGEN_CONSTEXPR const bool tinyness_before = std::numeric_limits<float>::tinyness_before;
244
245 static EIGEN_CONSTEXPR Eigen::bfloat16(min)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0080); }
246 static EIGEN_CONSTEXPR Eigen::bfloat16 lowest() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0xff7f); }
247 static EIGEN_CONSTEXPR Eigen::bfloat16(max)() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f7f); }
248 static EIGEN_CONSTEXPR Eigen::bfloat16 epsilon() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3c00); }
249 static EIGEN_CONSTEXPR Eigen::bfloat16 round_error() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x3f00); }
250 static EIGEN_CONSTEXPR Eigen::bfloat16 infinity() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7f80); }
251 static EIGEN_CONSTEXPR Eigen::bfloat16 quiet_NaN() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0); }
252 static EIGEN_CONSTEXPR Eigen::bfloat16 signaling_NaN() {
253 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x7fa0);
254 }
255 static EIGEN_CONSTEXPR Eigen::bfloat16 denorm_min() { return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(0x0001); }
256};
257
258// Redundant out-of-class definitions are required pre-C++17 but deprecated since.
259#if EIGEN_COMP_CXXVER < 17
260template <typename T>
261EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_specialized;
262template <typename T>
263EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_signed;
264template <typename T>
265EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_integer;
266template <typename T>
267EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_exact;
268template <typename T>
269EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_infinity;
270template <typename T>
271EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_quiet_NaN;
272template <typename T>
273EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_signaling_NaN;
274EIGEN_DIAGNOSTICS(push)
275EIGEN_DISABLE_DEPRECATED_WARNING
276template <typename T>
277EIGEN_CONSTEXPR const std::float_denorm_style numeric_limits_bfloat16_impl<T>::has_denorm;
278template <typename T>
279EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::has_denorm_loss;
280EIGEN_DIAGNOSTICS(pop)
281template <typename T>
282EIGEN_CONSTEXPR const std::float_round_style numeric_limits_bfloat16_impl<T>::round_style;
283template <typename T>
284EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_iec559;
285template <typename T>
286EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_bounded;
287template <typename T>
288EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::is_modulo;
289template <typename T>
290EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::digits;
291template <typename T>
292EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::digits10;
293template <typename T>
294EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::max_digits10;
295template <typename T>
296EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::radix;
297template <typename T>
298EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::min_exponent;
299template <typename T>
300EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::min_exponent10;
301template <typename T>
302EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::max_exponent;
303template <typename T>
304EIGEN_CONSTEXPR const int numeric_limits_bfloat16_impl<T>::max_exponent10;
305template <typename T>
306EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::traps;
307template <typename T>
308EIGEN_CONSTEXPR const bool numeric_limits_bfloat16_impl<T>::tinyness_before;
309#endif
310} // end namespace bfloat16_impl
311} // end namespace Eigen
312
313namespace std {
314// If std::numeric_limits<T> is specialized, should also specialize
315// std::numeric_limits<const T>, std::numeric_limits<volatile T>, and
316// std::numeric_limits<const volatile T>
317// https://stackoverflow.com/a/16519653/
318template <>
319class numeric_limits<Eigen::bfloat16> : public Eigen::bfloat16_impl::numeric_limits_bfloat16_impl<> {};
320template <>
321class numeric_limits<const Eigen::bfloat16> : public numeric_limits<Eigen::bfloat16> {};
322template <>
323class numeric_limits<volatile Eigen::bfloat16> : public numeric_limits<Eigen::bfloat16> {};
324template <>
325class numeric_limits<const volatile Eigen::bfloat16> : public numeric_limits<Eigen::bfloat16> {};
326} // end namespace std
327
328namespace Eigen {
329
330namespace bfloat16_impl {
331
332// We need to distinguish ‘clang as the CUDA compiler’ from ‘clang as the host compiler,
333// invoked by NVCC’ (e.g. on MacOS). The former needs to see both host and device implementation
334// of the functions, while the latter can only deal with one of them.
335#if !defined(EIGEN_HAS_NATIVE_BF16) || (EIGEN_COMP_CLANG && !EIGEN_COMP_NVCC) // Emulate support for bfloat16 floats
336
337#if EIGEN_COMP_CLANG && defined(EIGEN_GPUCC)
338// We need to provide emulated *host-side* BF16 operators for clang.
339#pragma push_macro("EIGEN_DEVICE_FUNC")
340#undef EIGEN_DEVICE_FUNC
341#if (defined(EIGEN_HAS_GPU_BF16) && defined(EIGEN_HAS_NATIVE_BF16))
342#define EIGEN_DEVICE_FUNC __host__
343#else // both host and device need emulated ops.
344#define EIGEN_DEVICE_FUNC __host__ __device__
345#endif
346#endif
347
348// Definitions for CPUs, mostly working through conversion
349// to/from fp32.
350
351EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(const bfloat16& a, const bfloat16& b) {
352 return bfloat16(float(a) + float(b));
353}
354EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(const bfloat16& a, const int& b) {
355 return bfloat16(float(a) + static_cast<float>(b));
356}
357EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator+(const int& a, const bfloat16& b) {
358 return bfloat16(static_cast<float>(a) + float(b));
359}
360EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator*(const bfloat16& a, const bfloat16& b) {
361 return bfloat16(float(a) * float(b));
362}
363EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator-(const bfloat16& a, const bfloat16& b) {
364 return bfloat16(float(a) - float(b));
365}
366EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator/(const bfloat16& a, const bfloat16& b) {
367 return bfloat16(float(a) / float(b));
368}
369EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator-(const bfloat16& a) {
370 numext::uint16_t x = numext::bit_cast<uint16_t>(a) ^ 0x8000;
371 return numext::bit_cast<bfloat16>(x);
372}
373EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator+=(bfloat16& a, const bfloat16& b) {
374 a = bfloat16(float(a) + float(b));
375 return a;
376}
377EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator*=(bfloat16& a, const bfloat16& b) {
378 a = bfloat16(float(a) * float(b));
379 return a;
380}
381EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator-=(bfloat16& a, const bfloat16& b) {
382 a = bfloat16(float(a) - float(b));
383 return a;
384}
385EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16& operator/=(bfloat16& a, const bfloat16& b) {
386 a = bfloat16(float(a) / float(b));
387 return a;
388}
389EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a) {
390 a += bfloat16(1);
391 return a;
392}
393EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a) {
394 a -= bfloat16(1);
395 return a;
396}
397EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator++(bfloat16& a, int) {
398 bfloat16 original_value = a;
399 ++a;
400 return original_value;
401}
402EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator--(bfloat16& a, int) {
403 bfloat16 original_value = a;
404 --a;
405 return original_value;
406}
407#if defined(EIGEN_GPU_COMPILE_PHASE)
408// On GPU, widening bfloat16 to float is a shift left by 16 and the comparison is
409// a native float compare, which is much cheaper than the 16-bit integer path. It
410// also avoids an LLVM 18 NVPTX bug on sm_90 that lowers SLP-vectorized i16
411// negation to the non-existent 'sub.s16x2' PTX instruction. Unlike the host
412// path, these follow float FTZ and fast-math semantics: under flush-to-zero,
413// subnormal operands compare equal to zero.
414EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator==(const bfloat16& a, const bfloat16& b) {
415 return numext::equal_strict(float(a), float(b));
416}
417EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const bfloat16& a, const bfloat16& b) {
418 return numext::not_equal_strict(float(a), float(b));
419}
420EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const bfloat16& a, const bfloat16& b) {
421 return float(a) < float(b);
422}
423EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const bfloat16& a, const bfloat16& b) {
424 return float(a) <= float(b);
425}
426EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const bfloat16& a, const bfloat16& b) {
427 return float(a) > float(b);
428}
429EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const bfloat16& a, const bfloat16& b) {
430 return float(a) >= float(b);
431}
432#else
433EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC int16_t bfloat16_map_to_signed(numext::uint16_t bits) {
434 constexpr numext::uint16_t kAbsMask = 0x7fff;
435 return (bits >> 15) ? -(bits & kAbsMask) : bits;
436}
437EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool bfloat16_is_ordered(numext::uint16_t a, numext::uint16_t b) {
438 constexpr numext::uint16_t kAbsMask = 0x7fff;
439 constexpr numext::uint16_t kInf = 0x7f80;
440 return numext::maxi(a & kAbsMask, b & kAbsMask) <= kInf;
441}
442// Evaluate both predicates to keep comparison loops branch-free. Integer operands avoid Clang's
443// -Wbitwise-instead-of-logical warning.
444EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator==(const bfloat16& a, const bfloat16& b) {
445 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
446 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
447 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) == bfloat16_map_to_signed(b_bits)) &
448 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
449}
450EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator!=(const bfloat16& a, const bfloat16& b) { return !(a == b); }
451EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<(const bfloat16& a, const bfloat16& b) {
452 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
453 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
454 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) < bfloat16_map_to_signed(b_bits)) &
455 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
456}
457EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator<=(const bfloat16& a, const bfloat16& b) {
458 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
459 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
460 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) <= bfloat16_map_to_signed(b_bits)) &
461 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
462}
463EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>(const bfloat16& a, const bfloat16& b) {
464 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
465 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
466 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) > bfloat16_map_to_signed(b_bits)) &
467 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
468}
469EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool operator>=(const bfloat16& a, const bfloat16& b) {
470 const numext::uint16_t a_bits = numext::bit_cast<numext::uint16_t>(a);
471 const numext::uint16_t b_bits = numext::bit_cast<numext::uint16_t>(b);
472 return static_cast<unsigned int>(bfloat16_map_to_signed(a_bits) >= bfloat16_map_to_signed(b_bits)) &
473 static_cast<unsigned int>(bfloat16_is_ordered(a_bits, b_bits));
474}
475#endif
476
477#if EIGEN_COMP_CLANG && defined(EIGEN_CUDACC)
478#pragma pop_macro("EIGEN_DEVICE_FUNC")
479#endif
480#endif // Emulate support for bfloat16 floats
481
482// Division by an index. Do it in full float precision to avoid accuracy
483// issues in converting the denominator to bfloat16.
484EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 operator/(const bfloat16& a, Index b) {
485 return bfloat16(static_cast<float>(a) / static_cast<float>(b));
486}
487
488EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw truncate_to_bfloat16(const float v) {
489#if defined(EIGEN_USE_HIP_BF16)
490 return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(v, __bfloat16_raw::truncate));
491#else
492 __bfloat16_raw output;
493 if (numext::isnan EIGEN_NOT_A_MACRO(v)) {
494 output.value = std::signbit(v) ? 0xFFC0 : 0x7FC0;
495 return output;
496 }
497 output.value = static_cast<numext::uint16_t>(numext::bit_cast<numext::uint32_t>(v) >> 16);
498 return output;
499#endif
500}
501
502EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(numext::uint16_t value) {
503#if defined(EIGEN_USE_HIP_BF16)
504 __bfloat16_raw bf;
505 bf.data = value;
506 return bf;
507#else
508 return __bfloat16_raw(value);
509#endif
510}
511
512EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR numext::uint16_t raw_bfloat16_as_uint16(
513 const __bfloat16_raw& bf) {
514#if defined(EIGEN_USE_HIP_BF16)
515 return bf.data;
516#else
517 return bf.value;
518#endif
519}
520
521// float_to_bfloat16_rtne template specialization that does not make any
522// assumption about the value of its function argument (ff).
523template <>
524EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<false>(float ff) {
525#if defined(EIGEN_USE_HIP_BF16)
526 return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff));
527#else
528 __bfloat16_raw output;
529
530 if (numext::isnan EIGEN_NOT_A_MACRO(ff)) {
531 // If the value is a NaN, squash it to a qNaN with msb of fraction set,
532 // this makes sure after truncation we don't end up with an inf.
533 //
534 // qNaN magic: All exponent bits set + most significant bit of fraction
535 // set.
536 output.value = std::signbit(ff) ? 0xFFC0 : 0x7FC0;
537 } else {
538 // Fast rounding algorithm that rounds a half value to nearest even. This
539 // reduces expected error when we convert a large number of floats. Here
540 // is how it works:
541 //
542 // Definitions:
543 // To convert a float 32 to bfloat16, a float 32 can be viewed as 32 bits
544 // with the following tags:
545 //
546 // Sign | Exp (8 bits) | Frac (23 bits)
547 // S EEEEEEEE FFFFFFLRTTTTTTTTTTTTTTT
548 //
549 // S: Sign bit.
550 // E: Exponent bits.
551 // F: First 6 bits of fraction.
552 // L: Least significant bit of resulting bfloat16 if we truncate away the
553 // rest of the float32. This is also the 7th bit of fraction
554 // R: Rounding bit, 8th bit of fraction.
555 // T: Sticky bits, rest of fraction, 15 bits.
556 //
557 // To round half to nearest even, there are 3 cases where we want to round
558 // down (simply truncate the result of the bits away, which consists of
559 // rounding bit and sticky bits) and two cases where we want to round up
560 // (truncate then add one to the result).
561 //
562 // The fast converting algorithm simply adds lsb (L) to 0x7fff (15 bits of
563 // 1s) as the rounding bias, adds the rounding bias to the input, then
564 // truncates the last 16 bits away.
565 //
566 // To understand how it works, we can analyze this algorithm case by case:
567 //
568 // 1. L = 0, R = 0:
569 // Expect: round down, this is less than half value.
570 //
571 // Algorithm:
572 // - Rounding bias: 0x7fff + 0 = 0x7fff
573 // - Adding rounding bias to input may create any carry, depending on
574 // whether there is any value set to 1 in T bits.
575 // - R may be set to 1 if there is a carry.
576 // - L remains 0.
577 // - Note that this case also handles Inf and -Inf, where all fraction
578 // bits, including L, R and Ts are all 0. The output remains Inf after
579 // this algorithm.
580 //
581 // 2. L = 1, R = 0:
582 // Expect: round down, this is less than half value.
583 //
584 // Algorithm:
585 // - Rounding bias: 0x7fff + 1 = 0x8000
586 // - Adding rounding bias to input doesn't change sticky bits but
587 // adds 1 to rounding bit.
588 // - L remains 1.
589 //
590 // 3. L = 0, R = 1, all of T are 0:
591 // Expect: round down, this is exactly at half, the result is already
592 // even (L=0).
593 //
594 // Algorithm:
595 // - Rounding bias: 0x7fff + 0 = 0x7fff
596 // - Adding rounding bias to input sets all sticky bits to 1, but
597 // doesn't create a carry.
598 // - R remains 1.
599 // - L remains 0.
600 //
601 // 4. L = 1, R = 1:
602 // Expect: round up, this is exactly at half, the result needs to be
603 // round to the next even number.
604 //
605 // Algorithm:
606 // - Rounding bias: 0x7fff + 1 = 0x8000
607 // - Adding rounding bias to input doesn't change sticky bits, but
608 // creates a carry from rounding bit.
609 // - The carry sets L to 0, creates another carry bit and propagate
610 // forward to F bits.
611 // - If all the F bits are 1, a carry then propagates to the exponent
612 // bits, which then creates the minimum value with the next exponent
613 // value. Note that we won't have the case where exponents are all 1,
614 // since that's either a NaN (handled in the other if condition) or inf
615 // (handled in case 1).
616 //
617 // 5. L = 0, R = 1, any of T is 1:
618 // Expect: round up, this is greater than half.
619 //
620 // Algorithm:
621 // - Rounding bias: 0x7fff + 0 = 0x7fff
622 // - Adding rounding bias to input creates a carry from sticky bits,
623 // sets rounding bit to 0, then create another carry.
624 // - The second carry sets L to 1.
625 //
626 // Examples:
627 //
628 // Exact half value that is already even:
629 // Input:
630 // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit)
631 // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT
632 // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1000000000000000
633 //
634 // This falls into case 3. We truncate the rest of 16 bits and no
635 // carry is created into F and L:
636 //
637 // Output:
638 // Sign | Exp (8 bit) | Frac (first 7 bit)
639 // S E E E E E E E E F F F F F F L
640 // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0
641 //
642 // Exact half value, round to next even number:
643 // Input:
644 // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit)
645 // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT
646 // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1000000000000000
647 //
648 // This falls into case 4. We create a carry from R and T,
649 // which then propagates into L and F:
650 //
651 // Output:
652 // Sign | Exp (8 bit) | Frac (first 7 bit)
653 // S E E E E E E E E F F F F F F L
654 // 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0
655 //
656 //
657 // Max denormal value round to min normal value:
658 // Input:
659 // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit)
660 // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT
661 // 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1111111111111111
662 //
663 // This falls into case 4. We create a carry from R and T,
664 // propagate into L and F, which then propagates into exponent
665 // bits:
666 //
667 // Output:
668 // Sign | Exp (8 bit) | Frac (first 7 bit)
669 // S E E E E E E E E F F F F F F L
670 // 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0
671 //
672 // Max normal value round to Inf:
673 // Input:
674 // Sign | Exp (8 bit) | Frac (first 7 bit) | Frac (last 16 bit)
675 // S E E E E E E E E F F F F F F L RTTTTTTTTTTTTTTT
676 // 0 1 1 1 1 1 1 1 0 1 1 1 1 1 1 1 1111111111111111
677 //
678 // This falls into case 4. We create a carry from R and T,
679 // propagate into L and F, which then propagates into exponent
680 // bits:
681 //
682 // Sign | Exp (8 bit) | Frac (first 7 bit)
683 // S E E E E E E E E F F F F F F L
684 // 0 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0
685
686 // At this point, ff must be either a normal float, or +/-infinity.
687 output = float_to_bfloat16_rtne<true>(ff);
688 }
689 return output;
690#endif
691}
692
693// float_to_bfloat16_rtne template specialization that assumes that its function
694// argument (ff) is either a normal floating point number, or +/-infinity, or
695// zero. Used to improve the runtime performance of conversion from an integer
696// type to bfloat16.
697template <>
698EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __bfloat16_raw float_to_bfloat16_rtne<true>(float ff) {
699#if defined(EIGEN_USE_HIP_BF16)
700 return __bfloat16_raw(__bfloat16_raw::round_to_bfloat16(ff));
701#else
702 numext::uint32_t input = numext::bit_cast<numext::uint32_t>(ff);
703 __bfloat16_raw output;
704
705 // Least significant bit of resulting bfloat.
706 numext::uint32_t lsb = (input >> 16) & 1;
707 numext::uint32_t rounding_bias = 0x7fff + lsb;
708 input += rounding_bias;
709 output.value = static_cast<numext::uint16_t>(input >> 16);
710 return output;
711#endif
712}
713
714EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC float bfloat16_to_float(__bfloat16_raw h) {
715#if defined(EIGEN_USE_HIP_BF16)
716 return static_cast<float>(h);
717#else
718 return numext::bit_cast<float>(static_cast<numext::uint32_t>(h.value) << 16);
719#endif
720}
721
722// --- standard functions ---
723
724EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isinf)(const bfloat16& a) {
725 return (raw_bfloat16_as_uint16(a) & 0x7fff) == 0x7f80;
726}
727EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isnan)(const bfloat16& a) {
728 return (raw_bfloat16_as_uint16(a) & 0x7fff) > 0x7f80;
729}
730EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool(isfinite)(const bfloat16& a) {
731 return (raw_bfloat16_as_uint16(a) & 0x7fff) < 0x7f80;
732}
733
734EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 abs(const bfloat16& a) {
735 numext::uint16_t x = numext::bit_cast<numext::uint16_t>(a) & 0x7FFF;
736 return numext::bit_cast<bfloat16>(x);
737}
738EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp(const bfloat16& a) { return bfloat16(numext::exp(float(a))); }
739EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exp2(const bfloat16& a) { return bfloat16(numext::exp2(float(a))); }
740EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 expm1(const bfloat16& a) { return bfloat16(numext::expm1(float(a))); }
741// float covers the bfloat16 range, so the single conversion back rounds correctly.
742EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 ldexp(const bfloat16& a, int exponent) {
743 return bfloat16(numext::ldexp(float(a), exponent));
744}
745EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log(const bfloat16& a) { return bfloat16(numext::log(float(a))); }
746EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log1p(const bfloat16& a) { return bfloat16(numext::log1p(float(a))); }
747EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log10(const bfloat16& a) { return bfloat16(numext::log10(float(a))); }
748EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 log2(const bfloat16& a) { return bfloat16(numext::log2(float(a))); }
749EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sqrt(const bfloat16& a) { return bfloat16(numext::sqrt(float(a))); }
750EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cbrt(const bfloat16& a) { return bfloat16(numext::cbrt(float(a))); }
751EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 pow(const bfloat16& a, const bfloat16& b) {
752 return bfloat16(numext::pow(float(a), float(b)));
753}
754EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan2(const bfloat16& a, const bfloat16& b) {
755 return bfloat16(numext::atan2(float(a), float(b)));
756}
757EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sin(const bfloat16& a) { return bfloat16(numext::sin(float(a))); }
758EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cos(const bfloat16& a) { return bfloat16(numext::cos(float(a))); }
759EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tan(const bfloat16& a) { return bfloat16(numext::tan(float(a))); }
760EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asin(const bfloat16& a) { return bfloat16(numext::asin(float(a))); }
761EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acos(const bfloat16& a) { return bfloat16(numext::acos(float(a))); }
762EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atan(const bfloat16& a) { return bfloat16(numext::atan(float(a))); }
763EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 sinh(const bfloat16& a) { return bfloat16(numext::sinh(float(a))); }
764EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 cosh(const bfloat16& a) { return bfloat16(numext::cosh(float(a))); }
765EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 tanh(const bfloat16& a) {
766 // Select the scalar template, not the host fast-math overload, to preserve bfloat16 rounding.
767 return bfloat16(numext::tanh<float>(float(a)));
768}
769EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 asinh(const bfloat16& a) { return bfloat16(numext::asinh(float(a))); }
770EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 acosh(const bfloat16& a) { return bfloat16(numext::acosh(float(a))); }
771EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 atanh(const bfloat16& a) { return bfloat16(numext::atanh(float(a))); }
772EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 exact_float_to_bfloat16(float f) {
773 return raw_uint16_to_bfloat16(static_cast<numext::uint16_t>(numext::bit_cast<numext::uint32_t>(f) >> 16));
774}
775EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 floor(const bfloat16& a) {
776 return exact_float_to_bfloat16(numext::floor(float(a)));
777}
778EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 ceil(const bfloat16& a) {
779 return exact_float_to_bfloat16(numext::ceil(float(a)));
780}
781EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 rint(const bfloat16& a) {
782 return exact_float_to_bfloat16(numext::rint(float(a)));
783}
784EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 round(const bfloat16& a) {
785 return exact_float_to_bfloat16(numext::round(float(a)));
786}
787EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 trunc(const bfloat16& a) {
788 return exact_float_to_bfloat16(numext::trunc(float(a)));
789}
790// fmod is exact: a - n*b is either a itself or a multiple of the ulp of b that is smaller than |b|.
791EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmod(const bfloat16& a, const bfloat16& b) {
792 return exact_float_to_bfloat16(numext::fmod(float(a), float(b)));
793}
794
795EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16(min)(const bfloat16& a, const bfloat16& b) {
796 const float f1 = static_cast<float>(a);
797 const float f2 = static_cast<float>(b);
798 return f2 < f1 ? b : a;
799}
800
801EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16(max)(const bfloat16& a, const bfloat16& b) {
802 const float f1 = static_cast<float>(a);
803 const float f2 = static_cast<float>(b);
804 return f1 < f2 ? b : a;
805}
806
807EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmin(const bfloat16& a, const bfloat16& b) {
808 const float f1 = static_cast<float>(a);
809 const float f2 = static_cast<float>(b);
810 EIGEN_USING_STD(fmin);
811 return exact_float_to_bfloat16(fmin(f1, f2));
812}
813
814EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 fmax(const bfloat16& a, const bfloat16& b) {
815 const float f1 = static_cast<float>(a);
816 const float f2 = static_cast<float>(b);
817 EIGEN_USING_STD(fmax);
818 return exact_float_to_bfloat16(fmax(f1, f2));
819}
820
821EIGEN_DEVICE_FUNC inline bfloat16 fma(const bfloat16& a, const bfloat16& b, const bfloat16& c) {
822#if defined(EIGEN_GPU_COMPILE_PHASE) || defined(__SYCL_DEVICE_ONLY__)
823 // Rounds twice; see half_impl::fma_round_to_odd.
824 return bfloat16(numext::fma(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c)));
825#else
826 return bfloat16(half_impl::fma_round_to_odd(static_cast<float>(a), static_cast<float>(b), static_cast<float>(c)));
827#endif
828}
829
830#ifndef EIGEN_NO_IO
831EIGEN_ALWAYS_INLINE std::ostream& operator<<(std::ostream& os, const bfloat16& v) {
832 os << static_cast<float>(v);
833 return os;
834}
835#endif
836
837} // namespace bfloat16_impl
838
839namespace internal {
840
841template <>
842struct is_arithmetic<bfloat16> : std::true_type {};
843
844template <>
845struct random_impl<bfloat16> {
846 enum : int { MantissaBits = 7 };
847 using Impl = random_impl<float>;
848 static EIGEN_DEVICE_FUNC inline bfloat16 run(const bfloat16& x, const bfloat16& y) {
849 float result = Impl::run(x, y, MantissaBits);
850 return bfloat16(result);
851 }
852 static EIGEN_DEVICE_FUNC inline bfloat16 run() {
853 float result = Impl::run(MantissaBits);
854 return bfloat16_impl::exact_float_to_bfloat16(result);
855 }
856};
857
858} // namespace internal
859
860template <>
861struct NumTraits<Eigen::bfloat16> : GenericNumTraits<Eigen::bfloat16> {
862 enum { IsSigned = true, IsInteger = false, IsComplex = false, RequireInitialization = false };
863
864 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 epsilon() {
865 return bfloat16_impl::raw_uint16_to_bfloat16(0x3c00);
866 }
867 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 dummy_precision() {
868 return bfloat16_impl::raw_uint16_to_bfloat16(0x3D4D); // bfloat16(5e-2f);
869 }
870 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 highest() {
871 return bfloat16_impl::raw_uint16_to_bfloat16(0x7F7F);
872 }
873 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 lowest() {
874 return bfloat16_impl::raw_uint16_to_bfloat16(0xFF7F);
875 }
876 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 infinity() {
877 return bfloat16_impl::raw_uint16_to_bfloat16(0x7f80);
878 }
879 EIGEN_DEVICE_FUNC EIGEN_CONSTEXPR static EIGEN_STRONG_INLINE Eigen::bfloat16 quiet_NaN() {
880 return bfloat16_impl::raw_uint16_to_bfloat16(0x7fc0);
881 }
882};
883
884} // namespace Eigen
885
886#if defined(EIGEN_HAS_HIP_BF16)
887#pragma pop_macro("EIGEN_CONSTEXPR")
888#endif
889
890namespace Eigen {
891namespace numext {
892
893template <>
894EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isnan)(const Eigen::bfloat16& h) {
895 return (bfloat16_impl::isnan)(h);
896}
897
898template <>
899EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isinf)(const Eigen::bfloat16& h) {
900 return (bfloat16_impl::isinf)(h);
901}
902
903template <>
904EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool(isfinite)(const Eigen::bfloat16& h) {
905 return (bfloat16_impl::isfinite)(h);
906}
907
908template <>
909EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 bit_cast<Eigen::bfloat16, uint16_t>(const uint16_t& src) {
910 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(src);
911}
912
913template <>
914EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC uint16_t bit_cast<uint16_t, Eigen::bfloat16>(const Eigen::bfloat16& src) {
915 return Eigen::bfloat16_impl::raw_bfloat16_as_uint16(src);
916}
917
918EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bfloat16 nextafter(const bfloat16& from, const bfloat16& to) {
919 if (numext::isnan EIGEN_NOT_A_MACRO(from)) {
920 return from;
921 }
922 if (numext::isnan EIGEN_NOT_A_MACRO(to)) {
923 return to;
924 }
925 if (from == to) {
926 return to;
927 }
928 uint16_t from_bits = numext::bit_cast<uint16_t>(from);
929 bool from_sign = from_bits >> 15;
930 if ((from_bits & 0x7fff) == 0) {
931 // From ±0 toward a nonzero value: the neighbor is the smallest subnormal
932 // carrying the sign of the direction (IEEE-754 nextUp/nextDown of zero).
933 from_bits = (to > from) ? uint16_t(0x0001) : uint16_t(0x8001);
934 } else if ((to > from) != from_sign) {
935 // Toward the infinity with the same sign as from: increase the magnitude.
936 ++from_bits;
937 } else {
938 --from_bits;
939 }
940 return numext::bit_cast<bfloat16>(from_bits);
941}
942
943// Specialize multiply-add to match packet operations and reduce conversions to/from float.
944template <>
945EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::bfloat16 madd<Eigen::bfloat16>(const Eigen::bfloat16& x,
946 const Eigen::bfloat16& y,
947 const Eigen::bfloat16& z) {
948 return Eigen::bfloat16(static_cast<float>(x) * static_cast<float>(y) + static_cast<float>(z));
949}
950
951} // namespace numext
952} // namespace Eigen
953
954#if EIGEN_HAS_STD_HASH
955namespace std {
956template <>
957struct hash<Eigen::bfloat16> {
958 EIGEN_STRONG_INLINE std::size_t operator()(const Eigen::bfloat16& a) const {
959 // -0 == +0, so both must hash alike.
960 const Eigen::numext::uint16_t bits = Eigen::numext::bit_cast<Eigen::numext::uint16_t>(a);
961 return static_cast<std::size_t>((bits & 0x7fff) == 0 ? 0 : bits);
962 }
963};
964} // namespace std
965#endif
966
967// Warp shuffle overloads for Eigen::bfloat16.
968// HIP uses non-sync __shfl variants. Eigen::bfloat16 has no conversion to the CUDA __nv_bfloat16 type, so on CUDA the
969// 16 raw bits travel through the 32-bit __shfl_*_sync overloads. Without these overloads a bfloat16 argument converts
970// to float, and a shuffle of a bfloat16 value returns a float.
971// Note that the following are __device__ - only functions.
972#if defined(EIGEN_HIPCC)
973
974#if defined(EIGEN_HAS_HIP_BF16)
975
976__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl(Eigen::bfloat16 var, int srcLane, int width = warpSize) {
977 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
978 return Eigen::numext::bit_cast<Eigen::bfloat16>(static_cast<Eigen::numext::uint16_t>(__shfl(ivar, srcLane, width)));
979}
980
981__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_up(Eigen::bfloat16 var, unsigned int delta,
982 int width = warpSize) {
983 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
984 return Eigen::numext::bit_cast<Eigen::bfloat16>(static_cast<Eigen::numext::uint16_t>(__shfl_up(ivar, delta, width)));
985}
986
987__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_down(Eigen::bfloat16 var, unsigned int delta,
988 int width = warpSize) {
989 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
990 return Eigen::numext::bit_cast<Eigen::bfloat16>(
991 static_cast<Eigen::numext::uint16_t>(__shfl_down(ivar, delta, width)));
992}
993
994__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_xor(Eigen::bfloat16 var, int laneMask, int width = warpSize) {
995 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
996 return Eigen::numext::bit_cast<Eigen::bfloat16>(
997 static_cast<Eigen::numext::uint16_t>(__shfl_xor(ivar, laneMask, width)));
998}
999
1000#endif // HIP
1001
1002#elif defined(EIGEN_CUDACC)
1003
1004__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_sync(unsigned mask, Eigen::bfloat16 var, int srcLane,
1005 int width = warpSize) {
1006 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1007 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1008 static_cast<Eigen::numext::uint16_t>(__shfl_sync(mask, ivar, srcLane, width)));
1009}
1010
1011__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_up_sync(unsigned mask, Eigen::bfloat16 var, unsigned int delta,
1012 int width = warpSize) {
1013 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1014 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1015 static_cast<Eigen::numext::uint16_t>(__shfl_up_sync(mask, ivar, delta, width)));
1016}
1017
1018__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_down_sync(unsigned mask, Eigen::bfloat16 var, unsigned int delta,
1019 int width = warpSize) {
1020 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1021 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1022 static_cast<Eigen::numext::uint16_t>(__shfl_down_sync(mask, ivar, delta, width)));
1023}
1024
1025__device__ EIGEN_STRONG_INLINE Eigen::bfloat16 __shfl_xor_sync(unsigned mask, Eigen::bfloat16 var, int laneMask,
1026 int width = warpSize) {
1027 const int ivar = static_cast<int>(Eigen::numext::bit_cast<Eigen::numext::uint16_t>(var));
1028 return Eigen::numext::bit_cast<Eigen::bfloat16>(
1029 static_cast<Eigen::numext::uint16_t>(__shfl_xor_sync(mask, ivar, laneMask, width)));
1030}
1031
1032#endif // HIP vs CUDA
1033
1034#if defined(EIGEN_HIPCC)
1035EIGEN_STRONG_INLINE __device__ Eigen::bfloat16 __ldg(const Eigen::bfloat16* ptr) {
1036 return Eigen::bfloat16_impl::raw_uint16_to_bfloat16(
1037 __ldg(Eigen::numext::bit_cast<const Eigen::numext::uint16_t*>(ptr)));
1038}
1039#endif // __ldg
1040
1041#undef EIGEN_BF16_DEFAULT_CTOR_CONSTEXPR
1042
1043#endif // EIGEN_BFLOAT16_H
Holds information about the various numeric (i.e. scalar) types allowed by Eigen.
Definition NumTraits.h:267