Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMathBF16.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// This Source Code Form is subject to the terms of the Mozilla
5// Public License v. 2.0. If a copy of the MPL was not distributed
6// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
7// SPDX-FileCopyrightText: The Eigen Authors
8// SPDX-License-Identifier: MPL-2.0
9
10#ifndef EIGEN_PACKET_MATH_BF16_NEON_H
11#define EIGEN_PACKET_MATH_BF16_NEON_H
12
13// IWYU pragma: private
14#include "../../InternalHeaderCheck.h"
15
16namespace Eigen {
17
18namespace internal {
19
20// TODO: Add support for native armv8.6-a bfloat16_t
21
22// TODO: Guard if we have native bfloat16 support
23using Packet4bf = eigen_packet_wrapper<uint16x4_t, 19>;
24using Packet8bf = eigen_packet_wrapper<uint16x8_t, 20>;
25
26template <>
27struct is_arithmetic<Packet4bf> : std::true_type {};
28template <>
29struct is_arithmetic<Packet8bf> : std::true_type {};
30
31// Packet8bf is the full packet: every bfloat16 op widens to float, and the float work of two
32// Packet4f halves overlaps where one Packet4bf's chain would leave the pipes idle.
33template <>
34struct packet_traits<bfloat16> : default_packet_traits {
35 using type = Packet8bf;
36 using half = Packet4bf;
37 enum {
38 Vectorizable = 1,
39 AlignedOnScalar = 1,
40 size = 8,
41
42 HasCmp = 1,
43 HasAdd = 1,
44 HasSub = 1,
45 HasShift = 1,
46 HasMul = 1,
47 HasNegate = 1,
48 HasAbs = 1,
49 HasArg = 0,
50 HasAbsDiff = 1,
51 HasMin = 1,
52 HasMax = 1,
53 HasConj = 1,
54 HasSetLinear = 1,
55 HasDiv = 1,
56 HasSin = EIGEN_FAST_MATH,
57 HasCos = EIGEN_FAST_MATH,
58 HasLog = 1,
59 HasExp = 1,
60 HasSqrt = 1,
61 HasRsqrt = 1,
62 HasTanh = EIGEN_FAST_MATH,
63 HasErf = EIGEN_FAST_MATH,
64 HasBessel = 0, // Issues with accuracy.
65 HasNdtri = 0
66 };
67};
68
69template <>
70struct unpacket_traits<Packet4bf> : neon_unpacket_default<Packet4bf, bfloat16> {};
71template <>
72struct unpacket_traits<Packet8bf> : neon_unpacket_default<Packet8bf, bfloat16> {
73 using half = Packet4bf;
74};
75
76namespace detail {
77template <>
78EIGEN_ALWAYS_INLINE void zip_in_place<Packet4bf>(Packet4bf& p1, Packet4bf& p2) {
79 const uint16x4x2_t tmp = vzip_u16(p1, p2);
80 p1 = tmp.val[0];
81 p2 = tmp.val[1];
82}
83template <>
84EIGEN_ALWAYS_INLINE void zip_in_place<Packet8bf>(Packet8bf& p1, Packet8bf& p2) {
85 const uint16x8x2_t tmp = vzipq_u16(p1, p2);
86 p1 = tmp.val[0];
87 p2 = tmp.val[1];
88}
89} // namespace detail
90
91// Round to nearest even on the bfloat16 boundary. With lsb the lowest bit kept, the scalar
92// formula is (p + 0x7fff + lsb) >> 16, and RADDHN computes (p + d + 0x8000) >> 16, so d = lsb - 1,
93// that is -1 where lsb is clear. Written as an add and RSHRN, the add stays a separate op in
94// loops with GCC and clang on both targets. The caller guarantees that every NaN lane has a zero low
95// half, which the FPU's NaN propagation does for bfloat16 operands: the rounding then leaves the
96// NaN's high half alone. A NaN with a nonzero low half can instead round to +-inf or carry into a
97// zero.
98EIGEN_STRONG_INLINE Packet4bf F32ToBf16KeepNaN(const Packet4f& p) {
99 Packet4ui bits = vreinterpretq_u32_f32(p);
100 Packet4ui round_down = vceqq_u32(vandq_u32(bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0));
101 return vraddhn_u32(bits, round_down);
102}
103
104// F32ToBf16KeepNaN for any float: NaN lanes become the bfloat16 NaN 0x7fc0, as on AVX. The scalar
105// conversion keeps the NaN's sign; doing that here costs an op on the critical path of every round
106// trip, 25% of a cast loop and 6% of exp on a Cortex-A72, for a bit that carries no value.
107// |p| <= inf is false only for NaN. The NaN lanes are selected after narrowing, so the rounding
108// reads the raw bits. Selecting first costs clang an extra AND and MOV.
109EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p) {
110 uint16x4_t not_nan = vmovn_u32(vcageq_f32(pinf<Packet4f>(), p));
111 return vbsl_u16(not_nan, F32ToBf16KeepNaN(p), vdup_n_u16(0x7fc0));
112}
113
114// Discard the low 16 bits of each float. Only valid when every lane already holds an exact
115// bfloat16 value, in which case this agrees with F32ToBf16 but skips its rounding and NaN
116// canonicalization. pmin/pmax qualify: they return one of their operands bit for bit.
117EIGEN_STRONG_INLINE Packet4bf F32ToBf16Truncate(const Packet4f& p) { return vshrn_n_u32(vreinterpretq_u32_f32(p), 16); }
118
119EIGEN_STRONG_INLINE Packet4f Bf16ToF32(const Packet4bf& p) {
120 return Packet4f(vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(p), 16)));
121}
122
123EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f& p) { return vmovn_u32(vreinterpretq_u32_f32(p)); }
124
125// Packet8bf conversions work on two Packet4f halves, lanes 0-3 and 4-7, with the same rounding as
126// the Packet4bf ones.
127EIGEN_STRONG_INLINE Packet4f Bf16ToF32Low(const Packet8bf& p) {
128 return vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(p), 16));
129}
130
131EIGEN_STRONG_INLINE Packet4f Bf16ToF32High(const Packet8bf& p) {
132#if EIGEN_ARCH_ARM64
133 return vreinterpretq_f32_u32(vshll_high_n_u16(p, 16));
134#else
135 return vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(p), 16));
136#endif
137}
138
139// (v + 0x8000) >> 16 on each lane of lo and hi.
140EIGEN_STRONG_INLINE Packet8bf RoundingNarrowBf16(const Packet4ui& lo, const Packet4ui& hi) {
141#if EIGEN_ARCH_ARM64
142 return vrshrn_high_n_u32(vrshrn_n_u32(lo, 16), hi, 16);
143#else
144 return vcombine_u16(vrshrn_n_u32(lo, 16), vrshrn_n_u32(hi, 16));
145#endif
146}
147
148EIGEN_STRONG_INLINE Packet8bf F32ToBf16KeepNaN(const Packet4f& lo, const Packet4f& hi) {
149 Packet4ui lo_bits = vreinterpretq_u32_f32(lo);
150 Packet4ui hi_bits = vreinterpretq_u32_f32(hi);
151 Packet4ui lo_round_down = vceqq_u32(vandq_u32(lo_bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0));
152 Packet4ui hi_round_down = vceqq_u32(vandq_u32(hi_bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0));
153#if EIGEN_ARCH_ARM64
154 return vraddhn_high_u32(vraddhn_u32(lo_bits, lo_round_down), hi_bits, hi_round_down);
155#else
156 return vcombine_u16(vraddhn_u32(lo_bits, lo_round_down), vraddhn_u32(hi_bits, hi_round_down));
157#endif
158}
159
160// As the Packet4bf F32ToBf16, with one UZP1 narrowing both NaN masks.
161EIGEN_STRONG_INLINE Packet8bf F32ToBf16(const Packet4f& lo, const Packet4f& hi) {
162 Packet4f inf = pinf<Packet4f>();
163 uint16x8_t not_nan =
164 vuzpq_u16(vreinterpretq_u16_u32(vcageq_f32(inf, lo)), vreinterpretq_u16_u32(vcageq_f32(inf, hi))).val[0];
165 return vbslq_u16(not_nan, F32ToBf16KeepNaN(lo, hi), vdupq_n_u16(0x7fc0));
166}
167
168// The high and low 16 bits of every float: one UZP2 or UZP1 on AArch64, one VUZP on ARMv7.
169EIGEN_STRONG_INLINE Packet8bf F32ToBf16Truncate(const Packet4f& lo, const Packet4f& hi) {
170 return vuzpq_u16(vreinterpretq_u16_f32(lo), vreinterpretq_u16_f32(hi)).val[1];
171}
172
173EIGEN_STRONG_INLINE Packet8bf F32MaskToBf16Mask(const Packet4f& lo, const Packet4f& hi) {
174 return vuzpq_u16(vreinterpretq_u16_f32(lo), vreinterpretq_u16_f32(hi)).val[0];
175}
176
177template <>
178EIGEN_STRONG_INLINE Packet4bf pset1<Packet4bf>(const bfloat16& from) {
179 return Packet4bf(pset1<Packet4us>(from.value));
180}
181
182template <>
183EIGEN_STRONG_INLINE bfloat16 pfirst<Packet4bf>(const Packet4bf& from) {
184 return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet4us>(Packet4us(from))));
185}
186
187template <>
188EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(const bfloat16* from) {
189 return Packet4bf(
190 pload<Packet4us>(reinterpret_cast<const uint16_t*>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(from))));
191}
192
193template <>
194EIGEN_STRONG_INLINE Packet4bf ploadu<Packet4bf>(const bfloat16* from) {
195 return Packet4bf(ploadu<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
196}
197
198template <>
199EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet4bf& from) {
200 EIGEN_DEBUG_ALIGNED_STORE vst1_u16(
201 reinterpret_cast<uint16_t*>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(to)), from);
202}
203
204template <>
205EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet4bf& from) {
206 EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
207}
208
209template <>
210EIGEN_STRONG_INLINE Packet4bf ploaddup<Packet4bf>(const bfloat16* from) {
211 return Packet4bf(ploaddup<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
212}
213
214template <>
215EIGEN_STRONG_INLINE Packet4bf pabs(const Packet4bf& a) {
216 return Packet4bf(vand_u16(a, vdup_n_u16(0x7fff)));
217}
218
219template <>
220EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNumbers, Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
221 return F32ToBf16Truncate(pmin<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
222}
223template <>
224EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNaN, Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
225 return F32ToBf16Truncate(pmin<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
226}
227
228template <>
229EIGEN_STRONG_INLINE Packet4bf pmin<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
230 return F32ToBf16Truncate(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
231}
232
233template <>
234EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNumbers, Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
235 return F32ToBf16Truncate(pmax<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
236}
237template <>
238EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNaN, Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
239 return F32ToBf16Truncate(pmax<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
240}
241
242template <>
243EIGEN_STRONG_INLINE Packet4bf pmax<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
244 return F32ToBf16Truncate(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
245}
246
247template <>
248EIGEN_STRONG_INLINE Packet4bf plset<Packet4bf>(const bfloat16& a) {
249 return F32ToBf16(plset<Packet4f>(static_cast<float>(a)));
250}
251
252template <>
253EIGEN_STRONG_INLINE Packet4bf por(const Packet4bf& a, const Packet4bf& b) {
254 return Packet4bf(por<Packet4us>(Packet4us(a), Packet4us(b)));
255}
256
257template <>
258EIGEN_STRONG_INLINE Packet4bf pxor(const Packet4bf& a, const Packet4bf& b) {
259 return Packet4bf(pxor<Packet4us>(Packet4us(a), Packet4us(b)));
260}
261
262template <>
263EIGEN_STRONG_INLINE Packet4bf pand(const Packet4bf& a, const Packet4bf& b) {
264 return Packet4bf(pand<Packet4us>(Packet4us(a), Packet4us(b)));
265}
266
267template <>
268EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a, const Packet4bf& b) {
269 return Packet4bf(pandnot<Packet4us>(Packet4us(a), Packet4us(b)));
270}
271
272template <>
273EIGEN_STRONG_INLINE Packet4bf pnot(const Packet4bf& a) {
274 return Packet4bf(pnot<Packet4us>(Packet4us(a)));
275}
276
277template <>
278EIGEN_STRONG_INLINE Packet4bf ptrue(const Packet4bf& /*a*/) {
279 return Packet4bf(vdup_n_u16(0xffff));
280}
281
282template <>
283EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a, const Packet4bf& b) {
284 return Packet4bf(pselect<Packet4us>(Packet4us(mask), Packet4us(a), Packet4us(b)));
285}
286
287template <>
288EIGEN_STRONG_INLINE Packet4bf print<Packet4bf>(const Packet4bf& a) {
289 return F32ToBf16Truncate(print<Packet4f>(Bf16ToF32(a)));
290}
291
292template <>
293EIGEN_STRONG_INLINE Packet4bf pfloor<Packet4bf>(const Packet4bf& a) {
294 return F32ToBf16Truncate(pfloor<Packet4f>(Bf16ToF32(a)));
295}
296
297template <>
298EIGEN_STRONG_INLINE Packet4bf pceil<Packet4bf>(const Packet4bf& a) {
299 return F32ToBf16Truncate(pceil<Packet4f>(Bf16ToF32(a)));
300}
301
302template <>
303EIGEN_STRONG_INLINE Packet4bf pround<Packet4bf>(const Packet4bf& a) {
304 return F32ToBf16Truncate(pround<Packet4f>(Bf16ToF32(a)));
305}
306
307template <>
308EIGEN_STRONG_INLINE Packet4bf ptrunc<Packet4bf>(const Packet4bf& a) {
309 return F32ToBf16Truncate(ptrunc<Packet4f>(Bf16ToF32(a)));
310}
311
312template <>
313EIGEN_STRONG_INLINE Packet4bf padd<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
314 return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
315}
316
317template <>
318EIGEN_STRONG_INLINE Packet4bf psub<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
319 return F32ToBf16KeepNaN(psub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
320}
321
322template <>
323EIGEN_STRONG_INLINE Packet4bf pmul<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
324 return F32ToBf16KeepNaN(pmul<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
325}
326
327template <>
328EIGEN_STRONG_INLINE Packet4bf pmadd<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) {
329 return F32ToBf16KeepNaN(pmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
330}
331
332template <>
333EIGEN_STRONG_INLINE Packet4bf pmsub<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) {
334 return F32ToBf16KeepNaN(pmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
335}
336
337template <>
338EIGEN_STRONG_INLINE Packet4bf pnmadd<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) {
339 return F32ToBf16KeepNaN(pnmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
340}
341
342template <>
343EIGEN_STRONG_INLINE Packet4bf pnmsub<Packet4bf>(const Packet4bf& a, const Packet4bf& b, const Packet4bf& c) {
344 return F32ToBf16KeepNaN(pnmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
345}
346
347template <>
348EIGEN_STRONG_INLINE Packet4bf pdiv<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
349 return F32ToBf16KeepNaN(pdiv<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
350}
351
352template <>
353EIGEN_STRONG_INLINE Packet4bf pgather<bfloat16, Packet4bf>(const bfloat16* from, Index stride) {
354 return Packet4bf(pgather<uint16_t, Packet4us>(reinterpret_cast<const uint16_t*>(from), stride));
355}
356
357template <>
358EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet4bf>(bfloat16* to, const Packet4bf& from, Index stride) {
359 pscatter<uint16_t, Packet4us>(reinterpret_cast<uint16_t*>(to), Packet4us(from), stride);
360}
361
362template <>
363EIGEN_STRONG_INLINE bfloat16 predux<Packet4bf>(const Packet4bf& a) {
364 return static_cast<bfloat16>(predux<Packet4f>(Bf16ToF32(a)));
365}
366
367template <>
368EIGEN_STRONG_INLINE bfloat16 predux_max<Packet4bf>(const Packet4bf& a) {
369 return bfloat16_impl::exact_float_to_bfloat16(predux_max<Packet4f>(Bf16ToF32(a)));
370}
371
372template <>
373EIGEN_STRONG_INLINE bfloat16 predux_min<Packet4bf>(const Packet4bf& a) {
374 return bfloat16_impl::exact_float_to_bfloat16(predux_min<Packet4f>(Bf16ToF32(a)));
375}
376
377template <>
378EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet4bf>(const Packet4bf& a) {
379 return static_cast<bfloat16>(predux_mul<Packet4f>(Bf16ToF32(a)));
380}
381
382template <>
383EIGEN_STRONG_INLINE bool predux_any(const Packet4bf& a) {
384 return vget_lane_u64(vreinterpret_u64_u16(Packet4us(a)), 0) != 0;
385}
386
387template <>
388EIGEN_STRONG_INLINE Packet4bf preverse<Packet4bf>(const Packet4bf& a) {
389 return Packet4bf(preverse<Packet4us>(Packet4us(a)));
390}
391
392EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4bf, 4>& kernel) {
393 detail::ptranspose_impl(kernel);
394}
395
396template <>
397EIGEN_STRONG_INLINE Packet4bf pabsdiff<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
398 return F32ToBf16KeepNaN(pabsdiff<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
399}
400
401template <>
402EIGEN_STRONG_INLINE Packet4bf pcmp_eq<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
403 return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
404}
405
406template <>
407EIGEN_STRONG_INLINE Packet4bf pcmp_lt<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
408 return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
409}
410
411template <>
412EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
413 return F32MaskToBf16Mask(pcmp_lt_or_nan<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
414}
415
416// Classify on the raw bits, as the scalar isinf/isnan/isfinite do: |a| ==, >, < 0x7f80.
417template <>
418EIGEN_STRONG_INLINE Packet4bf pisinf<Packet4bf>(const Packet4bf& a) {
419 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
420 constexpr uint16_t kAbsMask = (1 << 15) - 1;
421 return vceq_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf));
422}
423
424template <>
425EIGEN_STRONG_INLINE Packet4bf pisnan<Packet4bf>(const Packet4bf& a) {
426 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
427 constexpr uint16_t kAbsMask = (1 << 15) - 1;
428 return vcgt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf));
429}
430
431// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ.
432template <>
433EIGEN_STRONG_INLINE Packet4bf psign<Packet4bf>(const Packet4bf& a) {
434 const uint16x4_t abs_mask = vdup_n_u16(0x7fff);
435 const uint16x4_t is_nan = vcgt_u16(vand_u16(a, abs_mask), vdup_n_u16(0x7f80));
436 const uint16x4_t keep = vorr_u16(is_nan, vdup_n_u16(0x8000));
437 return vand_u16(vbsl_u16(keep, a, vdup_n_u16(0x3f80)), vtst_u16(a, abs_mask));
438}
439
440template <>
441EIGEN_STRONG_INLINE Packet4bf pisfinite<Packet4bf>(const Packet4bf& a) {
442 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
443 constexpr uint16_t kAbsMask = (1 << 15) - 1;
444 return vclt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf));
445}
446
447template <>
448EIGEN_STRONG_INLINE Packet4bf pcmp_le<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
449 return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
450}
451
452template <>
453EIGEN_STRONG_INLINE Packet4bf pnegate<Packet4bf>(const Packet4bf& a) {
454 return Packet4bf(pxor<Packet4us>(Packet4us(a), pset1<Packet4us>(static_cast<uint16_t>(0x8000))));
455}
456
457//---------- Packet8bf ----------
458
459template <>
460EIGEN_STRONG_INLINE Packet8bf pset1<Packet8bf>(const bfloat16& from) {
461 return Packet8bf(pset1<Packet8us>(from.value));
462}
463
464template <>
465EIGEN_STRONG_INLINE bfloat16 pfirst<Packet8bf>(const Packet8bf& from) {
466 return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet8us>(Packet8us(from))));
467}
468
469template <>
470EIGEN_STRONG_INLINE Packet8bf pload<Packet8bf>(const bfloat16* from) {
471 return Packet8bf(
472 pload<Packet8us>(reinterpret_cast<const uint16_t*>(assume_aligned<unpacket_traits<Packet8bf>::alignment>(from))));
473}
474
475template <>
476EIGEN_STRONG_INLINE Packet8bf ploadu<Packet8bf>(const bfloat16* from) {
477 return Packet8bf(ploadu<Packet8us>(reinterpret_cast<const uint16_t*>(from)));
478}
479
480template <>
481EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet8bf& from) {
482 EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(
483 reinterpret_cast<uint16_t*>(assume_aligned<unpacket_traits<Packet8bf>::alignment>(to)), from);
484}
485
486template <>
487EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet8bf& from) {
488 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(reinterpret_cast<uint16_t*>(to), from);
489}
490
491template <>
492EIGEN_STRONG_INLINE Packet8bf ploaddup<Packet8bf>(const bfloat16* from) {
493 return Packet8bf(ploaddup<Packet8us>(reinterpret_cast<const uint16_t*>(from)));
494}
495
496template <>
497EIGEN_STRONG_INLINE Packet8bf ploadquad<Packet8bf>(const bfloat16* from) {
498 return Packet8bf(ploadquad<Packet8us>(reinterpret_cast<const uint16_t*>(from)));
499}
500
501template <>
502EIGEN_STRONG_INLINE Packet8bf pabs(const Packet8bf& a) {
503 return Packet8bf(vandq_u16(a, vdupq_n_u16(0x7fff)));
504}
505
506template <>
507EIGEN_STRONG_INLINE Packet8bf pmin<PropagateNumbers, Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
508 return F32ToBf16Truncate(pmin<PropagateNumbers, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
509 pmin<PropagateNumbers, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
510}
511template <>
512EIGEN_STRONG_INLINE Packet8bf pmin<PropagateNaN, Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
513 return F32ToBf16Truncate(pmin<PropagateNaN, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
514 pmin<PropagateNaN, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
515}
516
517template <>
518EIGEN_STRONG_INLINE Packet8bf pmin<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
519 return F32ToBf16Truncate(pmin<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
520 pmin<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
521}
522
523template <>
524EIGEN_STRONG_INLINE Packet8bf pmax<PropagateNumbers, Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
525 return F32ToBf16Truncate(pmax<PropagateNumbers, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
526 pmax<PropagateNumbers, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
527}
528template <>
529EIGEN_STRONG_INLINE Packet8bf pmax<PropagateNaN, Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
530 return F32ToBf16Truncate(pmax<PropagateNaN, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
531 pmax<PropagateNaN, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
532}
533
534template <>
535EIGEN_STRONG_INLINE Packet8bf pmax<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
536 return F32ToBf16Truncate(pmax<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
537 pmax<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
538}
539
540template <>
541EIGEN_STRONG_INLINE Packet8bf plset<Packet8bf>(const bfloat16& a) {
542 Packet4f lo = plset<Packet4f>(static_cast<float>(a));
543 return F32ToBf16(lo, padd(lo, pset1<Packet4f>(4.0f)));
544}
545
546template <>
547EIGEN_STRONG_INLINE Packet8bf por(const Packet8bf& a, const Packet8bf& b) {
548 return Packet8bf(por<Packet8us>(Packet8us(a), Packet8us(b)));
549}
550
551template <>
552EIGEN_STRONG_INLINE Packet8bf pxor(const Packet8bf& a, const Packet8bf& b) {
553 return Packet8bf(pxor<Packet8us>(Packet8us(a), Packet8us(b)));
554}
555
556template <>
557EIGEN_STRONG_INLINE Packet8bf pand(const Packet8bf& a, const Packet8bf& b) {
558 return Packet8bf(pand<Packet8us>(Packet8us(a), Packet8us(b)));
559}
560
561template <>
562EIGEN_STRONG_INLINE Packet8bf pandnot(const Packet8bf& a, const Packet8bf& b) {
563 return Packet8bf(pandnot<Packet8us>(Packet8us(a), Packet8us(b)));
564}
565
566template <>
567EIGEN_STRONG_INLINE Packet8bf pnot(const Packet8bf& a) {
568 return Packet8bf(pnot<Packet8us>(Packet8us(a)));
569}
570
571template <>
572EIGEN_STRONG_INLINE Packet8bf ptrue(const Packet8bf& /*a*/) {
573 return Packet8bf(vdupq_n_u16(0xffff));
574}
575
576template <>
577EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8bf pselect(const Packet8bf& mask, const Packet8bf& a, const Packet8bf& b) {
578 return Packet8bf(pselect<Packet8us>(Packet8us(mask), Packet8us(a), Packet8us(b)));
579}
580
581template <>
582EIGEN_STRONG_INLINE Packet8bf print<Packet8bf>(const Packet8bf& a) {
583 return F32ToBf16Truncate(print<Packet4f>(Bf16ToF32Low(a)), print<Packet4f>(Bf16ToF32High(a)));
584}
585
586template <>
587EIGEN_STRONG_INLINE Packet8bf pfloor<Packet8bf>(const Packet8bf& a) {
588 return F32ToBf16Truncate(pfloor<Packet4f>(Bf16ToF32Low(a)), pfloor<Packet4f>(Bf16ToF32High(a)));
589}
590
591template <>
592EIGEN_STRONG_INLINE Packet8bf pceil<Packet8bf>(const Packet8bf& a) {
593 return F32ToBf16Truncate(pceil<Packet4f>(Bf16ToF32Low(a)), pceil<Packet4f>(Bf16ToF32High(a)));
594}
595
596template <>
597EIGEN_STRONG_INLINE Packet8bf pround<Packet8bf>(const Packet8bf& a) {
598 return F32ToBf16Truncate(pround<Packet4f>(Bf16ToF32Low(a)), pround<Packet4f>(Bf16ToF32High(a)));
599}
600
601template <>
602EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(const Packet8bf& a) {
603 return F32ToBf16Truncate(ptrunc<Packet4f>(Bf16ToF32Low(a)), ptrunc<Packet4f>(Bf16ToF32High(a)));
604}
605
606template <>
607EIGEN_STRONG_INLINE Packet8bf padd<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
608 return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
609 padd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
610}
611
612template <>
613EIGEN_STRONG_INLINE Packet8bf psub<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
614 return F32ToBf16KeepNaN(psub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
615 psub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
616}
617
618template <>
619EIGEN_STRONG_INLINE Packet8bf pmul<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
620 return F32ToBf16KeepNaN(pmul<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
621 pmul<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
622}
623
624template <>
625EIGEN_STRONG_INLINE Packet8bf pmadd<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
626 return F32ToBf16KeepNaN(pmadd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
627 pmadd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
628}
629
630template <>
631EIGEN_STRONG_INLINE Packet8bf pmsub<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
632 return F32ToBf16KeepNaN(pmsub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
633 pmsub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
634}
635
636template <>
637EIGEN_STRONG_INLINE Packet8bf pnmadd<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
638 return F32ToBf16KeepNaN(pnmadd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
639 pnmadd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
640}
641
642template <>
643EIGEN_STRONG_INLINE Packet8bf pnmsub<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
644 return F32ToBf16KeepNaN(pnmsub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
645 pnmsub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
646}
647
648template <>
649EIGEN_STRONG_INLINE Packet8bf pdiv<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
650 return F32ToBf16KeepNaN(pdiv<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
651 pdiv<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
652}
653
654template <>
655EIGEN_STRONG_INLINE Packet8bf pgather<bfloat16, Packet8bf>(const bfloat16* from, Index stride) {
656 return Packet8bf(pgather<uint16_t, Packet8us>(reinterpret_cast<const uint16_t*>(from), stride));
657}
658
659template <>
660EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet8bf>(bfloat16* to, const Packet8bf& from, Index stride) {
661 pscatter<uint16_t, Packet8us>(reinterpret_cast<uint16_t*>(to), Packet8us(from), stride);
662}
663
664template <>
665EIGEN_STRONG_INLINE bfloat16 predux<Packet8bf>(const Packet8bf& a) {
666 return static_cast<bfloat16>(predux<Packet4f>(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
667}
668
669template <>
670EIGEN_STRONG_INLINE bfloat16 predux_max<Packet8bf>(const Packet8bf& a) {
671 return bfloat16_impl::exact_float_to_bfloat16(
672 predux_max<Packet4f>(pmax<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
673}
674
675template <>
676EIGEN_STRONG_INLINE bfloat16 predux_min<Packet8bf>(const Packet8bf& a) {
677 return bfloat16_impl::exact_float_to_bfloat16(
678 predux_min<Packet4f>(pmin<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
679}
680
681template <>
682EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet8bf>(const Packet8bf& a) {
683 return static_cast<bfloat16>(predux_mul<Packet4f>(pmul<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
684}
685
686template <>
687EIGEN_STRONG_INLINE Packet4bf predux_half<Packet8bf>(const Packet8bf& a) {
688 return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a)));
689}
690
691template <>
692EIGEN_STRONG_INLINE bool predux_any(const Packet8bf& a) {
693 return predux_any(Packet8us(a));
694}
695
696template <>
697EIGEN_STRONG_INLINE Packet8bf preverse<Packet8bf>(const Packet8bf& a) {
698 return Packet8bf(preverse<Packet8us>(Packet8us(a)));
699}
700
701EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8bf, 4>& kernel) {
702 detail::ptranspose_impl(kernel);
703}
704
705EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8bf, 8>& kernel) {
706 detail::ptranspose_impl(kernel);
707}
708
709template <>
710EIGEN_STRONG_INLINE Packet8bf pabsdiff<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
711 return F32ToBf16KeepNaN(pabsdiff<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
712 pabsdiff<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
713}
714
715template <>
716EIGEN_STRONG_INLINE Packet8bf pcmp_eq<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
717 return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
718 pcmp_eq<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
719}
720
721template <>
722EIGEN_STRONG_INLINE Packet8bf pcmp_lt<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
723 return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
724 pcmp_lt<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
725}
726
727template <>
728EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
729 // Negate a >= b once on the narrowed mask rather than on each float half.
730 return vmvnq_u16(F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32Low(b), Bf16ToF32Low(a)),
731 pcmp_le<Packet4f>(Bf16ToF32High(b), Bf16ToF32High(a))));
732}
733
734template <>
735EIGEN_STRONG_INLINE Packet8bf pcmp_le<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
736 return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
737 pcmp_le<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
738}
739
740template <>
741EIGEN_STRONG_INLINE Packet8bf pisinf<Packet8bf>(const Packet8bf& a) {
742 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
743 constexpr uint16_t kAbsMask = (1 << 15) - 1;
744 return vceqq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf));
745}
746
747template <>
748EIGEN_STRONG_INLINE Packet8bf pisnan<Packet8bf>(const Packet8bf& a) {
749 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
750 constexpr uint16_t kAbsMask = (1 << 15) - 1;
751 return vcgtq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf));
752}
753
754template <>
755EIGEN_STRONG_INLINE Packet8bf pisfinite<Packet8bf>(const Packet8bf& a) {
756 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
757 constexpr uint16_t kAbsMask = (1 << 15) - 1;
758 return vcltq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf));
759}
760
761template <>
762EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(const Packet8bf& a) {
763 uint16x8_t abs_mask = vdupq_n_u16(0x7fff);
764 uint16x8_t is_nan = vcgtq_u16(vandq_u16(a, abs_mask), vdupq_n_u16(0x7f80));
765 uint16x8_t keep = vorrq_u16(is_nan, vdupq_n_u16(0x8000));
766 return vandq_u16(vbslq_u16(keep, a, vdupq_n_u16(0x3f80)), vtstq_u16(a, abs_mask));
767}
768
769template <>
770EIGEN_STRONG_INLINE Packet8bf pnegate<Packet8bf>(const Packet8bf& a) {
771 return Packet8bf(pxor<Packet8us>(Packet8us(a), pset1<Packet8us>(static_cast<uint16_t>(0x8000))));
772}
773
774// psqrt and prsqrt have their own kernels below.
775EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(Packet4f, Packet4bf)
776EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION_HALVES, Packet4f, Packet8bf)
777
778template <>
779EIGEN_STRONG_INLINE Packet4bf psqrt<Packet4bf>(const Packet4bf& a) {
780 // y = x r with r = FRSQRTE(x), then one FRSQRTS step y <- y (3 - y r) / 2. FRSQRTE is specified
781 // bit for bit, and rounding y to bfloat16 equals bfloat16(sqrtf(x)) for all 65536 inputs
782 // (packetmath checks them all); the raw estimate alone misrounds 7366 of them. RSHRN adds half a
783 // bfloat16 ulp and truncates. That is round half up, which differs from F32ToBf16 only on an
784 // exact float midpoint, and no input produces one; NaNs stay NaN since their low half is zero.
785 Packet4f x = Bf16ToF32(a);
786 Packet4f r = vrsqrteq_f32(x);
787 Packet4f y = pmul(x, r);
788 y = pmul(y, vrsqrtsq_f32(y, r));
789 Packet4bf ybf = vrshrn_n_u32(vreinterpretq_u32_f32(y), 16);
790 // y is NaN where r is +-inf (x is +-0, or a subnormal the FPU flushes) or +0 (x is +inf), and
791 // the answer there is the sign and exponent of a. Reading r rather than a keeps that right under
792 // FPCR.FZ and on ARMv7. The high half of r plus 0x80 is 0x80, 0 or 0x8000 on exactly those
793 // lanes; every other r, a positive normal or a NaN, has a bit set under 0x7f7f.
794 uint16x4_t u = vadd_u16(vshrn_n_u32(vreinterpretq_u32_f32(r), 16), vdup_n_u16(0x80));
795 uint16x4_t step_lane = vtst_u16(u, vdup_n_u16(0x7f7f));
796 return vbsl_u16(step_lane, ybf, vand_u16(a, vdup_n_u16(0xff80)));
797}
798
799template <>
800EIGEN_STRONG_INLINE Packet4bf prsqrt<Packet4bf>(const Packet4bf& a) {
801 // r = FRSQRTE(x) refined by one FRSQRTS step, r1 = r (3 - x r r) / 2, then the same RSHRN
802 // narrowing as psqrt. r1 rounds to the correctly rounded bfloat16 rsqrt for all 65536 inputs
803 // (packetmath checks them all). FRSQRTE has 8 fraction bits, so the raw estimate always lands on
804 // a bfloat16 value or midpoint and misrounds 1270 of them.
805 Packet4f x = Bf16ToF32(a);
806 Packet4f r = vrsqrteq_f32(x);
807#if EIGEN_ARCH_ARM64
808 // FMULX gives 2 for 0 * inf where FMUL gives NaN, so no lane needs a select. Where the FPU reads
809 // x as +-0 (a subnormal too under FPCR.FZ), r is +-inf, y = 2 and the step is -+inf, and |step|
810 // restores r1 = +-inf. Where x is +inf, r is 0, y = 2 and r1 = 0 * 1.5. Negative and NaN x stay
811 // NaN. On every other lane FMULX is FMUL and the step is positive, so r1 is the plain step.
812 Packet4f y = vmulxq_f32(x, r);
813 Packet4f r1 = pmul(r, vabsq_f32(vrsqrtsq_f32(y, r)));
814 return vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16);
815#else
816 // ARMv7 has no FMULX. r1 is NaN where x is +-0, +inf or a subnormal, which ARMv7 NEON flushes,
817 // and there r is already the answer (+-inf, 0, +-inf) with a zero low half, so those lanes take
818 // r truncated. They are the bit patterns with a - 0x80 >= 0x7f00 unsigned: the exponent-0 range
819 // wraps around, and +inf, NaN and every negative value sit above.
820 Packet4f r1 = pmul(r, vrsqrtsq_f32(pmul(x, r), r));
821 Packet4bf refined = vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16);
822 Packet4bf raw = vshrn_n_u32(vreinterpretq_u32_f32(r), 16);
823 uint16x4_t special = vcge_u16(vsub_u16(a, vdup_n_u16(0x80)), vdup_n_u16(0x7f00));
824 return vbsl_u16(special, raw, refined);
825#endif
826}
827
828template <>
829EIGEN_STRONG_INLINE Packet4bf pfrexp(const Packet4bf& a, Packet4bf& exponent) {
830 // Both results are exact: the mantissa keeps the input's significand, and the exponent is a
831 // small integer.
832 Packet4f fexponent;
833 const Packet4bf out = F32ToBf16Truncate(pfrexp<Packet4f>(Bf16ToF32(a), fexponent));
834 exponent = F32ToBf16Truncate(fexponent);
835 return out;
836}
837
838template <>
839EIGEN_STRONG_INLINE Packet4bf pldexp(const Packet4bf& a, const Packet4bf& exponent) {
840 return F32ToBf16(pldexp<Packet4f>(Bf16ToF32(a), Bf16ToF32(exponent)));
841}
842
843template <>
844EIGEN_STRONG_INLINE Packet8bf psqrt<Packet8bf>(const Packet8bf& a) {
845 // psqrt<Packet4bf> on each float half, narrowed and selected once over all 8 lanes.
846 Packet4f x_lo = Bf16ToF32Low(a);
847 Packet4f x_hi = Bf16ToF32High(a);
848 Packet4f r_lo = vrsqrteq_f32(x_lo);
849 Packet4f r_hi = vrsqrteq_f32(x_hi);
850 Packet4f y_lo = pmul(x_lo, r_lo);
851 Packet4f y_hi = pmul(x_hi, r_hi);
852 y_lo = pmul(y_lo, vrsqrtsq_f32(y_lo, r_lo));
853 y_hi = pmul(y_hi, vrsqrtsq_f32(y_hi, r_hi));
854 Packet8bf ybf = RoundingNarrowBf16(vreinterpretq_u32_f32(y_lo), vreinterpretq_u32_f32(y_hi));
855 uint16x8_t u = vaddq_u16(F32ToBf16Truncate(r_lo, r_hi), vdupq_n_u16(0x80));
856 uint16x8_t step_lane = vtstq_u16(u, vdupq_n_u16(0x7f7f));
857 return vbslq_u16(step_lane, ybf, vandq_u16(a, vdupq_n_u16(0xff80)));
858}
859
860template <>
861EIGEN_STRONG_INLINE Packet8bf prsqrt<Packet8bf>(const Packet8bf& a) {
862 // prsqrt<Packet4bf> on each float half.
863 Packet4f x_lo = Bf16ToF32Low(a);
864 Packet4f x_hi = Bf16ToF32High(a);
865 Packet4f r_lo = vrsqrteq_f32(x_lo);
866 Packet4f r_hi = vrsqrteq_f32(x_hi);
867#if EIGEN_ARCH_ARM64
868 Packet4f r1_lo = pmul(r_lo, vabsq_f32(vrsqrtsq_f32(vmulxq_f32(x_lo, r_lo), r_lo)));
869 Packet4f r1_hi = pmul(r_hi, vabsq_f32(vrsqrtsq_f32(vmulxq_f32(x_hi, r_hi), r_hi)));
870 return RoundingNarrowBf16(vreinterpretq_u32_f32(r1_lo), vreinterpretq_u32_f32(r1_hi));
871#else
872 Packet4f r1_lo = pmul(r_lo, vrsqrtsq_f32(pmul(x_lo, r_lo), r_lo));
873 Packet4f r1_hi = pmul(r_hi, vrsqrtsq_f32(pmul(x_hi, r_hi), r_hi));
874 Packet8bf refined = RoundingNarrowBf16(vreinterpretq_u32_f32(r1_lo), vreinterpretq_u32_f32(r1_hi));
875 Packet8bf raw = F32ToBf16Truncate(r_lo, r_hi);
876 uint16x8_t special = vcgeq_u16(vsubq_u16(a, vdupq_n_u16(0x80)), vdupq_n_u16(0x7f00));
877 return vbslq_u16(special, raw, refined);
878#endif
879}
880
881template <>
882EIGEN_STRONG_INLINE Packet8bf pfrexp(const Packet8bf& a, Packet8bf& exponent) {
883 Packet4f fexponent_lo;
884 Packet4f fexponent_hi;
885 Packet4f lo = pfrexp<Packet4f>(Bf16ToF32Low(a), fexponent_lo);
886 Packet4f hi = pfrexp<Packet4f>(Bf16ToF32High(a), fexponent_hi);
887 exponent = F32ToBf16Truncate(fexponent_lo, fexponent_hi);
888 return F32ToBf16Truncate(lo, hi);
889}
890
891template <>
892EIGEN_STRONG_INLINE Packet8bf pldexp(const Packet8bf& a, const Packet8bf& exponent) {
893 return F32ToBf16(pldexp<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(exponent)),
894 pldexp<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(exponent)));
895}
896
897template <>
898struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {};
899template <>
900struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
901
902template <>
903EIGEN_STRONG_INLINE Packet4f pcast<Packet4bf, Packet4f>(const Packet4bf& a) {
904 return Bf16ToF32(a);
905}
906
907template <>
908EIGEN_STRONG_INLINE Packet4bf pcast<Packet4f, Packet4bf>(const Packet4f& a) {
909 return F32ToBf16(a);
910}
911
912template <>
913EIGEN_STRONG_INLINE Packet4f pcast<Packet8bf, Packet4f>(const Packet8bf& a) {
914 return Bf16ToF32Low(a);
915}
916
917template <>
918EIGEN_STRONG_INLINE Packet8bf pcast<Packet4f, Packet8bf>(const Packet4f& a, const Packet4f& b) {
919 return F32ToBf16(a, b);
920}
921
922template <>
923EIGEN_STRONG_INLINE Packet2f pcast<Packet4bf, Packet2f>(const Packet4bf& a) {
924 return vget_low_f32(Bf16ToF32(a));
925}
926
927template <>
928EIGEN_STRONG_INLINE Packet4bf pcast<Packet2f, Packet4bf>(const Packet2f& a, const Packet2f& b) {
929 return F32ToBf16(vcombine_f32(a, b));
930}
931} // end namespace internal
932
933} // end namespace Eigen
934
935#endif // EIGEN_PACKET_MATH_BF16_NEON_H