10#ifndef EIGEN_PACKET_MATH_BF16_NEON_H
11#define EIGEN_PACKET_MATH_BF16_NEON_H
14#include "../../InternalHeaderCheck.h"
23using Packet4bf = eigen_packet_wrapper<uint16x4_t, 19>;
24using Packet8bf = eigen_packet_wrapper<uint16x8_t, 20>;
27struct is_arithmetic<Packet4bf> : std::true_type {};
29struct is_arithmetic<Packet8bf> : std::true_type {};
34struct packet_traits<bfloat16> : default_packet_traits {
35 using type = Packet8bf;
36 using half = Packet4bf;
56 HasSin = EIGEN_FAST_MATH,
57 HasCos = EIGEN_FAST_MATH,
62 HasTanh = EIGEN_FAST_MATH,
63 HasErf = EIGEN_FAST_MATH,
70struct unpacket_traits<Packet4bf> : neon_unpacket_default<Packet4bf, bfloat16> {};
72struct unpacket_traits<Packet8bf> : neon_unpacket_default<Packet8bf, bfloat16> {
73 using half = Packet4bf;
78EIGEN_ALWAYS_INLINE
void zip_in_place<Packet4bf>(Packet4bf& p1, Packet4bf& p2) {
79 const uint16x4x2_t tmp = vzip_u16(p1, p2);
84EIGEN_ALWAYS_INLINE
void zip_in_place<Packet8bf>(Packet8bf& p1, Packet8bf& p2) {
85 const uint16x8x2_t tmp = vzipq_u16(p1, p2);
98EIGEN_STRONG_INLINE Packet4bf F32ToBf16KeepNaN(
const Packet4f& p) {
99 Packet4ui bits = vreinterpretq_u32_f32(p);
100 Packet4ui round_down = vceqq_u32(vandq_u32(bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0));
101 return vraddhn_u32(bits, round_down);
109EIGEN_STRONG_INLINE Packet4bf F32ToBf16(
const Packet4f& p) {
110 uint16x4_t not_nan = vmovn_u32(vcageq_f32(pinf<Packet4f>(), p));
111 return vbsl_u16(not_nan, F32ToBf16KeepNaN(p), vdup_n_u16(0x7fc0));
117EIGEN_STRONG_INLINE Packet4bf F32ToBf16Truncate(
const Packet4f& p) {
return vshrn_n_u32(vreinterpretq_u32_f32(p), 16); }
119EIGEN_STRONG_INLINE Packet4f Bf16ToF32(
const Packet4bf& p) {
120 return Packet4f(vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(p), 16)));
123EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(
const Packet4f& p) {
return vmovn_u32(vreinterpretq_u32_f32(p)); }
127EIGEN_STRONG_INLINE Packet4f Bf16ToF32Low(
const Packet8bf& p) {
128 return vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(p), 16));
131EIGEN_STRONG_INLINE Packet4f Bf16ToF32High(
const Packet8bf& p) {
133 return vreinterpretq_f32_u32(vshll_high_n_u16(p, 16));
135 return vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(p), 16));
140EIGEN_STRONG_INLINE Packet8bf RoundingNarrowBf16(
const Packet4ui& lo,
const Packet4ui& hi) {
142 return vrshrn_high_n_u32(vrshrn_n_u32(lo, 16), hi, 16);
144 return vcombine_u16(vrshrn_n_u32(lo, 16), vrshrn_n_u32(hi, 16));
148EIGEN_STRONG_INLINE Packet8bf F32ToBf16KeepNaN(
const Packet4f& lo,
const Packet4f& hi) {
149 Packet4ui lo_bits = vreinterpretq_u32_f32(lo);
150 Packet4ui hi_bits = vreinterpretq_u32_f32(hi);
151 Packet4ui lo_round_down = vceqq_u32(vandq_u32(lo_bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0));
152 Packet4ui hi_round_down = vceqq_u32(vandq_u32(hi_bits, vdupq_n_u32(0x10000)), vdupq_n_u32(0));
154 return vraddhn_high_u32(vraddhn_u32(lo_bits, lo_round_down), hi_bits, hi_round_down);
156 return vcombine_u16(vraddhn_u32(lo_bits, lo_round_down), vraddhn_u32(hi_bits, hi_round_down));
161EIGEN_STRONG_INLINE Packet8bf F32ToBf16(
const Packet4f& lo,
const Packet4f& hi) {
162 Packet4f inf = pinf<Packet4f>();
164 vuzpq_u16(vreinterpretq_u16_u32(vcageq_f32(inf, lo)), vreinterpretq_u16_u32(vcageq_f32(inf, hi))).val[0];
165 return vbslq_u16(not_nan, F32ToBf16KeepNaN(lo, hi), vdupq_n_u16(0x7fc0));
169EIGEN_STRONG_INLINE Packet8bf F32ToBf16Truncate(
const Packet4f& lo,
const Packet4f& hi) {
170 return vuzpq_u16(vreinterpretq_u16_f32(lo), vreinterpretq_u16_f32(hi)).val[1];
173EIGEN_STRONG_INLINE Packet8bf F32MaskToBf16Mask(
const Packet4f& lo,
const Packet4f& hi) {
174 return vuzpq_u16(vreinterpretq_u16_f32(lo), vreinterpretq_u16_f32(hi)).val[0];
178EIGEN_STRONG_INLINE Packet4bf pset1<Packet4bf>(
const bfloat16& from) {
179 return Packet4bf(pset1<Packet4us>(from.value));
183EIGEN_STRONG_INLINE bfloat16 pfirst<Packet4bf>(
const Packet4bf& from) {
184 return bfloat16_impl::raw_uint16_to_bfloat16(
static_cast<uint16_t
>(pfirst<Packet4us>(Packet4us(from))));
188EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(
const bfloat16* from) {
190 pload<Packet4us>(
reinterpret_cast<const uint16_t*
>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(from))));
194EIGEN_STRONG_INLINE Packet4bf ploadu<Packet4bf>(
const bfloat16* from) {
195 return Packet4bf(ploadu<Packet4us>(
reinterpret_cast<const uint16_t*
>(from)));
199EIGEN_STRONG_INLINE
void pstore<bfloat16>(bfloat16* to,
const Packet4bf& from) {
200 EIGEN_DEBUG_ALIGNED_STORE vst1_u16(
201 reinterpret_cast<uint16_t*
>(assume_aligned<unpacket_traits<Packet4bf>::alignment>(to)), from);
205EIGEN_STRONG_INLINE
void pstoreu<bfloat16>(bfloat16* to,
const Packet4bf& from) {
206 EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(
reinterpret_cast<uint16_t*
>(to), from);
210EIGEN_STRONG_INLINE Packet4bf ploaddup<Packet4bf>(
const bfloat16* from) {
211 return Packet4bf(ploaddup<Packet4us>(
reinterpret_cast<const uint16_t*
>(from)));
215EIGEN_STRONG_INLINE Packet4bf pabs(
const Packet4bf& a) {
216 return Packet4bf(vand_u16(a, vdup_n_u16(0x7fff)));
220EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNumbers, Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
221 return F32ToBf16Truncate(pmin<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
224EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNaN, Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
225 return F32ToBf16Truncate(pmin<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
229EIGEN_STRONG_INLINE Packet4bf pmin<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
230 return F32ToBf16Truncate(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
234EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNumbers, Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
235 return F32ToBf16Truncate(pmax<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
238EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNaN, Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
239 return F32ToBf16Truncate(pmax<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
243EIGEN_STRONG_INLINE Packet4bf pmax<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
244 return F32ToBf16Truncate(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
248EIGEN_STRONG_INLINE Packet4bf plset<Packet4bf>(
const bfloat16& a) {
249 return F32ToBf16(plset<Packet4f>(
static_cast<float>(a)));
253EIGEN_STRONG_INLINE Packet4bf por(
const Packet4bf& a,
const Packet4bf& b) {
254 return Packet4bf(por<Packet4us>(Packet4us(a), Packet4us(b)));
258EIGEN_STRONG_INLINE Packet4bf pxor(
const Packet4bf& a,
const Packet4bf& b) {
259 return Packet4bf(pxor<Packet4us>(Packet4us(a), Packet4us(b)));
263EIGEN_STRONG_INLINE Packet4bf pand(
const Packet4bf& a,
const Packet4bf& b) {
264 return Packet4bf(pand<Packet4us>(Packet4us(a), Packet4us(b)));
268EIGEN_STRONG_INLINE Packet4bf pandnot(
const Packet4bf& a,
const Packet4bf& b) {
269 return Packet4bf(pandnot<Packet4us>(Packet4us(a), Packet4us(b)));
273EIGEN_STRONG_INLINE Packet4bf pnot(
const Packet4bf& a) {
274 return Packet4bf(pnot<Packet4us>(Packet4us(a)));
278EIGEN_STRONG_INLINE Packet4bf ptrue(
const Packet4bf& ) {
279 return Packet4bf(vdup_n_u16(0xffff));
283EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(
const Packet4bf& mask,
const Packet4bf& a,
const Packet4bf& b) {
284 return Packet4bf(pselect<Packet4us>(Packet4us(mask), Packet4us(a), Packet4us(b)));
288EIGEN_STRONG_INLINE Packet4bf print<Packet4bf>(
const Packet4bf& a) {
289 return F32ToBf16Truncate(print<Packet4f>(Bf16ToF32(a)));
293EIGEN_STRONG_INLINE Packet4bf pfloor<Packet4bf>(
const Packet4bf& a) {
294 return F32ToBf16Truncate(pfloor<Packet4f>(Bf16ToF32(a)));
298EIGEN_STRONG_INLINE Packet4bf pceil<Packet4bf>(
const Packet4bf& a) {
299 return F32ToBf16Truncate(pceil<Packet4f>(Bf16ToF32(a)));
303EIGEN_STRONG_INLINE Packet4bf pround<Packet4bf>(
const Packet4bf& a) {
304 return F32ToBf16Truncate(pround<Packet4f>(Bf16ToF32(a)));
308EIGEN_STRONG_INLINE Packet4bf ptrunc<Packet4bf>(
const Packet4bf& a) {
309 return F32ToBf16Truncate(ptrunc<Packet4f>(Bf16ToF32(a)));
313EIGEN_STRONG_INLINE Packet4bf padd<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
314 return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
318EIGEN_STRONG_INLINE Packet4bf psub<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
319 return F32ToBf16KeepNaN(psub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
323EIGEN_STRONG_INLINE Packet4bf pmul<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
324 return F32ToBf16KeepNaN(pmul<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
328EIGEN_STRONG_INLINE Packet4bf pmadd<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b,
const Packet4bf& c) {
329 return F32ToBf16KeepNaN(pmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
333EIGEN_STRONG_INLINE Packet4bf pmsub<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b,
const Packet4bf& c) {
334 return F32ToBf16KeepNaN(pmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
338EIGEN_STRONG_INLINE Packet4bf pnmadd<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b,
const Packet4bf& c) {
339 return F32ToBf16KeepNaN(pnmadd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
343EIGEN_STRONG_INLINE Packet4bf pnmsub<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b,
const Packet4bf& c) {
344 return F32ToBf16KeepNaN(pnmsub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
348EIGEN_STRONG_INLINE Packet4bf pdiv<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
349 return F32ToBf16KeepNaN(pdiv<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
353EIGEN_STRONG_INLINE Packet4bf pgather<bfloat16, Packet4bf>(
const bfloat16* from, Index stride) {
354 return Packet4bf(pgather<uint16_t, Packet4us>(
reinterpret_cast<const uint16_t*
>(from), stride));
358EIGEN_STRONG_INLINE
void pscatter<bfloat16, Packet4bf>(bfloat16* to,
const Packet4bf& from, Index stride) {
359 pscatter<uint16_t, Packet4us>(
reinterpret_cast<uint16_t*
>(to), Packet4us(from), stride);
363EIGEN_STRONG_INLINE bfloat16 predux<Packet4bf>(
const Packet4bf& a) {
364 return static_cast<bfloat16
>(predux<Packet4f>(Bf16ToF32(a)));
368EIGEN_STRONG_INLINE bfloat16 predux_max<Packet4bf>(
const Packet4bf& a) {
369 return bfloat16_impl::exact_float_to_bfloat16(predux_max<Packet4f>(Bf16ToF32(a)));
373EIGEN_STRONG_INLINE bfloat16 predux_min<Packet4bf>(
const Packet4bf& a) {
374 return bfloat16_impl::exact_float_to_bfloat16(predux_min<Packet4f>(Bf16ToF32(a)));
378EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet4bf>(
const Packet4bf& a) {
379 return static_cast<bfloat16
>(predux_mul<Packet4f>(Bf16ToF32(a)));
383EIGEN_STRONG_INLINE
bool predux_any(
const Packet4bf& a) {
384 return vget_lane_u64(vreinterpret_u64_u16(Packet4us(a)), 0) != 0;
388EIGEN_STRONG_INLINE Packet4bf preverse<Packet4bf>(
const Packet4bf& a) {
389 return Packet4bf(preverse<Packet4us>(Packet4us(a)));
392EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4bf, 4>& kernel) {
393 detail::ptranspose_impl(kernel);
397EIGEN_STRONG_INLINE Packet4bf pabsdiff<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
398 return F32ToBf16KeepNaN(pabsdiff<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
402EIGEN_STRONG_INLINE Packet4bf pcmp_eq<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
403 return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
407EIGEN_STRONG_INLINE Packet4bf pcmp_lt<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
408 return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
412EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
413 return F32MaskToBf16Mask(pcmp_lt_or_nan<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
418EIGEN_STRONG_INLINE Packet4bf pisinf<Packet4bf>(
const Packet4bf& a) {
419 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
420 constexpr uint16_t kAbsMask = (1 << 15) - 1;
421 return vceq_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf));
425EIGEN_STRONG_INLINE Packet4bf pisnan<Packet4bf>(
const Packet4bf& a) {
426 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
427 constexpr uint16_t kAbsMask = (1 << 15) - 1;
428 return vcgt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf));
433EIGEN_STRONG_INLINE Packet4bf psign<Packet4bf>(
const Packet4bf& a) {
434 const uint16x4_t abs_mask = vdup_n_u16(0x7fff);
435 const uint16x4_t is_nan = vcgt_u16(vand_u16(a, abs_mask), vdup_n_u16(0x7f80));
436 const uint16x4_t keep = vorr_u16(is_nan, vdup_n_u16(0x8000));
437 return vand_u16(vbsl_u16(keep, a, vdup_n_u16(0x3f80)), vtst_u16(a, abs_mask));
441EIGEN_STRONG_INLINE Packet4bf pisfinite<Packet4bf>(
const Packet4bf& a) {
442 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
443 constexpr uint16_t kAbsMask = (1 << 15) - 1;
444 return vclt_u16(vand_u16(a, vdup_n_u16(kAbsMask)), vdup_n_u16(kInf));
448EIGEN_STRONG_INLINE Packet4bf pcmp_le<Packet4bf>(
const Packet4bf& a,
const Packet4bf& b) {
449 return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
453EIGEN_STRONG_INLINE Packet4bf pnegate<Packet4bf>(
const Packet4bf& a) {
454 return Packet4bf(pxor<Packet4us>(Packet4us(a), pset1<Packet4us>(
static_cast<uint16_t
>(0x8000))));
460EIGEN_STRONG_INLINE Packet8bf pset1<Packet8bf>(
const bfloat16& from) {
461 return Packet8bf(pset1<Packet8us>(from.value));
465EIGEN_STRONG_INLINE bfloat16 pfirst<Packet8bf>(
const Packet8bf& from) {
466 return bfloat16_impl::raw_uint16_to_bfloat16(
static_cast<uint16_t
>(pfirst<Packet8us>(Packet8us(from))));
470EIGEN_STRONG_INLINE Packet8bf pload<Packet8bf>(
const bfloat16* from) {
472 pload<Packet8us>(
reinterpret_cast<const uint16_t*
>(assume_aligned<unpacket_traits<Packet8bf>::alignment>(from))));
476EIGEN_STRONG_INLINE Packet8bf ploadu<Packet8bf>(
const bfloat16* from) {
477 return Packet8bf(ploadu<Packet8us>(
reinterpret_cast<const uint16_t*
>(from)));
481EIGEN_STRONG_INLINE
void pstore<bfloat16>(bfloat16* to,
const Packet8bf& from) {
482 EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(
483 reinterpret_cast<uint16_t*
>(assume_aligned<unpacket_traits<Packet8bf>::alignment>(to)), from);
487EIGEN_STRONG_INLINE
void pstoreu<bfloat16>(bfloat16* to,
const Packet8bf& from) {
488 EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(
reinterpret_cast<uint16_t*
>(to), from);
492EIGEN_STRONG_INLINE Packet8bf ploaddup<Packet8bf>(
const bfloat16* from) {
493 return Packet8bf(ploaddup<Packet8us>(
reinterpret_cast<const uint16_t*
>(from)));
497EIGEN_STRONG_INLINE Packet8bf ploadquad<Packet8bf>(
const bfloat16* from) {
498 return Packet8bf(ploadquad<Packet8us>(
reinterpret_cast<const uint16_t*
>(from)));
502EIGEN_STRONG_INLINE Packet8bf pabs(
const Packet8bf& a) {
503 return Packet8bf(vandq_u16(a, vdupq_n_u16(0x7fff)));
507EIGEN_STRONG_INLINE Packet8bf pmin<PropagateNumbers, Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
508 return F32ToBf16Truncate(pmin<PropagateNumbers, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
509 pmin<PropagateNumbers, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
512EIGEN_STRONG_INLINE Packet8bf pmin<PropagateNaN, Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
513 return F32ToBf16Truncate(pmin<PropagateNaN, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
514 pmin<PropagateNaN, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
518EIGEN_STRONG_INLINE Packet8bf pmin<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
519 return F32ToBf16Truncate(pmin<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
520 pmin<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
524EIGEN_STRONG_INLINE Packet8bf pmax<PropagateNumbers, Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
525 return F32ToBf16Truncate(pmax<PropagateNumbers, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
526 pmax<PropagateNumbers, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
529EIGEN_STRONG_INLINE Packet8bf pmax<PropagateNaN, Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
530 return F32ToBf16Truncate(pmax<PropagateNaN, Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
531 pmax<PropagateNaN, Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
535EIGEN_STRONG_INLINE Packet8bf pmax<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
536 return F32ToBf16Truncate(pmax<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
537 pmax<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
541EIGEN_STRONG_INLINE Packet8bf plset<Packet8bf>(
const bfloat16& a) {
542 Packet4f lo = plset<Packet4f>(
static_cast<float>(a));
543 return F32ToBf16(lo, padd(lo, pset1<Packet4f>(4.0f)));
547EIGEN_STRONG_INLINE Packet8bf por(
const Packet8bf& a,
const Packet8bf& b) {
548 return Packet8bf(por<Packet8us>(Packet8us(a), Packet8us(b)));
552EIGEN_STRONG_INLINE Packet8bf pxor(
const Packet8bf& a,
const Packet8bf& b) {
553 return Packet8bf(pxor<Packet8us>(Packet8us(a), Packet8us(b)));
557EIGEN_STRONG_INLINE Packet8bf pand(
const Packet8bf& a,
const Packet8bf& b) {
558 return Packet8bf(pand<Packet8us>(Packet8us(a), Packet8us(b)));
562EIGEN_STRONG_INLINE Packet8bf pandnot(
const Packet8bf& a,
const Packet8bf& b) {
563 return Packet8bf(pandnot<Packet8us>(Packet8us(a), Packet8us(b)));
567EIGEN_STRONG_INLINE Packet8bf pnot(
const Packet8bf& a) {
568 return Packet8bf(pnot<Packet8us>(Packet8us(a)));
572EIGEN_STRONG_INLINE Packet8bf ptrue(
const Packet8bf& ) {
573 return Packet8bf(vdupq_n_u16(0xffff));
577EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8bf pselect(
const Packet8bf& mask,
const Packet8bf& a,
const Packet8bf& b) {
578 return Packet8bf(pselect<Packet8us>(Packet8us(mask), Packet8us(a), Packet8us(b)));
582EIGEN_STRONG_INLINE Packet8bf print<Packet8bf>(
const Packet8bf& a) {
583 return F32ToBf16Truncate(print<Packet4f>(Bf16ToF32Low(a)), print<Packet4f>(Bf16ToF32High(a)));
587EIGEN_STRONG_INLINE Packet8bf pfloor<Packet8bf>(
const Packet8bf& a) {
588 return F32ToBf16Truncate(pfloor<Packet4f>(Bf16ToF32Low(a)), pfloor<Packet4f>(Bf16ToF32High(a)));
592EIGEN_STRONG_INLINE Packet8bf pceil<Packet8bf>(
const Packet8bf& a) {
593 return F32ToBf16Truncate(pceil<Packet4f>(Bf16ToF32Low(a)), pceil<Packet4f>(Bf16ToF32High(a)));
597EIGEN_STRONG_INLINE Packet8bf pround<Packet8bf>(
const Packet8bf& a) {
598 return F32ToBf16Truncate(pround<Packet4f>(Bf16ToF32Low(a)), pround<Packet4f>(Bf16ToF32High(a)));
602EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(
const Packet8bf& a) {
603 return F32ToBf16Truncate(ptrunc<Packet4f>(Bf16ToF32Low(a)), ptrunc<Packet4f>(Bf16ToF32High(a)));
607EIGEN_STRONG_INLINE Packet8bf padd<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
608 return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
609 padd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
613EIGEN_STRONG_INLINE Packet8bf psub<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
614 return F32ToBf16KeepNaN(psub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
615 psub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
619EIGEN_STRONG_INLINE Packet8bf pmul<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
620 return F32ToBf16KeepNaN(pmul<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
621 pmul<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
625EIGEN_STRONG_INLINE Packet8bf pmadd<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
626 return F32ToBf16KeepNaN(pmadd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
627 pmadd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
631EIGEN_STRONG_INLINE Packet8bf pmsub<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
632 return F32ToBf16KeepNaN(pmsub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
633 pmsub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
637EIGEN_STRONG_INLINE Packet8bf pnmadd<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
638 return F32ToBf16KeepNaN(pnmadd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
639 pnmadd<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
643EIGEN_STRONG_INLINE Packet8bf pnmsub<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b,
const Packet8bf& c) {
644 return F32ToBf16KeepNaN(pnmsub<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b), Bf16ToF32Low(c)),
645 pnmsub<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b), Bf16ToF32High(c)));
649EIGEN_STRONG_INLINE Packet8bf pdiv<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
650 return F32ToBf16KeepNaN(pdiv<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
651 pdiv<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
655EIGEN_STRONG_INLINE Packet8bf pgather<bfloat16, Packet8bf>(
const bfloat16* from, Index stride) {
656 return Packet8bf(pgather<uint16_t, Packet8us>(
reinterpret_cast<const uint16_t*
>(from), stride));
660EIGEN_STRONG_INLINE
void pscatter<bfloat16, Packet8bf>(bfloat16* to,
const Packet8bf& from, Index stride) {
661 pscatter<uint16_t, Packet8us>(
reinterpret_cast<uint16_t*
>(to), Packet8us(from), stride);
665EIGEN_STRONG_INLINE bfloat16 predux<Packet8bf>(
const Packet8bf& a) {
666 return static_cast<bfloat16
>(predux<Packet4f>(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
670EIGEN_STRONG_INLINE bfloat16 predux_max<Packet8bf>(
const Packet8bf& a) {
671 return bfloat16_impl::exact_float_to_bfloat16(
672 predux_max<Packet4f>(pmax<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
676EIGEN_STRONG_INLINE bfloat16 predux_min<Packet8bf>(
const Packet8bf& a) {
677 return bfloat16_impl::exact_float_to_bfloat16(
678 predux_min<Packet4f>(pmin<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
682EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet8bf>(
const Packet8bf& a) {
683 return static_cast<bfloat16
>(predux_mul<Packet4f>(pmul<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a))));
687EIGEN_STRONG_INLINE Packet4bf predux_half<Packet8bf>(
const Packet8bf& a) {
688 return F32ToBf16KeepNaN(padd<Packet4f>(Bf16ToF32Low(a), Bf16ToF32High(a)));
692EIGEN_STRONG_INLINE
bool predux_any(
const Packet8bf& a) {
693 return predux_any(Packet8us(a));
697EIGEN_STRONG_INLINE Packet8bf preverse<Packet8bf>(
const Packet8bf& a) {
698 return Packet8bf(preverse<Packet8us>(Packet8us(a)));
701EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8bf, 4>& kernel) {
702 detail::ptranspose_impl(kernel);
705EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8bf, 8>& kernel) {
706 detail::ptranspose_impl(kernel);
710EIGEN_STRONG_INLINE Packet8bf pabsdiff<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
711 return F32ToBf16KeepNaN(pabsdiff<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
712 pabsdiff<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
716EIGEN_STRONG_INLINE Packet8bf pcmp_eq<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
717 return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
718 pcmp_eq<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
722EIGEN_STRONG_INLINE Packet8bf pcmp_lt<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
723 return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
724 pcmp_lt<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
728EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
730 return vmvnq_u16(F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32Low(b), Bf16ToF32Low(a)),
731 pcmp_le<Packet4f>(Bf16ToF32High(b), Bf16ToF32High(a))));
735EIGEN_STRONG_INLINE Packet8bf pcmp_le<Packet8bf>(
const Packet8bf& a,
const Packet8bf& b) {
736 return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(b)),
737 pcmp_le<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(b)));
741EIGEN_STRONG_INLINE Packet8bf pisinf<Packet8bf>(
const Packet8bf& a) {
742 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
743 constexpr uint16_t kAbsMask = (1 << 15) - 1;
744 return vceqq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf));
748EIGEN_STRONG_INLINE Packet8bf pisnan<Packet8bf>(
const Packet8bf& a) {
749 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
750 constexpr uint16_t kAbsMask = (1 << 15) - 1;
751 return vcgtq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf));
755EIGEN_STRONG_INLINE Packet8bf pisfinite<Packet8bf>(
const Packet8bf& a) {
756 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
757 constexpr uint16_t kAbsMask = (1 << 15) - 1;
758 return vcltq_u16(vandq_u16(a, vdupq_n_u16(kAbsMask)), vdupq_n_u16(kInf));
762EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(
const Packet8bf& a) {
763 uint16x8_t abs_mask = vdupq_n_u16(0x7fff);
764 uint16x8_t is_nan = vcgtq_u16(vandq_u16(a, abs_mask), vdupq_n_u16(0x7f80));
765 uint16x8_t keep = vorrq_u16(is_nan, vdupq_n_u16(0x8000));
766 return vandq_u16(vbslq_u16(keep, a, vdupq_n_u16(0x3f80)), vtstq_u16(a, abs_mask));
770EIGEN_STRONG_INLINE Packet8bf pnegate<Packet8bf>(
const Packet8bf& a) {
771 return Packet8bf(pxor<Packet8us>(Packet8us(a), pset1<Packet8us>(
static_cast<uint16_t
>(0x8000))));
775EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(Packet4f, Packet4bf)
776EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16_WITH(BF16_PACKET_FUNCTION_HALVES, Packet4f, Packet8bf)
779EIGEN_STRONG_INLINE Packet4bf psqrt<Packet4bf>(
const Packet4bf& a) {
785 Packet4f x = Bf16ToF32(a);
786 Packet4f r = vrsqrteq_f32(x);
787 Packet4f y = pmul(x, r);
788 y = pmul(y, vrsqrtsq_f32(y, r));
789 Packet4bf ybf = vrshrn_n_u32(vreinterpretq_u32_f32(y), 16);
794 uint16x4_t u = vadd_u16(vshrn_n_u32(vreinterpretq_u32_f32(r), 16), vdup_n_u16(0x80));
795 uint16x4_t step_lane = vtst_u16(u, vdup_n_u16(0x7f7f));
796 return vbsl_u16(step_lane, ybf, vand_u16(a, vdup_n_u16(0xff80)));
800EIGEN_STRONG_INLINE Packet4bf prsqrt<Packet4bf>(
const Packet4bf& a) {
805 Packet4f x = Bf16ToF32(a);
806 Packet4f r = vrsqrteq_f32(x);
812 Packet4f y = vmulxq_f32(x, r);
813 Packet4f r1 = pmul(r, vabsq_f32(vrsqrtsq_f32(y, r)));
814 return vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16);
820 Packet4f r1 = pmul(r, vrsqrtsq_f32(pmul(x, r), r));
821 Packet4bf refined = vrshrn_n_u32(vreinterpretq_u32_f32(r1), 16);
822 Packet4bf raw = vshrn_n_u32(vreinterpretq_u32_f32(r), 16);
823 uint16x4_t special = vcge_u16(vsub_u16(a, vdup_n_u16(0x80)), vdup_n_u16(0x7f00));
824 return vbsl_u16(special, raw, refined);
829EIGEN_STRONG_INLINE Packet4bf pfrexp(
const Packet4bf& a, Packet4bf& exponent) {
833 const Packet4bf out = F32ToBf16Truncate(pfrexp<Packet4f>(Bf16ToF32(a), fexponent));
834 exponent = F32ToBf16Truncate(fexponent);
839EIGEN_STRONG_INLINE Packet4bf pldexp(
const Packet4bf& a,
const Packet4bf& exponent) {
840 return F32ToBf16(pldexp<Packet4f>(Bf16ToF32(a), Bf16ToF32(exponent)));
844EIGEN_STRONG_INLINE Packet8bf psqrt<Packet8bf>(
const Packet8bf& a) {
846 Packet4f x_lo = Bf16ToF32Low(a);
847 Packet4f x_hi = Bf16ToF32High(a);
848 Packet4f r_lo = vrsqrteq_f32(x_lo);
849 Packet4f r_hi = vrsqrteq_f32(x_hi);
850 Packet4f y_lo = pmul(x_lo, r_lo);
851 Packet4f y_hi = pmul(x_hi, r_hi);
852 y_lo = pmul(y_lo, vrsqrtsq_f32(y_lo, r_lo));
853 y_hi = pmul(y_hi, vrsqrtsq_f32(y_hi, r_hi));
854 Packet8bf ybf = RoundingNarrowBf16(vreinterpretq_u32_f32(y_lo), vreinterpretq_u32_f32(y_hi));
855 uint16x8_t u = vaddq_u16(F32ToBf16Truncate(r_lo, r_hi), vdupq_n_u16(0x80));
856 uint16x8_t step_lane = vtstq_u16(u, vdupq_n_u16(0x7f7f));
857 return vbslq_u16(step_lane, ybf, vandq_u16(a, vdupq_n_u16(0xff80)));
861EIGEN_STRONG_INLINE Packet8bf prsqrt<Packet8bf>(
const Packet8bf& a) {
863 Packet4f x_lo = Bf16ToF32Low(a);
864 Packet4f x_hi = Bf16ToF32High(a);
865 Packet4f r_lo = vrsqrteq_f32(x_lo);
866 Packet4f r_hi = vrsqrteq_f32(x_hi);
868 Packet4f r1_lo = pmul(r_lo, vabsq_f32(vrsqrtsq_f32(vmulxq_f32(x_lo, r_lo), r_lo)));
869 Packet4f r1_hi = pmul(r_hi, vabsq_f32(vrsqrtsq_f32(vmulxq_f32(x_hi, r_hi), r_hi)));
870 return RoundingNarrowBf16(vreinterpretq_u32_f32(r1_lo), vreinterpretq_u32_f32(r1_hi));
872 Packet4f r1_lo = pmul(r_lo, vrsqrtsq_f32(pmul(x_lo, r_lo), r_lo));
873 Packet4f r1_hi = pmul(r_hi, vrsqrtsq_f32(pmul(x_hi, r_hi), r_hi));
874 Packet8bf refined = RoundingNarrowBf16(vreinterpretq_u32_f32(r1_lo), vreinterpretq_u32_f32(r1_hi));
875 Packet8bf raw = F32ToBf16Truncate(r_lo, r_hi);
876 uint16x8_t special = vcgeq_u16(vsubq_u16(a, vdupq_n_u16(0x80)), vdupq_n_u16(0x7f00));
877 return vbslq_u16(special, raw, refined);
882EIGEN_STRONG_INLINE Packet8bf pfrexp(
const Packet8bf& a, Packet8bf& exponent) {
883 Packet4f fexponent_lo;
884 Packet4f fexponent_hi;
885 Packet4f lo = pfrexp<Packet4f>(Bf16ToF32Low(a), fexponent_lo);
886 Packet4f hi = pfrexp<Packet4f>(Bf16ToF32High(a), fexponent_hi);
887 exponent = F32ToBf16Truncate(fexponent_lo, fexponent_hi);
888 return F32ToBf16Truncate(lo, hi);
892EIGEN_STRONG_INLINE Packet8bf pldexp(
const Packet8bf& a,
const Packet8bf& exponent) {
893 return F32ToBf16(pldexp<Packet4f>(Bf16ToF32Low(a), Bf16ToF32Low(exponent)),
894 pldexp<Packet4f>(Bf16ToF32High(a), Bf16ToF32High(exponent)));
898struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {};
900struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
903EIGEN_STRONG_INLINE Packet4f pcast<Packet4bf, Packet4f>(
const Packet4bf& a) {
908EIGEN_STRONG_INLINE Packet4bf pcast<Packet4f, Packet4bf>(
const Packet4f& a) {
913EIGEN_STRONG_INLINE Packet4f pcast<Packet8bf, Packet4f>(
const Packet8bf& a) {
914 return Bf16ToF32Low(a);
918EIGEN_STRONG_INLINE Packet8bf pcast<Packet4f, Packet8bf>(
const Packet4f& a,
const Packet4f& b) {
919 return F32ToBf16(a, b);
923EIGEN_STRONG_INLINE Packet2f pcast<Packet4bf, Packet2f>(
const Packet4bf& a) {
924 return vget_low_f32(Bf16ToF32(a));
928EIGEN_STRONG_INLINE Packet4bf pcast<Packet2f, Packet4bf>(
const Packet2f& a,
const Packet2f& b) {
929 return F32ToBf16(vcombine_f32(a, b));