11#ifndef THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_
12#define THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_
15#include "../../InternalHeaderCheck.h"
20EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet16f)
21EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet8d)
24EIGEN_STRONG_INLINE Packet16h pfrexp(
const Packet16h& a, Packet16h& exponent) {
26 const Packet16h out = float2half(pfrexp<Packet16f>(half2float(a), fexponent));
27 exponent = float2half(fexponent);
32EIGEN_STRONG_INLINE Packet16h pldexp(
const Packet16h& a,
const Packet16h& exponent) {
33 return float2half(pldexp<Packet16f>(half2float(a), half2float(exponent)));
37EIGEN_STRONG_INLINE Packet16bf pfrexp(
const Packet16bf& a, Packet16bf& exponent) {
41 const Packet16bf out = F32ToBf16Truncate(pfrexp<Packet16f>(Bf16ToF32(a), fexponent));
42 exponent = F32ToBf16Truncate(fexponent);
47EIGEN_STRONG_INLINE Packet16bf pldexp(
const Packet16bf& a,
const Packet16bf& exponent) {
48 return F32ToBf16(pldexp<Packet16f>(Bf16ToF32(a), Bf16ToF32(exponent)));
59EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f psqrt<Packet16f>(
const Packet16f& x) {
60 const Packet16f r = _mm512_rsqrt14_ps(x);
61 const Packet16f y = _mm512_mul_ps(x, r);
64 const Packet16f half_r = _mm512_castsi512_ps(_mm512_sub_epi32(_mm512_castps_si512(r), _mm512_set1_epi32(0x00800000)));
65 const Packet16f s = _mm512_fmadd_ps(half_r, _mm512_fnmadd_ps(y, y, x), y);
68 const __mmask16 nonzero = _mm512_cmp_ps_mask(x, _mm512_setzero_ps(), _CMP_NEQ_UQ);
69 const __mmask16 keep = _mm512_mask_cmp_ps_mask(nonzero, x, _mm512_set1_ps(NumTraits<float>::infinity()), _CMP_NEQ_UQ);
70 const Packet16f res = _mm512_mask_mov_ps(x, keep, s);
75 const __mmask16 redo = _mm512_mask_cmplt_epu32_mask(nonzero, _mm512_castps_si512(x), _mm512_set1_epi32(0x11800000));
76 return EIGEN_PREDICT_TRUE(redo == 0) ? res : _mm512_mask_sqrt_ps(res, redo, x);
80EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d psqrt<Packet8d>(
const Packet8d& x) {
81#ifdef EIGEN_VECTORIZE_AVX512ER
82 return generic_sqrt_newton_step<Packet8d, 1>::run(x, _mm512_rsqrt28_pd(x));
88 const Packet8d r = _mm512_rsqrt14_pd(x);
89 const Packet8d g0 = _mm512_mul_pd(x, r);
91 _mm512_castsi512_pd(_mm512_sub_epi64(_mm512_castpd_si512(r), _mm512_set1_epi64(0x0010000000000000ll)));
92 const Packet8d e = _mm512_fnmadd_pd(g0, h0, _mm512_set1_pd(0.5));
93 const Packet8d g = _mm512_fmadd_pd(g0, e, g0);
94 const Packet8d h = _mm512_fmadd_pd(h0, e, h0);
95 const Packet8d s = _mm512_fmadd_pd(h, _mm512_fnmadd_pd(g, g, x), g);
96 const __mmask8 nonzero = _mm512_cmp_pd_mask(x, _mm512_setzero_pd(), _CMP_NEQ_UQ);
97 const __mmask8 keep = _mm512_mask_cmp_pd_mask(nonzero, x, _mm512_set1_pd(NumTraits<double>::infinity()), _CMP_NEQ_UQ);
98 const Packet8d res = _mm512_mask_mov_pd(x, keep, s);
100 const __mmask8 redo =
101 _mm512_mask_cmplt_epu64_mask(nonzero, _mm512_castpd_si512(x), _mm512_set1_epi64(0x0430000000000000ll));
102 return EIGEN_PREDICT_TRUE(redo == 0) ? res : _mm512_mask_sqrt_pd(res, redo, x);
107EIGEN_STRONG_INLINE Packet16f psqrt<Packet16f>(
const Packet16f& x) {
108 return _mm512_sqrt_ps(x);
112EIGEN_STRONG_INLINE Packet8d psqrt<Packet8d>(
const Packet8d& x) {
113 return _mm512_sqrt_pd(x);
118#if defined(EIGEN_VECTORIZE_AVX512ER)
120EIGEN_STRONG_INLINE Packet16f prsqrt<Packet16f>(
const Packet16f& x) {
121 return _mm512_rsqrt28_ps(x);
126EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f prsqrt<Packet16f>(
const Packet16f& x) {
127 return generic_rsqrt_newton_step<Packet16f, 1>::run(x, _mm512_rsqrt14_ps(x));
134EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d prsqrt<Packet8d>(
const Packet8d& x) {
135#ifdef EIGEN_VECTORIZE_AVX512ER
136 return generic_rsqrt_newton_step<Packet8d, 1>::run(x, _mm512_rsqrt28_pd(x));
138 return generic_rsqrt_newton_step<Packet8d, 2>::run(x, _mm512_rsqrt14_pd(x));
143EIGEN_STRONG_INLINE Packet16f preciprocal<Packet16f>(
const Packet16f& a) {
144#ifdef EIGEN_VECTORIZE_AVX512ER
145 return _mm512_rcp28_ps(a);
150 const Packet16f one = pset1<Packet16f>(1.0f);
151 const Packet16f x = _mm512_rcp14_ps(a);
152 const Packet16f refined = pmadd(x, pnmadd(a, x, one), x);
153 const __mmask16 redo = _mm512_cmp_ps_mask(refined, _mm512_setzero_ps(), _CMP_EQ_UQ);
154 return redo == 0 ? refined : _mm512_mask_div_ps(refined, redo, one, a);
159EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(Packet16f, Packet16bf)
160EIGEN_INSTANTIATE_GENERIC_SQRT_FUNCS_BF16(Packet16f, Packet16bf)
162#ifndef EIGEN_VECTORIZE_AVX512FP16
163EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_F16(Packet16f, Packet16h)