Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
MathFunctions.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Pedro Gonnet (pedro.gonnet@gmail.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_
12#define THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_FLOAT(Packet16f)
21EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_DOUBLE(Packet8d)
22
23template <>
24EIGEN_STRONG_INLINE Packet16h pfrexp(const Packet16h& a, Packet16h& exponent) {
25 Packet16f fexponent;
26 const Packet16h out = float2half(pfrexp<Packet16f>(half2float(a), fexponent));
27 exponent = float2half(fexponent);
28 return out;
29}
30
31template <>
32EIGEN_STRONG_INLINE Packet16h pldexp(const Packet16h& a, const Packet16h& exponent) {
33 return float2half(pldexp<Packet16f>(half2float(a), half2float(exponent)));
34}
35
36template <>
37EIGEN_STRONG_INLINE Packet16bf pfrexp(const Packet16bf& a, Packet16bf& exponent) {
38 // Both results are exact: the mantissa keeps the input's significand, and the exponent is a
39 // small integer.
40 Packet16f fexponent;
41 const Packet16bf out = F32ToBf16Truncate(pfrexp<Packet16f>(Bf16ToF32(a), fexponent));
42 exponent = F32ToBf16Truncate(fexponent);
43 return out;
44}
45
46template <>
47EIGEN_STRONG_INLINE Packet16bf pldexp(const Packet16bf& a, const Packet16bf& exponent) {
48 return F32ToBf16(pldexp<Packet16f>(Bf16ToF32(a), Bf16ToF32(exponent)));
49}
50
51#if EIGEN_FAST_MATH
52// Square root from the estimate r of 1 / sqrt(x) that rsqrt14 gives to 14 bits, finished on the square root itself
53// (Karp and Markstein, High-precision division and square root, ACM TOMS 23(4), 1997): y = x r, then
54// y + (r / 2) (x - y^2). The fused residual x - y^2 is exact up to its own rounding, so the result is within 0.57 ulp:
55// correctly rounded for 99.5% of the floats and off by one ulp for the rest. A Newton step on r followed by x r, as in
56// generic_sqrt_newton_step, rounds three times after the residual is formed, which left 37% of the results off by 1
57// to 3 ulps for one operation more.
58template <>
59EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f psqrt<Packet16f>(const Packet16f& x) {
60 const Packet16f r = _mm512_rsqrt14_ps(x);
61 const Packet16f y = _mm512_mul_ps(x, r);
62 // r / 2 by its exponent field, which keeps the halving off the multiplier: r is normal wherever the result below is
63 // used.
64 const Packet16f half_r = _mm512_castsi512_ps(_mm512_sub_epi32(_mm512_castps_si512(r), _mm512_set1_epi32(0x00800000)));
65 const Packet16f s = _mm512_fmadd_ps(half_r, _mm512_fnmadd_ps(y, y, x), y);
66 // x r is NaN for x = +-0 and x = +inf, whose square root is x. A masked compare in place of two compares and an OR
67 // of their masks leaves the multiplier's port to the arithmetic.
68 const __mmask16 nonzero = _mm512_cmp_ps_mask(x, _mm512_setzero_ps(), _CMP_NEQ_UQ);
69 const __mmask16 keep = _mm512_mask_cmp_ps_mask(nonzero, x, _mm512_set1_ps(NumTraits<float>::infinity()), _CMP_NEQ_UQ);
70 const Packet16f res = _mm512_mask_mov_ps(x, keep, s);
71 // For a small x the residual, of the order of 2^-13 x, is subnormal: it loses the bits the correction needs, and
72 // all of them when subnormal results are flushed to zero (FTZ), which left the result with the 14 bits of x r.
73 // Below 2^-92 a flushed residual could move the result by more than 2^-11 ulp: those lanes take the square root
74 // instruction. Zeros, which are common, are masked out and stay on the fast path.
75 const __mmask16 redo = _mm512_mask_cmplt_epu32_mask(nonzero, _mm512_castps_si512(x), _mm512_set1_epi32(0x11800000));
76 return EIGEN_PREDICT_TRUE(redo == 0) ? res : _mm512_mask_sqrt_ps(res, redo, x);
77}
78
79template <>
80EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d psqrt<Packet8d>(const Packet8d& x) {
81#ifdef EIGEN_VECTORIZE_AVX512ER
82 return generic_sqrt_newton_step<Packet8d, /*Steps=*/1>::run(x, _mm512_rsqrt28_pd(x));
83#else
84 // As for float, after one step that takes g = x r ~ sqrt(x) and h = r / 2 ~ 1 / (2 sqrt(x)) from 14 to 28 bits
85 // together: with e = 1/2 - g h, g <- g + g e and h <- h + h e. The closing step, g + h (x - g^2), has the residual
86 // exact up to its rounding, and the result is within 0.78 ulp: correctly rounded for 99.4% of the doubles and off by
87 // one ulp for the rest. Two Newton steps on r followed by x r left 44% off by 1 to 3 ulps, for two operations more.
88 const Packet8d r = _mm512_rsqrt14_pd(x);
89 const Packet8d g0 = _mm512_mul_pd(x, r);
90 const Packet8d h0 =
91 _mm512_castsi512_pd(_mm512_sub_epi64(_mm512_castpd_si512(r), _mm512_set1_epi64(0x0010000000000000ll)));
92 const Packet8d e = _mm512_fnmadd_pd(g0, h0, _mm512_set1_pd(0.5));
93 const Packet8d g = _mm512_fmadd_pd(g0, e, g0);
94 const Packet8d h = _mm512_fmadd_pd(h0, e, h0);
95 const Packet8d s = _mm512_fmadd_pd(h, _mm512_fnmadd_pd(g, g, x), g);
96 const __mmask8 nonzero = _mm512_cmp_pd_mask(x, _mm512_setzero_pd(), _CMP_NEQ_UQ);
97 const __mmask8 keep = _mm512_mask_cmp_pd_mask(nonzero, x, _mm512_set1_pd(NumTraits<double>::infinity()), _CMP_NEQ_UQ);
98 const Packet8d res = _mm512_mask_mov_pd(x, keep, s);
99 // Below 2^-956 a residual flushed to zero could move the result by more than 2^-14 ulp.
100 const __mmask8 redo =
101 _mm512_mask_cmplt_epu64_mask(nonzero, _mm512_castpd_si512(x), _mm512_set1_epi64(0x0430000000000000ll));
102 return EIGEN_PREDICT_TRUE(redo == 0) ? res : _mm512_mask_sqrt_pd(res, redo, x);
103#endif
104}
105#else
106template <>
107EIGEN_STRONG_INLINE Packet16f psqrt<Packet16f>(const Packet16f& x) {
108 return _mm512_sqrt_ps(x);
109}
110
111template <>
112EIGEN_STRONG_INLINE Packet8d psqrt<Packet8d>(const Packet8d& x) {
113 return _mm512_sqrt_pd(x);
114}
115#endif
116
117// prsqrt for float.
118#if defined(EIGEN_VECTORIZE_AVX512ER)
119template <>
120EIGEN_STRONG_INLINE Packet16f prsqrt<Packet16f>(const Packet16f& x) {
121 return _mm512_rsqrt28_ps(x);
122}
123#elif EIGEN_FAST_MATH
124
125template <>
126EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet16f prsqrt<Packet16f>(const Packet16f& x) {
127 return generic_rsqrt_newton_step<Packet16f, /*Steps=*/1>::run(x, _mm512_rsqrt14_ps(x));
128}
129#endif
130
131// prsqrt for double.
132#if EIGEN_FAST_MATH
133template <>
134EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet8d prsqrt<Packet8d>(const Packet8d& x) {
135#ifdef EIGEN_VECTORIZE_AVX512ER
136 return generic_rsqrt_newton_step<Packet8d, /*Steps=*/1>::run(x, _mm512_rsqrt28_pd(x));
137#else
138 return generic_rsqrt_newton_step<Packet8d, /*Steps=*/2>::run(x, _mm512_rsqrt14_pd(x));
139#endif
140}
141
142template <>
143EIGEN_STRONG_INLINE Packet16f preciprocal<Packet16f>(const Packet16f& a) {
144#ifdef EIGEN_VECTORIZE_AVX512ER
145 return _mm512_rcp28_ps(a);
146#else
147 // generic_reciprocal_newton_step, with its NaN-or-zero test as one compare into a mask register (r == 0 or
148 // unordered), as for AVX: GCC compiles the generic predux_any(pcmp_lt_or_nan(...)) to a compare mask converted to a
149 // vector and back.
150 const Packet16f one = pset1<Packet16f>(1.0f);
151 const Packet16f x = _mm512_rcp14_ps(a);
152 const Packet16f refined = pmadd(x, pnmadd(a, x, one), x);
153 const __mmask16 redo = _mm512_cmp_ps_mask(refined, _mm512_setzero_ps(), _CMP_EQ_UQ);
154 return redo == 0 ? refined : _mm512_mask_div_ps(refined, redo, one, a);
155#endif
156}
157#endif
158
159EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_BF16(Packet16f, Packet16bf)
160EIGEN_INSTANTIATE_GENERIC_SQRT_FUNCS_BF16(Packet16f, Packet16bf)
161
162#ifndef EIGEN_VECTORIZE_AVX512FP16
163EIGEN_INSTANTIATE_GENERIC_MATH_FUNCS_F16(Packet16f, Packet16h)
164#endif // EIGEN_VECTORIZE_AVX512FP16
165
166} // end namespace internal
167
168} // end namespace Eigen
169
170#endif // THIRD_PARTY_EIGEN3_EIGEN_SRC_CORE_ARCH_AVX512_MATHFUNCTIONS_H_