11#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_DOUBLE_WORD_H
12#define EIGEN_ARCH_GENERIC_PACKET_MATH_DOUBLE_WORD_H
15#include "../../InternalHeaderCheck.h"
22template <
typename Packet>
23EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void absolute_split(
const Packet& x, Packet& n, Packet& r) {
29template <
typename Packet>
30EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twosum(
const Packet& x,
const Packet& y, Packet& s_hi, Packet& s_lo) {
33 Packet y_part = psub(s_hi, x);
35 Packet x_part = psub(s_hi, y_part);
37 Packet x_err = psub(x, x_part);
39 Packet y_err = psub(y, y_part);
41 s_lo = padd(x_err, y_err);
46template <
typename Packet>
47EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void fast_twosum(
const Packet& x,
const Packet& y, Packet& s_hi, Packet& s_lo) {
49 const Packet t = psub(s_hi, x);
53#ifdef EIGEN_VECTORIZE_FMA
55template <typename Packet, std::enable_if_t<!is_scalar<Packet>::value,
int> = 0>
56EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet twoprod_low(
const Packet& x,
const Packet& y,
const Packet& xy) {
57 return pmsub(x, y, xy);
60template <typename Scalar, std::enable_if_t<is_scalar<Scalar>::value,
int> = 0>
61EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar twoprod_low(
const Scalar& x,
const Scalar& y,
const Scalar& xy) {
63 return twoprod_low_fma(x, y, xy);
70template <
typename Packet,
bool PreventReassociation = false>
71EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twoprod(
const Packet& x,
const Packet& y, Packet& p_hi, Packet& p_lo) {
73 p_lo = twoprod_low(x, y, p_hi);
81template <typename Scalar, std::enable_if_t<has_fast_fma<Scalar>::value,
int> = 0>
82EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar twoprod_low(
const Scalar& x,
const Scalar& y,
const Scalar& xy) {
83 return twoprod_low_fma(x, y, xy);
86template <typename Scalar, bool PreventReassociation = false, std::enable_if_t<has_fast_fma<Scalar>::value,
int> = 0>
87EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twoprod(
const Scalar& x,
const Scalar& y, Scalar& p_hi, Scalar& p_lo) {
89 p_lo = twoprod_low(x, y, p_hi);
97template <
typename Packet,
bool PreventReassociation = false>
98EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void veltkamp_splitting(
const Packet& x, Packet& x_hi, Packet& x_lo) {
99 using Scalar =
typename unpacket_traits<Packet>::type;
100 constexpr int shift = (NumTraits<Scalar>::digits() + 1) / 2;
101 const Scalar shift_scale = Scalar(uint64_t(1) << shift);
102 Packet gamma = pmul(pset1<Packet>(shift_scale + Scalar(1)), x);
103 EIGEN_IF_CONSTEXPR (PreventReassociation) pbarrier(gamma);
104 Packet rho = psub(x, gamma);
105 EIGEN_IF_CONSTEXPR (PreventReassociation) pbarrier(rho);
106 x_hi = padd(rho, gamma);
107 EIGEN_IF_CONSTEXPR (PreventReassociation) pbarrier(x_hi);
108 x_lo = psub(x, x_hi);
113template <typename Packet, bool PreventReassociation = false, std::enable_if_t<!has_fast_fma<Packet>::value,
int> = 0>
114EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet twoprod_low(
const Packet& x,
const Packet& y,
const Packet& xy) {
115 Packet x_hi, x_lo, y_hi, y_lo;
116 veltkamp_splitting<Packet, PreventReassociation>(x, x_hi, x_lo);
117 veltkamp_splitting<Packet, PreventReassociation>(y, y_hi, y_lo);
119 Packet p_lo = pmsub(x_hi, y_hi, xy);
120 EIGEN_IF_CONSTEXPR (PreventReassociation) pbarrier(p_lo);
121 p_lo = pmadd(x_hi, y_lo, p_lo);
122 EIGEN_IF_CONSTEXPR (PreventReassociation) pbarrier(p_lo);
123 p_lo = pmadd(x_lo, y_hi, p_lo);
124 EIGEN_IF_CONSTEXPR (PreventReassociation) pbarrier(p_lo);
125 p_lo = pmadd(x_lo, y_lo, p_lo);
132template <typename Packet, bool PreventReassociation = false, std::enable_if_t<!has_fast_fma<Packet>::value,
int> = 0>
133EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twoprod(
const Packet& x,
const Packet& y, Packet& p_hi, Packet& p_lo) {
134 Packet hi = pmul(x, y);
135 EIGEN_IF_CONSTEXPR (PreventReassociation) pbarrier(hi);
136 const Packet lo = twoprod_low<Packet, PreventReassociation>(x, y, hi);
147#if defined(EIGEN_VECTORIZE_FMA) || defined(EIGEN_GPUCC) || defined(EIGEN_USE_SYCL)
148template <
typename Scalar>
149struct twoprod_full_range_uses_dekker : std::false_type {};
151template <
typename Scalar>
152struct twoprod_full_range_uses_dekker
153 : bool_constant<!has_fast_fma<Scalar>::value && std::numeric_limits<Scalar>::is_iec559 &&
154 std::numeric_limits<Scalar>::radix == 2 &&
155 std::numeric_limits<Scalar>::min_exponent == 3 - std::numeric_limits<Scalar>::max_exponent &&
156 std::numeric_limits<Scalar>::max_exponent - std::numeric_limits<Scalar>::min_exponent >=
157 6 * std::numeric_limits<Scalar>::digits> {};
163template <typename Scalar, std::enable_if_t<twoprod_full_range_uses_dekker<Scalar>::value,
int> = 0>
164EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twoprod_full_range_impl(
const Scalar& x,
const Scalar& y, Scalar& p_hi,
166 const Scalar eps2 = NumTraits<Scalar>::epsilon() * NumTraits<Scalar>::epsilon();
167 const Scalar abs_x = numext::abs(x);
171 if (abs_x > NumTraits<Scalar>::highest() * eps2) {
172 p_lo = twoprod_low<Scalar, true>(Scalar(x * eps2), y, Scalar(hi * eps2)) / eps2;
173 }
else if (abs_x < (std::numeric_limits<Scalar>::min)() / eps2) {
174 p_lo = twoprod_low<Scalar, true>(Scalar(x / eps2), y, Scalar(hi / eps2)) * eps2;
176 p_lo = twoprod_low<Scalar, true>(x, y, hi);
181template <typename Scalar, std::enable_if_t<!twoprod_full_range_uses_dekker<Scalar>::value,
int> = 0>
182EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twoprod_full_range_impl(
const Scalar& x,
const Scalar& y, Scalar& p_hi,
185 p_lo = twoprod_low_fma(x, y, p_hi);
189template <
typename Scalar>
190EIGEN_DEVICE_FUNC
void twoprod_full_range(
const Scalar& x,
const Scalar& y, Scalar& p_hi, Scalar& p_lo) {
191 twoprod_full_range_impl(x, y, p_hi, p_lo);
202template <
typename Packet>
203EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twosum(
const Packet& x_hi,
const Packet& x_lo,
const Packet& y_hi,
204 const Packet& y_lo, Packet& s_hi, Packet& s_lo) {
205 Packet s = padd(x_hi, y_hi);
206 Packet y_part = psub(s, x_hi);
207 Packet err = padd(psub(x_hi, psub(s, y_part)), psub(y_hi, y_part));
208 fast_twosum(s, padd(err, padd(x_lo, y_lo)), s_hi, s_lo);
213template <
typename Packet>
214EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twodiff(
const Packet& x_hi,
const Packet& x_lo,
const Packet& y_hi,
215 const Packet& y_lo, Packet& s_hi, Packet& s_lo) {
216 Packet s = psub(x_hi, y_hi);
217 Packet y_part = psub(x_hi, s);
218 Packet err = psub(psub(x_hi, padd(s, y_part)), psub(y_hi, y_part));
219 fast_twosum(s, padd(err, psub(x_lo, y_lo)), s_hi, s_lo);
224template <
typename Packet>
225EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void fast_twosum(
const Packet& x_hi,
const Packet& x_lo,
const Packet& y_hi,
226 const Packet& y_lo, Packet& s_hi, Packet& s_lo) {
228 fast_twosum(x_hi, y_hi, r_hi, r_lo);
229 const Packet s = padd(padd(y_lo, r_lo), x_lo);
230 fast_twosum(r_hi, s, s_hi, s_lo);
236template <
typename Packet>
237EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void fast_twosum(
const Packet& x,
const Packet& y_hi,
const Packet& y_lo,
238 Packet& s_hi, Packet& s_lo) {
240 fast_twosum(x, y_hi, r_hi, r_lo);
241 const Packet s = padd(y_lo, r_lo);
242 fast_twosum(r_hi, s, s_hi, s_lo);
253template <
typename Packet>
254EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twoprod(
const Packet& x_hi,
const Packet& x_lo,
const Packet& y,
255 Packet& p_hi, Packet& p_lo) {
257 twoprod(x_hi, y, c_hi, c_lo1);
258 const Packet c_lo2 = pmul(x_lo, y);
260 fast_twosum(c_hi, c_lo2, t_hi, t_lo1);
261 const Packet t_lo2 = padd(t_lo1, c_lo1);
262 fast_twosum(t_hi, t_lo2, p_hi, p_lo);
276template <
typename Packet>
277EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void twoprod(
const Packet& x_hi,
const Packet& x_lo,
const Packet& y_hi,
278 const Packet& y_lo, Packet& p_hi, Packet& p_lo) {
279 Packet p_hi_hi, p_hi_lo;
280 twoprod(x_hi, x_lo, y_hi, p_hi_hi, p_hi_lo);
281 Packet p_lo_hi, p_lo_lo;
282 twoprod(x_hi, x_lo, y_lo, p_lo_hi, p_lo_lo);
283 fast_twosum(p_hi_hi, p_hi_lo, p_lo_hi, p_lo_lo, p_hi, p_lo);
304template <
typename Packet>
305EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void fast_twoprod(
const Packet& x_hi,
const Packet& x_lo,
const Packet& y_hi,
306 const Packet& y_lo, Packet& p_hi, Packet& p_lo) {
308 twoprod(x_hi, y_hi, c_hi, c_lo);
309#ifdef EIGEN_VECTORIZE_FMA
311 constexpr bool kScalarFused = std::is_same<Packet, float>::value || std::is_same<Packet, double>::value;
313 constexpr bool kScalarFused =
false;
315 EIGEN_IF_CONSTEXPR (is_fused_madd<Packet>::value || kScalarFused) {
316 c_lo = pmadd(x_hi, y_lo, pmadd(x_lo, y_hi, c_lo));
318 c_lo = padd(c_lo, padd(pmul(x_hi, y_lo), pmul(x_lo, y_hi)));
320 fast_twosum(c_hi, c_lo, p_hi, p_lo);
327template <
typename Packet>
328EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void doubleword_div_fp(
const Packet& x_hi,
const Packet& x_lo,
const Packet& y,
329 Packet& z_hi, Packet& z_lo) {
330 const Packet t_hi = pdiv(x_hi, y);
332 twoprod(t_hi, y, pi_hi, pi_lo);
333 const Packet delta_hi = psub(x_hi, pi_hi);
334 const Packet delta_t = psub(delta_hi, pi_lo);
335 const Packet delta = padd(delta_t, x_lo);
336 const Packet t_lo = pdiv(delta, y);
337 fast_twosum(t_hi, t_lo, z_hi, z_lo);