11#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_POW_H
12#define EIGEN_ARCH_GENERIC_PACKET_MATH_POW_H
15#include "../../InternalHeaderCheck.h"
27template <
typename Packet>
28EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_halley_iteration_step(
const Packet& x_k,
30 using Scalar =
typename unpacket_traits<Packet>::type;
31 Packet x_k_cb = pmul(x_k, pmul(x_k, x_k));
32 Packet denom = pmadd(pset1<Packet>(Scalar(2)), x_k_cb, y);
33 Packet num = psub(x_k_cb, y);
34 Packet r = pdiv(num, denom);
35 return pnmadd(x_k, r, x_k);
43template <
typename Packet>
44EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_last_halley_iteration_step(
const Packet& x_k,
46 EIGEN_IF_CONSTEXPR (is_fused_madd<Packet>::value) {
47 using Scalar =
typename unpacket_traits<Packet>::type;
48 const Packet x_k_sq = pmul(x_k, x_k);
49 const Packet x_k_sq_lo = pmsub(x_k, x_k, x_k_sq);
50 const Packet x_k_cb = pmul(x_k_sq, x_k);
51 const Packet denom = pmadd(pset1<Packet>(Scalar(2)), x_k_cb, y);
52 const Packet num = pmadd(x_k_sq_lo, x_k, pmsub(x_k_sq, x_k, y));
53 const Packet r = pdiv(num, denom);
54 return pnmadd(x_k, r, x_k);
56 return cbrt_halley_iteration_step(x_k, y);
62template <
typename Packet>
63EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_decompose(
const Packet& x, Packet& e_div3) {
64 using Scalar =
typename unpacket_traits<Packet>::type;
72 constexpr Scalar kOneThird = Scalar(1) / 3;
73 e_div3 = pceil(pmul(e, pset1<Packet>(kOneThird)));
74 Packet e_mod3 = pnmadd(pset1<Packet>(Scalar(3)), e_div3, e);
77 return pldexp_fast(s, e_mod3);
80template <
typename Packet>
81EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_special_cases_and_sign(
const Packet& x,
82 const Packet& abs_root) {
84 const Packet sign_mask = psignmask<Packet>();
85 const Packet x_sign = pand(sign_mask, x);
86 Packet root = por(x_sign, abs_root);
89 const Packet is_not_finite = por(pisinf(x), pisnan(x));
90 const Packet is_zero = pcmp_eq(pzero(x), x);
91 const Packet use_x = por(is_not_finite, is_zero);
92 return pselect(use_x, x, root);
124template <
typename Packet>
125EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcbrt_float(
const Packet& x) {
126 using Scalar =
typename unpacket_traits<Packet>::type;
127 static_assert(std::is_same<Scalar, float>::value,
"Scalar type must be float");
132 const Packet y = cbrt_decompose(pabs(x), e_div3);
136 constexpr float alpha[] = {-0.884942114353179931640625f, 2.4534566402435302734375f, -2.6808016300201416015625f,
137 1.7967746257781982421875f, 0.31364738941192626953125f};
138 Packet r = ppolevl<Packet, 4>::run(y, alpha);
141 r = cbrt_last_halley_iteration_step(r, y);
144 r = pldexp_fast(r, e_div3);
146 return cbrt_special_cases_and_sign(x, r);
156template <
typename Packet>
157EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcbrt_double(
const Packet& x) {
158 using Scalar =
typename unpacket_traits<Packet>::type;
159 static_assert(std::is_same<Scalar, double>::value,
"Scalar type must be double");
164 const Packet y = cbrt_decompose(pabs(x), e_div3);
168 constexpr double alpha[] = {
169 0.59218535776880332210936330739059485495090484619140625, -1.3859252584188996859637654779362492263317108154296875,
170 1.4581131468387165117661652402603067457675933837890625, 0.340841899750747245878557123432983644306659698486328125};
171 Packet r = ppolevl<Packet, 3>::run(y, alpha);
174 r = cbrt_halley_iteration_step(r, y);
175 r = cbrt_last_halley_iteration_step(r, y);
178 r = pldexp_fast(r, e_div3);
179 return cbrt_special_cases_and_sign(x, r);
187template <
typename Scalar>
188struct accurate_log2 {
189 template <
typename Packet>
190 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void operator()(
const Packet& x, Packet& log2_x_hi, Packet& log2_x_lo)
const {
191 log2_x_hi = plog2(x);
192 log2_x_lo = pzero(x);
209struct accurate_log2<float> {
210 template <
typename Packet>
211 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void operator()(
const Packet& z, Packet& log2_x_hi, Packet& log2_x_lo)
const {
213 constexpr double kC0 = 1.442695041742110273474963832995854318141937255859375e+00;
214 constexpr float kC0_hi =
static_cast<float>(kC0);
215 constexpr float kC0_lo =
static_cast<float>(kC0 -
static_cast<double>(kC0_hi));
216 const Packet c0_hi = pset1<Packet>(kC0_hi);
217 const Packet c0_lo = pset1<Packet>(kC0_lo);
219 constexpr double kC1 = -7.2134751588268664068692714863573201000690460205078125e-01;
220 constexpr float kC1_hi =
static_cast<float>(kC1);
221 constexpr float kC1_lo =
static_cast<float>(kC1 -
static_cast<double>(kC1_hi));
222 const Packet c1_hi = pset1<Packet>(kC1_hi);
223 const Packet c1_lo = pset1<Packet>(kC1_lo);
225 constexpr float c[] = {
226 9.7010828554630279541015625e-02, -1.6896486282348632812500000e-01, 1.7200836539268493652343750e-01,
227 -1.7892272770404815673828125e-01, 2.0505344867706298828125000e-01, -2.4046677350997924804687500e-01,
228 2.8857553005218505859375000e-01, -3.6067414283752441406250000e-01, 4.8089790344238281250000000e-01};
232 const Packet one = pset1<Packet>(1.0f);
233 const Packet x = psub(z, one);
234 Packet p = ppolevl<Packet, 8>::run(x, c);
238 twoprod(x, p, p_hi, p_lo);
239 fast_twosum(c1_hi, c1_lo, p_hi, p_lo, p_hi, p_lo);
240 twoprod(p_hi, p_lo, x, p_hi, p_lo);
241 fast_twosum(c0_hi, c0_lo, p_hi, p_lo, p_hi, p_lo);
243 twoprod(p_hi, p_lo, x, log2_x_hi, log2_x_lo);
255struct accurate_log2<double> {
256 template <
typename Packet>
257 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void operator()(
const Packet& x, Packet& log2_x_hi, Packet& log2_x_lo)
const {
279 const Packet q12 = pset1<Packet>(2.87074255468000586e-9);
280 const Packet q10 = pset1<Packet>(2.38957980901884082e-8);
281 const Packet q8 = pset1<Packet>(2.31032094540014656e-7);
282 const Packet q6 = pset1<Packet>(2.27279857398537278e-6);
283 const Packet q4 = pset1<Packet>(2.31271023278625638e-5);
284 const Packet q2 = pset1<Packet>(2.47556738444535513e-4);
285 const Packet q0 = pset1<Packet>(2.88543873228900172e-3);
286 const Packet C_hi = pset1<Packet>(0.0400377511598501157);
287 const Packet C_lo = pset1<Packet>(-4.77726582251425391e-19);
288 const Packet one = pset1<Packet>(1.0);
290 const Packet cst_2_log2e_hi = pset1<Packet>(2.88539008177792677);
291 const Packet cst_2_log2e_lo = pset1<Packet>(4.07660016854549667e-17);
295 twoprod(cst_2_log2e_hi, cst_2_log2e_lo, psub(x, one), t_hi, t_lo);
299 fast_twosum(one, x, s_hi, s_lo);
301 doubleword_div_fp(t_hi, t_lo, s_hi, q_hi, q_lo);
303 fast_twosum(q_hi, pnmadd(q_hi, pdiv(s_lo, s_hi), q_lo), r_hi, r_lo);
307 twoprod(r_hi, r_lo, r_hi, r_lo, r2_hi, r2_lo);
310 twoprod(r2_hi, r2_lo, r2_hi, r2_lo, r4_hi, r4_lo);
314 Packet q_even = pmadd(q12, r4_hi, q8);
315 Packet q_odd = pmadd(q10, r4_hi, q6);
316 q_even = pmadd(q_even, r4_hi, q4);
317 q_odd = pmadd(q_odd, r4_hi, q2);
318 q_even = pmadd(q_even, r4_hi, q0);
319 Packet q = pmadd(q_odd, r2_hi, q_even);
327 twoprod(r2_hi, r2_lo, q, p_hi, p_lo);
330 fast_twosum(C_hi, C_lo, p_hi, p_lo, p1_hi, p1_lo);
333 twoprod(r2_hi, r2_lo, p1_hi, p1_lo, p2_hi, p2_lo);
336 fast_twosum(one, p2_hi, p2_lo, p3_hi, p3_lo);
339 twoprod(p3_hi, p3_lo, r_hi, r_lo, log2_x_hi, log2_x_lo);
348template <
typename Packet>
349EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet generic_pow_impl(
const Packet& x,
const Packet& y) {
350 using Scalar =
typename unpacket_traits<Packet>::type;
353 Packet m_x = pfrexp(x, e_x);
356 constexpr Scalar sqrt_half = Scalar(0.70710678118654752440);
357 const Packet m_x_scale_mask = pcmp_lt(m_x, pset1<Packet>(sqrt_half));
358 m_x = pselect(m_x_scale_mask, pmul(pset1<Packet>(Scalar(2)), m_x), m_x);
359 e_x = pselect(m_x_scale_mask, psub(e_x, pset1<Packet>(Scalar(1))), e_x);
363 accurate_log2<Scalar>()(m_x, rx_hi, rx_lo);
367 Packet f1_hi, f1_lo, f2_hi, f2_lo;
368 twoprod(e_x, y, f1_hi, f1_lo);
369 twoprod(rx_hi, rx_lo, y, f2_hi, f2_lo);
377 fast_twosum(f1_hi, f1_lo, f2_hi, f2_lo, f_hi, f_lo);
381 absolute_split(f_hi, n_z, r_z);
382 r_z = padd(r_z, f_lo);
384 absolute_split(r_z, n_r, r_z);
385 n_z = padd(n_z, n_r);
391 const Packet e_r = generic_exp2_reduced(r_z);
395 constexpr Scalar kPldExpThresh = std::numeric_limits<Scalar>::max_exponent - 2;
396 const Packet pldexp_fast_unsafe = pcmp_lt(pset1<Packet>(kPldExpThresh), pabs(n_z));
397 if (predux_any(pldexp_fast_unsafe)) {
398 return pldexp(e_r, n_z);
400 return pldexp_fast(e_r, n_z);
404template <
typename Packet>
405EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS std::enable_if_t<!is_scalar<Packet>::value, Packet> generic_pow(
406 const Packet& x,
const Packet& y) {
407 using Scalar =
typename unpacket_traits<Packet>::type;
409 const Packet cst_inf = pinf<Packet>();
410 const Packet cst_zero = pset1<Packet>(Scalar(0));
411 const Packet cst_one = pset1<Packet>(Scalar(1));
413 const Packet x_abs = pabs(x);
414 Packet result = generic_pow_impl(x_abs, y);
420 const Packet x_is_negative = pcmp_lt(x, cst_zero);
421 const Packet x_is_zero = pcmp_eq(x, cst_zero);
422 const Packet x_is_one = pcmp_eq(x, cst_one);
423 const Packet x_has_signbit = psignbit(x);
424 const Packet x_abs_gt_one = pcmp_lt(cst_one, x_abs);
425 const Packet x_abs_is_inf = pcmp_eq(x_abs, cst_inf);
428 const Packet y_abs = pabs(y);
429 const Packet y_abs_is_inf = pcmp_eq(y_abs, cst_inf);
430 const Packet y_is_negative = pcmp_lt(y, cst_zero);
431 const Packet y_is_zero = pcmp_eq(y, cst_zero);
432 const Packet y_is_one = pcmp_eq(y, cst_one);
434 const Packet y_is_int = pandnot(pcmp_eq(pfloor(y), y), y_abs_is_inf);
435 const Packet y_div_2 = pmul(y, pset1<Packet>(Scalar(0.5)));
436 const Packet y_is_even = pcmp_eq(pround(y_div_2), y_div_2);
437 const Packet y_is_odd_int = pandnot(y_is_int, y_is_even);
441 constexpr Scalar huge_exponent =
442 Scalar(2 * (numext::numeric_limits<Scalar>::digits - numext::numeric_limits<Scalar>::min_exponent + 1)) *
443 Scalar(EIGEN_LN2) / NumTraits<Scalar>::epsilon();
444 const Packet y_abs_is_huge = pcmp_le(pset1<Packet>(huge_exponent), y_abs);
450 result = por(pandnot(x_is_negative, y_is_int), result);
461 result = pselect(x_is_zero, pand(y_is_negative, cst_inf), result);
465 result = pselect(pand(x_has_signbit, y_is_odd_int), pnegate(result), result);
473 Packet inf_y_val = pand(pxor(y_is_negative, x_abs_gt_one), cst_inf);
474 inf_y_val = pselect(pcmp_eq(x, pset1<Packet>(Scalar(-1.0))), cst_one, inf_y_val);
475 result = pselect(y_abs_is_huge, inf_y_val, result);
485 auto x_pos_inf_value = pandnot(cst_inf, y_is_negative);
486 auto x_neg_inf_value = pselect(y_is_odd_int, pnegate(x_pos_inf_value), x_pos_inf_value);
487 result = pselect(x_abs_is_inf, pselect(x_is_negative, x_neg_inf_value, x_pos_inf_value), result);
490 result = por(por(pisnan(x), pisnan(y)), result);
495 result = pselect(y_is_one, x, pselect(por(x_is_one, y_is_zero), cst_one, result));
500template <
typename Scalar>
501EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS std::enable_if_t<is_scalar<Scalar>::value, Scalar> generic_pow(
502 const Scalar& x,
const Scalar& y) {
503 return numext::pow(x, y);
513template <
typename Scalar>
514constexpr numext::uint64_t max_squaring_exponent() {
515 return numext::uint64_t(1) << (std::is_same<typename NumTraits<Scalar>::Real,
double>::value ? 20 : 12);
518template <typename ScalarExponent, bool IsInteger = NumTraits<ScalarExponent>::IsInteger>
519struct exponent_helper {
520 using safe_abs_type = numext::uint64_t;
523 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type safe_abs(
const ScalarExponent& exp) {
524 eigen_assert(((numext::isfinite)(exp) && exp == numext::floor(exp)) &&
"exp must be an integer");
525 return static_cast<safe_abs_type
>(numext::abs(exp));
529template <
typename ScalarExponent>
530struct exponent_helper<ScalarExponent, true> {
533 using safe_abs_type =
typename numext::get_integer_by_size<
sizeof(ScalarExponent)>::unsigned_type;
534 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type safe_abs(
const ScalarExponent& exp) {
535 ScalarExponent mask = numext::signbit(exp);
536 safe_abs_type result = safe_abs_type(exp ^ mask);
537 return result + safe_abs_type(ScalarExponent(1) & mask);
541template <typename ScalarExponent, bool IsSigned = NumTraits<ScalarExponent>::IsSigned>
542struct exponent_is_negative {
543 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool run(
const ScalarExponent& exponent) {
544 return exponent < ScalarExponent(0);
548template <
typename ScalarExponent>
549struct exponent_is_negative<ScalarExponent, false> {
550 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool run(
const ScalarExponent&) {
return false; }
555template <
typename AbsExponentType>
556EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AbsExponentType highest_set_bit(AbsExponentType m) {
557 AbsExponentType bit = AbsExponentType(1);
558 while ((m >> 1) >= bit) bit <<= 1;
563template <
typename Packet,
typename ScalarExponent>
564EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet int_pow_wrapping(
const Packet& x,
const ScalarExponent& exponent) {
565 using Scalar =
typename unpacket_traits<Packet>::type;
566 using ExponentHelper = exponent_helper<ScalarExponent>;
567 using AbsExponentType =
typename ExponentHelper::safe_abs_type;
568 Packet cst_pos_one = pset1<Packet>(Scalar(1));
569 if (exponent == ScalarExponent(0))
return cst_pos_one;
570 eigen_assert(!exponent_is_negative<ScalarExponent>::run(exponent));
572 AbsExponentType m = ExponentHelper::safe_abs(exponent);
574 for (AbsExponentType bit = highest_set_bit(m) >> 1; bit != 0; bit >>= 1) {
576 if ((m & bit) != 0) y = pmul(y, x);
583template <
typename Scalar>
584struct is_double_word_base
585 : bool_constant<(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) &&
586 std::numeric_limits<Scalar>::is_iec559> {};
587template <
typename RealScalar>
588struct is_double_word_base<std::complex<RealScalar>> : is_double_word_base<RealScalar> {};
592template <
typename Packet,
typename>
593struct has_exponent_bit_ops : false_type {};
594template <
typename Packet>
595struct has_exponent_bit_ops<Packet, void_t<typename unpacket_traits<Packet>::integer_packet>> : true_type {};
597template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex,
598 bool IsScalar = is_scalar<Packet>::value>
602template <
typename Packet>
603struct real_view<Packet, true, false> {
604 using type =
typename unpacket_traits<Packet>::as_real;
606template <
typename Scalar>
607struct real_view<Scalar, true, true> {
608 using type =
typename NumTraits<Scalar>::Real;
611template <
typename Packet>
612struct use_double_word : bool_constant<is_double_word_base<typename unpacket_traits<Packet>::type>::value &&
613 has_exponent_bit_ops<typename real_view<Packet>::type>::value> {};
618template <
typename Packet>
619struct flushes_subnormals
620 : bool_constant<EIGEN_ARCH_ARM && !is_scalar<Packet>::value &&
621 std::is_same<typename NumTraits<typename unpacket_traits<Packet>::type>::Real, float>::value> {};
624template <
typename Packet>
625EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_shift_right(
const Packet& bits) {
626 using PacketI =
typename unpacket_traits<Packet>::integer_packet;
627 constexpr int kMantissaBits = numext::numeric_limits<typename unpacket_traits<Packet>::type>::digits - 1;
628 return preinterpret<Packet>(plogical_shift_right<kMantissaBits>(preinterpret<PacketI>(bits)));
630template <
typename Packet>
631EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_sub(
const Packet& a_bits,
const Packet& b_bits) {
632 using PacketI =
typename unpacket_traits<Packet>::integer_packet;
633 return preinterpret<Packet>(psub(preinterpret<PacketI>(a_bits), preinterpret<PacketI>(b_bits)));
639template <
typename Packet>
640struct binary_exponent_scaling {
641 using Scalar =
typename unpacket_traits<Packet>::type;
642 using Bits = std::make_unsigned_t<typename make_integer<Scalar>::type>;
643 static constexpr int kMantissaBits = numext::numeric_limits<Scalar>::digits - 1;
644 static constexpr int kBias = numext::numeric_limits<Scalar>::max_exponent - 1;
645 static constexpr Bits kExponentMask = ((Bits(1) << (CHAR_BIT *
sizeof(Scalar) - kMantissaBits - 1)) - Bits(1))
648 static constexpr Bits kMagicBits = Bits(kBias + kMantissaBits) << kMantissaBits;
652 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet inverse_scale(
const Packet& x, Packet& e) {
653 Packet exponent_bits = pand(x, pset1frombits<Packet>(kExponentMask));
654 Packet biased = exponent_bits_shift_right(exponent_bits);
655 Packet magic = pset1frombits<Packet>(kMagicBits);
656 e = psub(por(biased, magic), padd(magic, pset1<Packet>(Scalar(kBias + 1))));
658 Packet two_bias_plus_one = pset1frombits<Packet>(Bits(2 * kBias + 1) << kMantissaBits);
659 return exponent_bits_sub(two_bias_plus_one, exponent_bits);
666 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet scale_result(
const Packet& x,
const Packet& e) {
667 constexpr int kLimit = 4 * numext::numeric_limits<Scalar>::max_exponent;
668 constexpr int kZeroBelow =
669 numext::numeric_limits<Scalar>::min_exponent - numext::numeric_limits<Scalar>::digits - 64;
670 Packet zero_below = pset1<Packet>(Scalar(kZeroBelow));
671 Packet value = pselect(pcmp_lt(e, zero_below), pmul(x, pzero(x)), x);
672 Packet exponent = pmin(pmax(e, zero_below), pset1<Packet>(Scalar(kLimit)));
674 return with_subnormals(value, exponent, pldexp(value, exponent), flushes_subnormals<Packet>());
676 return pldexp(value, exponent);
680 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_subnormals(
const Packet&,
const Packet&,
const Packet& r,
689 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_subnormals(
const Packet& x,
const Packet& e,
const Packet& r,
691 using PacketI =
typename unpacket_traits<Packet>::integer_packet;
692 constexpr int kShift = numext::numeric_limits<Scalar>::digits - numext::numeric_limits<Scalar>::min_exponent;
693 Packet below = pcmp_lt(pabs(r), pset1<Packet>((numext::numeric_limits<Scalar>::min)()));
694 if (!predux_any(below))
return r;
695 Packet t = pldexp(pand(below, pabs(x)), padd(e, pset1<Packet>(Scalar(kShift))));
696 Packet k = preinterpret<Packet>(pcast<Packet, PacketI>(print(t)));
697 Packet rebuilt = pand(below, pcmp_lt(t, pset1<Packet>(Scalar(Bits(1) << kMantissaBits))));
698 return pselect(rebuilt, por(k, pand(x, pset1<Packet>(Scalar(-0.0)))), r);
702 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool any_subnormal(
const Packet& x) {
703 using PacketI =
typename unpacket_traits<Packet>::integer_packet;
704 using Int =
typename unpacket_traits<PacketI>::type;
705 PacketI magnitude = preinterpret<PacketI>(pandnot(x, pset1<Packet>(Scalar(-0.0))));
707 pand(pcmp_lt(pzero(magnitude), magnitude), pcmp_lt(magnitude, pset1<PacketI>(Int(1) << kMantissaBits)));
708 return predux_any(preinterpret<Packet>(subnormal));
714 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void input_scale(
const Packet& x, Packet& lift, Packet& scale,
716 constexpr int kDigits = numext::numeric_limits<Scalar>::digits;
717 Packet is_subnormal = pcmp_lt(pabs(x), pset1<Packet>((numext::numeric_limits<Scalar>::min)()));
718 lift = pselect(is_subnormal, pset1<Packet>(Scalar(Bits(1) << kDigits)), pset1<Packet>(Scalar(1)));
719 scale = inverse_scale(pmul(x, lift), e);
720 e = psub(e, pselect(is_subnormal, pset1<Packet>(Scalar(kDigits)), pzero(x)));
730template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex>
731struct repeated_squaring_ops {
732 using Scalar =
typename unpacket_traits<Packet>::type;
733 using Scaling = binary_exponent_scaling<Packet>;
734 using Bound = Packet;
736 Packet hi, lo, exponent, special;
738 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool any_subnormal(
const Packet& x) {
return Scaling::any_subnormal(x); }
740 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool any_below_normal(
const Packet& x,
const Packet& r) {
741 Packet abs_x = pabs(x);
742 Packet regular = pand(pcmp_lt(pzero(x), abs_x), pcmp_lt(abs_x, pset1<Packet>(NumTraits<Scalar>::infinity())));
743 return predux_any(pand(regular, pcmp_lt(pabs(r), pset1<Packet>((numext::numeric_limits<Scalar>::min)()))));
746 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool in_range(
const Packet& x,
const Packet& bound) {
747 Packet abs_x = pabs(x);
748 Packet out = por(pcmp_lt(pmul(abs_x, bound), pset1<Packet>(Scalar(1))), pcmp_lt_or_nan(bound, abs_x));
749 return !predux_any(out);
754 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void power_base(
const Packet& m,
bool reciprocal, Packet& hi,
761 Packet cst_pos_one = pset1<Packet>(Scalar(1));
762 Packet q = pdiv(cst_pos_one, m);
764 twoprod(q, m, p_hi, p_lo);
765 fast_twosum(q, pdiv(psub(psub(cst_pos_one, p_hi), p_lo), m), hi, lo);
767 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE State base(
const Packet& x,
bool reciprocal,
bool scaled,
771 power_base(x, reciprocal, b.hi, b.lo);
772 b.exponent = pzero(x);
777 Scaling::input_scale(x, lift, scale, b.exponent);
778 power_base(pmul(pmul(x, lift), scale), reciprocal, b.hi, b.lo);
780 b.exponent = pnegate(b.exponent);
781 b.special = pdiv(pset1<Packet>(Scalar(1)), x);
787 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void multiply(State& y,
const State& b) {
788 fast_twoprod(y.hi, y.lo, b.hi, b.lo, y.hi, y.lo);
789 y.exponent = padd(y.exponent, b.exponent);
791 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void square(State& y) { multiply(y, y); }
792 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void renormalize(State& y) {
794 Packet scale = Scaling::inverse_scale(y.hi, e);
795 y.hi = pmul(y.hi, scale);
796 y.lo = pmul(y.lo, scale);
797 y.exponent = padd(y.exponent, e);
799 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet result(
const Packet&,
const State& y,
const State& b,
bool odd,
801 if (!scaled)
return y.hi;
802 Packet use_special = por(pisnan(y.hi), pcmp_eq(y.hi, pzero(y.hi)));
803 return pselect(use_special, odd ? b.special : pabs(b.special), Scaling::scale_result(y.hi, y.exponent));
807 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
int renormalization_steps(
const Scalar&) {
return 4; }
809 struct SpecialBases {
810 static constexpr bool any =
false;
812 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SpecialBases special_bases(
const Packet&,
bool) {
813 return SpecialBases();
815 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet without_special_bases(
const Packet& x,
const SpecialBases&) {
818 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_special_bases(
const Packet& r,
const SpecialBases&,
bool) {
826template <typename Packet, bool IsScalar = is_scalar<Packet>::value>
827struct complex_components {
828 using Scalar =
typename unpacket_traits<Packet>::type;
829 using RealScalar =
typename NumTraits<Scalar>::Real;
830 using R =
typename unpacket_traits<Packet>::as_real;
831 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R flip(
const R& x) {
return pcplxflip(Packet(x)).v; }
832 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R odd_lanes() {
833 return pcmp_eq(pset1<Packet>(Scalar(0, 1)).v, pset1<R>(RealScalar(1)));
835 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void split(
const Packet& z, R& re, R& im) {
837 re = pselect(odd, flip(z.v), z.v);
838 im = pselect(odd, z.v, flip(z.v));
840 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet join(
const R& re,
const R& im) {
841 return Packet(pselect(odd_lanes(), im, re));
844 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R magnitude(
const Packet& z) {
846 return pmax(abs_z, flip(abs_z));
849 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R minor_magnitude(
const Packet& z) {
851 return pmin(abs_z, flip(abs_z));
854 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R exceeds(
const Packet& z,
const R& bound) {
855 return pcmp_lt_or_nan(bound, pabs(z.v));
857 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool any_zero(
const Packet& z) {
858 return predux_any(pcmp_eq(z.v, pzero(z.v)));
861 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void special_masks(
const Packet& z, R& zero, R& inf, R& nan) {
862 R zero_lane = pcmp_eq(z.v, pzero(z.v));
863 R inf_lane = pcmp_eq(pabs(z.v), pset1<R>(NumTraits<RealScalar>::infinity()));
864 R nan_lane = pisnan(z.v);
865 zero = pand(zero_lane, flip(zero_lane));
866 inf = por(inf_lane, flip(inf_lane));
867 nan = pandnot(por(nan_lane, flip(nan_lane)), inf);
869 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet replace(
const R& mask,
const Scalar& value,
const Packet& z) {
870 return Packet(pselect(mask, pset1<Packet>(value).v, z.v));
874template <
typename Scalar>
875struct complex_components<Scalar, true> {
876 using R =
typename NumTraits<Scalar>::Real;
877 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void split(
const Scalar& z, R& re, R& im) {
878 re = numext::real(z);
879 im = numext::imag(z);
881 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar join(
const R& re,
const R& im) {
return Scalar(re, im); }
882 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R magnitude(
const Scalar& z) {
883 return numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z)));
885 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R minor_magnitude(
const Scalar& z) {
886 return numext::mini(numext::abs(numext::real(z)), numext::abs(numext::imag(z)));
888 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R exceeds(
const Scalar& z,
const R& bound) {
889 return por(pcmp_lt_or_nan(bound, numext::abs(numext::real(z))),
890 pcmp_lt_or_nan(bound, numext::abs(numext::imag(z))));
892 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool any_zero(
const Scalar& z) {
893 return numext::real(z) == R(0) || numext::imag(z) == R(0);
895 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void special_masks(
const Scalar& z, R& zero, R& inf, R& nan) {
896 R re = numext::real(z), im = numext::imag(z);
897 bool is_inf = (numext::isinf)(re) || (numext::isinf)(im);
898 zero = re == R(0) && im == R(0) ? R(1) : R(0);
899 inf = is_inf ? R(1) : R(0);
900 nan = !is_inf && ((numext::isnan)(re) || (numext::isnan)(im)) ? R(1) : R(0);
902 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar replace(
const R& mask,
const Scalar& value,
const Scalar& z) {
903 return mask != R(0) ? value : z;
915template <
typename Packet>
916struct repeated_squaring_ops<Packet, true> {
917 using Scalar =
typename unpacket_traits<Packet>::type;
918 using Real =
typename NumTraits<Scalar>::Real;
919 using Components = complex_components<Packet>;
920 using R =
typename Components::R;
921 using Scaling = binary_exponent_scaling<R>;
926 R re_hi, re_lo, im_hi, im_lo, exponent;
927 R separated, coefficient_hi, coefficient_lo, coefficient_exponent;
929 bool negative, any_zero;
931 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool any_subnormal(
const Packet& x) {
932 return Scaling::any_subnormal(x.v);
935 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool any_below_normal(
const Packet&,
const Packet&) {
return false; }
942 static constexpr int kMinExponent = numext::numeric_limits<Real>::min_exponent;
943 static constexpr int kDigits = numext::numeric_limits<Real>::digits;
944 static constexpr int kSeparationExponent = kMinExponent + kDigits + 64;
945 static constexpr int kFirstOrderLog2Limit = (1 - kDigits - 2 * kSeparationExponent) / 2;
946 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool every_step(
const Real& count) {
947 return kFirstOrderLog2Limit < 64 &&
948 count > Real(numext::uint64_t(1) << (kFirstOrderLog2Limit < 64 ? kFirstOrderLog2Limit : 0));
950 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
int renormalization_steps(
const Real& count) {
951 return every_step(count) ? 1 : 4;
955 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
bool in_range(
const Packet& x,
const R& bound) {
956 R magnitude = Components::magnitude(x);
957 R minor = Components::minor_magnitude(x);
958 R one = pset1<R>(Real(1));
959 R out = por(pcmp_lt(pmul(magnitude, bound), one), Components::exceeds(x, bound));
960 out = por(out, pandnot(pcmp_lt(pmul(minor, bound), one), pcmp_eq(minor, pzero(minor))));
961 return predux_any(out) ==
false;
963 template <
typename Count>
964 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE State base(
const Packet& x,
bool reciprocal,
bool scaled,
967 b.any_separated =
false;
969 Components::split(x, re, im);
970 b.negative = reciprocal;
971 b.any_zero = Components::any_zero(x);
973 power_base(re, im, reciprocal, b);
974 b.exponent = pzero(re);
978 Scaling::input_scale(Components::magnitude(x), lift, scale, b.exponent);
979 R wr = pmul(pmul(re, lift), scale), wi = pmul(pmul(im, lift), scale);
984 int separation_exponent = every_step(Real(m)) ? kSeparationExponent - 56 : kSeparationExponent;
986 pset1frombits<R>(
typename Scaling::Bits(Scaling::kBias + separation_exponent) << Scaling::kMantissaBits);
988 R re_separated = pandnot(pcmp_lt(pabs(wr), separation), pcmp_eq(re, zero));
989 R im_separated = pandnot(pcmp_lt(pabs(wi), separation), pcmp_eq(im, zero));
990 b.separated = por(re_separated, im_separated);
991 b.any_separated = predux_any(b.separated);
992 if (b.any_separated) {
993 R s_lift, s_scale, s_exponent, count_exponent;
994 Scaling::input_scale(pselect(im_separated, im, re), s_lift, s_scale, s_exponent);
995 R s = pmul(pmul(pselect(im_separated, im, re), s_lift), s_scale);
998 numext::uint64_t n = numext::uint64_t(m);
999 numext::uint64_t unit = highest_set_bit(n) >> (kDigits - 1);
1000 numext::uint64_t n_hi = unit > 1 ? n & ~(unit - 1) : n;
1001 R count_hi = pset1<R>(Real(n_hi)), count_lo = pset1<R>(Real(n - n_hi));
1002 R count_scale = Scaling::inverse_scale(count_hi, count_exponent);
1006 R l = pselect(im_separated, wr, wi);
1008 R p_hi, p_lo, c_hi, c_lo;
1009 twoprod(q, l, p_hi, p_lo);
1010 R q_lo = pdiv(psub(psub(s, p_hi), p_lo), l);
1011 R scaled_count = pmul(count_hi, count_scale);
1012 R scaled_count_lo = pmul(count_lo, count_scale);
1013 twoprod(scaled_count, q, c_hi, c_lo);
1014 fast_twosum(c_hi, pmadd(scaled_count, q_lo, pmadd(scaled_count_lo, q, c_lo)), p_hi, p_lo);
1015 R sign = pselect(im_separated, pset1<R>(Real(reciprocal ? -0.125 : 0.125)),
1016 pset1<R>(Real(reciprocal ? 0.125 : -0.125)));
1017 b.coefficient_hi = pmul(p_hi, sign);
1018 b.coefficient_lo = pmul(p_lo, sign);
1019 b.coefficient_exponent = padd(padd(psub(s_exponent, b.exponent), count_exponent), pset1<R>(Real(3)));
1020 wr = pselect(re_separated, zero, wr);
1021 wi = pselect(im_separated, zero, wi);
1023 power_base(wr, wi, reciprocal, b);
1024 if (reciprocal) b.exponent = pnegate(b.exponent);
1031 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void power_base(
const R& wr,
const R& wi,
bool reciprocal, State& b) {
1032 b.re_lo = b.im_lo = pzero(wr);
1038 R inv = pdiv(pset1<R>(
typename NumTraits<Scalar>::Real(1)), pmadd(wr, wr, pmul(wi, wi)));
1039 R qr = pmul(wr, inv), qi = pnegate(pmul(wi, inv));
1040 R a_hi, a_lo, c_hi, c_lo, s_hi, s_lo, t_hi, t_lo;
1041 twoprod(qr, wr, a_hi, a_lo);
1042 twoprod(qi, wi, c_hi, c_lo);
1043 twodiff(a_hi, a_lo, c_hi, c_lo, s_hi, s_lo);
1044 twoprod(qr, wi, a_hi, a_lo);
1045 twoprod(qi, wr, c_hi, c_lo);
1046 twosum(a_hi, a_lo, c_hi, c_lo, t_hi, t_lo);
1047 R er = psub(psub(pset1<R>(
typename NumTraits<Scalar>::Real(1)), s_hi), s_lo);
1048 R t = padd(t_hi, t_lo);
1049 fast_twosum(qr, pmadd(er, qr, pmul(t, qi)), b.re_hi, b.re_lo);
1050 fast_twosum(qi, pmsub(er, qi, pmul(t, qr)), b.im_hi, b.im_lo);
1052 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void square(State& y) {
1053 R a_hi, a_lo, c_hi, c_lo, p_hi, p_lo;
1054 fast_twoprod(y.re_hi, y.re_lo, y.re_hi, y.re_lo, a_hi, a_lo);
1055 fast_twoprod(y.im_hi, y.im_lo, y.im_hi, y.im_lo, c_hi, c_lo);
1056 fast_twoprod(y.re_hi, y.re_lo, y.im_hi, y.im_lo, p_hi, p_lo);
1057 twodiff(a_hi, a_lo, c_hi, c_lo, y.re_hi, y.re_lo);
1058 y.im_hi = padd(p_hi, p_hi);
1059 y.im_lo = padd(p_lo, p_lo);
1060 y.exponent = padd(y.exponent, y.exponent);
1062 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void multiply(State& y,
const State& b) {
1063 R ac_hi, ac_lo, bd_hi, bd_lo, ad_hi, ad_lo, bc_hi, bc_lo;
1064 fast_twoprod(y.re_hi, y.re_lo, b.re_hi, b.re_lo, ac_hi, ac_lo);
1065 fast_twoprod(y.im_hi, y.im_lo, b.im_hi, b.im_lo, bd_hi, bd_lo);
1066 fast_twoprod(y.re_hi, y.re_lo, b.im_hi, b.im_lo, ad_hi, ad_lo);
1067 fast_twoprod(y.im_hi, y.im_lo, b.re_hi, b.re_lo, bc_hi, bc_lo);
1068 twodiff(ac_hi, ac_lo, bd_hi, bd_lo, y.re_hi, y.re_lo);
1069 twosum(ad_hi, ad_lo, bc_hi, bc_lo, y.im_hi, y.im_lo);
1070 y.exponent = padd(y.exponent, b.exponent);
1072 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void renormalize(State& y) {
1074 R scale = Scaling::inverse_scale(pmax(pabs(y.re_hi), pabs(y.im_hi)), e);
1075 y.re_hi = pmul(y.re_hi, scale);
1076 y.re_lo = pmul(y.re_lo, scale);
1077 y.im_hi = pmul(y.im_hi, scale);
1078 y.im_lo = pmul(y.im_lo, scale);
1079 y.exponent = padd(y.exponent, e);
1081 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet result(
const Packet& x,
const State& y,
const State& b,
bool odd,
1084 R re = y.re_hi, im = y.im_hi;
1085 if (b.any_zero) zero_component_signs(x, b.negative, odd, re, im);
1086 return Components::join(re, im);
1088 R re = Scaling::scale_result(y.re_hi, y.exponent);
1089 R im = Scaling::scale_result(y.im_hi, y.exponent);
1090 if (b.any_separated) {
1092 R e = padd(y.exponent, b.coefficient_exponent);
1094 R re_hi, re_lo, im_hi, im_lo;
1095 fast_twoprod(b.coefficient_hi, b.coefficient_lo, y.im_hi, y.im_lo, re_hi, re_lo);
1096 fast_twoprod(b.coefficient_hi, b.coefficient_lo, y.re_hi, y.re_lo, im_hi, im_lo);
1097 re = pselect(pand(b.separated, pcmp_eq(y.re_hi, zero)), Scaling::scale_result(pnegate(re_hi), e), re);
1098 im = pselect(pand(b.separated, pcmp_eq(y.im_hi, zero)), Scaling::scale_result(im_hi, e), im);
1100 if (b.any_zero) zero_component_signs(x, b.negative, odd, re, im);
1101 return Components::join(re, im);
1108 struct SpecialBases {
1112 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SpecialBases special_bases(
const Packet& x,
bool scaled) {
1113 SpecialBases special;
1115 special.any =
false;
1118 Components::special_masks(x, zero, inf, nan);
1119 special.any = predux_any(por(por(zero, inf), nan));
1123 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet without_special_bases(
const Packet& x,
1124 const SpecialBases& special) {
1125 if (!special.any)
return x;
1127 Components::special_masks(x, zero, inf, nan);
1128 return Components::replace(por(por(zero, inf), nan), Scalar(1, 1), x);
1130 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_special_bases(
const Packet& r,
const SpecialBases& special,
1132 if (!special.any)
return r;
1134 Components::special_masks(special.x, zero, inf, nan);
1135 Real real_inf = NumTraits<Real>::infinity(), real_nan = NumTraits<Real>::quiet_NaN();
1136 Packet out = Components::replace(negative ? zero : inf, Scalar(real_inf, Real(0)), r);
1137 out = Components::replace(negative ? inf : zero, Scalar(0), out);
1138 return Components::replace(nan, Scalar(real_nan, real_nan), out);
1144 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
void zero_component_signs(
const Packet& x,
bool negative,
bool odd,
1148 Components::split(x, x_re, x_im);
1149 R imaginary = pcmp_eq(x_re, zero);
1150 R one_zero = pxor(imaginary, pcmp_eq(x_im, zero));
1151 R minus_zero = pset1<R>(Real(-0.0));
1153 R power_imaginary = odd ? imaginary : zero;
1154 R n_sign = negative ? minus_zero : zero;
1155 R extra = odd ? n_sign : pselect(imaginary, pxor(n_sign, minus_zero), n_sign);
1156 R sign = pxor(pand(pxor(pxor(x_re, x_im), pselect(power_imaginary, im, re)), minus_zero), extra);
1157 re = pselect(pand(one_zero, power_imaginary), sign, re);
1158 im = pselect(pandnot(one_zero, power_imaginary), sign, im);
1163template <
typename Packet,
typename ScalarExponent>
1164EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_plain(
const Packet& x,
const ScalarExponent& exponent) {
1165 using Scalar =
typename unpacket_traits<Packet>::type;
1166 using ExponentHelper = exponent_helper<ScalarExponent>;
1167 using AbsExponentType =
typename ExponentHelper::safe_abs_type;
1168 if (exponent == ScalarExponent(0))
return pset1<Packet>(Scalar(1));
1170 Packet base = exponent_is_negative<ScalarExponent>::run(exponent) ? pdiv(pset1<Packet>(Scalar(1)), x) : x;
1171 AbsExponentType m = ExponentHelper::safe_abs(exponent);
1173 for (AbsExponentType bit = highest_set_bit(m) >> 1; bit != 0; bit >>= 1) {
1175 if ((m & bit) != 0) y = pmul(y, base);
1189template <
typename Packet,
typename ScalarExponent>
1190EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(
const Packet& x,
const ScalarExponent& exponent) {
1191 using Scalar =
typename unpacket_traits<Packet>::type;
1192 using ExponentHelper = exponent_helper<ScalarExponent>;
1193 using AbsExponentType =
typename ExponentHelper::safe_abs_type;
1194 using Ops = repeated_squaring_ops<Packet>;
1195 if (exponent == ScalarExponent(0))
return pset1<Packet>(Scalar(1));
1197 bool negative = exponent_is_negative<ScalarExponent>::run(exponent);
1198 AbsExponentType m = ExponentHelper::safe_abs(exponent);
1199 bool odd = (m & AbsExponentType(1)) != 0;
1200 if (m == AbsExponentType(1) && !negative)
return x;
1203 EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsComplex) {
1204 if (m == AbsExponentType(1) && !flushes_subnormals<Packet>::value)
return pdiv(pset1<Packet>(Scalar(1)), x);
1205 if (m == AbsExponentType(2) && !negative)
return pmul(x, x);
1207 AbsExponentType top = highest_set_bit(m);
1215 using Real =
typename NumTraits<Scalar>::Real;
1216 using RealBits = std::make_unsigned_t<typename make_integer<Real>::type>;
1217 constexpr int kBudget = -(numext::numeric_limits<Real>::min_exponent + numext::numeric_limits<Real>::digits);
1218 constexpr int kMantissaBits = numext::numeric_limits<Real>::digits - 1;
1219 constexpr RealBits kBiasBits = RealBits(numext::numeric_limits<Real>::max_exponent - 1);
1220 int b = numext::uint64_t(m) > numext::uint64_t(kBudget) ? 0 : kBudget / int(m);
1221 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex)
1222 b = numext::mini(b - 1, (numext::numeric_limits<Real>::max_exponent - 1) / 2);
1225 EIGEN_IF_CONSTEXPR (flushes_subnormals<Packet>::value) b = -1;
1226 typename Ops::Bound bound =
1227 pset1frombits<typename Ops::Bound>(RealBits(kBiasBits + RealBits(b < 0 ? 0 : b)) << kMantissaBits);
1228 bool scaled = b < 0 || !Ops::in_range(x, bound);
1230 typename Ops::SpecialBases special = Ops::special_bases(x, scaled);
1231 Packet x_regular = Ops::without_special_bases(x, special);
1232 if (special.any) scaled = b < 0 || !Ops::in_range(x_regular, bound);
1234 typename Ops::State base = Ops::base(x_regular, negative, scaled, m);
1235 typename Ops::State y = base;
1236 int renormalization_steps = Ops::renormalization_steps(Real(m));
1237 int steps_since_renormalization = 0;
1238 for (AbsExponentType bit = top >> 1; bit != 0; bit >>= 1) {
1240 if ((m & bit) != 0) Ops::multiply(y, base);
1241 if (scaled && ++steps_since_renormalization == renormalization_steps) {
1242 Ops::renormalize(y);
1243 steps_since_renormalization = 0;
1246 return Ops::with_special_bases(Ops::result(x_regular, y, base, odd, scaled), special, negative);
1249template <
typename Packet,
typename ScalarExponent>
1250EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE Packet int_pow_lanewise(
const Packet& x,
const ScalarExponent& exponent) {
1251 using Scalar =
typename unpacket_traits<Packet>::type;
1252 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar values[unpacket_traits<Packet>::size];
1254 for (Scalar& value : values) value = int_pow_double_word(value, exponent);
1255 return pload<Packet>(values);
1258template <
typename Packet,
typename ScalarExponent>
1259EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(
const Packet& x,
const ScalarExponent& exponent,
1261 return int_pow_double_word(x, exponent);
1267template <
typename Packet,
typename ScalarExponent>
1268EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(
const Packet& x,
const ScalarExponent& exponent,
1270 using Ops = repeated_squaring_ops<Packet>;
1271 if (Ops::any_subnormal(x))
return int_pow_lanewise(x, exponent);
1272 Packet r = int_pow_double_word(x, exponent);
1273 if (exponent_helper<ScalarExponent>::safe_abs(exponent) <= 2 && Ops::any_below_normal(x, r))
1274 return int_pow_lanewise(x, exponent);
1278template <
typename Packet,
typename ScalarExponent>
1279EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(
const Packet& x,
const ScalarExponent& exponent, true_type) {
1281 return int_pow_double_word(x, exponent, flushes_subnormals<Packet>());
1283 return int_pow_double_word(x, exponent);
1287template <
typename Packet,
typename ScalarExponent>
1288EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(
const Packet& x,
const ScalarExponent& exponent, false_type) {
1289 return int_pow_plain(x, exponent);
1292template <
typename Packet,
typename ScalarExponent>
1293EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(
const Packet& x,
const ScalarExponent& exponent) {
1294 return int_pow(x, exponent, use_double_word<Packet>());
1299template <
typename Packet,
typename ScalarExponent>
1300EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool use_repeated_squaring(
const ScalarExponent& exponent) {
1301 using Scalar =
typename unpacket_traits<Packet>::type;
1302 return use_double_word<Packet>::value ? numext::abs(exponent) <= ScalarExponent(max_squaring_exponent<Scalar>())
1303 : (exponent <= ScalarExponent(7) && exponent >= ScalarExponent(-3));
1308template <
typename Packet,
typename ScalarExponent>
1309EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool use_repeated_squaring_for_integer(
const ScalarExponent& exponent) {
1310 using Scalar =
typename unpacket_traits<Packet>::type;
1311 constexpr numext::uint64_t kExactLimit = numext::uint64_t(1) << numext::numeric_limits<Scalar>::digits;
1312 return exponent_helper<ScalarExponent>::safe_abs(exponent) > kExactLimit ||
1313 use_repeated_squaring<Packet>(
static_cast<Scalar
>(exponent));
1316template <
typename Packet>
1317EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!is_scalar<Packet>::value, Packet> gen_pow(
1318 const Packet& x,
const typename unpacket_traits<Packet>::type& exponent) {
1319 const Packet exponent_packet = pset1<Packet>(exponent);
1321 return generic_pow_impl(pabs(x), exponent_packet);
1324template <
typename Scalar>
1325EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<is_scalar<Scalar>::value, Scalar> gen_pow(
1326 const Scalar& x,
const Scalar& exponent) {
1327 return numext::pow(x, exponent);
1334template <
typename Packet,
typename ScalarExponent>
1335EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_nonint_nonint_errors(
const Packet& x,
const Packet& powx,
1336 const ScalarExponent& exponent) {
1337 using Scalar =
typename unpacket_traits<Packet>::type;
1338 const Packet cst_zero = pzero(x);
1339 const Packet cst_one = pset1<Packet>(Scalar(1));
1340 const Packet cst_inf = pinf<Packet>();
1341 const Packet cst_nan = pnan<Packet>();
1343 const Packet abs_x = pabs(x);
1346 Packet result = por(pcmp_lt(x, cst_zero), powx);
1348 if (!(numext::isfinite)(exponent)) {
1349 if (exponent != exponent) {
1351 result = pselect(pcmp_eq(x, cst_one), cst_one, cst_nan);
1354 const Packet abs_x_is_one = pcmp_eq(abs_x, cst_one);
1355 if (exponent > ScalarExponent(0)) {
1357 result = pand(pcmp_lt(cst_one, abs_x), cst_inf);
1360 result = pand(pcmp_lt(abs_x, cst_one), cst_inf);
1363 result = pselect(abs_x_is_one, cst_one, result);
1367 const Packet x_is_zero = pcmp_eq(x, cst_zero);
1368 const Packet abs_x_is_inf = pcmp_eq(abs_x, cst_inf);
1369 if (exponent < ScalarExponent(0)) {
1371 result = pselect(x_is_zero, cst_inf, result);
1372 result = pselect(abs_x_is_inf, cst_zero, result);
1375 result = pselect(x_is_zero, cst_zero, result);
1376 result = pselect(abs_x_is_inf, cst_inf, result);
1383 result = por(pisnan(x), result);
1388template <
typename Packet,
typename ScalarExponent,
1389 std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsSigned,
bool> =
true>
1390EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_negative_exponent(
const Packet& x,
const ScalarExponent& exponent) {
1391 using Scalar =
typename unpacket_traits<Packet>::type;
1397 Packet cst_pos_one = pset1<Packet>(Scalar(1));
1398 const bool exponent_is_odd = exponent % ScalarExponent(2) != ScalarExponent(0);
1399 const Packet exp_is_odd = exponent_is_odd ? ptrue<Packet>(x) : pzero<Packet>(x);
1401 const Packet abs_x = pabs(x);
1402 const Packet abs_x_is_one = pcmp_eq(abs_x, cst_pos_one);
1404 Packet result = pselect(exp_is_odd, x, abs_x);
1405 result = pselect(abs_x_is_one, result, pzero<Packet>(x));
1409template <
typename Packet,
typename ScalarExponent,
1410 std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsSigned,
bool> =
true>
1411EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_negative_exponent(
const Packet& x,
const ScalarExponent&) {
1412 using Scalar =
typename unpacket_traits<Packet>::type;
1419 const Scalar pos_one = Scalar(1);
1421 const Packet cst_pos_one = pset1<Packet>(pos_one);
1423 const Packet x_is_one = pcmp_eq(x, cst_pos_one);
1425 return pand(x_is_one, x);
1430template <
typename Packet,
typename ScalarExponent,
1431 bool BaseIsIntegerType = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger,
1432 bool ExponentIsIntegerType = NumTraits<ScalarExponent>::IsInteger,
1433 bool ExponentIsSigned = NumTraits<ScalarExponent>::IsSigned>
1434struct unary_pow_impl;
1436template <
typename Packet,
typename ScalarExponent,
bool ExponentIsSigned>
1437struct unary_pow_impl<Packet, ScalarExponent, false, false, ExponentIsSigned> {
1438 using Scalar =
typename unpacket_traits<Packet>::type;
1439 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(
const Packet& x,
const ScalarExponent& exponent) {
1440 const bool exponent_is_integer = (numext::isfinite)(exponent) && numext::round(exponent) == exponent;
1441 if (exponent_is_integer) {
1442 return unary_pow::use_repeated_squaring<Packet>(exponent) ? unary_pow::int_pow(x, exponent)
1443 : generic_pow(x, pset1<Packet>(exponent));
1445 Packet result = unary_pow::gen_pow(x, exponent);
1446 result = unary_pow::handle_nonint_nonint_errors(x, result, exponent);
1452template <
typename Packet,
typename ScalarExponent,
bool ExponentIsSigned>
1453struct unary_pow_impl<Packet, ScalarExponent, false, true, ExponentIsSigned> {
1454 using Scalar =
typename unpacket_traits<Packet>::type;
1459 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(
const Packet& x,
const ScalarExponent& exponent) {
1460 return run(x, exponent,
1461 bool_constant < unary_pow::use_double_word<Packet>::value && !NumTraits<Scalar>::IsComplex > ());
1463 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(
const Packet& x,
const ScalarExponent& exponent, true_type) {
1464 return unary_pow::use_repeated_squaring_for_integer<Packet>(exponent)
1465 ? unary_pow::int_pow(x, exponent)
1466 : generic_pow(x, pset1<Packet>(static_cast<Scalar>(exponent)));
1468 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(
const Packet& x,
const ScalarExponent& exponent, false_type) {
1469 return unary_pow::int_pow(x, exponent);
1473template <
typename Packet,
typename ScalarExponent>
1474struct unary_pow_impl<Packet, ScalarExponent, true, true, true> {
1475 using Scalar =
typename unpacket_traits<Packet>::type;
1476 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(
const Packet& x,
const ScalarExponent& exponent) {
1477 if (exponent < ScalarExponent(0)) {
1478 return unary_pow::handle_negative_exponent(x, exponent);
1480 return unary_pow::int_pow_wrapping(x, exponent);
1485template <
typename Packet,
typename ScalarExponent>
1486struct unary_pow_impl<Packet, ScalarExponent, true, true, false> {
1487 using Scalar =
typename unpacket_traits<Packet>::type;
1488 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(
const Packet& x,
const ScalarExponent& exponent) {
1489 return unary_pow::int_pow_wrapping(x, exponent);