Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GenericPacketMathPow.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018-2025 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_POW_H
12#define EIGEN_ARCH_GENERIC_PACKET_MATH_POW_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18namespace internal {
19
20//----------------------------------------------------------------------
21// Cubic Root Functions
22//----------------------------------------------------------------------
23
24// This function implements a single step of Halley's iteration for
25// computing x = y^(1/3):
26// x_{k+1} = x_k - (x_k^3 - y) x_k / (2x_k^3 + y)
27template <typename Packet>
28EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_halley_iteration_step(const Packet& x_k,
29 const Packet& y) {
30 using Scalar = typename unpacket_traits<Packet>::type;
31 Packet x_k_cb = pmul(x_k, pmul(x_k, x_k));
32 Packet denom = pmadd(pset1<Packet>(Scalar(2)), x_k_cb, y);
33 Packet num = psub(x_k_cb, y);
34 Packet r = pdiv(num, denom);
35 return pnmadd(x_k, r, x_k);
36}
37
38// The last Halley step, the only one whose rounding error reaches the result. Where pmadd is fused,
39// x_k_sq + x_k_sq_lo = x_k^2 exactly and num = fl(fl(x_k_sq * x_k - y) + x_k_sq_lo * x_k) is within
40// 2u |x_k^3 - y| + u^2 y of x_k^3 - y for the unit roundoff u; cbrt_halley_iteration_step loses up to 2u y to the
41// two roundings of x_k^3. The result's relative error is then 2/3 e^3 for a relative error e in x_k, plus the
42// rounding of the final pnmadd.
43template <typename Packet>
44EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_last_halley_iteration_step(const Packet& x_k,
45 const Packet& y) {
46 EIGEN_IF_CONSTEXPR (is_fused_madd<Packet>::value) {
47 using Scalar = typename unpacket_traits<Packet>::type;
48 const Packet x_k_sq = pmul(x_k, x_k);
49 const Packet x_k_sq_lo = pmsub(x_k, x_k, x_k_sq);
50 const Packet x_k_cb = pmul(x_k_sq, x_k);
51 const Packet denom = pmadd(pset1<Packet>(Scalar(2)), x_k_cb, y);
52 const Packet num = pmadd(x_k_sq_lo, x_k, pmsub(x_k_sq, x_k, y));
53 const Packet r = pdiv(num, denom);
54 return pnmadd(x_k, r, x_k);
55 } else {
56 return cbrt_halley_iteration_step(x_k, y);
57 }
58}
59
60// Decompose the input such that x^(1/3) = y^(1/3) * 2^e_div3, and y is in the
61// interval [0.125,1].
62template <typename Packet>
63EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_decompose(const Packet& x, Packet& e_div3) {
64 using Scalar = typename unpacket_traits<Packet>::type;
65 // Extract the significand s in the range [0.5,1) and exponent e, such that
66 // x = 2^e * s.
67 Packet e, s;
68 s = pfrexp(x, e);
69
70 // Split the exponent into a part divisible by 3 and the remainder.
71 // e = 3*e_div3 + e_mod3.
72 constexpr Scalar kOneThird = Scalar(1) / 3;
73 e_div3 = pceil(pmul(e, pset1<Packet>(kOneThird)));
74 Packet e_mod3 = pnmadd(pset1<Packet>(Scalar(3)), e_div3, e);
75
76 // Replace s by y = (s * 2^e_mod3).
77 return pldexp_fast(s, e_mod3);
78}
79
80template <typename Packet>
81EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet cbrt_special_cases_and_sign(const Packet& x,
82 const Packet& abs_root) {
83 // Set sign.
84 const Packet sign_mask = psignmask<Packet>();
85 const Packet x_sign = pand(sign_mask, x);
86 Packet root = por(x_sign, abs_root);
87
88 // Pass non-finite and zero values of x straight through.
89 const Packet is_not_finite = por(pisinf(x), pisnan(x));
90 const Packet is_zero = pcmp_eq(pzero(x), x);
91 const Packet use_x = por(is_not_finite, is_zero);
92 return pselect(use_x, x, root);
93}
94
95// Generic implementation of cbrt(x) for float.
96//
97// The algorithm computes the cubic root of the input by first
98// decomposing it into an exponent and significand
99// x = s * 2^e.
100//
101// We can then write the cube root as
102//
103// x^(1/3) = 2^(e/3) * s^(1/3)
104// = 2^((3*e_div3 + e_mod3)/3) * s^(1/3)
105// = 2^(e_div3) * 2^(e_mod3/3) * s^(1/3)
106// = 2^(e_div3) * (s * 2^e_mod3)^(1/3)
107//
108// where e_div3 = ceil(e/3) and e_mod3 = e - 3*e_div3.
109//
110// The cube root of the second term y = (s * 2^e_mod3)^(1/3) is coarsely
111// approximated using a quartic polynomial and subsequently refined using a
112// single step of Halley's iteration, and finally the two terms are combined
113// using pldexp_fast.
114//
115// Note: Many alternatives exist for implementing cbrt. See, for example,
116// the excellent discussion in Kahan's note:
117// https://csclub.uwaterloo.ca/~pbarfuss/qbrt.pdf
118// This particular implementation was found to be very fast and accurate
119// among several alternatives tried, but is probably not "optimal" on all
120// platforms.
121//
122// Measured over every float in [1/8, 1), the error is at most 0.58 ULP where
123// pmadd is fused and 0.98 ULP elsewhere.
124template <typename Packet>
125EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcbrt_float(const Packet& x) {
126 using Scalar = typename unpacket_traits<Packet>::type;
127 static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
128
129 // Decompose the input such that x^(1/3) = y^(1/3) * 2^e_div3, and y is in the
130 // interval [0.125,1].
131 Packet e_div3;
132 const Packet y = cbrt_decompose(pabs(x), e_div3);
133
134 // Initial approximation with relative error <= 1.87e-3, which one Halley step reduces to 0.07 ULP. Sollya 8.0:
135 // fpminimax(x^(1/3), 4, [|single...|], [1/8; 1], relative, floating);
136 constexpr float alpha[] = {-0.884942114353179931640625f, 2.4534566402435302734375f, -2.6808016300201416015625f,
137 1.7967746257781982421875f, 0.31364738941192626953125f};
138 Packet r = ppolevl<Packet, 4>::run(y, alpha);
139
140 // Take one step of Halley's iteration.
141 r = cbrt_last_halley_iteration_step(r, y);
142
143 // Finally multiply by 2^(e_div3)
144 r = pldexp_fast(r, e_div3);
145
146 return cbrt_special_cases_and_sign(x, r);
147}
148
149// Generic implementation of cbrt(x) for double.
150//
151// The algorithm is identical to the one for float except that a different initial
152// approximation is used for y^(1/3) and two Halley iteration steps are performed.
153//
154// Measured, the result is within 1 ULP of the correctly rounded one, and where
155// pmadd is fused it is correctly rounded for all but about one input in a million.
156template <typename Packet>
157EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcbrt_double(const Packet& x) {
158 using Scalar = typename unpacket_traits<Packet>::type;
159 static_assert(std::is_same<Scalar, double>::value, "Scalar type must be double");
160
161 // Decompose the input such that x^(1/3) = y^(1/3) * 2^e_div3, and y is in the
162 // interval [0.125,1].
163 Packet e_div3;
164 const Packet y = cbrt_decompose(pabs(x), e_div3);
165
166 // Initial approximation with relative error <= 5.22e-3, which two Halley steps reduce to 6e-22. Sollya 8.0:
167 // fpminimax(x^(1/3), 3, [|double...|], [1/8; 1], relative, floating);
168 constexpr double alpha[] = {
169 0.59218535776880332210936330739059485495090484619140625, -1.3859252584188996859637654779362492263317108154296875,
170 1.4581131468387165117661652402603067457675933837890625, 0.340841899750747245878557123432983644306659698486328125};
171 Packet r = ppolevl<Packet, 3>::run(y, alpha);
172
173 // Take two steps of Halley's iteration.
174 r = cbrt_halley_iteration_step(r, y);
175 r = cbrt_last_halley_iteration_step(r, y);
176
177 // Finally multiply by 2^(e_div3).
178 r = pldexp_fast(r, e_div3);
179 return cbrt_special_cases_and_sign(x, r);
180}
181
182//----------------------------------------------------------------------
183// Power Functions (accurate_log2, generic_pow, unary_pow)
184//----------------------------------------------------------------------
185
186// This function computes log2(x) and returns the result as a double word.
187template <typename Scalar>
188struct accurate_log2 {
189 template <typename Packet>
190 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(const Packet& x, Packet& log2_x_hi, Packet& log2_x_lo) const {
191 log2_x_hi = plog2(x);
192 log2_x_lo = pzero(x);
193 }
194};
195
196// This specialization uses a more accurate algorithm to compute log2(x) for
197// floats in [1/sqrt(2);sqrt(2)] with a relative accuracy of ~6.56508e-10.
198// This additional accuracy is needed to counter the error-magnification
199// inherent in multiplying by a potentially large exponent in pow(x,y).
200// The minimax polynomial used was calculated using the Rminimax tool,
201// see https://gitlab.inria.fr/sfilip/rminimax.
202// Command line:
203// $ ratapprox --function="log2(1+x)/x" --dom='[-0.2929,0.41422]'
204// --type=[10,0]
205// --numF="[D,D,SG]" --denF="[SG]" --log --dispCoeff="dec"
206//
207// The resulting implementation of pow(x,y) is accurate to 3 ulps.
208template <>
209struct accurate_log2<float> {
210 template <typename Packet>
211 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(const Packet& z, Packet& log2_x_hi, Packet& log2_x_lo) const {
212 // Split the two lowest order constant coefficient into double-word representation.
213 constexpr double kC0 = 1.442695041742110273474963832995854318141937255859375e+00;
214 constexpr float kC0_hi = static_cast<float>(kC0);
215 constexpr float kC0_lo = static_cast<float>(kC0 - static_cast<double>(kC0_hi));
216 const Packet c0_hi = pset1<Packet>(kC0_hi);
217 const Packet c0_lo = pset1<Packet>(kC0_lo);
218
219 constexpr double kC1 = -7.2134751588268664068692714863573201000690460205078125e-01;
220 constexpr float kC1_hi = static_cast<float>(kC1);
221 constexpr float kC1_lo = static_cast<float>(kC1 - static_cast<double>(kC1_hi));
222 const Packet c1_hi = pset1<Packet>(kC1_hi);
223 const Packet c1_lo = pset1<Packet>(kC1_lo);
224
225 constexpr float c[] = {
226 9.7010828554630279541015625e-02, -1.6896486282348632812500000e-01, 1.7200836539268493652343750e-01,
227 -1.7892272770404815673828125e-01, 2.0505344867706298828125000e-01, -2.4046677350997924804687500e-01,
228 2.8857553005218505859375000e-01, -3.6067414283752441406250000e-01, 4.8089790344238281250000000e-01};
229
230 // Evaluate the higher order terms in the polynomial using
231 // standard arithmetic.
232 const Packet one = pset1<Packet>(1.0f);
233 const Packet x = psub(z, one);
234 Packet p = ppolevl<Packet, 8>::run(x, c);
235 // Evaluate the final two steps in Horner's rule using double-word
236 // arithmetic.
237 Packet p_hi, p_lo;
238 twoprod(x, p, p_hi, p_lo);
239 fast_twosum(c1_hi, c1_lo, p_hi, p_lo, p_hi, p_lo);
240 twoprod(p_hi, p_lo, x, p_hi, p_lo);
241 fast_twosum(c0_hi, c0_lo, p_hi, p_lo, p_hi, p_lo);
242 // Multiply by x to recover log2(z).
243 twoprod(p_hi, p_lo, x, log2_x_hi, log2_x_lo);
244 }
245};
246
247// This specialization uses a more accurate algorithm to compute log2(x) for
248// floats in [1/sqrt(2);sqrt(2)] with a relative accuracy of ~1.27e-18.
249// This additional accuracy is needed to counter the error-magnification
250// inherent in multiplying by a potentially large exponent in pow(x,y).
251// The minimax polynomial used was calculated using the Sollya tool.
252// See sollya.org.
253
254template <>
255struct accurate_log2<double> {
256 template <typename Packet>
257 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void operator()(const Packet& x, Packet& log2_x_hi, Packet& log2_x_lo) const {
258 // We use a transformation of variables:
259 // r = c * (x-1) / (x+1),
260 // such that
261 // log2(x) = log2((1 + r/c) / (1 - r/c)) = f(r).
262 // The function f(r) can be approximated well using an odd polynomial
263 // of the form
264 // P(r) = ((Q(r^2) * r^2 + C) * r^2 + 1) * r,
265 // For the implementation of log2<double> here, Q is of degree 6 with
266 // coefficient represented in working precision (double), while C is a
267 // constant represented in extra precision as a double word to achieve
268 // full accuracy.
269 //
270 // The polynomial coefficients were computed by the Sollya script:
271 //
272 // c = 2 / log(2);
273 // trans = c * (x-1)/(x+1);
274 // itrans = (1+x/c)/(1-x/c);
275 // interval=[trans(sqrt(0.5)); trans(sqrt(2))];
276 // print(interval);
277 // f = log2(itrans(x));
278 // p=fpminimax(f,[|1,3,5,7,9,11,13,15,17|],[|1,DD,double...|],interval,relative,floating);
279 const Packet q12 = pset1<Packet>(2.87074255468000586e-9);
280 const Packet q10 = pset1<Packet>(2.38957980901884082e-8);
281 const Packet q8 = pset1<Packet>(2.31032094540014656e-7);
282 const Packet q6 = pset1<Packet>(2.27279857398537278e-6);
283 const Packet q4 = pset1<Packet>(2.31271023278625638e-5);
284 const Packet q2 = pset1<Packet>(2.47556738444535513e-4);
285 const Packet q0 = pset1<Packet>(2.88543873228900172e-3);
286 const Packet C_hi = pset1<Packet>(0.0400377511598501157);
287 const Packet C_lo = pset1<Packet>(-4.77726582251425391e-19);
288 const Packet one = pset1<Packet>(1.0);
289
290 const Packet cst_2_log2e_hi = pset1<Packet>(2.88539008177792677);
291 const Packet cst_2_log2e_lo = pset1<Packet>(4.07660016854549667e-17);
292 // c * (x - 1)
293 Packet t_hi, t_lo;
294 // t = c * (x-1)
295 twoprod(cst_2_log2e_hi, cst_2_log2e_lo, psub(x, one), t_hi, t_lo);
296 // r = c * (x-1) / (x+1). x + 1 = s_hi + s_lo exactly (fast_twosum, as x < 2): rounding it would limit r, and with
297 // it y * log2(x) in pow, to working precision. t / (s_hi + s_lo) = (t / s_hi) (1 - s_lo / s_hi) to relative u^2.
298 Packet s_hi, s_lo;
299 fast_twosum(one, x, s_hi, s_lo);
300 Packet q_hi, q_lo;
301 doubleword_div_fp(t_hi, t_lo, s_hi, q_hi, q_lo);
302 Packet r_hi, r_lo;
303 fast_twosum(q_hi, pnmadd(q_hi, pdiv(s_lo, s_hi), q_lo), r_hi, r_lo);
304
305 // r2 = r * r
306 Packet r2_hi, r2_lo;
307 twoprod(r_hi, r_lo, r_hi, r_lo, r2_hi, r2_lo);
308 // r4 = r2 * r2
309 Packet r4_hi, r4_lo;
310 twoprod(r2_hi, r2_lo, r2_hi, r2_lo, r4_hi, r4_lo);
311
312 // Evaluate Q(r^2) in working precision. We evaluate it in two parts
313 // (even and odd in r^2) to improve instruction level parallelism.
314 Packet q_even = pmadd(q12, r4_hi, q8);
315 Packet q_odd = pmadd(q10, r4_hi, q6);
316 q_even = pmadd(q_even, r4_hi, q4);
317 q_odd = pmadd(q_odd, r4_hi, q2);
318 q_even = pmadd(q_even, r4_hi, q0);
319 Packet q = pmadd(q_odd, r2_hi, q_even);
320
321 // Now evaluate the low order terms of P(x) in double word precision.
322 // In the following, due to the increasing magnitude of the coefficients
323 // and r being constrained to [-0.5, 0.5] we can use fast_twosum instead
324 // of the slower twosum.
325 // Q(r^2) * r^2
326 Packet p_hi, p_lo;
327 twoprod(r2_hi, r2_lo, q, p_hi, p_lo);
328 // Q(r^2) * r^2 + C
329 Packet p1_hi, p1_lo;
330 fast_twosum(C_hi, C_lo, p_hi, p_lo, p1_hi, p1_lo);
331 // (Q(r^2) * r^2 + C) * r^2
332 Packet p2_hi, p2_lo;
333 twoprod(r2_hi, r2_lo, p1_hi, p1_lo, p2_hi, p2_lo);
334 // ((Q(r^2) * r^2 + C) * r^2 + 1)
335 Packet p3_hi, p3_lo;
336 fast_twosum(one, p2_hi, p2_lo, p3_hi, p3_lo);
337
338 // log2(x) ~= ((Q(r^2) * r^2 + C) * r^2 + 1) * r
339 twoprod(p3_hi, p3_lo, r_hi, r_lo, log2_x_hi, log2_x_lo);
340 }
341};
342
343// This function implements the non-trivial case of pow(x,y) where x is
344// positive and y is (possibly) non-integer.
345// Formally, pow(x,y) = exp2(y * log2(x)), where exp2(x) is shorthand for 2^x.
346// TODO(rmlarsen): We should probably add this as a packet op 'ppow', to make it
347// easier to specialize or turn off for specific types and/or backends.
348template <typename Packet>
349EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet generic_pow_impl(const Packet& x, const Packet& y) {
350 using Scalar = typename unpacket_traits<Packet>::type;
351 // Split x into exponent e_x and mantissa m_x.
352 Packet e_x;
353 Packet m_x = pfrexp(x, e_x);
354
355 // Adjust m_x to lie in [1/sqrt(2):sqrt(2)] to minimize absolute error in log2(m_x).
356 constexpr Scalar sqrt_half = Scalar(0.70710678118654752440);
357 const Packet m_x_scale_mask = pcmp_lt(m_x, pset1<Packet>(sqrt_half));
358 m_x = pselect(m_x_scale_mask, pmul(pset1<Packet>(Scalar(2)), m_x), m_x);
359 e_x = pselect(m_x_scale_mask, psub(e_x, pset1<Packet>(Scalar(1))), e_x);
360
361 // Compute log2(m_x) with 6 extra bits of accuracy.
362 Packet rx_hi, rx_lo;
363 accurate_log2<Scalar>()(m_x, rx_hi, rx_lo);
364
365 // Compute the two terms {y * e_x, y * r_x} in f = y * log2(x) with doubled
366 // precision using double word arithmetic.
367 Packet f1_hi, f1_lo, f2_hi, f2_lo;
368 twoprod(e_x, y, f1_hi, f1_lo);
369 twoprod(rx_hi, rx_lo, y, f2_hi, f2_lo);
370 // Sum the two terms in f using double word arithmetic. We know
371 // that |e_x| > |log2(m_x)|, except for the case where e_x==0.
372 // This means that we can use fast_twosum(f1,f2).
373 // In the case e_x == 0, e_x * y = f1 = 0, so we don't lose any
374 // accuracy by violating the assumption of fast_twosum, because
375 // it's a no-op.
376 Packet f_hi, f_lo;
377 fast_twosum(f1_hi, f1_lo, f2_hi, f2_lo, f_hi, f_lo);
378
379 // Split f into integer and fractional parts.
380 Packet n_z, r_z;
381 absolute_split(f_hi, n_z, r_z);
382 r_z = padd(r_z, f_lo);
383 Packet n_r;
384 absolute_split(r_z, n_r, r_z);
385 n_z = padd(n_z, n_r);
386
387 // We now have an accurate split of f = n_z + r_z and can compute
388 // x^y = 2**{n_z + r_z) = exp2(r_z) * 2**{n_z}.
389 // Multiplication by the second factor can be done exactly using pldexp(), since
390 // it is an integer power of 2.
391 const Packet e_r = generic_exp2_reduced(r_z);
392
393 // Since we know that e_r is in [1/sqrt(2); sqrt(2)], we can use the fast version
394 // of pldexp to multiply by 2**{n_z} when |n_z| is sufficiently small.
395 constexpr Scalar kPldExpThresh = std::numeric_limits<Scalar>::max_exponent - 2;
396 const Packet pldexp_fast_unsafe = pcmp_lt(pset1<Packet>(kPldExpThresh), pabs(n_z));
397 if (predux_any(pldexp_fast_unsafe)) {
398 return pldexp(e_r, n_z);
399 }
400 return pldexp_fast(e_r, n_z);
401}
402
403// Generic implementation of pow(x,y).
404template <typename Packet>
405EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS std::enable_if_t<!is_scalar<Packet>::value, Packet> generic_pow(
406 const Packet& x, const Packet& y) {
407 using Scalar = typename unpacket_traits<Packet>::type;
408
409 const Packet cst_inf = pinf<Packet>();
410 const Packet cst_zero = pset1<Packet>(Scalar(0));
411 const Packet cst_one = pset1<Packet>(Scalar(1));
412
413 const Packet x_abs = pabs(x);
414 Packet result = generic_pow_impl(x_abs, y);
415
416 // In the following we enforce the special case handling prescribed in
417 // https://en.cppreference.com/w/cpp/numeric/math/pow.
418
419 // Predicates for sign and magnitude of x.
420 const Packet x_is_negative = pcmp_lt(x, cst_zero);
421 const Packet x_is_zero = pcmp_eq(x, cst_zero);
422 const Packet x_is_one = pcmp_eq(x, cst_one);
423 const Packet x_has_signbit = psignbit(x);
424 const Packet x_abs_gt_one = pcmp_lt(cst_one, x_abs);
425 const Packet x_abs_is_inf = pcmp_eq(x_abs, cst_inf);
426
427 // Predicates for sign and magnitude of y.
428 const Packet y_abs = pabs(y);
429 const Packet y_abs_is_inf = pcmp_eq(y_abs, cst_inf);
430 const Packet y_is_negative = pcmp_lt(y, cst_zero);
431 const Packet y_is_zero = pcmp_eq(y, cst_zero);
432 const Packet y_is_one = pcmp_eq(y, cst_one);
433 // Predicates for whether y is integer and odd/even.
434 const Packet y_is_int = pandnot(pcmp_eq(pfloor(y), y), y_abs_is_inf);
435 const Packet y_div_2 = pmul(y, pset1<Packet>(Scalar(0.5)));
436 const Packet y_is_even = pcmp_eq(pround(y_div_2), y_div_2);
437 const Packet y_is_odd_int = pandnot(y_is_int, y_is_even);
438 // From this |y| on, x^y is 0 or inf for every |x| != 1. The |x| nearest 1 is 1 - eps/2, with |log(x)| > eps/2,
439 // and |y| eps/2 >= (digits - min_exponent + 1) ln(2) takes x^y below half the smallest subnormal or past the
440 // largest finite value.
441 constexpr Scalar huge_exponent =
442 Scalar(2 * (numext::numeric_limits<Scalar>::digits - numext::numeric_limits<Scalar>::min_exponent + 1)) *
443 Scalar(EIGEN_LN2) / NumTraits<Scalar>::epsilon();
444 const Packet y_abs_is_huge = pcmp_le(pset1<Packet>(huge_exponent), y_abs);
445
446 // * pow(base, exp) returns NaN if base is finite and negative
447 // and exp is finite and non-integer.
448 // A packet comparison mask is all ones, a NaN bit pattern, so por selects NaN and pand selects a constant
449 // against zero. Scalar masks hold the value one instead, but this overload takes packets only.
450 result = por(pandnot(x_is_negative, y_is_int), result);
451
452 // * pow(±0, exp), where exp is negative, finite, and is an even integer or
453 // a non-integer, returns +∞
454 // * pow(±0, exp), where exp is positive non-integer or a positive even
455 // integer, returns +0
456 // * pow(+0, exp), where exp is a negative odd integer, returns +∞
457 // * pow(-0, exp), where exp is a negative odd integer, returns -∞
458 // * pow(+0, exp), where exp is a positive odd integer, returns +0
459 // * pow(-0, exp), where exp is a positive odd integer, returns -0
460 // Sign is flipped by the rule below.
461 result = pselect(x_is_zero, pand(y_is_negative, cst_inf), result);
462
463 // pow(base, exp) returns -pow(abs(base), exp) if base has the sign bit set,
464 // and exp is an odd integer exponent.
465 result = pselect(pand(x_has_signbit, y_is_odd_int), pnegate(result), result);
466
467 // * pow(base, -∞) returns +∞ for any |base|<1
468 // * pow(base, -∞) returns +0 for any |base|>1
469 // * pow(base, +∞) returns +0 for any |base|<1
470 // * pow(base, +∞) returns +∞ for any |base|>1
471 // * pow(±0, -∞) returns +∞
472 // * pow(-1, +-∞) = 1
473 Packet inf_y_val = pand(pxor(y_is_negative, x_abs_gt_one), cst_inf);
474 inf_y_val = pselect(pcmp_eq(x, pset1<Packet>(Scalar(-1.0))), cst_one, inf_y_val);
475 result = pselect(y_abs_is_huge, inf_y_val, result);
476
477 // * pow(+∞, exp) returns +0 for any negative exp
478 // * pow(+∞, exp) returns +∞ for any positive exp
479 // * pow(-∞, exp) returns -0 if exp is a negative odd integer.
480 // * pow(-∞, exp) returns +0 if exp is a negative non-integer or negative
481 // even integer.
482 // * pow(-∞, exp) returns -∞ if exp is a positive odd integer.
483 // * pow(-∞, exp) returns +∞ if exp is a positive non-integer or positive
484 // even integer.
485 auto x_pos_inf_value = pandnot(cst_inf, y_is_negative);
486 auto x_neg_inf_value = pselect(y_is_odd_int, pnegate(x_pos_inf_value), x_pos_inf_value);
487 result = pselect(x_abs_is_inf, pselect(x_is_negative, x_neg_inf_value, x_pos_inf_value), result);
488
489 // All cases of NaN inputs return NaN, except the two below.
490 result = por(por(pisnan(x), pisnan(y)), result);
491
492 // * pow(base, 1) returns base.
493 // * pow(base, +/-0) returns 1, regardless of base, even NaN.
494 // * pow(+1, exp) returns 1, regardless of exponent, even NaN.
495 result = pselect(y_is_one, x, pselect(por(x_is_one, y_is_zero), cst_one, result));
496
497 return result;
498}
499
500template <typename Scalar>
501EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS std::enable_if_t<is_scalar<Scalar>::value, Scalar> generic_pow(
502 const Scalar& x, const Scalar& y) {
503 return numext::pow(x, y);
504}
505
506namespace unary_pow {
507
508// Integer exponents up to this magnitude use repeated squaring; larger ones take the log/exp path of generic_pow
509// where that can represent them. The crossover is where a squaring step per exponent bit stops being cheaper
510// than generic_pow, whose cost per element is about four times higher for double than for float (half the
511// lanes, longer polynomials): measured at 2^12 for float and 2^20 for double on AVX2 with FMA. Complex bases
512// take their real scalar's value; it only limits their scalar path, since no vectorized complex pow exists.
513template <typename Scalar>
514constexpr numext::uint64_t max_squaring_exponent() {
515 return numext::uint64_t(1) << (std::is_same<typename NumTraits<Scalar>::Real, double>::value ? 20 : 12);
516}
517
518template <typename ScalarExponent, bool IsInteger = NumTraits<ScalarExponent>::IsInteger>
519struct exponent_helper {
520 using safe_abs_type = numext::uint64_t;
521 // this routine assumes that exp is an integer of magnitude at most max_squaring_exponent() stored as a floating
522 // point type
523 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type safe_abs(const ScalarExponent& exp) {
524 eigen_assert(((numext::isfinite)(exp) && exp == numext::floor(exp)) && "exp must be an integer");
525 return static_cast<safe_abs_type>(numext::abs(exp));
526 }
527};
528
529template <typename ScalarExponent>
530struct exponent_helper<ScalarExponent, true> {
531 // if `exp` is a signed integer type, cast it to its unsigned counterpart to safely store its absolute value
532 // consider the (rare) case where `exp` is an int32_t: abs(-2147483648) != 2147483648
533 using safe_abs_type = typename numext::get_integer_by_size<sizeof(ScalarExponent)>::unsigned_type;
534 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE safe_abs_type safe_abs(const ScalarExponent& exp) {
535 ScalarExponent mask = numext::signbit(exp);
536 safe_abs_type result = safe_abs_type(exp ^ mask);
537 return result + safe_abs_type(ScalarExponent(1) & mask);
538 }
539};
540
541template <typename ScalarExponent, bool IsSigned = NumTraits<ScalarExponent>::IsSigned>
542struct exponent_is_negative {
543 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const ScalarExponent& exponent) {
544 return exponent < ScalarExponent(0);
545 }
546};
547
548template <typename ScalarExponent>
549struct exponent_is_negative<ScalarExponent, false> {
550 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool run(const ScalarExponent&) { return false; }
551};
552
553// Left-to-right binary exponentiation, so that the base is multiplied in as is: each step squares the running
554// power and multiplies by the base when the next exponent bit is set.
555template <typename AbsExponentType>
556EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE AbsExponentType highest_set_bit(AbsExponentType m) {
557 AbsExponentType bit = AbsExponentType(1);
558 while ((m >> 1) >= bit) bit <<= 1;
559 return bit;
560}
561
562// Repeated squaring for integer bases, wrapping on overflow like the underlying multiplication.
563template <typename Packet, typename ScalarExponent>
564EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet int_pow_wrapping(const Packet& x, const ScalarExponent& exponent) {
565 using Scalar = typename unpacket_traits<Packet>::type;
566 using ExponentHelper = exponent_helper<ScalarExponent>;
567 using AbsExponentType = typename ExponentHelper::safe_abs_type;
568 Packet cst_pos_one = pset1<Packet>(Scalar(1));
569 if (exponent == ScalarExponent(0)) return cst_pos_one;
570 eigen_assert(!exponent_is_negative<ScalarExponent>::run(exponent));
571
572 AbsExponentType m = ExponentHelper::safe_abs(exponent);
573 Packet y = x;
574 for (AbsExponentType bit = highest_set_bit(m) >> 1; bit != 0; bit >>= 1) {
575 y = pmul(y, y);
576 if ((m & bit) != 0) y = pmul(y, x);
577 }
578 return y;
579}
580
581// Double-word repeated squaring needs an exact two-product and the binary layout below; it serves binary32 and
582// binary64 bases, real or complex. Other bases keep plain repeated squaring.
583template <typename Scalar>
584struct is_double_word_base
585 : bool_constant<(std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) &&
586 std::numeric_limits<Scalar>::is_iec559> {};
587template <typename RealScalar>
588struct is_double_word_base<std::complex<RealScalar>> : is_double_word_base<RealScalar> {};
589
590// Whether exponent_bits_shift_right/exponent_bits_sub are available, not whether integer_packet itself is:
591// AVX/PacketMath.h supplies all three for Packet4d without AVX2, where there is no Packet4l.
592template <typename Packet, typename>
593struct has_exponent_bit_ops : false_type {};
594template <typename Packet>
595struct has_exponent_bit_ops<Packet, void_t<typename unpacket_traits<Packet>::integer_packet>> : true_type {};
596
597template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex,
598 bool IsScalar = is_scalar<Packet>::value>
599struct real_view {
600 using type = Packet;
601};
602template <typename Packet>
603struct real_view<Packet, true, false> {
604 using type = typename unpacket_traits<Packet>::as_real;
605};
606template <typename Scalar>
607struct real_view<Scalar, true, true> {
608 using type = typename NumTraits<Scalar>::Real;
609};
610
611template <typename Packet>
612struct use_double_word : bool_constant<is_double_word_base<typename unpacket_traits<Packet>::type>::value &&
613 has_exponent_bit_ops<typename real_view<Packet>::type>::value> {};
614
615// ARMv7 NEON flushes subnormal float operands and results to zero in its arithmetic and comparisons; the scalar VFP
616// unit, which runs the scalar path, does not. Its dispatch is also under #if EIGEN_ARCH_ARM: elsewhere, although it
617// selects the unchanged path, GCC inlines the pow kernels differently.
618template <typename Packet>
619struct flushes_subnormals
620 : bool_constant<EIGEN_ARCH_ARM && !is_scalar<Packet>::value &&
621 std::is_same<typename NumTraits<typename unpacket_traits<Packet>::type>::Real, float>::value> {};
622
623// Declared in GenericPacketMathFunctionsFwd.h so that a backend header can override them.
624template <typename Packet>
625EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_shift_right(const Packet& bits) {
626 using PacketI = typename unpacket_traits<Packet>::integer_packet;
627 constexpr int kMantissaBits = numext::numeric_limits<typename unpacket_traits<Packet>::type>::digits - 1;
628 return preinterpret<Packet>(plogical_shift_right<kMantissaBits>(preinterpret<PacketI>(bits)));
629}
630template <typename Packet>
631EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet exponent_bits_sub(const Packet& a_bits, const Packet& b_bits) {
632 using PacketI = typename unpacket_traits<Packet>::integer_packet;
633 return preinterpret<Packet>(psub(preinterpret<PacketI>(a_bits), preinterpret<PacketI>(b_bits)));
634}
635
636// Power-of-two scaling of a real packet through its exponent bits. A finite value's magnitude is never let far
637// from one, so the residuals of the double-word arithmetic stay normal (a subnormal residual costs a microcode
638// assist on x86 for every operation that touches it) and no intermediate overflows or underflows.
639template <typename Packet>
640struct binary_exponent_scaling {
641 using Scalar = typename unpacket_traits<Packet>::type;
642 using Bits = std::make_unsigned_t<typename make_integer<Scalar>::type>;
643 static constexpr int kMantissaBits = numext::numeric_limits<Scalar>::digits - 1;
644 static constexpr int kBias = numext::numeric_limits<Scalar>::max_exponent - 1;
645 static constexpr Bits kExponentMask = ((Bits(1) << (CHAR_BIT * sizeof(Scalar) - kMantissaBits - 1)) - Bits(1))
646 << kMantissaBits;
647 // 2^kMantissaBits: or-ing an integer below 2^kMantissaBits into its low bits adds that integer to it.
648 static constexpr Bits kMagicBits = Bits(kBias + kMantissaBits) << kMantissaBits;
649
650 // For a normal x = m * 2^e with 2 <= |m| < 4, returns 2^-e and sets e (as a floating-point value). The
651 // target [2, 4) rather than [1, 2) keeps 2^-e a normal number for every x.
652 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet inverse_scale(const Packet& x, Packet& e) {
653 Packet exponent_bits = pand(x, pset1frombits<Packet>(kExponentMask));
654 Packet biased = exponent_bits_shift_right(exponent_bits);
655 Packet magic = pset1frombits<Packet>(kMagicBits);
656 e = psub(por(biased, magic), padd(magic, pset1<Packet>(Scalar(kBias + 1))));
657 // 2^(bias + 1 - E) has biased exponent 2 * bias + 1 - E, in [1, 2 * bias] for a normal x.
658 Packet two_bias_plus_one = pset1frombits<Packet>(Bits(2 * kBias + 1) << kMantissaBits);
659 return exponent_bits_sub(two_bias_plus_one, exponent_bits);
660 }
661
662 // x * 2^e for the scaled power, whose |x| lies within 2^(+-62): beyond the clamp the result is infinite or zero
663 // either way, and the scalar pldexp converts the exponent to int. Below kZeroBelow the result rounds to zero,
664 // which multiplying by zero gives exactly, whereas pldexp would underflow through several subnormal
665 // intermediates, each a microcode assist on x86.
666 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet scale_result(const Packet& x, const Packet& e) {
667 constexpr int kLimit = 4 * numext::numeric_limits<Scalar>::max_exponent;
668 constexpr int kZeroBelow =
669 numext::numeric_limits<Scalar>::min_exponent - numext::numeric_limits<Scalar>::digits - 64;
670 Packet zero_below = pset1<Packet>(Scalar(kZeroBelow));
671 Packet value = pselect(pcmp_lt(e, zero_below), pmul(x, pzero(x)), x);
672 Packet exponent = pmin(pmax(e, zero_below), pset1<Packet>(Scalar(kLimit)));
673#if EIGEN_ARCH_ARM
674 return with_subnormals(value, exponent, pldexp(value, exponent), flushes_subnormals<Packet>());
675#else
676 return pldexp(value, exponent);
677#endif
678 }
679
680 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_subnormals(const Packet&, const Packet&, const Packet& r,
681 false_type) {
682 return r;
683 }
684 // A flushing pldexp returns zero where x * 2^e lies below the smallest normal, i.e. t = |x| * 2^(e - min_exponent +
685 // digits) < 2^kMantissaBits. Rounded once, as pldexp rounds, that is k * 2^(min_exponent - digits) for k = rint(t) <=
686 // 2^kMantissaBits, the bit pattern of its magnitude. Only lanes with |r| < min are scaled, as elsewhere the scaling
687 // can overflow and its conversion raise FE_INVALID, and of those only lanes with t < 2^kMantissaBits are rebuilt:
688 // pldexp also zeroes some normal x * 2^e through a flushed partial product (2^-124 * 2^-1 in float).
689 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_subnormals(const Packet& x, const Packet& e, const Packet& r,
690 true_type) {
691 using PacketI = typename unpacket_traits<Packet>::integer_packet;
692 constexpr int kShift = numext::numeric_limits<Scalar>::digits - numext::numeric_limits<Scalar>::min_exponent;
693 Packet below = pcmp_lt(pabs(r), pset1<Packet>((numext::numeric_limits<Scalar>::min)()));
694 if (!predux_any(below)) return r;
695 Packet t = pldexp(pand(below, pabs(x)), padd(e, pset1<Packet>(Scalar(kShift))));
696 Packet k = preinterpret<Packet>(pcast<Packet, PacketI>(print(t)));
697 Packet rebuilt = pand(below, pcmp_lt(t, pset1<Packet>(Scalar(Bits(1) << kMantissaBits))));
698 return pselect(rebuilt, por(k, pand(x, pset1<Packet>(Scalar(-0.0)))), r);
699 }
700
701 // Whether a lane is subnormal, tested on its bits, as a flushing comparison reads a subnormal as zero.
702 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_subnormal(const Packet& x) {
703 using PacketI = typename unpacket_traits<Packet>::integer_packet;
704 using Int = typename unpacket_traits<PacketI>::type;
705 PacketI magnitude = preinterpret<PacketI>(pandnot(x, pset1<Packet>(Scalar(-0.0))));
706 PacketI subnormal =
707 pand(pcmp_lt(pzero(magnitude), magnitude), pcmp_lt(magnitude, pset1<PacketI>(Int(1) << kMantissaBits)));
708 return predux_any(preinterpret<Packet>(subnormal));
709 }
710
711 // Factors x = m * 2^e with 2 <= |m| < 4 for a finite, nonzero x, where m = (x * lift) * scale: lift is 2^digits
712 // for a subnormal x and one otherwise, and both products are exact. Zero and infinity make m NaN (their scale
713 // is infinite, respectively zero), which the callers resolve at the end; NaN stays NaN.
714 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void input_scale(const Packet& x, Packet& lift, Packet& scale,
715 Packet& e) {
716 constexpr int kDigits = numext::numeric_limits<Scalar>::digits;
717 Packet is_subnormal = pcmp_lt(pabs(x), pset1<Packet>((numext::numeric_limits<Scalar>::min)()));
718 lift = pselect(is_subnormal, pset1<Packet>(Scalar(Bits(1) << kDigits)), pset1<Packet>(Scalar(1)));
719 scale = inverse_scale(pmul(x, lift), e);
720 e = psub(e, pselect(is_subnormal, pset1<Packet>(Scalar(kDigits)), pzero(x)));
721 }
722};
723
724// The running power of repeated squaring: a double word {hi, lo} times 2^exponent, with the exponent kept as a
725// floating-point value that is exact while the result is finite, plus what a zero or infinite base turns into:
726// itself, or its reciprocal, with the sign dropped for an even exponent. Such a base makes hi NaN from the input
727// scaling on, and a finite nonzero base cannot make hi NaN or zero since the scaled power stays within 2^(+-62),
728// so result() substitutes the special value exactly where hi is NaN or zero. (A zero hi would also lose its sign
729// in the double-word sums.) NaN needs nothing: it propagates.
730template <typename Packet, bool IsComplex = NumTraits<typename unpacket_traits<Packet>::type>::IsComplex>
731struct repeated_squaring_ops {
732 using Scalar = typename unpacket_traits<Packet>::type;
733 using Scaling = binary_exponent_scaling<Packet>;
734 using Bound = Packet;
735 struct State {
736 Packet hi, lo, exponent, special;
737 };
738 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_subnormal(const Packet& x) { return Scaling::any_subnormal(x); }
739 // Whether a finite nonzero base has a power r below the smallest normal.
740 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_below_normal(const Packet& x, const Packet& r) {
741 Packet abs_x = pabs(x);
742 Packet regular = pand(pcmp_lt(pzero(x), abs_x), pcmp_lt(abs_x, pset1<Packet>(NumTraits<Scalar>::infinity())));
743 return predux_any(pand(regular, pcmp_lt(pabs(r), pset1<Packet>((numext::numeric_limits<Scalar>::min)()))));
744 }
745 // Whether every lane lies within [1/bound, bound], where the power and its residuals stay normal without scaling.
746 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool in_range(const Packet& x, const Packet& bound) {
747 Packet abs_x = pabs(x);
748 Packet out = por(pcmp_lt(pmul(abs_x, bound), pset1<Packet>(Scalar(1))), pcmp_lt_or_nan(bound, abs_x));
749 return !predux_any(out);
750 }
751 // The double word of m or of 1/m = q + (1 - q*m)/m, where 1 - q*m is formed from the exact product
752 // q*m = p_hi + p_lo (1 - p_hi is exact as p_hi is within rounding of 1); renormalizing makes hi the correctly
753 // rounded reciprocal.
754 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void power_base(const Packet& m, bool reciprocal, Packet& hi,
755 Packet& lo) {
756 if (!reciprocal) {
757 hi = m;
758 lo = pzero(m);
759 return;
760 }
761 Packet cst_pos_one = pset1<Packet>(Scalar(1));
762 Packet q = pdiv(cst_pos_one, m);
763 Packet p_hi, p_lo;
764 twoprod(q, m, p_hi, p_lo);
765 fast_twosum(q, pdiv(psub(psub(cst_pos_one, p_hi), p_lo), m), hi, lo);
766 }
767 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE State base(const Packet& x, bool reciprocal, bool scaled,
768 const Scalar&) {
769 State b;
770 if (!scaled) {
771 power_base(x, reciprocal, b.hi, b.lo);
772 b.exponent = pzero(x);
773 b.special = x;
774 return b;
775 }
776 Packet lift, scale;
777 Scaling::input_scale(x, lift, scale, b.exponent);
778 power_base(pmul(pmul(x, lift), scale), reciprocal, b.hi, b.lo);
779 if (reciprocal) {
780 b.exponent = pnegate(b.exponent);
781 b.special = pdiv(pset1<Packet>(Scalar(1)), x);
782 } else {
783 b.special = x;
784 }
785 return b;
786 }
787 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void multiply(State& y, const State& b) {
788 fast_twoprod(y.hi, y.lo, b.hi, b.lo, y.hi, y.lo);
789 y.exponent = padd(y.exponent, b.exponent);
790 }
791 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void square(State& y) { multiply(y, y); }
792 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void renormalize(State& y) {
793 Packet e;
794 Packet scale = Scaling::inverse_scale(y.hi, e);
795 y.hi = pmul(y.hi, scale);
796 y.lo = pmul(y.lo, scale);
797 y.exponent = padd(y.exponent, e);
798 }
799 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet result(const Packet&, const State& y, const State& b, bool odd,
800 bool scaled) {
801 if (!scaled) return y.hi;
802 Packet use_special = por(pisnan(y.hi), pcmp_eq(y.hi, pzero(y.hi)));
803 return pselect(use_special, odd ? b.special : pabs(b.special), Scaling::scale_result(y.hi, y.exponent));
804 }
805 // With a scaled |base| in [1/4, 4) a step at most cubes the magnitude bound, so four steps keep it within 2^(+-62)
806 // and the residuals, u times smaller, normal.
807 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int renormalization_steps(const Scalar&) { return 4; }
808 // Zero, infinite and NaN bases are resolved by the special value in result().
809 struct SpecialBases {
810 static constexpr bool any = false;
811 };
812 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SpecialBases special_bases(const Packet&, bool) {
813 return SpecialBases();
814 }
815 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet without_special_bases(const Packet& x, const SpecialBases&) {
816 return x;
817 }
818 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_special_bases(const Packet& r, const SpecialBases&, bool) {
819 return r;
820 }
821};
822
823// The real and imaginary parts of a complex value or packet as two values of a real representation R, on which
824// the complex algorithm below runs component-wise. For a complex packet R is its interleaved real view with both
825// lanes of a pair holding the same component, so every lane operation applies to the pair at once.
826template <typename Packet, bool IsScalar = is_scalar<Packet>::value>
827struct complex_components {
828 using Scalar = typename unpacket_traits<Packet>::type;
829 using RealScalar = typename NumTraits<Scalar>::Real;
830 using R = typename unpacket_traits<Packet>::as_real;
831 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R flip(const R& x) { return pcplxflip(Packet(x)).v; }
832 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R odd_lanes() {
833 return pcmp_eq(pset1<Packet>(Scalar(0, 1)).v, pset1<R>(RealScalar(1)));
834 }
835 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void split(const Packet& z, R& re, R& im) {
836 R odd = odd_lanes();
837 re = pselect(odd, flip(z.v), z.v);
838 im = pselect(odd, z.v, flip(z.v));
839 }
840 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet join(const R& re, const R& im) {
841 return Packet(pselect(odd_lanes(), im, re));
842 }
843 // max(|re|, |im|) in both lanes of each pair.
844 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R magnitude(const Packet& z) {
845 R abs_z = pabs(z.v);
846 return pmax(abs_z, flip(abs_z));
847 }
848 // min(|re|, |im|) in both lanes of each pair.
849 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R minor_magnitude(const Packet& z) {
850 R abs_z = pabs(z.v);
851 return pmin(abs_z, flip(abs_z));
852 }
853 // Whether a component exceeds bound or is NaN, in its own lane: pmax need not propagate NaN.
854 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R exceeds(const Packet& z, const R& bound) {
855 return pcmp_lt_or_nan(bound, pabs(z.v));
856 }
857 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_zero(const Packet& z) {
858 return predux_any(pcmp_eq(z.v, pzero(z.v)));
859 }
860 // In both lanes of each pair: both components zero; a component infinite; a component NaN and none infinite.
861 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void special_masks(const Packet& z, R& zero, R& inf, R& nan) {
862 R zero_lane = pcmp_eq(z.v, pzero(z.v));
863 R inf_lane = pcmp_eq(pabs(z.v), pset1<R>(NumTraits<RealScalar>::infinity()));
864 R nan_lane = pisnan(z.v);
865 zero = pand(zero_lane, flip(zero_lane));
866 inf = por(inf_lane, flip(inf_lane));
867 nan = pandnot(por(nan_lane, flip(nan_lane)), inf);
868 }
869 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet replace(const R& mask, const Scalar& value, const Packet& z) {
870 return Packet(pselect(mask, pset1<Packet>(value).v, z.v));
871 }
872};
873
874template <typename Scalar>
875struct complex_components<Scalar, true> {
876 using R = typename NumTraits<Scalar>::Real;
877 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void split(const Scalar& z, R& re, R& im) {
878 re = numext::real(z);
879 im = numext::imag(z);
880 }
881 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar join(const R& re, const R& im) { return Scalar(re, im); }
882 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R magnitude(const Scalar& z) {
883 return numext::maxi(numext::abs(numext::real(z)), numext::abs(numext::imag(z)));
884 }
885 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R minor_magnitude(const Scalar& z) {
886 return numext::mini(numext::abs(numext::real(z)), numext::abs(numext::imag(z)));
887 }
888 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE R exceeds(const Scalar& z, const R& bound) {
889 return por(pcmp_lt_or_nan(bound, numext::abs(numext::real(z))),
890 pcmp_lt_or_nan(bound, numext::abs(numext::imag(z))));
891 }
892 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_zero(const Scalar& z) {
893 return numext::real(z) == R(0) || numext::imag(z) == R(0);
894 }
895 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void special_masks(const Scalar& z, R& zero, R& inf, R& nan) {
896 R re = numext::real(z), im = numext::imag(z);
897 bool is_inf = (numext::isinf)(re) || (numext::isinf)(im);
898 zero = re == R(0) && im == R(0) ? R(1) : R(0);
899 inf = is_inf ? R(1) : R(0);
900 nan = !is_inf && ((numext::isnan)(re) || (numext::isnan)(im)) ? R(1) : R(0);
901 }
902 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar replace(const R& mask, const Scalar& value, const Scalar& z) {
903 return mask != R(0) ? value : z;
904 }
905};
906
907// Complex bases keep the real and imaginary parts as separate double words sharing one exponent, scaled by the
908// larger component: z^2 = (a^2 - b^2) + 2ab i is three products and a product (a + bi)(c + di) four. Either
909// component may legitimately be zero. Zero and infinite bases are the two points 0 and infinity of the extended
910// complex plane, as std::proj sees it: every value with an infinite component, a NaN one included, is the same
911// infinity, +inf + 0i. So a zero base gives +0 + 0i for n > 0 and +inf + 0i for n < 0, an infinite base the reverse,
912// whatever the signs of their zeros; a base with a NaN component and no infinite one gives NaN + NaN i. std::pow
913// leaves all of these to the implementation, and libstdc++'s follow its evaluation order: 1/0 = inf + nan i,
914// (-inf + 2i)^3 = nan + inf i.
915template <typename Packet>
916struct repeated_squaring_ops<Packet, true> {
917 using Scalar = typename unpacket_traits<Packet>::type;
918 using Real = typename NumTraits<Scalar>::Real;
919 using Components = complex_components<Packet>;
920 using R = typename Components::R;
921 using Scaling = binary_exponent_scaling<R>;
922 using Bound = R;
923 // Lanes marked `separated` hold only the larger component L (a, or bi) of z = L (1 + delta); the power is
924 // L^n (1 + i c) with i c = n delta, c = (coefficient_hi + coefficient_lo) 2^coefficient_exponent (see base()).
925 struct State {
926 R re_hi, re_lo, im_hi, im_lo, exponent;
927 R separated, coefficient_hi, coefficient_lo, coefficient_exponent;
928 bool any_separated;
929 bool negative, any_zero;
930 };
931 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_subnormal(const Packet& x) {
932 return Scaling::any_subnormal(x.v);
933 }
934 // A complex power has no single-operation form for the scalar path to round differently.
935 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool any_below_normal(const Packet&, const Packet&) { return false; }
936 // Four steps between renormalizations let the power fall to 2^-62 of the scale when the base is a reciprocal in
937 // [1/4, 1/2), and the smaller component, of order r = |s/L| times the power, keeps normal residuals only for
938 // r >= 2^(min_exponent - 1 + digits + 62). Below that the base is separated (see base()), and first order is
939 // exact to within (n r)^2 / 2 <= u/4 for |n| <= 2^kFirstOrderLog2Limit: every n for double, 2^25 for float.
940 // Larger n renormalize at every step instead, which keeps the power above 2^-6 (a step from [2, 4), or from the
941 // base itself) and lets the separation drop 56 binades, where first order holds for any n.
942 static constexpr int kMinExponent = numext::numeric_limits<Real>::min_exponent;
943 static constexpr int kDigits = numext::numeric_limits<Real>::digits;
944 static constexpr int kSeparationExponent = kMinExponent + kDigits + 64;
945 static constexpr int kFirstOrderLog2Limit = (1 - kDigits - 2 * kSeparationExponent) / 2;
946 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool every_step(const Real& count) {
947 return kFirstOrderLog2Limit < 64 &&
948 count > Real(numext::uint64_t(1) << (kFirstOrderLog2Limit < 64 ? kFirstOrderLog2Limit : 0));
949 }
950 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE int renormalization_steps(const Real& count) {
951 return every_step(count) ? 1 : 4;
952 }
953 // Whether every lane's max(|re|, |im|), and min(|re|, |im|) unless zero, lie within [1/bound, bound], where the
954 // power and its residuals stay normal without scaling. A zero component stays exactly zero in every power.
955 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE bool in_range(const Packet& x, const R& bound) {
956 R magnitude = Components::magnitude(x);
957 R minor = Components::minor_magnitude(x);
958 R one = pset1<R>(Real(1));
959 R out = por(pcmp_lt(pmul(magnitude, bound), one), Components::exceeds(x, bound));
960 out = por(out, pandnot(pcmp_lt(pmul(minor, bound), one), pcmp_eq(minor, pzero(minor))));
961 return predux_any(out) == false;
962 }
963 template <typename Count>
964 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE State base(const Packet& x, bool reciprocal, bool scaled,
965 const Count& m) {
966 State b;
967 b.any_separated = false;
968 R re, im;
969 Components::split(x, re, im);
970 b.negative = reciprocal;
971 b.any_zero = Components::any_zero(x);
972 if (!scaled) {
973 power_base(re, im, reciprocal, b);
974 b.exponent = pzero(re);
975 return b;
976 }
977 R lift, scale;
978 Scaling::input_scale(Components::magnitude(x), lift, scale, b.exponent);
979 R wr = pmul(pmul(re, lift), scale), wi = pmul(pmul(im, lift), scale);
980 // Both components take the larger one's scale, under which a nonzero smaller component s far below the larger
981 // one L would lose its residuals (see kSeparationExponent). Then z = L (1 + delta) with |delta| tiny and
982 // z^n = L^n (1 + n delta) to working precision: the power runs on L alone and result() adds L^n n delta, with
983 // delta = i b/a, or a/(bi) = -i a/b, and -delta for the reciprocal base.
984 int separation_exponent = every_step(Real(m)) ? kSeparationExponent - 56 : kSeparationExponent;
985 R separation =
986 pset1frombits<R>(typename Scaling::Bits(Scaling::kBias + separation_exponent) << Scaling::kMantissaBits);
987 R zero = pzero(wr);
988 R re_separated = pandnot(pcmp_lt(pabs(wr), separation), pcmp_eq(re, zero));
989 R im_separated = pandnot(pcmp_lt(pabs(wi), separation), pcmp_eq(im, zero));
990 b.separated = por(re_separated, im_separated);
991 b.any_separated = predux_any(b.separated);
992 if (b.any_separated) {
993 R s_lift, s_scale, s_exponent, count_exponent;
994 Scaling::input_scale(pselect(im_separated, im, re), s_lift, s_scale, s_exponent);
995 R s = pmul(pmul(pselect(im_separated, im, re), s_lift), s_scale);
996 // The count n is a double word too, as Real(n) is inexact beyond 2^digits: its leading digits, which Real
997 // holds exactly, and the rest.
998 numext::uint64_t n = numext::uint64_t(m);
999 numext::uint64_t unit = highest_set_bit(n) >> (kDigits - 1);
1000 numext::uint64_t n_hi = unit > 1 ? n & ~(unit - 1) : n;
1001 R count_hi = pset1<R>(Real(n_hi)), count_lo = pset1<R>(Real(n - n_hi));
1002 R count_scale = Scaling::inverse_scale(count_hi, count_exponent);
1003 // The coefficient n s / L is a double word so that result() rounds the added component once: the ratio
1004 // q + (s - q L) / L, where s - q L = (s - p_hi) - p_lo is exact, times the count. The scaled |s / L| is in
1005 // (1/2, 2) and the scaled count in [2, 4); an eighth keeps |c hi| within 2^62.
1006 R l = pselect(im_separated, wr, wi);
1007 R q = pdiv(s, l);
1008 R p_hi, p_lo, c_hi, c_lo;
1009 twoprod(q, l, p_hi, p_lo);
1010 R q_lo = pdiv(psub(psub(s, p_hi), p_lo), l);
1011 R scaled_count = pmul(count_hi, count_scale);
1012 R scaled_count_lo = pmul(count_lo, count_scale);
1013 twoprod(scaled_count, q, c_hi, c_lo);
1014 fast_twosum(c_hi, pmadd(scaled_count, q_lo, pmadd(scaled_count_lo, q, c_lo)), p_hi, p_lo);
1015 R sign = pselect(im_separated, pset1<R>(Real(reciprocal ? -0.125 : 0.125)),
1016 pset1<R>(Real(reciprocal ? 0.125 : -0.125)));
1017 b.coefficient_hi = pmul(p_hi, sign);
1018 b.coefficient_lo = pmul(p_lo, sign);
1019 b.coefficient_exponent = padd(padd(psub(s_exponent, b.exponent), count_exponent), pset1<R>(Real(3)));
1020 wr = pselect(re_separated, zero, wr);
1021 wi = pselect(im_separated, zero, wi);
1022 }
1023 power_base(wr, wi, reciprocal, b);
1024 if (reciprocal) b.exponent = pnegate(b.exponent);
1025 return b;
1026 }
1027 // The double words of w or of 1/w = q + e*q with q = conj(w)/|w|^2 and e = 1 - q*w = er - t i. The norm cannot
1028 // over- or underflow for max(|wr|, |wi|) in [2, 4) or within the unscaled range, and the few ulps of error in q
1029 // are what the residual corrects: e is of order u and is formed from exact products so that e*q carries the
1030 // correction to order u^2. 1 - s_hi is exact as s_hi is within rounding of 1.
1031 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void power_base(const R& wr, const R& wi, bool reciprocal, State& b) {
1032 b.re_lo = b.im_lo = pzero(wr);
1033 if (!reciprocal) {
1034 b.re_hi = wr;
1035 b.im_hi = wi;
1036 return;
1037 }
1038 R inv = pdiv(pset1<R>(typename NumTraits<Scalar>::Real(1)), pmadd(wr, wr, pmul(wi, wi)));
1039 R qr = pmul(wr, inv), qi = pnegate(pmul(wi, inv));
1040 R a_hi, a_lo, c_hi, c_lo, s_hi, s_lo, t_hi, t_lo;
1041 twoprod(qr, wr, a_hi, a_lo);
1042 twoprod(qi, wi, c_hi, c_lo);
1043 twodiff(a_hi, a_lo, c_hi, c_lo, s_hi, s_lo);
1044 twoprod(qr, wi, a_hi, a_lo);
1045 twoprod(qi, wr, c_hi, c_lo);
1046 twosum(a_hi, a_lo, c_hi, c_lo, t_hi, t_lo);
1047 R er = psub(psub(pset1<R>(typename NumTraits<Scalar>::Real(1)), s_hi), s_lo);
1048 R t = padd(t_hi, t_lo);
1049 fast_twosum(qr, pmadd(er, qr, pmul(t, qi)), b.re_hi, b.re_lo);
1050 fast_twosum(qi, pmsub(er, qi, pmul(t, qr)), b.im_hi, b.im_lo);
1051 }
1052 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void square(State& y) {
1053 R a_hi, a_lo, c_hi, c_lo, p_hi, p_lo;
1054 fast_twoprod(y.re_hi, y.re_lo, y.re_hi, y.re_lo, a_hi, a_lo);
1055 fast_twoprod(y.im_hi, y.im_lo, y.im_hi, y.im_lo, c_hi, c_lo);
1056 fast_twoprod(y.re_hi, y.re_lo, y.im_hi, y.im_lo, p_hi, p_lo);
1057 twodiff(a_hi, a_lo, c_hi, c_lo, y.re_hi, y.re_lo);
1058 y.im_hi = padd(p_hi, p_hi);
1059 y.im_lo = padd(p_lo, p_lo);
1060 y.exponent = padd(y.exponent, y.exponent);
1061 }
1062 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void multiply(State& y, const State& b) {
1063 R ac_hi, ac_lo, bd_hi, bd_lo, ad_hi, ad_lo, bc_hi, bc_lo;
1064 fast_twoprod(y.re_hi, y.re_lo, b.re_hi, b.re_lo, ac_hi, ac_lo);
1065 fast_twoprod(y.im_hi, y.im_lo, b.im_hi, b.im_lo, bd_hi, bd_lo);
1066 fast_twoprod(y.re_hi, y.re_lo, b.im_hi, b.im_lo, ad_hi, ad_lo);
1067 fast_twoprod(y.im_hi, y.im_lo, b.re_hi, b.re_lo, bc_hi, bc_lo);
1068 twodiff(ac_hi, ac_lo, bd_hi, bd_lo, y.re_hi, y.re_lo);
1069 twosum(ad_hi, ad_lo, bc_hi, bc_lo, y.im_hi, y.im_lo);
1070 y.exponent = padd(y.exponent, b.exponent);
1071 }
1072 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void renormalize(State& y) {
1073 R e;
1074 R scale = Scaling::inverse_scale(pmax(pabs(y.re_hi), pabs(y.im_hi)), e);
1075 y.re_hi = pmul(y.re_hi, scale);
1076 y.re_lo = pmul(y.re_lo, scale);
1077 y.im_hi = pmul(y.im_hi, scale);
1078 y.im_lo = pmul(y.im_lo, scale);
1079 y.exponent = padd(y.exponent, e);
1080 }
1081 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet result(const Packet& x, const State& y, const State& b, bool odd,
1082 bool scaled) {
1083 if (!scaled) {
1084 R re = y.re_hi, im = y.im_hi;
1085 if (b.any_zero) zero_component_signs(x, b.negative, odd, re, im);
1086 return Components::join(re, im);
1087 }
1088 R re = Scaling::scale_result(y.re_hi, y.exponent);
1089 R im = Scaling::scale_result(y.im_hi, y.exponent);
1090 if (b.any_separated) {
1091 // L^n is real or imaginary, and i c L^n fills the component it leaves exactly zero.
1092 R e = padd(y.exponent, b.coefficient_exponent);
1093 R zero = pzero(re);
1094 R re_hi, re_lo, im_hi, im_lo;
1095 fast_twoprod(b.coefficient_hi, b.coefficient_lo, y.im_hi, y.im_lo, re_hi, re_lo);
1096 fast_twoprod(b.coefficient_hi, b.coefficient_lo, y.re_hi, y.re_lo, im_hi, im_lo);
1097 re = pselect(pand(b.separated, pcmp_eq(y.re_hi, zero)), Scaling::scale_result(pnegate(re_hi), e), re);
1098 im = pselect(pand(b.separated, pcmp_eq(y.im_hi, zero)), Scaling::scale_result(im_hi, e), im);
1099 }
1100 if (b.any_zero) zero_component_signs(x, b.negative, odd, re, im);
1101 return Components::join(re, im);
1102 }
1103 // Zero, infinite and NaN bases are replaced by 1 + i before the loop, so that they neither force the scaled loop on
1104 // their packet nor come back NaN, and their results are set at the end (see the comment above the struct). 1 + i
1105 // has no zero component, which would take the packet through zero_component_signs. Every one of them fails
1106 // in_range, so only a packet bound for the scaled loop is checked, and only whether there are any is carried through
1107 // the loop.
1108 struct SpecialBases {
1109 Packet x;
1110 bool any;
1111 };
1112 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE SpecialBases special_bases(const Packet& x, bool scaled) {
1113 SpecialBases special;
1114 special.x = x;
1115 special.any = false;
1116 if (scaled) {
1117 R zero, inf, nan;
1118 Components::special_masks(x, zero, inf, nan);
1119 special.any = predux_any(por(por(zero, inf), nan));
1120 }
1121 return special;
1122 }
1123 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet without_special_bases(const Packet& x,
1124 const SpecialBases& special) {
1125 if (!special.any) return x;
1126 R zero, inf, nan;
1127 Components::special_masks(x, zero, inf, nan);
1128 return Components::replace(por(por(zero, inf), nan), Scalar(1, 1), x);
1129 }
1130 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet with_special_bases(const Packet& r, const SpecialBases& special,
1131 bool negative) {
1132 if (!special.any) return r;
1133 R zero, inf, nan;
1134 Components::special_masks(special.x, zero, inf, nan);
1135 Real real_inf = NumTraits<Real>::infinity(), real_nan = NumTraits<Real>::quiet_NaN();
1136 Packet out = Components::replace(negative ? zero : inf, Scalar(real_inf, Real(0)), r);
1137 out = Components::replace(negative ? inf : zero, Scalar(0), out);
1138 return Components::replace(nan, Scalar(real_nan, real_nan), out);
1139 }
1140 // A base with one component s = +-0 is the limit of L (1 + i t), with t = s/L for a real L and -s/b for L = bi,
1141 // so z^n = L^n (1 + i n t): the zero component of the power has the sign of i n t L^n, which the double-word sums
1142 // lose (-0 + 0 = +0). Sign bits combine by xor; a scalar comparison mask is one rather than all ones, so masks
1143 // select.
1144 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void zero_component_signs(const Packet& x, bool negative, bool odd,
1145 R& re, R& im) {
1146 R zero = pzero(re);
1147 R x_re, x_im;
1148 Components::split(x, x_re, x_im);
1149 R imaginary = pcmp_eq(x_re, zero);
1150 R one_zero = pxor(imaginary, pcmp_eq(x_im, zero));
1151 R minus_zero = pset1<R>(Real(-0.0));
1152 // i^2 = -1 from t for L = bi and from i t L^n for an imaginary power cancel for odd n.
1153 R power_imaginary = odd ? imaginary : zero;
1154 R n_sign = negative ? minus_zero : zero;
1155 R extra = odd ? n_sign : pselect(imaginary, pxor(n_sign, minus_zero), n_sign);
1156 R sign = pxor(pand(pxor(pxor(x_re, x_im), pselect(power_imaginary, im, re)), minus_zero), extra);
1157 re = pselect(pand(one_zero, power_imaginary), sign, re);
1158 im = pselect(pandnot(one_zero, power_imaginary), sign, im);
1159 }
1160};
1161
1162// Plain repeated squaring for the remaining floating-point bases.
1163template <typename Packet, typename ScalarExponent>
1164EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_plain(const Packet& x, const ScalarExponent& exponent) {
1165 using Scalar = typename unpacket_traits<Packet>::type;
1166 using ExponentHelper = exponent_helper<ScalarExponent>;
1167 using AbsExponentType = typename ExponentHelper::safe_abs_type;
1168 if (exponent == ScalarExponent(0)) return pset1<Packet>(Scalar(1));
1169
1170 Packet base = exponent_is_negative<ScalarExponent>::run(exponent) ? pdiv(pset1<Packet>(Scalar(1)), x) : x;
1171 AbsExponentType m = ExponentHelper::safe_abs(exponent);
1172 Packet y = base;
1173 for (AbsExponentType bit = highest_set_bit(m) >> 1; bit != 0; bit >>= 1) {
1174 y = pmul(y, y);
1175 if ((m & bit) != 0) y = pmul(y, base);
1176 }
1177 return y;
1178}
1179
1180// Repeated squaring for a binary32 or binary64 base, real or complex, carried in double-word arithmetic. Plain
1181// repeated squaring doubles the accumulated rounding error at every squaring, so its error grows like n * u for
1182// x^n. Keeping the running power as an unevaluated sum {hi, lo} bounds each step's relative error by about
1183// 5 * u^2 (fast_twoprod; measured at most 4.98 * u^2 per squaring), so the result is correctly rounded unless the
1184// exact power lies within about 5 * n * u^2 of a rounding boundary, and stays within 1 ulp up to
1185// n = max_squaring_exponent() for float. For a complex base the bound is normwise, relative to |z^n|, as
1186// a^2 - b^2 and ac - bd can cancel, and the n u^2 term passes u/2 as |n| approaches 1/u: (0.6 + 0.8i)^n in
1187// complex<float> is 0.2 u off at n = 4096, 0.5 u at 2^24 and 4.2 u at 2^31 - 1. The power is scaled by powers of
1188// two throughout and only the final pldexp can overflow or underflow.
1189template <typename Packet, typename ScalarExponent>
1190EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(const Packet& x, const ScalarExponent& exponent) {
1191 using Scalar = typename unpacket_traits<Packet>::type;
1192 using ExponentHelper = exponent_helper<ScalarExponent>;
1193 using AbsExponentType = typename ExponentHelper::safe_abs_type;
1194 using Ops = repeated_squaring_ops<Packet>;
1195 if (exponent == ScalarExponent(0)) return pset1<Packet>(Scalar(1));
1196
1197 bool negative = exponent_is_negative<ScalarExponent>::run(exponent);
1198 AbsExponentType m = ExponentHelper::safe_abs(exponent);
1199 bool odd = (m & AbsExponentType(1)) != 0;
1200 if (m == AbsExponentType(1) && !negative) return x;
1201 // A real x^-1 and x^2 are a single correctly rounded operation, which is what the double word would produce. On
1202 // ARMv7 NEON pdiv is a refined reciprocal estimate instead.
1203 EIGEN_IF_CONSTEXPR (!NumTraits<Scalar>::IsComplex) {
1204 if (m == AbsExponentType(1) && !flushes_subnormals<Packet>::value) return pdiv(pset1<Packet>(Scalar(1)), x);
1205 if (m == AbsExponentType(2) && !negative) return pmul(x, x);
1206 }
1207 AbsExponentType top = highest_set_bit(m);
1208
1209 // Bases whose magnitude lies within 2^(+-B) for B = budget / |n| keep every power up to x^n and its residuals
1210 // normal: |x^n| <= 2^budget < max and u * |x^n| >= 2^(min_exponent - 1), the smallest normal, so the loop
1211 // needs neither scaling nor exponents and the result is hi itself. A complex base is tested on both nonzero
1212 // components, as every term a^k b^(n-k) of its power must stay normal too; its magnitude exceeds the larger one
1213 // by up to sqrt(2), so B is one less to absorb sqrt(2)^n (an exhausted budget therefore admits nothing), and its
1214 // reciprocal forms |w|^2, which bounds B by half the exponent range. Zero, infinity and NaN fail the test.
1215 using Real = typename NumTraits<Scalar>::Real;
1216 using RealBits = std::make_unsigned_t<typename make_integer<Real>::type>;
1217 constexpr int kBudget = -(numext::numeric_limits<Real>::min_exponent + numext::numeric_limits<Real>::digits);
1218 constexpr int kMantissaBits = numext::numeric_limits<Real>::digits - 1;
1219 constexpr RealBits kBiasBits = RealBits(numext::numeric_limits<Real>::max_exponent - 1);
1220 int b = numext::uint64_t(m) > numext::uint64_t(kBudget) ? 0 : kBudget / int(m);
1221 EIGEN_IF_CONSTEXPR (NumTraits<Scalar>::IsComplex)
1222 b = numext::mini(b - 1, (numext::numeric_limits<Real>::max_exponent - 1) / 2);
1223 // A flushing packet always scales: the budget keeps the unscaled power normal, but not all of its residuals, nor the
1224 // components and the norm of a complex reciprocal.
1225 EIGEN_IF_CONSTEXPR (flushes_subnormals<Packet>::value) b = -1;
1226 typename Ops::Bound bound =
1227 pset1frombits<typename Ops::Bound>(RealBits(kBiasBits + RealBits(b < 0 ? 0 : b)) << kMantissaBits);
1228 bool scaled = b < 0 || !Ops::in_range(x, bound);
1229 // Zero, infinite and NaN bases are never in range, so only a packet bound for the scaled loop can hold one.
1230 typename Ops::SpecialBases special = Ops::special_bases(x, scaled);
1231 Packet x_regular = Ops::without_special_bases(x, special);
1232 if (special.any) scaled = b < 0 || !Ops::in_range(x_regular, bound);
1233
1234 typename Ops::State base = Ops::base(x_regular, negative, scaled, m);
1235 typename Ops::State y = base;
1236 int renormalization_steps = Ops::renormalization_steps(Real(m));
1237 int steps_since_renormalization = 0;
1238 for (AbsExponentType bit = top >> 1; bit != 0; bit >>= 1) {
1239 Ops::square(y);
1240 if ((m & bit) != 0) Ops::multiply(y, base);
1241 if (scaled && ++steps_since_renormalization == renormalization_steps) {
1242 Ops::renormalize(y);
1243 steps_since_renormalization = 0;
1244 }
1245 }
1246 return Ops::with_special_bases(Ops::result(x_regular, y, base, odd, scaled), special, negative);
1247}
1248
1249template <typename Packet, typename ScalarExponent>
1250EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE Packet int_pow_lanewise(const Packet& x, const ScalarExponent& exponent) {
1251 using Scalar = typename unpacket_traits<Packet>::type;
1252 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Scalar values[unpacket_traits<Packet>::size];
1253 pstore(values, x);
1254 for (Scalar& value : values) value = int_pow_double_word(value, exponent);
1255 return pload<Packet>(values);
1256}
1257
1258template <typename Packet, typename ScalarExponent>
1259EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(const Packet& x, const ScalarExponent& exponent,
1260 false_type) {
1261 return int_pow_double_word(x, exponent);
1262}
1263
1264// A flushing packet is computed lane by lane on scalars, whose arithmetic does not flush, where it holds a subnormal
1265// base, which the comparisons, magnitudes and scalings of the base would all read as zero, or where a real x^-1 or x^2
1266// falls below the smallest normal: the scalar path rounds those once, as 1/x and x * x do.
1267template <typename Packet, typename ScalarExponent>
1268EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow_double_word(const Packet& x, const ScalarExponent& exponent,
1269 true_type) {
1270 using Ops = repeated_squaring_ops<Packet>;
1271 if (Ops::any_subnormal(x)) return int_pow_lanewise(x, exponent);
1272 Packet r = int_pow_double_word(x, exponent);
1273 if (exponent_helper<ScalarExponent>::safe_abs(exponent) <= 2 && Ops::any_below_normal(x, r))
1274 return int_pow_lanewise(x, exponent);
1275 return r;
1276}
1277
1278template <typename Packet, typename ScalarExponent>
1279EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent, true_type) {
1280#if EIGEN_ARCH_ARM
1281 return int_pow_double_word(x, exponent, flushes_subnormals<Packet>());
1282#else
1283 return int_pow_double_word(x, exponent);
1284#endif
1285}
1286
1287template <typename Packet, typename ScalarExponent>
1288EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent, false_type) {
1289 return int_pow_plain(x, exponent);
1290}
1291
1292template <typename Packet, typename ScalarExponent>
1293EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet int_pow(const Packet& x, const ScalarExponent& exponent) {
1294 return int_pow(x, exponent, use_double_word<Packet>());
1295}
1296
1297// The largest integer-valued floating-point exponent that repeated squaring handles; beyond it generic_pow takes
1298// over. Plain squaring is accurate to a few ulps only for small exponents.
1299template <typename Packet, typename ScalarExponent>
1300EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool use_repeated_squaring(const ScalarExponent& exponent) {
1301 using Scalar = typename unpacket_traits<Packet>::type;
1302 return use_double_word<Packet>::value ? numext::abs(exponent) <= ScalarExponent(max_squaring_exponent<Scalar>())
1303 : (exponent <= ScalarExponent(7) && exponent >= ScalarExponent(-3));
1304}
1305
1306// The same for an exponent of integer type, which repeated squaring handles exactly whatever its size: generic_pow
1307// only takes over where the exponent converts exactly to the base type.
1308template <typename Packet, typename ScalarExponent>
1309EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool use_repeated_squaring_for_integer(const ScalarExponent& exponent) {
1310 using Scalar = typename unpacket_traits<Packet>::type;
1311 constexpr numext::uint64_t kExactLimit = numext::uint64_t(1) << numext::numeric_limits<Scalar>::digits;
1312 return exponent_helper<ScalarExponent>::safe_abs(exponent) > kExactLimit ||
1313 use_repeated_squaring<Packet>(static_cast<Scalar>(exponent));
1314}
1315
1316template <typename Packet>
1317EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<!is_scalar<Packet>::value, Packet> gen_pow(
1318 const Packet& x, const typename unpacket_traits<Packet>::type& exponent) {
1319 const Packet exponent_packet = pset1<Packet>(exponent);
1320 // generic_pow_impl requires positive x; sign/error handling is done by the caller.
1321 return generic_pow_impl(pabs(x), exponent_packet);
1322}
1323
1324template <typename Scalar>
1325EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::enable_if_t<is_scalar<Scalar>::value, Scalar> gen_pow(
1326 const Scalar& x, const Scalar& exponent) {
1327 return numext::pow(x, exponent);
1328}
1329
1330// Handle special cases for pow(x, exponent) where both base and exponent are
1331// floating point and the exponent is a non-integer scalar (uniform across all
1332// SIMD lanes). This allows us to use scalar branches on exponent properties.
1333// Reached from packetOp only, so the comparison masks are all ones and select a NaN or an infinity bitwise.
1334template <typename Packet, typename ScalarExponent>
1335EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_nonint_nonint_errors(const Packet& x, const Packet& powx,
1336 const ScalarExponent& exponent) {
1337 using Scalar = typename unpacket_traits<Packet>::type;
1338 const Packet cst_zero = pzero(x);
1339 const Packet cst_one = pset1<Packet>(Scalar(1));
1340 const Packet cst_inf = pinf<Packet>();
1341 const Packet cst_nan = pnan<Packet>();
1342
1343 const Packet abs_x = pabs(x);
1344
1345 // x < 0 with non-integer exponent -> NaN.
1346 Packet result = por(pcmp_lt(x, cst_zero), powx);
1347
1348 if (!(numext::isfinite)(exponent)) {
1349 if (exponent != exponent) {
1350 // pow(x, NaN) = NaN, except pow(+1, NaN) = 1.
1351 result = pselect(pcmp_eq(x, cst_one), cst_one, cst_nan);
1352 } else {
1353 // Exponent is +inf or -inf.
1354 const Packet abs_x_is_one = pcmp_eq(abs_x, cst_one);
1355 if (exponent > ScalarExponent(0)) {
1356 // pow(x, +inf): |x| > 1 -> +inf, |x| < 1 -> 0, |x| == 1 -> 1.
1357 result = pand(pcmp_lt(cst_one, abs_x), cst_inf);
1358 } else {
1359 // pow(x, -inf): |x| < 1 -> +inf, |x| > 1 -> 0, |x| == 1 -> 1.
1360 result = pand(pcmp_lt(abs_x, cst_one), cst_inf);
1361 }
1362 // pow(+-1, +-inf) = 1.
1363 result = pselect(abs_x_is_one, cst_one, result);
1364 }
1365 } else {
1366 // Finite non-integer exponent.
1367 const Packet x_is_zero = pcmp_eq(x, cst_zero);
1368 const Packet abs_x_is_inf = pcmp_eq(abs_x, cst_inf);
1369 if (exponent < ScalarExponent(0)) {
1370 // pow(+-0, negative non-integer) = +inf. pow(+-inf, negative) = +0.
1371 result = pselect(x_is_zero, cst_inf, result);
1372 result = pselect(abs_x_is_inf, cst_zero, result);
1373 } else {
1374 // pow(+-0, positive non-integer) = +0. pow(+-inf, positive) = +inf.
1375 result = pselect(x_is_zero, cst_zero, result);
1376 result = pselect(abs_x_is_inf, cst_inf, result);
1377 }
1378 }
1379
1380 // NaN base produces NaN. This overrides all cases above, but pow(NaN, 0) = 1
1381 // and pow(NaN, integer) are handled by the integer exponent path and never
1382 // reach this function.
1383 result = por(pisnan(x), result);
1384
1385 return result;
1386}
1387
1388template <typename Packet, typename ScalarExponent,
1389 std::enable_if_t<NumTraits<typename unpacket_traits<Packet>::type>::IsSigned, bool> = true>
1390EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_negative_exponent(const Packet& x, const ScalarExponent& exponent) {
1391 using Scalar = typename unpacket_traits<Packet>::type;
1392
1393 // signed integer base, signed integer exponent case
1394
1395 // This routine handles negative exponents.
1396 // The return value is either 0, 1, or -1.
1397 Packet cst_pos_one = pset1<Packet>(Scalar(1));
1398 const bool exponent_is_odd = exponent % ScalarExponent(2) != ScalarExponent(0);
1399 const Packet exp_is_odd = exponent_is_odd ? ptrue<Packet>(x) : pzero<Packet>(x);
1400
1401 const Packet abs_x = pabs(x);
1402 const Packet abs_x_is_one = pcmp_eq(abs_x, cst_pos_one);
1403
1404 Packet result = pselect(exp_is_odd, x, abs_x);
1405 result = pselect(abs_x_is_one, result, pzero<Packet>(x));
1406 return result;
1407}
1408
1409template <typename Packet, typename ScalarExponent,
1410 std::enable_if_t<!NumTraits<typename unpacket_traits<Packet>::type>::IsSigned, bool> = true>
1411EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet handle_negative_exponent(const Packet& x, const ScalarExponent&) {
1412 using Scalar = typename unpacket_traits<Packet>::type;
1413
1414 // unsigned integer base, signed integer exponent case
1415
1416 // This routine handles negative exponents.
1417 // The return value is either 0 or 1
1418
1419 const Scalar pos_one = Scalar(1);
1420
1421 const Packet cst_pos_one = pset1<Packet>(pos_one);
1422
1423 const Packet x_is_one = pcmp_eq(x, cst_pos_one);
1424
1425 return pand(x_is_one, x);
1426}
1427
1428} // end namespace unary_pow
1429
1430template <typename Packet, typename ScalarExponent,
1431 bool BaseIsIntegerType = NumTraits<typename unpacket_traits<Packet>::type>::IsInteger,
1432 bool ExponentIsIntegerType = NumTraits<ScalarExponent>::IsInteger,
1433 bool ExponentIsSigned = NumTraits<ScalarExponent>::IsSigned>
1434struct unary_pow_impl;
1435
1436template <typename Packet, typename ScalarExponent, bool ExponentIsSigned>
1437struct unary_pow_impl<Packet, ScalarExponent, false, false, ExponentIsSigned> {
1438 using Scalar = typename unpacket_traits<Packet>::type;
1439 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) {
1440 const bool exponent_is_integer = (numext::isfinite)(exponent) && numext::round(exponent) == exponent;
1441 if (exponent_is_integer) {
1442 return unary_pow::use_repeated_squaring<Packet>(exponent) ? unary_pow::int_pow(x, exponent)
1443 : generic_pow(x, pset1<Packet>(exponent));
1444 } else {
1445 Packet result = unary_pow::gen_pow(x, exponent);
1446 result = unary_pow::handle_nonint_nonint_errors(x, result, exponent);
1447 return result;
1448 }
1449 }
1450};
1451
1452template <typename Packet, typename ScalarExponent, bool ExponentIsSigned>
1453struct unary_pow_impl<Packet, ScalarExponent, false, true, ExponentIsSigned> {
1454 using Scalar = typename unpacket_traits<Packet>::type;
1455 // Only real float and double bases with double-word support fall back to generic_pow for large exponents:
1456 // complex bases have no vectorized generic_pow, and the bases left without double-word support (half,
1457 // bfloat16, the GPU packets, the MSA and HVX float packets) have no packet generic_pow either, so their
1458 // squaring loop -- at most 64 steps -- is all there is.
1459 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) {
1460 return run(x, exponent,
1461 bool_constant < unary_pow::use_double_word<Packet>::value && !NumTraits<Scalar>::IsComplex > ());
1462 }
1463 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent, true_type) {
1464 return unary_pow::use_repeated_squaring_for_integer<Packet>(exponent)
1465 ? unary_pow::int_pow(x, exponent)
1466 : generic_pow(x, pset1<Packet>(static_cast<Scalar>(exponent)));
1467 }
1468 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet run(const Packet& x, const ScalarExponent& exponent, false_type) {
1469 return unary_pow::int_pow(x, exponent);
1470 }
1471};
1472
1473template <typename Packet, typename ScalarExponent>
1474struct unary_pow_impl<Packet, ScalarExponent, true, true, true> {
1475 using Scalar = typename unpacket_traits<Packet>::type;
1476 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) {
1477 if (exponent < ScalarExponent(0)) {
1478 return unary_pow::handle_negative_exponent(x, exponent);
1479 } else {
1480 return unary_pow::int_pow_wrapping(x, exponent);
1481 }
1482 }
1483};
1484
1485template <typename Packet, typename ScalarExponent>
1486struct unary_pow_impl<Packet, ScalarExponent, true, true, false> {
1487 using Scalar = typename unpacket_traits<Packet>::type;
1488 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& x, const ScalarExponent& exponent) {
1489 return unary_pow::int_pow_wrapping(x, exponent);
1490 }
1491};
1492
1493} // end namespace internal
1494} // end namespace Eigen
1495
1496#endif // EIGEN_ARCH_GENERIC_PACKET_MATH_POW_H