Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GenericPacketMathTrig.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2007 Julien Pommier
5// Copyright (C) 2009-2019 Gael Guennebaud <gael.guennebaud@inria.fr>
6// Copyright (C) 2018-2025 Rasmus Munk Larsen <rmlarsen@gmail.com>
7//
8// This Source Code Form is subject to the terms of the Mozilla
9// Public License v. 2.0. If a copy of the MPL was not distributed
10// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11// SPDX-License-Identifier: MPL-2.0
12
13#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_TRIG_H
14#define EIGEN_ARCH_GENERIC_PACKET_MATH_TRIG_H
15
16// IWYU pragma: private
17#include "../../InternalHeaderCheck.h"
18
19namespace Eigen {
20namespace internal {
21
22//----------------------------------------------------------------------
23// Trigonometric Functions
24//----------------------------------------------------------------------
25
26// Enum for selecting which function to compute. SinCos is intended to compute
27// pairs of Sin and Cos of the even entries in the packet, e.g.
28// SinCos([a, *, b, *]) = [sin(a), cos(a), sin(b), cos(b)].
29enum class TrigFunction : uint8_t { Sin, Cos, Tan, SinCos };
30
31// The following code is inspired by the following stack-overflow answer:
32// https://stackoverflow.com/questions/30463616/payne-hanek-algorithm-implementation-in-c/30465751#30465751
33// It has been largely optimized:
34// - By-pass calls to frexp.
35// - Aligned loads of required 96 bits of 2/pi. This is accomplished by
36// (1) balancing the mantissa and exponent to the required bits of 2/pi are
37// aligned on 8-bits, and (2) replicating the storage of the bits of 2/pi.
38// - Avoid a branch in rounding and extraction of the remaining fractional part.
39// Overall, I measured a speed up higher than x2 on x86-64.
40inline float trig_reduce_huge(float xf, Eigen::numext::int32_t* quadrant) {
41 using Eigen::numext::int32_t;
42 using Eigen::numext::int64_t;
43 using Eigen::numext::uint32_t;
44 using Eigen::numext::uint64_t;
45
46 const double pio2_62 = 3.4061215800865545e-19; // pi/2 * 2^-62
47 const uint64_t zero_dot_five = uint64_t(1) << 61; // 0.5 in 2.62-bit fixed-point format
48
49 // 192 bits of 2/pi for Payne-Hanek reduction
50 // Bits are introduced by packet of 8 to enable aligned reads.
51 static const uint32_t two_over_pi[] = {
52 0x00000028, 0x000028be, 0x0028be60, 0x28be60db, 0xbe60db93, 0x60db9391, 0xdb939105, 0x9391054a, 0x91054a7f,
53 0x054a7f09, 0x4a7f09d5, 0x7f09d5f4, 0x09d5f47d, 0xd5f47d4d, 0xf47d4d37, 0x7d4d3770, 0x4d377036, 0x377036d8,
54 0x7036d8a5, 0x36d8a566, 0xd8a5664f, 0xa5664f10, 0x664f10e4, 0x4f10e410, 0x10e41000, 0xe4100000};
55
56 uint32_t xi = numext::bit_cast<uint32_t>(xf);
57 // Below, -118 = -126 + 8.
58 // -126 is to get the exponent,
59 // +8 is to enable alignment of 2/pi's bits on 8 bits.
60 // This is possible because the fractional part of x has only 24 meaningful bits.
61 uint32_t e = (xi >> 23) - 118;
62 // Extract the mantissa and shift it to align it wrt the exponent
63 xi = ((xi & 0x007fffffu) | 0x00800000u) << (e & 0x7);
64
65 uint32_t i = e >> 3;
66 uint32_t twoopi_1 = two_over_pi[i - 1];
67 uint32_t twoopi_2 = two_over_pi[i + 3];
68 uint32_t twoopi_3 = two_over_pi[i + 7];
69
70 // Compute x * 2/pi in 2.62-bit fixed-point format.
71 uint64_t p;
72 p = uint64_t(xi) * twoopi_3;
73 p = uint64_t(xi) * twoopi_2 + (p >> 32);
74 p = (uint64_t(xi * twoopi_1) << 32) + p;
75
76 // Round to nearest: add 0.5 and extract integral part.
77 uint64_t q = (p + zero_dot_five) >> 62;
78 *quadrant = int(q);
79 // Now it remains to compute "r = x - q*pi/2" with high accuracy,
80 // since we have p=x/(pi/2) with high accuracy, we can more efficiently compute r as:
81 // r = (p-q)*pi/2,
82 // where the product can be carried out with sufficient accuracy using double precision.
83 p -= q << 62;
84 return float(double(int64_t(p)) * pio2_62);
85}
86
87template <TrigFunction Func, typename Packet>
88EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psincos_float(const Packet& _x) {
89 using PacketI = typename unpacket_traits<Packet>::integer_packet;
90
91 const Packet cst_2oPI = pset1<Packet>(0.636619746685028076171875f); // 2/PI
92 const Packet cst_rounding_magic = pset1<Packet>(12582912); // 2^23 for rounding
93 const PacketI csti_1 = pset1<PacketI>(1);
94 const Packet cst_sign_mask = psignmask<Packet>();
95
96 Packet x = pabs(_x);
97
98 // Scale x by 2/Pi to find x's octant.
99 Packet y = pmul(x, cst_2oPI);
100
101 // Rounding trick to find nearest integer:
102 Packet y_round = padd(y, cst_rounding_magic);
103 pbarrier(y_round);
104 PacketI y_int = preinterpret<PacketI>(y_round); // last 23 digits represent integer (if abs(x)<2^24)
105 y = psub(y_round, cst_rounding_magic); // nearest integer to x * (2/pi)
106
107// Subtract y * Pi/2 to reduce x to the interval -Pi/4 <= x <= +Pi/4
108// using "Extended precision modular arithmetic"
109#if defined(EIGEN_VECTORIZE_FMA)
110 // This version requires true FMA for high accuracy.
111 // It provides a max error of 1ULP up to (with absolute_error < 5.9605e-08):
112 constexpr float huge_th = (Func == TrigFunction::Sin) ? 117435.992f : 71476.0625f;
113 x = pmadd(y, pset1<Packet>(-1.57079601287841796875f), x);
114 x = pmadd(y, pset1<Packet>(-3.1391647326017846353352069854736328125e-07f), x);
115 x = pmadd(y, pset1<Packet>(-5.390302529957764765544681040410068817436695098876953125e-15f), x);
116#else
117 // Without true FMA, the previous set of coefficients maintain 1ULP accuracy
118 // up to x<15.7 (for sin), but accuracy is immediately lost for x>15.7.
119 // We thus use one more iteration to maintain 2ULPs up to reasonably large inputs.
120
121 // The following set of coefficients maintain 1ULP up to 9.43 and 14.16 for sin and cos respectively.
122 // and 2 ULP up to:
123 constexpr float huge_th = (Func == TrigFunction::Sin) ? 25966.f : 18838.f;
124 x = pmadd(y, pset1<Packet>(-1.5703125), x); // = 0xbfc90000
125 pbarrier(x);
126 x = pmadd(y, pset1<Packet>(-0.000483989715576171875), x); // = 0xb9fdc000
127 pbarrier(x);
128 x = pmadd(y, pset1<Packet>(1.62865035235881805419921875e-07), x); // = 0x342ee000
129 pbarrier(x);
130 x = pmadd(y, pset1<Packet>(5.5644315544167710640977020375430583953857421875e-11), x); // = 0x2e74b9ee
131
132// For the record, the following set of coefficients maintain 2ULP up
133// to a slightly larger range:
134// const float huge_th = ComputeSine ? 51981.f : 39086.125f;
135// but it slightly fails to maintain 1ULP for two values of sin below pi.
136// x = pmadd(y, pset1<Packet>(-3.140625/2.), x);
137// x = pmadd(y, pset1<Packet>(-0.00048351287841796875), x);
138// x = pmadd(y, pset1<Packet>(-3.13855707645416259765625e-07), x);
139// x = pmadd(y, pset1<Packet>(-6.0771006282767103812147979624569416046142578125e-11), x);
140
141// For the record, with only 3 iterations it is possible to maintain
142// 1 ULP up to 3PI (maybe more) and 2ULP up to 255.
143// The coefficients are: 0xbfc90f80, 0xb7354480, 0x2e74b9ee
144#endif
145
146 if (predux_any(pcmp_le(pset1<Packet>(huge_th), pabs(_x)))) {
147 const int PacketSize = unpacket_traits<Packet>::size;
148 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) float vals[PacketSize];
149 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) float x_cpy[PacketSize];
150 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Eigen::numext::int32_t y_int2[PacketSize];
151 pstoreu(vals, pabs(_x));
152 pstoreu(x_cpy, x);
153 pstoreu(y_int2, y_int);
154 for (int k = 0; k < PacketSize; ++k) {
155 float val = vals[k];
156 if (val >= huge_th && (numext::isfinite)(val)) x_cpy[k] = trig_reduce_huge(val, &y_int2[k]);
157 }
158 x = ploadu<Packet>(x_cpy);
159 y_int = ploadu<PacketI>(y_int2);
160 }
161
162 // Get the polynomial selection mask from the second bit of y_int
163 // We'll calculate both (sin and cos) polynomials and then select from the two.
164 Packet poly_mask = preinterpret<Packet>(pcmp_eq(pand(y_int, csti_1), pzero(y_int)));
165
166 Packet x2 = pmul(x, x);
167
168 // Evaluate the cos(x) polynomial. (-Pi/4 <= x <= Pi/4)
169 Packet y1 = pset1<Packet>(2.4372266125283204019069671630859375e-05f);
170 y1 = pmadd(y1, x2, pset1<Packet>(-0.00138865201734006404876708984375f));
171 y1 = pmadd(y1, x2, pset1<Packet>(0.041666619479656219482421875f));
172 y1 = pmadd(y1, x2, pset1<Packet>(-0.5f));
173 y1 = pmadd(y1, x2, pset1<Packet>(1.f));
174
175 // Evaluate the sin(x) polynomial. (-Pi/4 <= x <= Pi/4)
176 // octave/matlab code to compute those coefficients:
177 // x = (0:0.0001:pi/4)';
178 // A = [x.^3 x.^5 x.^7];
179 // w = ((1.-(x/(pi/4)).^2).^5)*2000+1; # weights trading relative accuracy
180 // c = (A'*diag(w)*A)\‍(A'*diag(w)*(sin(x)-x)); # weighted LS, linear coeff forced to 1
181 // printf('%.64f\n %.64f\n%.64f\n', c(3), c(2), c(1))
182 //
183 Packet y2 = pset1<Packet>(-0.0001959234114083702898469196984621021329076029360294342041015625f);
184 y2 = pmadd(y2, x2, pset1<Packet>(0.0083326873655616851693794799871284340042620897293090820312500000f));
185 y2 = pmadd(y2, x2, pset1<Packet>(-0.1666666203982298255503735617821803316473960876464843750000000000f));
186 y2 = pmul(y2, x2);
187 y2 = pmadd(y2, x, x);
188
189 // Select the correct result from the two polynomials.
190 // Compute the sign to apply to the polynomial.
191 // sin: sign = second_bit(y_int) xor signbit(_x)
192 // cos: sign = second_bit(y_int+1)
193 Packet sign_bit = (Func == TrigFunction::Sin) ? pxor(_x, preinterpret<Packet>(plogical_shift_left<30>(y_int)))
194 : preinterpret<Packet>(plogical_shift_left<30>(padd(y_int, csti_1)));
195 sign_bit = pand(sign_bit, cst_sign_mask); // clear all but left most bit
196
197 if ((Func == TrigFunction::SinCos) || (Func == TrigFunction::Tan)) {
198 Packet peven = peven_mask(x);
199 Packet ysin = pselect(poly_mask, y2, y1);
200 Packet ycos = pselect(poly_mask, y1, y2);
201 Packet sign_bit_sin = pxor(_x, preinterpret<Packet>(plogical_shift_left<30>(y_int)));
202 Packet sign_bit_cos = preinterpret<Packet>(plogical_shift_left<30>(padd(y_int, csti_1)));
203 sign_bit_sin = pand(sign_bit_sin, cst_sign_mask); // clear all but left most bit
204 sign_bit_cos = pand(sign_bit_cos, cst_sign_mask); // clear all but left most bit
205 y = (Func == TrigFunction::SinCos) ? pselect(peven, pxor(ysin, sign_bit_sin), pxor(ycos, sign_bit_cos))
206 : pdiv(pxor(ysin, sign_bit_sin), pxor(ycos, sign_bit_cos));
207 } else {
208 y = (Func == TrigFunction::Sin) ? pselect(poly_mask, y2, y1) : pselect(poly_mask, y1, y2);
209 y = pxor(y, sign_bit);
210 }
211 return y;
212}
213
214template <typename Packet>
215EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psin_float(const Packet& x) {
216 return psincos_float<TrigFunction::Sin>(x);
217}
218
219template <typename Packet>
220EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcos_float(const Packet& x) {
221 return psincos_float<TrigFunction::Cos>(x);
222}
223
224template <typename Packet>
225EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptan_float(const Packet& x) {
226 return psincos_float<TrigFunction::Tan>(x);
227}
228
229// Pi/2 split into 3 double-precision parts (triple-double).
230// c1 + c2 + c3 = pi/2 to ~159 bits. Computed by Sollya.
231// c1 = RD(pi/2), c2 = RD(pi/2 - c1), c3 = RD(pi/2 - c1 - c2).
232template <typename Packet>
233Packet cst_pio2_1() {
234 return pset1<Packet>(-1.5707963267948965579989817342720925807952880859375); // -0x1.921fb54442d18p0
235}
236template <typename Packet>
237Packet cst_pio2_2() {
238 return pset1<Packet>(-6.12323399573676603586882014729198302312846062338790e-17); // -0x1.1a62633145c07p-54
239}
240template <typename Packet>
241Packet cst_pio2_3() {
242 return pset1<Packet>(1.4973849048591698329435081771059920083527504761695190e-33); // 0x1.f1976b7ed8fbcp-110
243}
244
245// Trigonometric argument reduction for double.
246// Reduces x to t such that x + q * pi/2 = t, where |t| <= pi/4.
247// Uses a triple-double split of pi/2 (cst_pio2_{1,2,3}).
248template <typename Packet>
249Packet trig_reduce_small_double(const Packet& x, const Packet& q) {
250#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
251 // With FMA, pmadd(a, b, c) = fl(a*b + c) in a single rounding,
252 // so Cody-Waite reduction is accurate even under catastrophic cancellation.
253 Packet t;
254 t = pmadd(cst_pio2_1<Packet>(), q, x);
255 t = pmadd(cst_pio2_2<Packet>(), q, t);
256 t = pmadd(cst_pio2_3<Packet>(), q, t);
257 return t;
258#else
259 // Without FMA, pmadd is mul + add (two roundings). For large q,
260 // pmul(pio2_1, q) rounds before the cancellation with x, losing
261 // catastrophic amounts of precision (observed: ~10 digits lost).
262 // Use error-free transformations to preserve accuracy.
263
264 // Compute q * pio2_1 exactly as a double-word using Dekker's algorithm.
265 Packet qp_hi, qp_lo;
266 twoprod<Packet, true>(cst_pio2_1<Packet>(), q, qp_hi, qp_lo);
267
268 // Error-free addition of x and qp_hi using Knuth's 2sum.
269 // Returns t_hi + t_lo = x + qp_hi exactly, with t_hi = fl(x + qp_hi).
270 Packet t_hi, t_lo;
271 twosum(x, qp_hi, t_hi, t_lo);
272
273 // Near a multiple of pi/2, t_hi + qp_lo cancels against q * pio2_2 too, so that product is formed exactly as well
274 // and the two large terms are added by 2sum: only terms of order u * |q * pio2_2| are rounded.
275 Packet qp2_hi, qp2_lo;
276 twoprod<Packet, true>(cst_pio2_2<Packet>(), q, qp2_hi, qp2_lo);
277 Packet a_hi, a_lo, b_lo;
278 twosum(t_hi, qp_lo, a_hi, a_lo);
279 twosum(a_hi, qp2_hi, t_hi, b_lo);
280 t_lo = padd(t_lo, padd(a_lo, b_lo));
281
282 // Accumulate the low part of the product and the remaining pi/2 term.
283 t_lo = padd(t_lo, qp2_lo);
284 t_lo = pmadd(cst_pio2_3<Packet>(), q, t_lo);
285
286 return padd(t_hi, t_lo);
287#endif
288}
289
290template <TrigFunction Func, typename Packet>
291EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psincos_double(const Packet& x) {
292 using PacketI = typename unpacket_traits<Packet>::integer_packet;
293 using ScalarI = typename unpacket_traits<PacketI>::type;
294
295 const Packet cst_sign_mask = psignmask<Packet>();
296
297 // If the argument is smaller than this value, use a simpler argument reduction
298 const double small_th = 15;
299 // If the argument is bigger than this value, use the non-vectorized std version
300 const double huge_th = 1e14;
301
302 // 2/PI as a double-word: hi + lo = 2/pi to ~107 bits. Computed by Sollya.
303 const Packet cst_2oPI_hi =
304 pset1<Packet>(0.63661977236758138243288840385503135621547698974609375); // 0x1.45f306dc9c883p-1
305 const Packet cst_2oPI_lo =
306 pset1<Packet>(-3.9357353350364971763790381828183628368294820823718866e-17); // -0x1.6b01ec5417056p-55
307 // Integer Packet constants
308 const PacketI cst_one = pset1<PacketI>(ScalarI(1));
309
310 Packet x_abs = pabs(x);
311
312 // Scale x by 2/Pi
313 PacketI q_int;
314 Packet s;
315
316 if (EIGEN_PREDICT_FALSE(predux_any(pcmp_le(pset1<Packet>(small_th), x_abs)))) {
317 // Medium path: use double-word product x * (2/pi) for precise quadrant computation.
318 Packet prod_hi, prod_lo;
319 twoprod<Packet, true>(x_abs, cst_2oPI_hi, prod_hi, prod_lo);
320 // Correction for 2/pi truncation: add x * lo(2/pi)
321 prod_lo = pmadd(x_abs, cst_2oPI_lo, prod_lo);
322
323 // Round the double-word (prod_hi, prod_lo) to the nearest integer.
324 Packet q = pround(prod_hi);
325 // Compute exact fractional part to check if rounding was correct.
326 Packet frac = padd(psub(prod_hi, q), prod_lo);
327 // Correct if fractional part crossed +-0.5 boundary.
328 q = padd(q, pand(pcmp_lt(pset1<Packet>(0.5), frac), pset1<Packet>(1.0)));
329 q = padd(q, pand(pcmp_lt(frac, pset1<Packet>(-0.5)), pset1<Packet>(-1.0)));
330
331 q_int = pcast<Packet, PacketI>(q);
332 s = trig_reduce_small_double(x_abs, q);
333 } else {
334 // Small path: simple reduction with triple-double pi/2 split.
335 Packet qval_noround = pmul(x_abs, cst_2oPI_hi);
336#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
337 q_int = pcast<Packet, PacketI>(padd(qval_noround, pset1<Packet>(0.5)));
338 Packet q = pcast<PacketI, Packet>(q_int);
339 s = trig_reduce_small_double(x_abs, q);
340#else
341 // Only q_int & 3 is consumed below. In round-to-nearest mode, the bias leaves q in its low bits.
342 const Packet bias = pset1<Packet>(6755399441055744.0); // 1.5 * 2^52.
343 Packet biased_q = padd(qval_noround, bias);
344 pbarrier(biased_q);
345 q_int = preinterpret<PacketI>(biased_q);
346 const Packet q = psub(biased_q, bias);
347 // |x| < 15 implies q <= 10. The first two 49-bit parts give exact q*c products;
348 // the third leaves < 2^-155 error in pi/2. Generated with MPFR at 256 bits:
349 // c1 = trunc49(pi/2), c2 = trunc49(pi/2-c1), c3 = RN53(pi/2-c1-c2).
350 s = pmadd(q, pset1<Packet>(-1.5707963267948948), x_abs);
351 pbarrier(s);
352 s = pmadd(q, pset1<Packet>(-1.837589179357618e-15), s);
353 pbarrier(s);
354 s = pmadd(q, pset1<Packet>(-8.4784276603688996e-32), s);
355#endif
356 }
357
358 Packet ss = pmul(s, s);
359
360 // Minimax polynomial approximation of cos(x) on [-pi/4, pi/4].
361 // cos(x) = 1 + u * P(u), where u = x^2 and P is degree 6 (7 FMAs total).
362 // Coefficients computed by Sollya fpminimax. Max polynomial error ~1.3e-19.
363 Packet scos = pset1<Packet>(-1.1368926065317776472832699312119132152576472805094454088248312473297119140625e-11);
364 scos = pmadd(scos, ss, pset1<Packet>(2.0875905481768720039634091158002593413556269297259859740734100341796875e-09));
365 scos = pmadd(scos, ss, pset1<Packet>(-2.7557315712466412785356544880299711763882442028261721134185791015625e-07));
366 scos = pmadd(scos, ss, pset1<Packet>(2.480158729424286522739599714082459058772656135261058807373046875e-05));
367 scos = pmadd(scos, ss, pset1<Packet>(-1.388888888888178789471350427220386336557567119598388671875e-03));
368 scos = pmadd(scos, ss, pset1<Packet>(4.166666666666664353702032030923874117434024810791015625e-02));
369 scos = pmadd(scos, ss, pset1<Packet>(-0.5));
370 scos = pmadd(scos, ss, pset1<Packet>(1.0));
371
372 // Minimax polynomial approximation of sin(x) on [-pi/4, pi/4].
373 // sin(x) = x * (1 + u * R(u)), where u = x^2 and R is degree 5.
374 // Computed as: x + x * u * R(u) (6 FMAs + 1 mul).
375 // Coefficients computed by Sollya fpminimax. Max polynomial error ~1.0e-17.
376 Packet ssin = pset1<Packet>(1.59193066075142890698150587293845624470289834562208852730691432952880859375e-10);
377 ssin = pmadd(ssin, ss, pset1<Packet>(-2.50511517945670206974594627392927126408039839589037001132965087890625e-08));
378 ssin = pmadd(ssin, ss, pset1<Packet>(2.755731622544328228235042954619160582296899519860744476318359375e-06));
379 ssin = pmadd(ssin, ss, pset1<Packet>(-1.9841269837089632013978068858506276228581555187702178955078125e-04));
380 ssin = pmadd(ssin, ss, pset1<Packet>(8.333333333331312264835588621281203813850879669189453125e-03));
381 ssin = pmadd(ssin, ss, pset1<Packet>(-0.1666666666666666574148081281236954964697360992431640625));
382 ssin = pmul(ssin, ss);
383 ssin = pmadd(ssin, s, s);
384
385 Packet poly_mask = preinterpret<Packet>(pcmp_eq(pand(q_int, cst_one), pzero(q_int)));
386
387 Packet sign_sin = pxor(x, preinterpret<Packet>(plogical_shift_left<62>(q_int)));
388 Packet sign_cos = preinterpret<Packet>(plogical_shift_left<62>(padd(q_int, cst_one)));
389 Packet sign_bit, sFinalRes;
390 if (Func == TrigFunction::Sin) {
391 sign_bit = sign_sin;
392 sFinalRes = pselect(poly_mask, ssin, scos);
393 } else if (Func == TrigFunction::Cos) {
394 sign_bit = sign_cos;
395 sFinalRes = pselect(poly_mask, scos, ssin);
396 } else if (Func == TrigFunction::Tan) {
397 sign_bit = pxor(sign_sin, sign_cos);
398 sFinalRes = pdiv(pselect(poly_mask, ssin, scos), pselect(poly_mask, scos, ssin));
399 } else if (Func == TrigFunction::SinCos) {
400 Packet peven = peven_mask(x);
401 sign_bit = pselect(peven, sign_sin, sign_cos);
402 sFinalRes = pselect(pxor(peven, poly_mask), scos, ssin);
403 }
404 sign_bit = pand(sign_bit, cst_sign_mask); // clear all but left most bit
405 sFinalRes = pxor(sFinalRes, sign_bit);
406
407 // For inputs above huge_th the medium-path reduction loses too much precision. A vectorized
408 // Payne-Hanek reduction was investigated and judged not worthwhile (high implementation cost
409 // for what is in practice a rare path), so these inputs fall back to the scalar libm.
410 if (EIGEN_PREDICT_FALSE(predux_any(pcmp_le(pset1<Packet>(huge_th), x_abs)))) {
411 const int PacketSize = unpacket_traits<Packet>::size;
412 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) double sincos_vals[PacketSize];
413 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) double x_cpy[PacketSize];
414 pstoreu(x_cpy, x);
415 pstoreu(sincos_vals, sFinalRes);
416 for (int k = 0; k < PacketSize; ++k) {
417 double val = x_cpy[k];
418 if (numext::abs(val) > huge_th && (numext::isfinite)(val)) {
419 if (Func == TrigFunction::Sin) {
420 sincos_vals[k] = numext::sin(val);
421 } else if (Func == TrigFunction::Cos) {
422 sincos_vals[k] = numext::cos(val);
423 } else if (Func == TrigFunction::Tan) {
424 sincos_vals[k] = numext::tan(val);
425 } else if (Func == TrigFunction::SinCos) {
426 sincos_vals[k] = k % 2 == 0 ? numext::sin(val) : numext::cos(val);
427 }
428 }
429 }
430 sFinalRes = ploadu<Packet>(sincos_vals);
431 }
432 return sFinalRes;
433}
434
435template <typename Packet>
436EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psin_double(const Packet& x) {
437 return psincos_double<TrigFunction::Sin>(x);
438}
439
440template <typename Packet>
441EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcos_double(const Packet& x) {
442 return psincos_double<TrigFunction::Cos>(x);
443}
444
445template <typename Packet>
446EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptan_double(const Packet& x) {
447 return psincos_double<TrigFunction::Tan>(x);
448}
449
450template <typename Packet>
451EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
452 std::enable_if_t<std::is_same<typename unpacket_traits<Packet>::type, float>::value, Packet>
453 psincos_selector(const Packet& x) {
454 return psincos_float<TrigFunction::SinCos, Packet>(x);
455}
456
457template <typename Packet>
458EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
459 std::enable_if_t<std::is_same<typename unpacket_traits<Packet>::type, double>::value, Packet>
460 psincos_selector(const Packet& x) {
461 return psincos_double<TrigFunction::SinCos, Packet>(x);
462}
463
464//----------------------------------------------------------------------
465// Inverse Trigonometric Functions
466//----------------------------------------------------------------------
467
468// Generic implementation of acos(x).
469template <typename Packet>
470EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacos_float(const Packet& x_in) {
471 using Scalar = typename unpacket_traits<Packet>::type;
472 static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
473
474 const Packet cst_one = pset1<Packet>(Scalar(1));
475 const Packet cst_pi = pset1<Packet>(Scalar(EIGEN_PI));
476 const Packet p6 = pset1<Packet>(Scalar(2.36423197202384471893310546875e-3));
477 const Packet p5 = pset1<Packet>(Scalar(-1.1368644423782825469970703125e-2));
478 const Packet p4 = pset1<Packet>(Scalar(2.717843465507030487060546875e-2));
479 const Packet p3 = pset1<Packet>(Scalar(-4.8969544470310211181640625e-2));
480 const Packet p2 = pset1<Packet>(Scalar(8.8804088532924652099609375e-2));
481 const Packet p1 = pset1<Packet>(Scalar(-0.214591205120086669921875));
482 const Packet p0 = pset1<Packet>(Scalar(1.57079637050628662109375));
483
484 // For x in [0:1], we approximate acos(x)/sqrt(1-x), which is a smooth
485 // function, by a 6'th order polynomial.
486 // For x in [-1:0) we use that acos(-x) = pi - acos(x).
487 const Packet neg_mask = psignbit(x_in);
488 const Packet abs_x = pabs(x_in);
489
490 // Evaluate the polynomial using Horner's rule:
491 // P(x) = p0 + x * (p1 + x * (p2 + ... (p5 + x * p6)) ... ) .
492 // We evaluate even and odd terms independently to increase
493 // instruction level parallelism.
494 Packet x2 = pmul(x_in, x_in);
495 Packet p_even = pmadd(p6, x2, p4);
496 Packet p_odd = pmadd(p5, x2, p3);
497 p_even = pmadd(p_even, x2, p2);
498 p_odd = pmadd(p_odd, x2, p1);
499 p_even = pmadd(p_even, x2, p0);
500 Packet p = pmadd(p_odd, abs_x, p_even);
501
502 // The polynomial approximates acos(x)/sqrt(1-x), so
503 // multiply by sqrt(1-x) to get acos(x).
504 // Conveniently returns NaN for arguments outside [-1:1].
505 Packet denom = psqrt(psub(cst_one, abs_x));
506 Packet result = pmul(denom, p);
507 // Undo mapping for negative arguments.
508 return pselect(neg_mask, psub(cst_pi, result), result);
509}
510
511// Generic implementation of asin(x).
512template <typename Packet>
513EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasin_float(const Packet& x_in) {
514 using Scalar = typename unpacket_traits<Packet>::type;
515 static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
516
517 constexpr float kPiOverTwo = static_cast<float>(EIGEN_PI / 2);
518
519 const Packet cst_half = pset1<Packet>(0.5f);
520 const Packet cst_two = pset1<Packet>(2.0f);
521 const Packet cst_pi_over_two = pset1<Packet>(kPiOverTwo);
522 // asin(z) = z + z u P(u) for u = z^2 <= 1/4, to a relative error of 2^-27.6, P = p0 + p1 u + ... + p4 u^4.
523 // Sollya 8.0: fpminimax(asin(x)/x, [|2,4,6,8,10|], [|24...|], [2^-20; 1/2], relative, floating, 1)
524 const Packet p4 = pset1<Packet>(4.21570129692554473876953125e-2f);
525 const Packet p3 = pset1<Packet>(2.418550290167331695556640625e-2f);
526 const Packet p2 = pset1<Packet>(4.54690195620059967041015625e-2f);
527 const Packet p1 = pset1<Packet>(7.4953101575374603271484375e-2f);
528 const Packet p0 = pset1<Packet>(0.166667520999908447265625f);
529
530 const Packet abs_x = pabs(x_in);
531 const Packet sign_mask = pandnot(x_in, abs_x);
532 const Packet large_mask = pcmp_lt(cst_half, abs_x);
533
534 // |x| <= 1/2: asin(|x|) = asin(z), z = |x|
535 // |x| > 1/2: asin(|x|) = pi/2 - 2 asin(z), z = sqrt((1 - |x|) / 2), which is NaN for |x| > 1
536 const Packet z = pselect(large_mask, psqrt(pnmadd(cst_half, abs_x, cst_half)), abs_x);
537 const Packet u = pmul(z, z);
538 // P(u) = (p0 + p1 u) + u^2 ((p2 + p3 u) + p4 u^2): two short chains in place of one Horner chain, to increase
539 // instruction level parallelism.
540 const Packet u2 = pmul(u, u);
541 const Packet p_low = pmadd(p1, u, p0);
542 const Packet p_high = pmadd(p4, u2, pmadd(p3, u, p2));
543 Packet p = pmadd(pmul(z, u), pmadd(p_high, u2, p_low), z);
544
545 const Packet p_large = pnmadd(cst_two, p, cst_pi_over_two);
546 p = pselect(large_mask, p_large, p);
547 // Flip the sign for negative arguments.
548 return pxor(p, sign_mask);
549}
550
551// Computes atan(x) for x in [0:1]. The double version also subtracts the low word of pi/2 from the lanes set in
552// `large_mask`, where the caller goes on to subtract the result from pi/2.
553template <typename Scalar>
554struct patan_reduced {
555 template <typename Packet>
556 static EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet run(const Packet& x, const Packet& large_mask);
557};
558
559// atan(x) = x + x u N(u) / Q(u), u = x^2. P / Q is a rational approximation of atan(x) / x calculated using Rminimax,
560// and N = (P - Q) / u takes the leading 1 out of it: the rounding errors of the rational function then enter the
561// result scaled by |atan(x) / x - 1| < 0.22.
562template <>
563template <typename Packet>
564EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patan_reduced<double>::run(const Packet& x,
565 const Packet& large_mask) {
566 constexpr double kPiOverTwoLo = 6.123233995736766e-17; // pi/2 - double(pi/2)
567 constexpr double alpha[] = {-2.4644487075790443e-04, -7.8073996177164305e-03, -6.2915029683194929e-02,
568 -1.9307140255021149e-01, -2.4634297465012978e-01, -1.1001453763093239e-01};
569
570 constexpr double beta[] = {
571 2.7311202462436667e-04, 1.0899150928962708e-02, 1.1548932646420353e-01, 4.9716458728465573e-01, 1.0,
572 9.3705509168587852e-01, 3.3004361289279920e-01};
573
574 Packet x2 = pmul(x, x);
575 Packet p = ppolevl<Packet, 5>::run(x2, alpha);
576 Packet q = ppolevl<Packet, 6>::run(x2, beta);
577 // The low word of pi/2 goes into the leading term.
578 const Packet x0 = psub(x, pand(large_mask, pset1<Packet>(kPiOverTwoLo)));
579 return pmadd(pmul(x, x2), pdiv(p, q), x0);
580}
581
582// Computes elementwise atan(x) for x in [-1:1] with 2 ulp accuracy.
583template <>
584template <typename Packet>
585EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patan_reduced<float>::run(const Packet& x, const Packet&) {
586 constexpr float alpha[] = {1.12026982009410858154296875e-01f, 7.296695709228515625e-01f, 8.109951019287109375e-01f};
587
588 constexpr float beta[] = {1.00917108356952667236328125e-02f, 2.8318560123443603515625e-01f, 1.0f,
589 8.109951019287109375e-01f};
590
591 Packet x2 = pmul(x, x);
592 Packet p = ppolevl<Packet, 2>::run(x2, alpha);
593 Packet q = ppolevl<Packet, 3>::run(x2, beta);
594 return pmul(x, pdiv(p, q));
595}
596
597template <typename Packet>
598EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_atan(const Packet& x_in) {
599 using Scalar = typename unpacket_traits<Packet>::type;
600
601 constexpr Scalar kPiOverTwo = static_cast<Scalar>(EIGEN_PI / 2);
602
603 const Packet cst_signmask = psignmask<Packet>();
604 const Packet cst_one = pset1<Packet>(Scalar(1));
605 const Packet cst_pi_over_two = pset1<Packet>(kPiOverTwo);
606
607 // "Large": For |x| > 1, use atan(1/x) = sign(x)*pi/2 - atan(x).
608 // "Small": For |x| <= 1, approximate atan(x) directly by a polynomial
609 // calculated using Rminimax.
610
611 const Packet abs_x = pabs(x_in);
612 const Packet x_signmask = pand(x_in, cst_signmask);
613 const Packet large_mask = pcmp_lt(cst_one, abs_x);
614 const Packet x = pselect(large_mask, preciprocal(abs_x), abs_x);
615 const Packet p = patan_reduced<Scalar>::run(x, large_mask);
616 // Apply transformations according to the range reduction masks.
617 Packet result = pselect(large_mask, psub(cst_pi_over_two, p), p);
618 // Return correct sign
619 return pxor(result, x_signmask);
620}
621
622//----------------------------------------------------------------------
623// Hyperbolic Functions
624//----------------------------------------------------------------------
625
626#if EIGEN_FAST_MATH
627
637template <typename T>
638EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T ptanh_float(const T& a_x) {
639 // Clamp the inputs to the range [-c, c] and set everything
640 // outside that range to 1.0. The value c is chosen as the smallest
641 // floating point argument such that the approximation is exactly 1.
642 // This saves clamping the value at the end.
643#ifdef EIGEN_VECTORIZE_FMA
644 constexpr float kClamp = 8.01773357391357422f;
645#else
646 constexpr bool kUsesFma = is_scalar<T>::value && scalar_madd_uses_fma<T>::value;
647 constexpr float kClamp = kUsesFma ? 8.01773357391357422f : 7.90738964080810547f;
648#endif
649 const T plus_clamp = pset1<T>(kClamp);
650 const T minus_clamp = pset1<T>(-kClamp);
651 const T x = pmax(pmin(a_x, plus_clamp), minus_clamp);
652
653 // The following rational approximation was generated by rminimax
654 // (https://gitlab.inria.fr/sfilip/rminimax) using the following
655 // command:
656 // $ ratapprox --function="tanh(x)" --dom='[-8.67,8.67]' --num="odd"
657 // --den="even" --type="[9,8]" --numF="[SG]" --denF="[SG]" --log
658 // --output=tanhf.sollya --dispCoeff="dec"
659
660 // The monomial coefficients of the numerator polynomial (odd).
661 constexpr float alpha[] = {1.394553628e-8f, 2.102733560e-5f, 3.520756727e-3f, 1.340216100e-1f};
662
663 // The monomial coefficients of the denominator polynomial (even).
664 constexpr float beta[] = {8.015776984e-7f, 3.326951409e-4f, 2.597254514e-2f, 4.673548340e-1f, 1.0f};
665
666 // Since the polynomials are odd/even, we need x^2.
667 const T x2 = pmul(x, x);
668 const T x3 = pmul(x2, x);
669
670 T p = ppolevl<T, 3>::run(x2, alpha);
671 T q = ppolevl<T, 4>::run(x2, beta);
672 // Take advantage of the fact that the constant term in p is 1 to compute
673 // x*(x^2*p + 1) = x^3 * p + x.
674 p = pmadd(x3, p, x);
675
676 // Divide the numerator by the denominator.
677 return pdiv(p, q);
678}
679
680#else
681
691template <typename T>
692EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T ptanh_float(const T& x) {
693 // The polynomial coefficients were computed using Rminimax:
694 // % ./ratapprox --function="tanh(x)-x" --dom='[-1.25,1.25]' --num="[x^3,x^5]" --den="even"
695 // --type="[3,4]" --numF="[SG]" --denF="[SG]" --log --dispCoeff="dec" --output=tanhf.solly
696 constexpr float alpha[] = {-1.46725140511989593505859375e-02f, -3.333333432674407958984375e-01f};
697 constexpr float beta[] = {1.570280082523822784423828125e-02, 4.4401752948760986328125e-01, 1.0f};
698 const T sign_mask = psignmask<T>();
699 const T x_sign = pand(sign_mask, x);
700 const T abs_x = pandnot(x, sign_mask);
701 constexpr float kSmallThreshold = 1.25f;
702 const T large_mask = pcmp_lt(pset1<T>(kSmallThreshold), abs_x);
703 // The large elements enter the polynomial as 0, so that x^2 cannot overflow nor P/Q become inf/inf.
704 const T x_small = pandnot(x, large_mask);
705 const T x2 = pmul(x_small, x_small);
706 const T x3 = pmul(x2, x_small);
707 const T p = ppolevl<T, 1>::run(x2, alpha);
708 const T q = ppolevl<T, 2>::run(x2, beta);
709 // x^3 P/Q has the sign opposite to x, so the sum rounds tanh(-0) to +0; the sign of x restores it.
710 const T small_tanh = por(x_sign, pmadd(x3, pdiv(p, q), x_small));
711 // Fast exit if all elements are small.
712 if (!predux_any(large_mask)) {
713 return small_tanh;
714 }
715
716 // Compute as 1 - (2 / (1 + exp(2*x))) for |x| clamped to 16, where it rounds to 1; the clamp keeps exp(2*x) from
717 // overflowing. The small elements, NaN included, enter exp as 0, which cannot raise FE_INVALID.
718 const T one = pset1<T>(1.0f);
719 const T two = pset1<T>(2.0f);
720 constexpr float kHugeThreshold = 16.0f;
721 const T s = pexp_float<T, true>(pmul(two, pmin(pselect(large_mask, abs_x, pzero(x)), pset1<T>(kHugeThreshold))));
722 const T large_tanh = por(x_sign, psub(one, pdiv(two, padd(s, one))));
723 return pselect(large_mask, large_tanh, small_tanh);
724}
725
726#endif // EIGEN_FAST_MATH
727
737template <typename T>
738EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T ptanh_double(const T& a_x) {
739 // Clamp the inputs to the range [-c, c] and set everything
740 // outside that range to 1.0. The value c is chosen as the smallest
741 // floating point argument such that the approximation is exactly 1.
742 // This saves clamping the value at the end.
743#ifdef EIGEN_VECTORIZE_FMA
744 constexpr double kClamp = 17.6610191624600077;
745#else
746 constexpr bool kUsesFma = is_scalar<T>::value && scalar_madd_uses_fma<T>::value;
747 constexpr double kClamp = kUsesFma ? 17.6610191624600077 : 17.714196154005176;
748#endif
749 const T plus_clamp = pset1<T>(kClamp);
750 const T minus_clamp = pset1<T>(-kClamp);
751 const T x = pmax(pmin(a_x, plus_clamp), minus_clamp);
752 // The following rational approximation was generated by rminimax
753 // (https://gitlab.inria.fr/sfilip/rminimax) using the following
754 // command:
755 // $ ./ratapprox --function="tanh(x)" --dom='[-18.72,18.72]'
756 // --num="odd" --den="even" --type="[19,18]" --numF="[D]"
757 // --denF="[D]" --log --output=tanh.sollya --dispCoeff="dec"
758
759 // The monomial coefficients of the numerator polynomial (odd).
760 constexpr double alpha[] = {2.6158007860482230e-23, 7.6534862268749319e-19, 3.1309488231386680e-15,
761 4.2303918148209176e-12, 2.4618379131293676e-09, 6.8644367682497074e-07,
762 9.3839087674268880e-05, 5.9809711724441161e-03, 1.5184719640284322e-01};
763
764 // The monomial coefficients of the denominator polynomial (even).
765 constexpr double beta[] = {6.463747022670968018e-21, 5.782506856739003571e-17,
766 1.293019623712687916e-13, 1.123643448069621992e-10,
767 4.492975677839633985e-08, 8.785185266237658698e-06,
768 8.295161192716231542e-04, 3.437448108450402717e-02,
769 4.851805297361760360e-01, 1.0};
770
771 // Since the polynomials are odd/even, we need x^2.
772 const T x2 = pmul(x, x);
773 const T x3 = pmul(x2, x);
774
775 // Interleave the evaluation of the numerator polynomial p and
776 // denominator polynomial q.
777 T p = ppolevl<T, 8>::run(x2, alpha);
778 T q = ppolevl<T, 9>::run(x2, beta);
779 // Take advantage of the fact that the constant term in p is 1 to compute
780 // x*(x^2*p + 1) = x^3 * p + x.
781 p = pmadd(x3, p, x);
782
783 // Divide the numerator by the denominator.
784 return pdiv(p, q);
785}
786
787template <typename Packet>
788EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patanh_float(const Packet& x) {
789 using Scalar = typename unpacket_traits<Packet>::type;
790 static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
791
792 // For |x| in [0:0.5] we use a polynomial approximation of the form
793 // P(x) = x + x^3*(alpha[4] + x^2 * (alpha[3] + x^2 * (... x^2 * alpha[0]) ... )).
794 constexpr float alpha[] = {0.1819281280040740966796875f, 8.2311116158962249755859375e-2f,
795 0.14672131836414337158203125f, 0.1997792422771453857421875f, 0.3333373963832855224609375f};
796 const Packet x2 = pmul(x, x);
797 const Packet x3 = pmul(x, x2);
798 Packet p = ppolevl<Packet, 4>::run(x2, alpha);
799 p = pmadd(x3, p, x);
800
801 const Packet half = pset1<Packet>(0.5f);
802 const Packet one = pset1<Packet>(1.0f);
803 const Packet x_gt_half = pcmp_le(half, pabs(x));
804 // Fast exit: if all |x| <= 0.5, skip the expensive plog/pdiv branch.
805 if (!predux_any(x_gt_half)) {
806 return p;
807 }
808
809 // For |x| in ]0.5:1.0] we use atanh = 0.5*ln((1+x)/(1-x));
810 Packet r = pdiv(padd(one, x), psub(one, x));
811 r = pmul(half, plog(r));
812
813 const Packet x_eq_one = pcmp_eq(one, pabs(x));
814 const Packet x_gt_one = pcmp_lt(one, pabs(x));
815 const Packet sign_mask = psignmask<Packet>();
816 const Packet x_sign = pand(sign_mask, x);
817 const Packet inf = pinf<Packet>();
818 return por(x_gt_one, pselect(x_eq_one, por(x_sign, inf), pselect(x_gt_half, r, p)));
819}
820
821template <typename Packet>
822EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patanh_double(const Packet& x) {
823 using Scalar = typename unpacket_traits<Packet>::type;
824 static_assert(std::is_same<Scalar, double>::value, "Scalar type must be double");
825 // For x in [-0.5:0.5] we use a rational approximation of the form
826 // R(x) = x + x^3*P(x^2)/Q(x^2), where P is or order 4 and Q is of order 5.
827 constexpr double alpha[] = {3.3071338469301391e-03, -4.7129526768798737e-02, 1.8185306179826699e-01,
828 -2.5949536095445679e-01, 1.2306328729812676e-01};
829
830 constexpr double beta[] = {-3.8679974580640881e-03, 7.6391885763341910e-02, -4.2828141436397615e-01,
831 9.8733495886883648e-01, -1.0000000000000000e+00, 3.6918986189438030e-01};
832
833 const Packet x2 = pmul(x, x);
834 const Packet x3 = pmul(x, x2);
835 Packet p = ppolevl<Packet, 4>::run(x2, alpha);
836 Packet q = ppolevl<Packet, 5>::run(x2, beta);
837 Packet y_small = pmadd(x3, pdiv(p, q), x);
838
839 const Packet half = pset1<Packet>(0.5);
840 const Packet one = pset1<Packet>(1.0);
841 const Packet x_gt_half = pcmp_le(half, pabs(x));
842 // Fast exit: if all |x| <= 0.5, skip the expensive plog/pdiv branch.
843 if (!predux_any(x_gt_half)) {
844 return y_small;
845 }
846
847 // For |x| in ]0.5:1.0] we use atanh = 0.5*ln((1+x)/(1-x));
848 Packet y_large = pdiv(padd(one, x), psub(one, x));
849 y_large = pmul(half, plog(y_large));
850
851 const Packet x_eq_one = pcmp_eq(one, pabs(x));
852 const Packet x_gt_one = pcmp_lt(one, pabs(x));
853 const Packet sign_mask = psignmask<Packet>();
854 const Packet x_sign = pand(sign_mask, x);
855 const Packet inf = pinf<Packet>();
856 return por(x_gt_one, pselect(x_eq_one, por(x_sign, inf), pselect(x_gt_half, y_large, y_small)));
857}
858
859//----------------------------------------------------------------------
860// sinh / cosh
861//----------------------------------------------------------------------
862
870template <typename Packet>
871EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psinh_float(const Packet& x) {
872 using Scalar = typename unpacket_traits<Packet>::type;
873 static_assert(std::is_same<Scalar, float>::value, "Scalar type must be float");
874
875 const Packet sign_mask = psignmask<Packet>();
876 const Packet abs_x = pandnot(x, sign_mask);
877 const Packet x_sign = pand(x, sign_mask);
878
879 // For |x| < 1, use a polynomial approximation to avoid
880 // cancellation in exp(x) - exp(-x).
881 constexpr float alpha[] = {2.7557314045e-06f, 1.9841270114e-04f, 8.3333335817e-03f, 1.6666666716e-01f};
882 const Packet x2 = pmul(x, x);
883 Packet p_small = ppolevl<Packet, 3>::run(x2, alpha);
884 p_small = pmadd(pmul(x2, x), p_small, x);
885
886 // e = exp(|x|) / 2, from one evaluation of the exponential.
887 const Packet one = pset1<Packet>(1.0f);
888 const Packet e = generic_half_exp(abs_x);
889
890 // Medium path (1 <= |x| <= 20):
891 // sinh(x) = (exp(|x|) - exp(-|x|)) / 2
892 // = (2*e - 1/(2*e)) / 2 = e - 1/(4*e)
893 const Packet quarter = pset1<Packet>(0.25f);
894 Packet p_medium = psub(e, pdiv(quarter, e));
895
896 // Large path (|x| > 20): exp(-|x|) is negligible, sinh(x) ~ exp(|x|)/2 = e.
897 const Packet large_threshold = pset1<Packet>(20.0f);
898 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
899 Packet p_large = pselect(large_mask, e, p_medium);
900 p_large = por(x_sign, p_large);
901
902 const Packet small_mask = pcmp_lt(abs_x, one);
903 return pselect(small_mask, p_small, p_large);
904}
905
906template <typename Packet>
907EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psinh_double(const Packet& x) {
908 using Scalar = typename unpacket_traits<Packet>::type;
909 static_assert(std::is_same<Scalar, double>::value, "Scalar type must be double");
910
911 const Packet sign_mask = psignmask<Packet>();
912 const Packet abs_x = pandnot(x, sign_mask);
913 const Packet x_sign = pand(x, sign_mask);
914
915 // Taylor series: sinh(x) = x + x^3/3! + x^5/5! + ... + x^19/19!
916 // Polynomial form: sinh(x) = x + x^3 * P(x^2) where P(t) = sum_{k=0}^{8} t^k/(2k+3)!
917 // ppolevl stores highest-degree coefficient first.
918 constexpr double alpha[] = {
919 8.2206352466243297e-18, // t^8: 1/19!
920 2.8114572543455206e-15, // t^7: 1/17!
921 7.6471637318198164e-13, // t^6: 1/15!
922 1.6059043836821613e-10, // t^5: 1/13!
923 2.5052108385441718e-08, // t^4: 1/11!
924 2.7557319223985893e-06, // t^3: 1/9!
925 1.9841269841269841e-04, // t^2: 1/7!
926 8.3333333333333332e-03, // t^1: 1/5!
927 1.6666666666666666e-01, // t^0: 1/3!
928 };
929 const Packet x2 = pmul(x, x);
930 Packet p_small = ppolevl<Packet, 8>::run(x2, alpha);
931 p_small = pmadd(pmul(x2, x), p_small, x);
932
933 // e = exp(|x|) / 2, from one evaluation of the exponential.
934 const Packet one = pset1<Packet>(1.0);
935 const Packet e = generic_half_exp(abs_x);
936
937 // Medium path (1 <= |x| <= 20):
938 // sinh(x) = (exp(|x|) - exp(-|x|)) / 2 = e - 1/(4*e)
939 const Packet quarter = pset1<Packet>(0.25);
940 Packet p_medium = psub(e, pdiv(quarter, e));
941
942 // Large path (|x| > 20): exp(-|x|) is negligible, sinh(x) ~ exp(|x|)/2 = e.
943 const Packet large_threshold = pset1<Packet>(20.0);
944 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
945 Packet p_large = pselect(large_mask, e, p_medium);
946 p_large = por(x_sign, p_large);
947 const Packet small_mask = pcmp_lt(abs_x, one);
948 return pselect(small_mask, p_small, p_large);
949}
950
957template <typename Packet>
958EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcosh_float(const Packet& x) {
959 const Packet abs_x = pabs(x);
960
961 // e = exp(|x|) / 2, from one evaluation of the exponential.
962 const Packet e = generic_half_exp(abs_x);
963
964 // Medium path: cosh(x) = (exp(|x|) + exp(-|x|)) / 2
965 // = (2*e + 1/(2*e)) / 2 = e + 1/(4*e)
966 const Packet quarter = pset1<Packet>(0.25f);
967 Packet p_medium = padd(e, pdiv(quarter, e));
968
969 // Large path (|x| > 20): exp(-|x|) is negligible, cosh(x) ~ exp(|x|)/2 = e.
970 const Packet large_threshold = pset1<Packet>(20.0f);
971 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
972 return pselect(large_mask, e, p_medium);
973}
974
975template <typename Packet>
976EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcosh_double(const Packet& x) {
977 const Packet abs_x = pabs(x);
978
979 // e = exp(|x|) / 2, from one evaluation of the exponential.
980 const Packet e = generic_half_exp(abs_x);
981
982 // Medium path: cosh(x) = (exp(|x|) + exp(-|x|)) / 2 = e + 1/(4*e)
983 const Packet quarter = pset1<Packet>(0.25);
984 Packet p_medium = padd(e, pdiv(quarter, e));
985
986 // Large path (|x| > 20): exp(-|x|) is negligible, cosh(x) ~ exp(|x|)/2 = e.
987 const Packet large_threshold = pset1<Packet>(20.0);
988 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
989 return pselect(large_mask, e, p_medium);
990}
991
992//----------------------------------------------------------------------
993// asinh / acosh
994//----------------------------------------------------------------------
995
1002template <typename Packet>
1003EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasinh_float(const Packet& x) {
1004 const Packet sign_mask = psignmask<Packet>();
1005 const Packet abs_x = pandnot(x, sign_mask);
1006 const Packet x_sign = pand(x, sign_mask);
1007 const Packet one = pset1<Packet>(1.0f);
1008
1009 // For |x| >= 1e10, use log(2|x|) = log1p(|x| - 1) + ln2 to avoid x^2 overflow.
1010 const Packet large_mask = pcmp_lt(pset1<Packet>(1e10f), abs_x);
1011 // Guard x^2 against overflow in the large case.
1012 const Packet x2 = pmul(abs_x, pselect(large_mask, pzero(abs_x), abs_x));
1013 // For |x| < 1e10: log1p(|x| + x^2 / (1 + sqrt(1 + x^2))).
1014 // Algebraically equivalent to log(|x| + sqrt(x^2 + 1))
1015 // but avoids cancellation for small |x|.
1016 Packet normal_arg = padd(abs_x, pdiv(x2, padd(one, psqrt(padd(one, x2)))));
1017 // For |x| >= 1e10: log1p(|x| - 1), then add ln2 after.
1018 Packet large_arg = psub(abs_x, one);
1019 // Select argument, then call log1p once.
1020 Packet result = plog1p(pselect(large_mask, large_arg, normal_arg));
1021 // Add ln2 for the large path: log(2|x|) = log(|x|) + ln2 = log1p(|x|-1) + ln2.
1022 const Packet ln2 = pset1<Packet>(0.6931471805599453f);
1023 result = pselect(large_mask, padd(result, ln2), result);
1024 // asinh(x) = x - x^3/6 + ...: below 2^-12 the result rounds to x.
1025 result = pselect(pcmp_lt(abs_x, pset1<Packet>(2.44140625e-4f)), abs_x, result);
1026 return por(x_sign, result);
1027}
1028
1029template <typename Packet>
1030EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasinh_double(const Packet& x) {
1031 const Packet sign_mask = psignmask<Packet>();
1032 const Packet abs_x = pandnot(x, sign_mask);
1033 const Packet x_sign = pand(x, sign_mask);
1034 const Packet one = pset1<Packet>(1.0);
1035
1036 const Packet large_mask = pcmp_lt(pset1<Packet>(1e150), abs_x);
1037 const Packet x2 = pmul(abs_x, pselect(large_mask, pzero(abs_x), abs_x));
1038 Packet normal_arg = padd(abs_x, pdiv(x2, padd(one, psqrt(padd(one, x2)))));
1039 Packet large_arg = psub(abs_x, one);
1040 Packet result = plog1p(pselect(large_mask, large_arg, normal_arg));
1041 const Packet ln2 = pset1<Packet>(0.6931471805599453);
1042 result = pselect(large_mask, padd(result, ln2), result);
1043 // asinh(x) = x - x^3/6 + ...: below 2^-26 the result rounds to x.
1044 result = pselect(pcmp_lt(abs_x, pset1<Packet>(1.4901161193847656e-8)), abs_x, result);
1045 return por(x_sign, result);
1046}
1047
1054template <typename Packet>
1055EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacosh_float(const Packet& x) {
1056 const Packet one = pset1<Packet>(1.0f);
1057 const Packet two = pset1<Packet>(2.0f);
1058 const Packet t = psub(x, one);
1059 const Packet huge_mask = pcmp_lt(pset1<Packet>(1e10f), x);
1060 // Guard t*(t+2) against overflow in the huge case.
1061 const Packet t_tp2 = pmul(pselect(huge_mask, pzero(t), t), padd(t, two));
1062 Packet normal_arg = padd(t, psqrt(t_tp2));
1063 // For huge x: acosh(x) = log(2x) = log1p(x - 1) + ln2.
1064 Packet huge_arg = t;
1065 // Select argument, then call log1p once.
1066 Packet result = plog1p(pselect(huge_mask, huge_arg, normal_arg));
1067 const Packet ln2 = pset1<Packet>(0.6931471805599453f);
1068 result = pselect(huge_mask, padd(result, ln2), result);
1069 // Return NaN for x < 1.
1070 const Packet invalid_mask = pcmp_lt(x, one);
1071 return por(invalid_mask, result);
1072}
1073
1074template <typename Packet>
1075EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacosh_double(const Packet& x) {
1076 const Packet one = pset1<Packet>(1.0);
1077 const Packet two = pset1<Packet>(2.0);
1078 const Packet t = psub(x, one);
1079 const Packet huge_mask = pcmp_lt(pset1<Packet>(1e150), x);
1080 const Packet t_tp2 = pmul(pselect(huge_mask, pzero(t), t), padd(t, two));
1081 Packet normal_arg = padd(t, psqrt(t_tp2));
1082 Packet huge_arg = t;
1083 Packet result = plog1p(pselect(huge_mask, huge_arg, normal_arg));
1084 const Packet ln2 = pset1<Packet>(0.6931471805599453);
1085 result = pselect(huge_mask, padd(result, ln2), result);
1086 const Packet invalid_mask = pcmp_lt(x, one);
1087 return por(invalid_mask, result);
1088}
1089
1090} // end namespace internal
1091} // end namespace Eigen
1092
1093#endif // EIGEN_ARCH_GENERIC_PACKET_MATH_TRIG_H