13#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_TRIG_H
14#define EIGEN_ARCH_GENERIC_PACKET_MATH_TRIG_H
17#include "../../InternalHeaderCheck.h"
29enum class TrigFunction : uint8_t { Sin, Cos, Tan, SinCos };
40inline float trig_reduce_huge(
float xf, Eigen::numext::int32_t* quadrant) {
41 using Eigen::numext::int32_t;
42 using Eigen::numext::int64_t;
43 using Eigen::numext::uint32_t;
44 using Eigen::numext::uint64_t;
46 const double pio2_62 = 3.4061215800865545e-19;
47 const uint64_t zero_dot_five = uint64_t(1) << 61;
51 static const uint32_t two_over_pi[] = {
52 0x00000028, 0x000028be, 0x0028be60, 0x28be60db, 0xbe60db93, 0x60db9391, 0xdb939105, 0x9391054a, 0x91054a7f,
53 0x054a7f09, 0x4a7f09d5, 0x7f09d5f4, 0x09d5f47d, 0xd5f47d4d, 0xf47d4d37, 0x7d4d3770, 0x4d377036, 0x377036d8,
54 0x7036d8a5, 0x36d8a566, 0xd8a5664f, 0xa5664f10, 0x664f10e4, 0x4f10e410, 0x10e41000, 0xe4100000};
56 uint32_t xi = numext::bit_cast<uint32_t>(xf);
61 uint32_t e = (xi >> 23) - 118;
63 xi = ((xi & 0x007fffffu) | 0x00800000u) << (e & 0x7);
66 uint32_t twoopi_1 = two_over_pi[i - 1];
67 uint32_t twoopi_2 = two_over_pi[i + 3];
68 uint32_t twoopi_3 = two_over_pi[i + 7];
72 p = uint64_t(xi) * twoopi_3;
73 p = uint64_t(xi) * twoopi_2 + (p >> 32);
74 p = (uint64_t(xi * twoopi_1) << 32) + p;
77 uint64_t q = (p + zero_dot_five) >> 62;
84 return float(
double(int64_t(p)) * pio2_62);
87template <TrigFunction Func,
typename Packet>
88EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psincos_float(
const Packet& _x) {
89 using PacketI =
typename unpacket_traits<Packet>::integer_packet;
91 const Packet cst_2oPI = pset1<Packet>(0.636619746685028076171875f);
92 const Packet cst_rounding_magic = pset1<Packet>(12582912);
93 const PacketI csti_1 = pset1<PacketI>(1);
94 const Packet cst_sign_mask = psignmask<Packet>();
99 Packet y = pmul(x, cst_2oPI);
102 Packet y_round = padd(y, cst_rounding_magic);
104 PacketI y_int = preinterpret<PacketI>(y_round);
105 y = psub(y_round, cst_rounding_magic);
109#if defined(EIGEN_VECTORIZE_FMA)
112 constexpr float huge_th = (Func == TrigFunction::Sin) ? 117435.992f : 71476.0625f;
113 x = pmadd(y, pset1<Packet>(-1.57079601287841796875f), x);
114 x = pmadd(y, pset1<Packet>(-3.1391647326017846353352069854736328125e-07f), x);
115 x = pmadd(y, pset1<Packet>(-5.390302529957764765544681040410068817436695098876953125e-15f), x);
123 constexpr float huge_th = (Func == TrigFunction::Sin) ? 25966.f : 18838.f;
124 x = pmadd(y, pset1<Packet>(-1.5703125), x);
126 x = pmadd(y, pset1<Packet>(-0.000483989715576171875), x);
128 x = pmadd(y, pset1<Packet>(1.62865035235881805419921875e-07), x);
130 x = pmadd(y, pset1<Packet>(5.5644315544167710640977020375430583953857421875e-11), x);
146 if (predux_any(pcmp_le(pset1<Packet>(huge_th), pabs(_x)))) {
147 const int PacketSize = unpacket_traits<Packet>::size;
148 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment)
float vals[PacketSize];
149 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment)
float x_cpy[PacketSize];
150 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment) Eigen::numext::int32_t y_int2[PacketSize];
151 pstoreu(vals, pabs(_x));
153 pstoreu(y_int2, y_int);
154 for (
int k = 0; k < PacketSize; ++k) {
156 if (val >= huge_th && (numext::isfinite)(val)) x_cpy[k] = trig_reduce_huge(val, &y_int2[k]);
158 x = ploadu<Packet>(x_cpy);
159 y_int = ploadu<PacketI>(y_int2);
164 Packet poly_mask = preinterpret<Packet>(pcmp_eq(pand(y_int, csti_1), pzero(y_int)));
166 Packet x2 = pmul(x, x);
169 Packet y1 = pset1<Packet>(2.4372266125283204019069671630859375e-05f);
170 y1 = pmadd(y1, x2, pset1<Packet>(-0.00138865201734006404876708984375f));
171 y1 = pmadd(y1, x2, pset1<Packet>(0.041666619479656219482421875f));
172 y1 = pmadd(y1, x2, pset1<Packet>(-0.5f));
173 y1 = pmadd(y1, x2, pset1<Packet>(1.f));
183 Packet y2 = pset1<Packet>(-0.0001959234114083702898469196984621021329076029360294342041015625f);
184 y2 = pmadd(y2, x2, pset1<Packet>(0.0083326873655616851693794799871284340042620897293090820312500000f));
185 y2 = pmadd(y2, x2, pset1<Packet>(-0.1666666203982298255503735617821803316473960876464843750000000000f));
187 y2 = pmadd(y2, x, x);
193 Packet sign_bit = (Func == TrigFunction::Sin) ? pxor(_x, preinterpret<Packet>(plogical_shift_left<30>(y_int)))
194 : preinterpret<Packet>(plogical_shift_left<30>(padd(y_int, csti_1)));
195 sign_bit = pand(sign_bit, cst_sign_mask);
197 if ((Func == TrigFunction::SinCos) || (Func == TrigFunction::Tan)) {
198 Packet peven = peven_mask(x);
199 Packet ysin = pselect(poly_mask, y2, y1);
200 Packet ycos = pselect(poly_mask, y1, y2);
201 Packet sign_bit_sin = pxor(_x, preinterpret<Packet>(plogical_shift_left<30>(y_int)));
202 Packet sign_bit_cos = preinterpret<Packet>(plogical_shift_left<30>(padd(y_int, csti_1)));
203 sign_bit_sin = pand(sign_bit_sin, cst_sign_mask);
204 sign_bit_cos = pand(sign_bit_cos, cst_sign_mask);
205 y = (Func == TrigFunction::SinCos) ? pselect(peven, pxor(ysin, sign_bit_sin), pxor(ycos, sign_bit_cos))
206 : pdiv(pxor(ysin, sign_bit_sin), pxor(ycos, sign_bit_cos));
208 y = (Func == TrigFunction::Sin) ? pselect(poly_mask, y2, y1) : pselect(poly_mask, y1, y2);
209 y = pxor(y, sign_bit);
214template <
typename Packet>
215EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psin_float(
const Packet& x) {
216 return psincos_float<TrigFunction::Sin>(x);
219template <
typename Packet>
220EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcos_float(
const Packet& x) {
221 return psincos_float<TrigFunction::Cos>(x);
224template <
typename Packet>
225EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptan_float(
const Packet& x) {
226 return psincos_float<TrigFunction::Tan>(x);
232template <
typename Packet>
234 return pset1<Packet>(-1.5707963267948965579989817342720925807952880859375);
236template <
typename Packet>
238 return pset1<Packet>(-6.12323399573676603586882014729198302312846062338790e-17);
240template <
typename Packet>
242 return pset1<Packet>(1.4973849048591698329435081771059920083527504761695190e-33);
248template <
typename Packet>
249Packet trig_reduce_small_double(
const Packet& x,
const Packet& q) {
250#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
254 t = pmadd(cst_pio2_1<Packet>(), q, x);
255 t = pmadd(cst_pio2_2<Packet>(), q, t);
256 t = pmadd(cst_pio2_3<Packet>(), q, t);
266 twoprod<Packet, true>(cst_pio2_1<Packet>(), q, qp_hi, qp_lo);
271 twosum(x, qp_hi, t_hi, t_lo);
275 Packet qp2_hi, qp2_lo;
276 twoprod<Packet, true>(cst_pio2_2<Packet>(), q, qp2_hi, qp2_lo);
277 Packet a_hi, a_lo, b_lo;
278 twosum(t_hi, qp_lo, a_hi, a_lo);
279 twosum(a_hi, qp2_hi, t_hi, b_lo);
280 t_lo = padd(t_lo, padd(a_lo, b_lo));
283 t_lo = padd(t_lo, qp2_lo);
284 t_lo = pmadd(cst_pio2_3<Packet>(), q, t_lo);
286 return padd(t_hi, t_lo);
290template <TrigFunction Func,
typename Packet>
291EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psincos_double(
const Packet& x) {
292 using PacketI =
typename unpacket_traits<Packet>::integer_packet;
293 using ScalarI =
typename unpacket_traits<PacketI>::type;
295 const Packet cst_sign_mask = psignmask<Packet>();
298 const double small_th = 15;
300 const double huge_th = 1e14;
303 const Packet cst_2oPI_hi =
304 pset1<Packet>(0.63661977236758138243288840385503135621547698974609375);
305 const Packet cst_2oPI_lo =
306 pset1<Packet>(-3.9357353350364971763790381828183628368294820823718866e-17);
308 const PacketI cst_one = pset1<PacketI>(ScalarI(1));
310 Packet x_abs = pabs(x);
316 if (EIGEN_PREDICT_FALSE(predux_any(pcmp_le(pset1<Packet>(small_th), x_abs)))) {
318 Packet prod_hi, prod_lo;
319 twoprod<Packet, true>(x_abs, cst_2oPI_hi, prod_hi, prod_lo);
321 prod_lo = pmadd(x_abs, cst_2oPI_lo, prod_lo);
324 Packet q = pround(prod_hi);
326 Packet frac = padd(psub(prod_hi, q), prod_lo);
328 q = padd(q, pand(pcmp_lt(pset1<Packet>(0.5), frac), pset1<Packet>(1.0)));
329 q = padd(q, pand(pcmp_lt(frac, pset1<Packet>(-0.5)), pset1<Packet>(-1.0)));
331 q_int = pcast<Packet, PacketI>(q);
332 s = trig_reduce_small_double(x_abs, q);
335 Packet qval_noround = pmul(x_abs, cst_2oPI_hi);
336#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
337 q_int = pcast<Packet, PacketI>(padd(qval_noround, pset1<Packet>(0.5)));
338 Packet q = pcast<PacketI, Packet>(q_int);
339 s = trig_reduce_small_double(x_abs, q);
342 const Packet bias = pset1<Packet>(6755399441055744.0);
343 Packet biased_q = padd(qval_noround, bias);
345 q_int = preinterpret<PacketI>(biased_q);
346 const Packet q = psub(biased_q, bias);
350 s = pmadd(q, pset1<Packet>(-1.5707963267948948), x_abs);
352 s = pmadd(q, pset1<Packet>(-1.837589179357618e-15), s);
354 s = pmadd(q, pset1<Packet>(-8.4784276603688996e-32), s);
358 Packet ss = pmul(s, s);
363 Packet scos = pset1<Packet>(-1.1368926065317776472832699312119132152576472805094454088248312473297119140625e-11);
364 scos = pmadd(scos, ss, pset1<Packet>(2.0875905481768720039634091158002593413556269297259859740734100341796875e-09));
365 scos = pmadd(scos, ss, pset1<Packet>(-2.7557315712466412785356544880299711763882442028261721134185791015625e-07));
366 scos = pmadd(scos, ss, pset1<Packet>(2.480158729424286522739599714082459058772656135261058807373046875e-05));
367 scos = pmadd(scos, ss, pset1<Packet>(-1.388888888888178789471350427220386336557567119598388671875e-03));
368 scos = pmadd(scos, ss, pset1<Packet>(4.166666666666664353702032030923874117434024810791015625e-02));
369 scos = pmadd(scos, ss, pset1<Packet>(-0.5));
370 scos = pmadd(scos, ss, pset1<Packet>(1.0));
376 Packet ssin = pset1<Packet>(1.59193066075142890698150587293845624470289834562208852730691432952880859375e-10);
377 ssin = pmadd(ssin, ss, pset1<Packet>(-2.50511517945670206974594627392927126408039839589037001132965087890625e-08));
378 ssin = pmadd(ssin, ss, pset1<Packet>(2.755731622544328228235042954619160582296899519860744476318359375e-06));
379 ssin = pmadd(ssin, ss, pset1<Packet>(-1.9841269837089632013978068858506276228581555187702178955078125e-04));
380 ssin = pmadd(ssin, ss, pset1<Packet>(8.333333333331312264835588621281203813850879669189453125e-03));
381 ssin = pmadd(ssin, ss, pset1<Packet>(-0.1666666666666666574148081281236954964697360992431640625));
382 ssin = pmul(ssin, ss);
383 ssin = pmadd(ssin, s, s);
385 Packet poly_mask = preinterpret<Packet>(pcmp_eq(pand(q_int, cst_one), pzero(q_int)));
387 Packet sign_sin = pxor(x, preinterpret<Packet>(plogical_shift_left<62>(q_int)));
388 Packet sign_cos = preinterpret<Packet>(plogical_shift_left<62>(padd(q_int, cst_one)));
389 Packet sign_bit, sFinalRes;
390 if (Func == TrigFunction::Sin) {
392 sFinalRes = pselect(poly_mask, ssin, scos);
393 }
else if (Func == TrigFunction::Cos) {
395 sFinalRes = pselect(poly_mask, scos, ssin);
396 }
else if (Func == TrigFunction::Tan) {
397 sign_bit = pxor(sign_sin, sign_cos);
398 sFinalRes = pdiv(pselect(poly_mask, ssin, scos), pselect(poly_mask, scos, ssin));
399 }
else if (Func == TrigFunction::SinCos) {
400 Packet peven = peven_mask(x);
401 sign_bit = pselect(peven, sign_sin, sign_cos);
402 sFinalRes = pselect(pxor(peven, poly_mask), scos, ssin);
404 sign_bit = pand(sign_bit, cst_sign_mask);
405 sFinalRes = pxor(sFinalRes, sign_bit);
410 if (EIGEN_PREDICT_FALSE(predux_any(pcmp_le(pset1<Packet>(huge_th), x_abs)))) {
411 const int PacketSize = unpacket_traits<Packet>::size;
412 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment)
double sincos_vals[PacketSize];
413 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet>::alignment)
double x_cpy[PacketSize];
415 pstoreu(sincos_vals, sFinalRes);
416 for (
int k = 0; k < PacketSize; ++k) {
417 double val = x_cpy[k];
418 if (numext::abs(val) > huge_th && (numext::isfinite)(val)) {
419 if (Func == TrigFunction::Sin) {
420 sincos_vals[k] = numext::sin(val);
421 }
else if (Func == TrigFunction::Cos) {
422 sincos_vals[k] = numext::cos(val);
423 }
else if (Func == TrigFunction::Tan) {
424 sincos_vals[k] = numext::tan(val);
425 }
else if (Func == TrigFunction::SinCos) {
426 sincos_vals[k] = k % 2 == 0 ? numext::sin(val) : numext::cos(val);
430 sFinalRes = ploadu<Packet>(sincos_vals);
435template <
typename Packet>
436EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psin_double(
const Packet& x) {
437 return psincos_double<TrigFunction::Sin>(x);
440template <
typename Packet>
441EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcos_double(
const Packet& x) {
442 return psincos_double<TrigFunction::Cos>(x);
445template <
typename Packet>
446EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet ptan_double(
const Packet& x) {
447 return psincos_double<TrigFunction::Tan>(x);
450template <
typename Packet>
451EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
452 std::enable_if_t<std::is_same<typename unpacket_traits<Packet>::type,
float>::value, Packet>
453 psincos_selector(
const Packet& x) {
454 return psincos_float<TrigFunction::SinCos, Packet>(x);
457template <
typename Packet>
458EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
459 std::enable_if_t<std::is_same<typename unpacket_traits<Packet>::type,
double>::value, Packet>
460 psincos_selector(
const Packet& x) {
461 return psincos_double<TrigFunction::SinCos, Packet>(x);
469template <
typename Packet>
470EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacos_float(
const Packet& x_in) {
471 using Scalar =
typename unpacket_traits<Packet>::type;
472 static_assert(std::is_same<Scalar, float>::value,
"Scalar type must be float");
474 const Packet cst_one = pset1<Packet>(Scalar(1));
475 const Packet cst_pi = pset1<Packet>(Scalar(EIGEN_PI));
476 const Packet p6 = pset1<Packet>(Scalar(2.36423197202384471893310546875e-3));
477 const Packet p5 = pset1<Packet>(Scalar(-1.1368644423782825469970703125e-2));
478 const Packet p4 = pset1<Packet>(Scalar(2.717843465507030487060546875e-2));
479 const Packet p3 = pset1<Packet>(Scalar(-4.8969544470310211181640625e-2));
480 const Packet p2 = pset1<Packet>(Scalar(8.8804088532924652099609375e-2));
481 const Packet p1 = pset1<Packet>(Scalar(-0.214591205120086669921875));
482 const Packet p0 = pset1<Packet>(Scalar(1.57079637050628662109375));
487 const Packet neg_mask = psignbit(x_in);
488 const Packet abs_x = pabs(x_in);
494 Packet x2 = pmul(x_in, x_in);
495 Packet p_even = pmadd(p6, x2, p4);
496 Packet p_odd = pmadd(p5, x2, p3);
497 p_even = pmadd(p_even, x2, p2);
498 p_odd = pmadd(p_odd, x2, p1);
499 p_even = pmadd(p_even, x2, p0);
500 Packet p = pmadd(p_odd, abs_x, p_even);
505 Packet denom = psqrt(psub(cst_one, abs_x));
506 Packet result = pmul(denom, p);
508 return pselect(neg_mask, psub(cst_pi, result), result);
512template <
typename Packet>
513EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasin_float(
const Packet& x_in) {
514 using Scalar =
typename unpacket_traits<Packet>::type;
515 static_assert(std::is_same<Scalar, float>::value,
"Scalar type must be float");
517 constexpr float kPiOverTwo =
static_cast<float>(EIGEN_PI / 2);
519 const Packet cst_half = pset1<Packet>(0.5f);
520 const Packet cst_two = pset1<Packet>(2.0f);
521 const Packet cst_pi_over_two = pset1<Packet>(kPiOverTwo);
524 const Packet p4 = pset1<Packet>(4.21570129692554473876953125e-2f);
525 const Packet p3 = pset1<Packet>(2.418550290167331695556640625e-2f);
526 const Packet p2 = pset1<Packet>(4.54690195620059967041015625e-2f);
527 const Packet p1 = pset1<Packet>(7.4953101575374603271484375e-2f);
528 const Packet p0 = pset1<Packet>(0.166667520999908447265625f);
530 const Packet abs_x = pabs(x_in);
531 const Packet sign_mask = pandnot(x_in, abs_x);
532 const Packet large_mask = pcmp_lt(cst_half, abs_x);
536 const Packet z = pselect(large_mask, psqrt(pnmadd(cst_half, abs_x, cst_half)), abs_x);
537 const Packet u = pmul(z, z);
540 const Packet u2 = pmul(u, u);
541 const Packet p_low = pmadd(p1, u, p0);
542 const Packet p_high = pmadd(p4, u2, pmadd(p3, u, p2));
543 Packet p = pmadd(pmul(z, u), pmadd(p_high, u2, p_low), z);
545 const Packet p_large = pnmadd(cst_two, p, cst_pi_over_two);
546 p = pselect(large_mask, p_large, p);
548 return pxor(p, sign_mask);
553template <
typename Scalar>
554struct patan_reduced {
555 template <
typename Packet>
556 static EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet run(
const Packet& x,
const Packet& large_mask);
563template <
typename Packet>
564EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patan_reduced<double>::run(
const Packet& x,
565 const Packet& large_mask) {
566 constexpr double kPiOverTwoLo = 6.123233995736766e-17;
567 constexpr double alpha[] = {-2.4644487075790443e-04, -7.8073996177164305e-03, -6.2915029683194929e-02,
568 -1.9307140255021149e-01, -2.4634297465012978e-01, -1.1001453763093239e-01};
570 constexpr double beta[] = {
571 2.7311202462436667e-04, 1.0899150928962708e-02, 1.1548932646420353e-01, 4.9716458728465573e-01, 1.0,
572 9.3705509168587852e-01, 3.3004361289279920e-01};
574 Packet x2 = pmul(x, x);
575 Packet p = ppolevl<Packet, 5>::run(x2, alpha);
576 Packet q = ppolevl<Packet, 6>::run(x2, beta);
578 const Packet x0 = psub(x, pand(large_mask, pset1<Packet>(kPiOverTwoLo)));
579 return pmadd(pmul(x, x2), pdiv(p, q), x0);
584template <
typename Packet>
585EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patan_reduced<float>::run(
const Packet& x,
const Packet&) {
586 constexpr float alpha[] = {1.12026982009410858154296875e-01f, 7.296695709228515625e-01f, 8.109951019287109375e-01f};
588 constexpr float beta[] = {1.00917108356952667236328125e-02f, 2.8318560123443603515625e-01f, 1.0f,
589 8.109951019287109375e-01f};
591 Packet x2 = pmul(x, x);
592 Packet p = ppolevl<Packet, 2>::run(x2, alpha);
593 Packet q = ppolevl<Packet, 3>::run(x2, beta);
594 return pmul(x, pdiv(p, q));
597template <
typename Packet>
598EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_atan(
const Packet& x_in) {
599 using Scalar =
typename unpacket_traits<Packet>::type;
601 constexpr Scalar kPiOverTwo =
static_cast<Scalar
>(EIGEN_PI / 2);
603 const Packet cst_signmask = psignmask<Packet>();
604 const Packet cst_one = pset1<Packet>(Scalar(1));
605 const Packet cst_pi_over_two = pset1<Packet>(kPiOverTwo);
611 const Packet abs_x = pabs(x_in);
612 const Packet x_signmask = pand(x_in, cst_signmask);
613 const Packet large_mask = pcmp_lt(cst_one, abs_x);
614 const Packet x = pselect(large_mask, preciprocal(abs_x), abs_x);
615 const Packet p = patan_reduced<Scalar>::run(x, large_mask);
617 Packet result = pselect(large_mask, psub(cst_pi_over_two, p), p);
619 return pxor(result, x_signmask);
638EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T ptanh_float(
const T& a_x) {
643#ifdef EIGEN_VECTORIZE_FMA
644 constexpr float kClamp = 8.01773357391357422f;
646 constexpr bool kUsesFma = is_scalar<T>::value && scalar_madd_uses_fma<T>::value;
647 constexpr float kClamp = kUsesFma ? 8.01773357391357422f : 7.90738964080810547f;
649 const T plus_clamp = pset1<T>(kClamp);
650 const T minus_clamp = pset1<T>(-kClamp);
651 const T x = pmax(pmin(a_x, plus_clamp), minus_clamp);
661 constexpr float alpha[] = {1.394553628e-8f, 2.102733560e-5f, 3.520756727e-3f, 1.340216100e-1f};
664 constexpr float beta[] = {8.015776984e-7f, 3.326951409e-4f, 2.597254514e-2f, 4.673548340e-1f, 1.0f};
667 const T x2 = pmul(x, x);
668 const T x3 = pmul(x2, x);
670 T p = ppolevl<T, 3>::run(x2, alpha);
671 T q = ppolevl<T, 4>::run(x2, beta);
692EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T ptanh_float(
const T& x) {
696 constexpr float alpha[] = {-1.46725140511989593505859375e-02f, -3.333333432674407958984375e-01f};
697 constexpr float beta[] = {1.570280082523822784423828125e-02, 4.4401752948760986328125e-01, 1.0f};
698 const T sign_mask = psignmask<T>();
699 const T x_sign = pand(sign_mask, x);
700 const T abs_x = pandnot(x, sign_mask);
701 constexpr float kSmallThreshold = 1.25f;
702 const T large_mask = pcmp_lt(pset1<T>(kSmallThreshold), abs_x);
704 const T x_small = pandnot(x, large_mask);
705 const T x2 = pmul(x_small, x_small);
706 const T x3 = pmul(x2, x_small);
707 const T p = ppolevl<T, 1>::run(x2, alpha);
708 const T q = ppolevl<T, 2>::run(x2, beta);
710 const T small_tanh = por(x_sign, pmadd(x3, pdiv(p, q), x_small));
712 if (!predux_any(large_mask)) {
718 const T one = pset1<T>(1.0f);
719 const T two = pset1<T>(2.0f);
720 constexpr float kHugeThreshold = 16.0f;
721 const T s = pexp_float<T, true>(pmul(two, pmin(pselect(large_mask, abs_x, pzero(x)), pset1<T>(kHugeThreshold))));
722 const T large_tanh = por(x_sign, psub(one, pdiv(two, padd(s, one))));
723 return pselect(large_mask, large_tanh, small_tanh);
738EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T ptanh_double(
const T& a_x) {
743#ifdef EIGEN_VECTORIZE_FMA
744 constexpr double kClamp = 17.6610191624600077;
746 constexpr bool kUsesFma = is_scalar<T>::value && scalar_madd_uses_fma<T>::value;
747 constexpr double kClamp = kUsesFma ? 17.6610191624600077 : 17.714196154005176;
749 const T plus_clamp = pset1<T>(kClamp);
750 const T minus_clamp = pset1<T>(-kClamp);
751 const T x = pmax(pmin(a_x, plus_clamp), minus_clamp);
760 constexpr double alpha[] = {2.6158007860482230e-23, 7.6534862268749319e-19, 3.1309488231386680e-15,
761 4.2303918148209176e-12, 2.4618379131293676e-09, 6.8644367682497074e-07,
762 9.3839087674268880e-05, 5.9809711724441161e-03, 1.5184719640284322e-01};
765 constexpr double beta[] = {6.463747022670968018e-21, 5.782506856739003571e-17,
766 1.293019623712687916e-13, 1.123643448069621992e-10,
767 4.492975677839633985e-08, 8.785185266237658698e-06,
768 8.295161192716231542e-04, 3.437448108450402717e-02,
769 4.851805297361760360e-01, 1.0};
772 const T x2 = pmul(x, x);
773 const T x3 = pmul(x2, x);
777 T p = ppolevl<T, 8>::run(x2, alpha);
778 T q = ppolevl<T, 9>::run(x2, beta);
787template <
typename Packet>
788EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patanh_float(
const Packet& x) {
789 using Scalar =
typename unpacket_traits<Packet>::type;
790 static_assert(std::is_same<Scalar, float>::value,
"Scalar type must be float");
794 constexpr float alpha[] = {0.1819281280040740966796875f, 8.2311116158962249755859375e-2f,
795 0.14672131836414337158203125f, 0.1997792422771453857421875f, 0.3333373963832855224609375f};
796 const Packet x2 = pmul(x, x);
797 const Packet x3 = pmul(x, x2);
798 Packet p = ppolevl<Packet, 4>::run(x2, alpha);
801 const Packet half = pset1<Packet>(0.5f);
802 const Packet one = pset1<Packet>(1.0f);
803 const Packet x_gt_half = pcmp_le(half, pabs(x));
805 if (!predux_any(x_gt_half)) {
810 Packet r = pdiv(padd(one, x), psub(one, x));
811 r = pmul(half, plog(r));
813 const Packet x_eq_one = pcmp_eq(one, pabs(x));
814 const Packet x_gt_one = pcmp_lt(one, pabs(x));
815 const Packet sign_mask = psignmask<Packet>();
816 const Packet x_sign = pand(sign_mask, x);
817 const Packet inf = pinf<Packet>();
818 return por(x_gt_one, pselect(x_eq_one, por(x_sign, inf), pselect(x_gt_half, r, p)));
821template <
typename Packet>
822EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet patanh_double(
const Packet& x) {
823 using Scalar =
typename unpacket_traits<Packet>::type;
824 static_assert(std::is_same<Scalar, double>::value,
"Scalar type must be double");
827 constexpr double alpha[] = {3.3071338469301391e-03, -4.7129526768798737e-02, 1.8185306179826699e-01,
828 -2.5949536095445679e-01, 1.2306328729812676e-01};
830 constexpr double beta[] = {-3.8679974580640881e-03, 7.6391885763341910e-02, -4.2828141436397615e-01,
831 9.8733495886883648e-01, -1.0000000000000000e+00, 3.6918986189438030e-01};
833 const Packet x2 = pmul(x, x);
834 const Packet x3 = pmul(x, x2);
835 Packet p = ppolevl<Packet, 4>::run(x2, alpha);
836 Packet q = ppolevl<Packet, 5>::run(x2, beta);
837 Packet y_small = pmadd(x3, pdiv(p, q), x);
839 const Packet half = pset1<Packet>(0.5);
840 const Packet one = pset1<Packet>(1.0);
841 const Packet x_gt_half = pcmp_le(half, pabs(x));
843 if (!predux_any(x_gt_half)) {
848 Packet y_large = pdiv(padd(one, x), psub(one, x));
849 y_large = pmul(half, plog(y_large));
851 const Packet x_eq_one = pcmp_eq(one, pabs(x));
852 const Packet x_gt_one = pcmp_lt(one, pabs(x));
853 const Packet sign_mask = psignmask<Packet>();
854 const Packet x_sign = pand(sign_mask, x);
855 const Packet inf = pinf<Packet>();
856 return por(x_gt_one, pselect(x_eq_one, por(x_sign, inf), pselect(x_gt_half, y_large, y_small)));
870template <
typename Packet>
871EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psinh_float(
const Packet& x) {
872 using Scalar =
typename unpacket_traits<Packet>::type;
873 static_assert(std::is_same<Scalar, float>::value,
"Scalar type must be float");
875 const Packet sign_mask = psignmask<Packet>();
876 const Packet abs_x = pandnot(x, sign_mask);
877 const Packet x_sign = pand(x, sign_mask);
881 constexpr float alpha[] = {2.7557314045e-06f, 1.9841270114e-04f, 8.3333335817e-03f, 1.6666666716e-01f};
882 const Packet x2 = pmul(x, x);
883 Packet p_small = ppolevl<Packet, 3>::run(x2, alpha);
884 p_small = pmadd(pmul(x2, x), p_small, x);
887 const Packet one = pset1<Packet>(1.0f);
888 const Packet e = generic_half_exp(abs_x);
893 const Packet quarter = pset1<Packet>(0.25f);
894 Packet p_medium = psub(e, pdiv(quarter, e));
897 const Packet large_threshold = pset1<Packet>(20.0f);
898 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
899 Packet p_large = pselect(large_mask, e, p_medium);
900 p_large = por(x_sign, p_large);
902 const Packet small_mask = pcmp_lt(abs_x, one);
903 return pselect(small_mask, p_small, p_large);
906template <
typename Packet>
907EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psinh_double(
const Packet& x) {
908 using Scalar =
typename unpacket_traits<Packet>::type;
909 static_assert(std::is_same<Scalar, double>::value,
"Scalar type must be double");
911 const Packet sign_mask = psignmask<Packet>();
912 const Packet abs_x = pandnot(x, sign_mask);
913 const Packet x_sign = pand(x, sign_mask);
918 constexpr double alpha[] = {
919 8.2206352466243297e-18,
920 2.8114572543455206e-15,
921 7.6471637318198164e-13,
922 1.6059043836821613e-10,
923 2.5052108385441718e-08,
924 2.7557319223985893e-06,
925 1.9841269841269841e-04,
926 8.3333333333333332e-03,
927 1.6666666666666666e-01,
929 const Packet x2 = pmul(x, x);
930 Packet p_small = ppolevl<Packet, 8>::run(x2, alpha);
931 p_small = pmadd(pmul(x2, x), p_small, x);
934 const Packet one = pset1<Packet>(1.0);
935 const Packet e = generic_half_exp(abs_x);
939 const Packet quarter = pset1<Packet>(0.25);
940 Packet p_medium = psub(e, pdiv(quarter, e));
943 const Packet large_threshold = pset1<Packet>(20.0);
944 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
945 Packet p_large = pselect(large_mask, e, p_medium);
946 p_large = por(x_sign, p_large);
947 const Packet small_mask = pcmp_lt(abs_x, one);
948 return pselect(small_mask, p_small, p_large);
957template <
typename Packet>
958EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcosh_float(
const Packet& x) {
959 const Packet abs_x = pabs(x);
962 const Packet e = generic_half_exp(abs_x);
966 const Packet quarter = pset1<Packet>(0.25f);
967 Packet p_medium = padd(e, pdiv(quarter, e));
970 const Packet large_threshold = pset1<Packet>(20.0f);
971 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
972 return pselect(large_mask, e, p_medium);
975template <
typename Packet>
976EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pcosh_double(
const Packet& x) {
977 const Packet abs_x = pabs(x);
980 const Packet e = generic_half_exp(abs_x);
983 const Packet quarter = pset1<Packet>(0.25);
984 Packet p_medium = padd(e, pdiv(quarter, e));
987 const Packet large_threshold = pset1<Packet>(20.0);
988 const Packet large_mask = pcmp_lt(large_threshold, abs_x);
989 return pselect(large_mask, e, p_medium);
1002template <
typename Packet>
1003EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasinh_float(
const Packet& x) {
1004 const Packet sign_mask = psignmask<Packet>();
1005 const Packet abs_x = pandnot(x, sign_mask);
1006 const Packet x_sign = pand(x, sign_mask);
1007 const Packet one = pset1<Packet>(1.0f);
1010 const Packet large_mask = pcmp_lt(pset1<Packet>(1e10f), abs_x);
1012 const Packet x2 = pmul(abs_x, pselect(large_mask, pzero(abs_x), abs_x));
1016 Packet normal_arg = padd(abs_x, pdiv(x2, padd(one, psqrt(padd(one, x2)))));
1018 Packet large_arg = psub(abs_x, one);
1020 Packet result = plog1p(pselect(large_mask, large_arg, normal_arg));
1022 const Packet ln2 = pset1<Packet>(0.6931471805599453f);
1023 result = pselect(large_mask, padd(result, ln2), result);
1025 result = pselect(pcmp_lt(abs_x, pset1<Packet>(2.44140625e-4f)), abs_x, result);
1026 return por(x_sign, result);
1029template <
typename Packet>
1030EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pasinh_double(
const Packet& x) {
1031 const Packet sign_mask = psignmask<Packet>();
1032 const Packet abs_x = pandnot(x, sign_mask);
1033 const Packet x_sign = pand(x, sign_mask);
1034 const Packet one = pset1<Packet>(1.0);
1036 const Packet large_mask = pcmp_lt(pset1<Packet>(1e150), abs_x);
1037 const Packet x2 = pmul(abs_x, pselect(large_mask, pzero(abs_x), abs_x));
1038 Packet normal_arg = padd(abs_x, pdiv(x2, padd(one, psqrt(padd(one, x2)))));
1039 Packet large_arg = psub(abs_x, one);
1040 Packet result = plog1p(pselect(large_mask, large_arg, normal_arg));
1041 const Packet ln2 = pset1<Packet>(0.6931471805599453);
1042 result = pselect(large_mask, padd(result, ln2), result);
1044 result = pselect(pcmp_lt(abs_x, pset1<Packet>(1.4901161193847656e-8)), abs_x, result);
1045 return por(x_sign, result);
1054template <
typename Packet>
1055EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacosh_float(
const Packet& x) {
1056 const Packet one = pset1<Packet>(1.0f);
1057 const Packet two = pset1<Packet>(2.0f);
1058 const Packet t = psub(x, one);
1059 const Packet huge_mask = pcmp_lt(pset1<Packet>(1e10f), x);
1061 const Packet t_tp2 = pmul(pselect(huge_mask, pzero(t), t), padd(t, two));
1062 Packet normal_arg = padd(t, psqrt(t_tp2));
1064 Packet huge_arg = t;
1066 Packet result = plog1p(pselect(huge_mask, huge_arg, normal_arg));
1067 const Packet ln2 = pset1<Packet>(0.6931471805599453f);
1068 result = pselect(huge_mask, padd(result, ln2), result);
1070 const Packet invalid_mask = pcmp_lt(x, one);
1071 return por(invalid_mask, result);
1074template <
typename Packet>
1075EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pacosh_double(
const Packet& x) {
1076 const Packet one = pset1<Packet>(1.0);
1077 const Packet two = pset1<Packet>(2.0);
1078 const Packet t = psub(x, one);
1079 const Packet huge_mask = pcmp_lt(pset1<Packet>(1e150), x);
1080 const Packet t_tp2 = pmul(pselect(huge_mask, pzero(t), t), padd(t, two));
1081 Packet normal_arg = padd(t, psqrt(t_tp2));
1082 Packet huge_arg = t;
1083 Packet result = plog1p(pselect(huge_mask, huge_arg, normal_arg));
1084 const Packet ln2 = pset1<Packet>(0.6931471805599453);
1085 result = pselect(huge_mask, padd(result, ln2), result);
1086 const Packet invalid_mask = pcmp_lt(x, one);
1087 return por(invalid_mask, result);