Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GenericPacketMathFunctions.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2007 Julien Pommier
5// Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com)
6// Copyright (C) 2009-2019 Gael Guennebaud <gael.guennebaud@inria.fr>
7// Copyright (C) 2018-2025 Rasmus Munk Larsen <rmlarsen@gmail.com>
8//
9// This Source Code Form is subject to the terms of the Mozilla
10// Public License v. 2.0. If a copy of the MPL was not distributed
11// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
12// SPDX-License-Identifier: MPL-2.0
13
14/* The exp and log functions of this file initially come from
15 * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/
16 */
17
18#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H
19#define EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H
20
21// IWYU pragma: private
22#include "../../InternalHeaderCheck.h"
23#include "GenericPacketMathPolynomials.h"
24#include "GenericPacketMathFrexpLdexp.h"
25#include "GenericPacketMathDoubleWord.h"
26
27namespace Eigen {
28namespace internal {
29
30//----------------------------------------------------------------------
31// Exponential and Logarithmic Functions
32//----------------------------------------------------------------------
33
34// Core range reduction and polynomial evaluation for float logarithm.
35//
36// Given a positive float value v (may be denormal), decomposes it as
37// v = 2^e * (1+f) with f in [sqrt(0.5)-1, sqrt(2)-1], then evaluates
38// log(1+f) ≈ f + f^2 * P(f) using a degree-7 minimax polynomial.
39//
40// Returns the approximation of log(v_mantissa) in log_mantissa and the
41// integer exponent in e. The caller combines these as appropriate
42// (e.g. e*ln2 + log_mantissa for natural log, or log_mantissa*log2e + e
43// for log2).
44//
45// Range reduction uses integer bit manipulation (musl-inspired) instead of the
46// heavier pfrexp_generic, saving ~12 ops. The minimax polynomial was found via
47// Sollya's fpminimax, giving faithfully-rounded results (max 1 ULP for log).
48//
49// Returns whether any lane of v is not a positive, normal, finite value. Such inputs are rare, so packets without
50// them skip the normalization of subnormals here and the special-value handling of the caller.
51template <typename Packet>
52EIGEN_STRONG_INLINE bool plog_core_float(const Packet v, Packet& log_mantissa, Packet& e) {
53 using PacketI = typename unpacket_traits<Packet>::integer_packet;
54
55 const PacketI cst_min_normal = pset1<PacketI>(0x00800000);
56 const PacketI cst_mant_mask = pset1<PacketI>(0x007fffff);
57 // Adding this offset to the integer representation biases the exponent so
58 // that values near 1 (0x3f800000) map to exponent 0, and values below
59 // sqrt(0.5) get folded into the previous exponent. The magic constant is
60 // 0x3f800000 - 0x3f3504f3 = 0x004afb0d, where 0x3f3504f3 ≈ sqrt(0.5).
61 const PacketI cst_sqrt_half_offset = pset1<PacketI>(0x004afb0d);
62 const PacketI cst_exp_bias = pset1<PacketI>(0x7f); // 127
63 const PacketI cst_half_mant = pset1<PacketI>(0x3f3504f3); // sqrt(0.5)
64
65 // The signed comparisons also catch zero and negative values (below the smallest normal), and inf and NaN.
66 PacketI vi = preinterpret<PacketI>(v);
67 const PacketI is_denormal = pcmp_lt(vi, cst_min_normal);
68 const bool any_special = predux_any(preinterpret<Packet>(por(is_denormal, pcmp_lt(pset1<PacketI>(0x7f7fffff), vi))));
69 // Normalize denormals by multiplying by 2^23.
70 if (any_special) {
71 Packet v_normalized = pmul(v, pset1<Packet>(8388608.0f)); // 2^23
72 vi = pselect(is_denormal, preinterpret<PacketI>(v_normalized), vi);
73 }
74
75 // Combined range reduction: bias integer representation so that exponent
76 // extraction automatically shifts mantissa to [sqrt(0.5), sqrt(2)).
77 PacketI vi_biased = padd(vi, cst_sqrt_half_offset);
78 // Extract exponent as integer, subtract bias and denormal adjustment (23).
79 PacketI e_int = psub(plogical_shift_right<23>(vi_biased), cst_exp_bias);
80 if (any_special) e_int = psub(e_int, pand(is_denormal, pset1<PacketI>(23)));
81 e = pcast<PacketI, Packet>(e_int);
82 // Reconstruct mantissa in [sqrt(0.5), sqrt(2)). The integer addition of the
83 // masked mantissa with 0x3f3504f3 (sqrt(0.5)) naturally produces carry into
84 // the exponent field, yielding values in [sqrt(0.5), 1) or [1, sqrt(2)).
85 // Then subtract 1 to center on 0 → f in [sqrt(0.5)-1, sqrt(2)-1].
86 Packet f = psub(preinterpret<Packet>(padd(pand(vi_biased, cst_mant_mask), cst_half_mant)), pset1<Packet>(1.0f));
87
88 // Minimax degree-7 polynomial for g(f) = (log(1+f) - f) / f^2 on
89 // [sqrt(0.5)-1, sqrt(2)-1], so log(1+f) ≈ f + f^2 * P(f).
90 // Generated by Sollya: fpminimax(g, 7, [|single...|], [lo;hi])
91 // Mathematical approximation error: max |log(1+f) - (f + f^2*P(f))| < 2.04e-8.
92 // Coefficients stored in reverse order for ppolevl (highest degree first).
93 constexpr float coeffs[] = {
94 8.8758550584316254e-02f, // c7 (x^7)
95 -1.4199858903884888e-01f, // c6 (x^6)
96 1.4824025332927704e-01f, // c5 (x^5)
97 -1.6583317518234253e-01f, // c4 (x^4)
98 1.9972395896911621e-01f, // c3 (x^3)
99 -2.5001299381256104e-01f, // c2 (x^2)
100 3.3333668112754822e-01f, // c1 (x^1)
101 -4.9999997019767761e-01f, // c0 (x^0)
102 };
103
104 // Evaluate P(f) via Horner's method, then log(1+f) ≈ f + f^2 * P(f).
105 Packet f2 = pmul(f, f);
106 Packet p = ppolevl<Packet, 7>::run(f, coeffs);
107 log_mantissa = pmadd(p, f2, f);
108 return any_special;
109}
110
111// Natural or base-2 logarithm for float packets.
112//
113// Computes log(x) as e*C + log(m), where x = 2^e * m with m in [sqrt(1/2), sqrt(2))
114// and C = ln(2) for natural log, C = 1 for log2.
115template <typename Packet, bool base2>
116EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_impl_float(const Packet _x) {
117 Packet log_mantissa, e;
118 const bool any_special = plog_core_float(_x, log_mantissa, e);
119
120 // Add the logarithm of the exponent back to the result.
121 Packet x;
122 if (base2) {
123 const Packet cst_log2e = pset1<Packet>(static_cast<float>(EIGEN_LOG2E));
124 x = pmadd(log_mantissa, cst_log2e, e);
125 } else {
126 const Packet cst_ln2 = pset1<Packet>(static_cast<float>(EIGEN_LN2));
127 x = pmadd(e, cst_ln2, log_mantissa);
128 }
129 if (!any_special) return x;
130
131 // Filter out invalid inputs:
132 // - negative arg → NAN
133 // - 0 → -INF
134 // - +INF → +INF
135 const Packet cst_minus_inf = por(psignmask<Packet>(), pinf<Packet>());
136 const Packet cst_pos_inf = pinf<Packet>();
137 Packet invalid_mask = pcmp_lt_or_nan(_x, pzero(_x));
138 Packet iszero_mask = pcmp_eq(_x, pzero(_x));
139 Packet pos_inf_mask = pcmp_eq(_x, cst_pos_inf);
140 return pselect(iszero_mask, cst_minus_inf, por(pselect(pos_inf_mask, cst_pos_inf, x), invalid_mask));
141}
142
143template <typename Packet>
144EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_float(const Packet _x) {
145 return plog_impl_float<Packet, /* base2 */ false>(_x);
146}
147
148template <typename Packet>
149EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog2_float(const Packet _x) {
150 return plog_impl_float<Packet, /* base2 */ true>(_x);
151}
152
153// -----------------------------------------------------------------------
154// Double logarithm: shared polynomial + two range-reduction backends
155// -----------------------------------------------------------------------
156
157// Cephes rational-polynomial approximation of log(1+f) for
158// f in [sqrt(0.5)-1, sqrt(2)-1].
159// Evaluates x - 0.5*x^2 + x^3 * P(x)/Q(x) where P and Q are degree-5.
160// See: http://www.netlib.org/cephes/
161template <typename Packet>
162EIGEN_STRONG_INLINE Packet plog_mantissa_double(const Packet x) {
163 const Packet cst_cephes_log_p0 = pset1<Packet>(1.01875663804580931796E-4);
164 const Packet cst_cephes_log_p1 = pset1<Packet>(4.97494994976747001425E-1);
165 const Packet cst_cephes_log_p2 = pset1<Packet>(4.70579119878881725854E0);
166 const Packet cst_cephes_log_p3 = pset1<Packet>(1.44989225341610930846E1);
167 const Packet cst_cephes_log_p4 = pset1<Packet>(1.79368678507819816313E1);
168 const Packet cst_cephes_log_p5 = pset1<Packet>(7.70838733755885391666E0);
169 // Q0 = 1.0; pmadd(1, x, q1) simplifies to padd(x, q1).
170 const Packet cst_cephes_log_q1 = pset1<Packet>(1.12873587189167450590E1);
171 const Packet cst_cephes_log_q2 = pset1<Packet>(4.52279145837532221105E1);
172 const Packet cst_cephes_log_q3 = pset1<Packet>(8.29875266912776603211E1);
173 const Packet cst_cephes_log_q4 = pset1<Packet>(7.11544750618563894466E1);
174 const Packet cst_cephes_log_q5 = pset1<Packet>(2.31251620126765340583E1);
175
176 Packet x2 = pmul(x, x);
177 Packet x3 = pmul(x2, x);
178
179 // Evaluate P and Q simultaneously for better ILP.
180 Packet y, y1, y_;
181 y = pmadd(cst_cephes_log_p0, x, cst_cephes_log_p1);
182 y1 = pmadd(cst_cephes_log_p3, x, cst_cephes_log_p4);
183 y = pmadd(y, x, cst_cephes_log_p2);
184 y1 = pmadd(y1, x, cst_cephes_log_p5);
185 y_ = pmadd(y, x3, y1);
186
187 y = padd(x, cst_cephes_log_q1);
188 y1 = pmadd(cst_cephes_log_q3, x, cst_cephes_log_q4);
189 y = pmadd(y, x, cst_cephes_log_q2);
190 y1 = pmadd(y1, x, cst_cephes_log_q5);
191 y = pmadd(y, x3, y1);
192
193 y_ = pmul(y_, x3);
194 y = pdiv(y_, y);
195 y = pnmadd(pset1<Packet>(0.5), x2, y);
196 return padd(x, y);
197}
198
199// Detect whether unpacket_traits<Packet>::integer_packet is defined.
200template <typename Packet, typename = void>
201struct packet_has_integer_packet : std::false_type {};
202template <typename Packet>
203struct packet_has_integer_packet<Packet, void_t<typename unpacket_traits<Packet>::integer_packet>> : std::true_type {};
204
205// Dispatch struct for double-precision range reduction. run() returns whether any lane of v may not be a positive,
206// normal, finite value, as plog_core_float does.
207// Primary template: pfrexp-based fallback (used when integer_packet is absent).
208template <typename Packet, bool UseIntegerPacket>
209struct plog_range_reduce_double {
210 EIGEN_STRONG_INLINE static bool run(const Packet v, Packet& f, Packet& e) {
211 const Packet one = pset1<Packet>(1.0);
212 const Packet cst_cephes_SQRTHF = pset1<Packet>(0.70710678118654752440E0);
213 // pfrexp: f in [0.5, 1), e = unbiased exponent as double.
214 f = pfrexp(v, e);
215 // Shift [0.5,1) -> [sqrt(0.5)-1, sqrt(2)-1] with exponent correction:
216 // if f < sqrt(0.5): f = f + f - 1, e -= 1 (giving f in [0, sqrt(2)-1))
217 // else: f = f - 1 (giving f in [sqrt(0.5)-1, 0))
218 Packet mask = pcmp_lt(f, cst_cephes_SQRTHF);
219 Packet tmp = pand(f, mask);
220 f = psub(f, one);
221 e = psub(e, pand(one, mask));
222 f = padd(f, tmp);
223 return true;
224 }
225};
226
227// Specialisation: fast integer-bit-manipulation path (musl-inspired).
228// Requires unpacket_traits<Packet>::integer_packet to be a 64-bit integer packet.
229template <typename Packet>
230struct plog_range_reduce_double<Packet, true> {
231 EIGEN_STRONG_INLINE static bool run(const Packet v, Packet& f, Packet& e) {
232 using PacketI = typename unpacket_traits<Packet>::integer_packet;
233 // 2^-1022: smallest positive normal double.
234 const PacketI cst_min_normal = pset1<PacketI>(static_cast<int64_t>(0x0010000000000000LL));
235 // Lower 52-bit mask (IEEE mantissa field).
236 const PacketI cst_mant_mask = pset1<PacketI>(static_cast<int64_t>(0x000FFFFFFFFFFFFFLL));
237 // Offset = 1.0_bits - sqrt(0.5)_bits. Adding this to the integer
238 // representation shifts the exponent field so that the [sqrt(0.5), sqrt(2))
239 // half-octave boundary falls on an exact biased-exponent boundary, letting
240 // us extract e with a single right shift. The constant is:
241 // 0x3FF0000000000000 - 0x3FE6A09E667F3BCD = 0x00095F619980C433
242 const PacketI cst_sqrt_half_offset =
243 pset1<PacketI>(static_cast<int64_t>(0x3FF0000000000000LL - 0x3FE6A09E667F3BCDLL));
244 // IEEE double exponent bias (1023).
245 const PacketI cst_exp_bias = pset1<PacketI>(static_cast<int64_t>(1023));
246 // sqrt(0.5) IEEE bits — used to reconstruct f from biased mantissa.
247 const PacketI cst_half_mant = pset1<PacketI>(static_cast<int64_t>(0x3FE6A09E667F3BCDLL));
248
249 // Reinterpret v as a 64-bit integer vector.
250 PacketI vi = preinterpret<PacketI>(v);
251
252 // The signed comparisons also catch zero and negative values (below the smallest normal), and inf and NaN.
253 const PacketI is_denormal = pcmp_lt(vi, cst_min_normal);
254 const bool any_special = predux_any(preinterpret<Packet>(
255 por(is_denormal, pcmp_lt(pset1<PacketI>(static_cast<int64_t>(0x7FEFFFFFFFFFFFFFLL)), vi))));
256 // Normalise denormals: multiply by 2^52 and correct the exponent by -52.
257 if (any_special) {
258 // 2^52 via bit pattern: biased exponent = 52 + 1023 = 0x433, mantissa = 0.
259 Packet v_norm = pmul(v, pset1frombits<Packet>(static_cast<uint64_t>(int64_t(52 + 0x3ff) << 52)));
260 vi = pselect(is_denormal, preinterpret<PacketI>(v_norm), vi);
261 }
262
263 // Bias the integer representation so the exponent field directly encodes
264 // the half-octave index.
265 PacketI vi_biased = padd(vi, cst_sqrt_half_offset);
266 // Extract unbiased exponent: shift out mantissa bits, subtract IEEE bias
267 // and denormal adjustment.
268 PacketI e_int = psub(plogical_shift_right<52>(vi_biased), cst_exp_bias);
269 if (any_special) e_int = psub(e_int, pand(is_denormal, pset1<PacketI>(static_cast<int64_t>(52))));
270 // Convert integer exponent to floating-point.
271 e = pcast<PacketI, Packet>(e_int);
272
273 // Reconstruct mantissa in [sqrt(0.5), sqrt(2)) via integer arithmetic.
274 // The integer addition of the masked mantissa bits and the sqrt(0.5) bit
275 // pattern carries into the exponent field, yielding a value in that range.
276 // Then subtract 1 to centre on 0: f in [sqrt(0.5)-1, sqrt(2)-1].
277 f = psub(preinterpret<Packet>(padd(pand(vi_biased, cst_mant_mask), cst_half_mant)), pset1<Packet>(1.0));
278 return any_special;
279 }
280};
281
282// Core range reduction and polynomial for double logarithm.
283// Input: v > 0 (zero / negative / inf / nan are handled by the caller).
284// Output: log_mantissa ≈ log(mantissa of v in [sqrt(0.5), sqrt(2))),
285// e = unbiased exponent of v as a double.
286// Returns whether the caller must handle special values (see plog_core_float).
287// Selects the fast integer path when integer_packet is available, otherwise
288// falls back to pfrexp.
289template <typename Packet>
290EIGEN_STRONG_INLINE bool plog_core_double(const Packet v, Packet& log_mantissa, Packet& e) {
291 Packet f;
292 const bool any_special = plog_range_reduce_double<Packet, packet_has_integer_packet<Packet>::value>::run(v, f, e);
293 log_mantissa = plog_mantissa_double(f);
294 return any_special;
295}
296
297/* Returns the base e (2.718...) or base 2 logarithm of x.
298 * The argument is separated into its exponent and fractional parts.
299 * The logarithm of the fraction in the interval [sqrt(1/2), sqrt(2)],
300 * is approximated by
301 *
302 * log(1+x) = x - 0.5 x**2 + x**3 P(x)/Q(x).
303 *
304 * for more detail see: http://www.netlib.org/cephes/
305 */
306template <typename Packet, bool base2>
307EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_impl_double(const Packet _x) {
308 const Packet cst_minus_inf = por(psignmask<Packet>(), pinf<Packet>());
309 const Packet cst_pos_inf = pinf<Packet>();
310
311 Packet log_mantissa, e;
312 const bool any_special = plog_core_double(_x, log_mantissa, e);
313
314 // Combine: log(x) = e * ln2 + log(mantissa), or log2(x) = log(mantissa)*log2e + e.
315 Packet x;
316 if (base2) {
317 const Packet cst_log2e = pset1<Packet>(static_cast<double>(EIGEN_LOG2E));
318 x = pmadd(log_mantissa, cst_log2e, e);
319 } else {
320 const Packet cst_ln2 = pset1<Packet>(static_cast<double>(EIGEN_LN2));
321 x = pmadd(e, cst_ln2, log_mantissa);
322 }
323 if (!any_special) return x;
324
325 Packet invalid_mask = pcmp_lt_or_nan(_x, pzero(_x));
326 Packet iszero_mask = pcmp_eq(_x, pzero(_x));
327 Packet pos_inf_mask = pcmp_eq(_x, cst_pos_inf);
328 // Filter out invalid inputs:
329 // - negative arg → NAN
330 // - 0 → -INF
331 // - +INF → +INF
332 return pselect(iszero_mask, cst_minus_inf, por(pselect(pos_inf_mask, cst_pos_inf, x), invalid_mask));
333}
334
335template <typename Packet>
336EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_double(const Packet _x) {
337 return plog_impl_double<Packet, /* base2 */ false>(_x);
338}
339
340template <typename Packet>
341EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog2_double(const Packet _x) {
342 return plog_impl_double<Packet, /* base2 */ true>(_x);
343}
344
351template <typename Packet>
352EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_log1p_float(const Packet& x) {
353 const Packet one = pset1<Packet>(1.0f);
354 const Packet cst_minus_inf = por(psignmask<Packet>(), pinf<Packet>());
355 const Packet cst_pos_inf = pinf<Packet>();
356
357 // u = 1 + x, with rounding. Recover the lost low bits: dx = x - (u - 1).
358 Packet u = padd(one, x);
359 Packet dx = psub(x, psub(u, one));
360
361 // For |x| tiny enough that u rounds to 1, return x directly.
362 Packet small_mask = pcmp_eq(u, one);
363
364 // Core range reduction and polynomial on u.
365 Packet log_u, e;
366 const bool any_special = plog_core_float(u, log_u, e);
367
368 // result = e * ln2 + log(u) + dx/u.
369 // The dx/u term corrects for the rounding error in u = fl(1+x).
370 const Packet cst_ln2 = pset1<Packet>(static_cast<float>(EIGEN_LN2));
371 Packet result = pmadd(e, cst_ln2, padd(log_u, pdiv(dx, u)));
372 result = pselect(small_mask, x, result);
373 // The remaining cases have u <= 0, u = +inf or u = NaN.
374 if (!any_special) return result;
375
376 // Handle special cases.
377 // For u = +inf (x very large), return +inf.
378 Packet inf_mask = pcmp_eq(u, cst_pos_inf);
379 Packet neg_mask = pcmp_lt(u, pzero(u));
380 Packet zero_mask = pcmp_eq(x, pset1<Packet>(-1.0f));
381 result = pselect(inf_mask, cst_pos_inf, result);
382 result = pselect(zero_mask, cst_minus_inf, result);
383 result = por(neg_mask, result); // NaN for x < -1
384 return result;
385}
386
393template <typename Packet>
394EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_log1p_double(const Packet& x) {
395 const Packet one = pset1<Packet>(1.0);
396 const Packet cst_minus_inf = por(psignmask<Packet>(), pinf<Packet>());
397 const Packet cst_pos_inf = pinf<Packet>();
398
399 // u = 1 + x, with rounding. Recover the lost low bits: dx = x - (u - 1).
400 Packet u = padd(one, x);
401 Packet dx = psub(x, psub(u, one));
402
403 // For |x| tiny enough that u rounds to 1, return x directly.
404 Packet small_mask = pcmp_eq(u, one);
405
406 // Core range reduction and polynomial on u.
407 Packet log_u, e;
408 const bool any_special = plog_core_double(u, log_u, e);
409
410 // result = e * ln2 + log(u) + dx/u.
411 // The dx/u term corrects for the rounding error in u = fl(1+x).
412 const Packet cst_ln2 = pset1<Packet>(static_cast<double>(EIGEN_LN2));
413 Packet result = pmadd(e, cst_ln2, padd(log_u, pdiv(dx, u)));
414 result = pselect(small_mask, x, result);
415 // The remaining cases have u <= 0, u = +inf or u = NaN.
416 if (!any_special) return result;
417
418 // Handle special cases.
419 // For u = +inf (x very large), return +inf.
420 Packet inf_mask = pcmp_eq(u, cst_pos_inf);
421 Packet neg_mask = pcmp_lt(u, pzero(u));
422 Packet zero_mask = pcmp_eq(x, pset1<Packet>(-1.0));
423 result = pselect(inf_mask, cst_pos_inf, result);
424 result = pselect(zero_mask, cst_minus_inf, result);
425 result = por(neg_mask, result); // NaN for x < -1
426 return result;
427}
428
433template <typename Packet>
434EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_log1p(const Packet& x) {
435 using ScalarType = typename unpacket_traits<Packet>::type;
436 const Packet one = pset1<Packet>(ScalarType(1));
437 Packet xp1 = padd(x, one);
438 Packet small_mask = pcmp_eq(xp1, one);
439 Packet log1 = plog(xp1);
440 Packet inf_mask = pcmp_eq(xp1, log1);
441 Packet log_large = pmul(x, pdiv(log1, psub(xp1, one)));
442 return pselect(por(small_mask, inf_mask), x, log_large);
443}
444
445// exp(r) for r in [-ln(2)/2, ln(2)/2], by a 6th order minimax polynomial.
446template <typename Packet>
447EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pexp_float_reduced(const Packet& r) {
448 const Packet cst_one = pset1<Packet>(1.0f);
449 const Packet cst_p2 = pset1<Packet>(0.49999988079071044921875f);
450 const Packet cst_p3 = pset1<Packet>(0.16666518151760101318359375f);
451 const Packet cst_p4 = pset1<Packet>(4.166965186595916748046875e-2f);
452 const Packet cst_p5 = pset1<Packet>(8.36894474923610687255859375e-3f);
453 const Packet cst_p6 = pset1<Packet>(1.37449637986719608306884765625e-3f);
454
455 const Packet r2 = pmul(r, r);
456 Packet p_even = pmadd(r2, cst_p6, cst_p4);
457 const Packet p_odd = pmadd(r2, cst_p5, cst_p3);
458 p_even = pmadd(r2, p_even, cst_p2);
459 const Packet p_low = padd(r, cst_one);
460 const Packet y = pmadd(r, p_odd, p_even);
461 return pmadd(r2, y, p_low);
462}
463
464// expm1(r) for r in [-ln(2)/2, ln(2)/2], as r + r^2 P(r). The 5th order polynomial P is Sollya's fpminimax for the
465// relative error of expm1(r), 2^-31; the kernel of pexp_float_reduced, fitted for exp(r), is off by 2^-25 here.
466template <typename Packet>
467EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pexpm1_float_reduced(const Packet& r) {
468 const Packet cst_p2 = pset1<Packet>(0.5f);
469 const Packet cst_p3 = pset1<Packet>(0.1666666567325592041015625f);
470 const Packet cst_p4 = pset1<Packet>(4.16664779186248779296875e-2f);
471 const Packet cst_p5 = pset1<Packet>(8.333566598594188690185546875e-3f);
472 const Packet cst_p6 = pset1<Packet>(1.393371261656284332275390625e-3f);
473 const Packet cst_p7 = pset1<Packet>(1.97434448637068271636962890625e-4f);
474
475 const Packet r2 = pmul(r, r);
476 Packet p_even = pmadd(r2, cst_p6, cst_p4);
477 Packet p_odd = pmadd(r2, cst_p7, cst_p5);
478 p_even = pmadd(r2, p_even, cst_p2);
479 p_odd = pmadd(r2, p_odd, cst_p3);
480 const Packet y = pmadd(r, p_odd, p_even);
481 return pmadd(r2, y, r);
482}
483
484// Exponential function. Works by writing "x = m*log(2) + r" where
485// "m = rint(x/log(2))" and "r" is the remainder. The result is then
486// "exp(x) = 2^m*exp(r)" where exp(r) is in the range [-1,1).
487// exp(r) is computed using a 6th order minimax polynomial approximation.
488template <typename Packet, bool IsFinite>
489EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp_float(const Packet _x) {
490 using PacketI = typename unpacket_traits<Packet>::integer_packet;
491
492 const Packet cst_exp_hi = pset1<Packet>(88.723f);
493 const Packet cst_exp_lo = pset1<Packet>(-104.f);
494
495 const Packet cst_cephes_LOG2EF = pset1<Packet>(1.44269504088896341f);
496
497 // Clamp x to prevent overflow/underflow.
498 Packet x = pmin(pmax(_x, cst_exp_lo), cst_exp_hi);
499
500 // Express exp(x) as exp(m*ln(2) + r), start by extracting
501 // m = rint(x/ln(2)).
502 Packet m = print(pmul(x, cst_cephes_LOG2EF));
503
504 // Get r = x - m*ln(2). m*ln(2) is subtracted out in two parts,
505 // m*C1+m*C2 = m*ln(2), to avoid accumulating truncation errors.
506 const Packet cst_cephes_exp_C1 = pset1<Packet>(-0.693359375f);
507 const Packet cst_cephes_exp_C2 = pset1<Packet>(2.12194440e-4f);
508 Packet r = pmadd(m, cst_cephes_exp_C1, x);
509 r = pmadd(m, cst_cephes_exp_C2, r);
510
511 Packet y = pexp_float_reduced(r);
512
513 // Construct the result y * 2^m via a 2-way exponent split. Writing
514 // 2^m = 2^floor(m/2) * 2^(m - floor(m/2))
515 // keeps each constructed power-of-two within the normal float range (since
516 // |m| <= 150 after the input clamp implies each half lies in [-75, 65]), so
517 // the IEEE-correct rounding of normal*normal handles the subnormal output
518 // range without special-casing.
519 //
520 // To minimize integer ops we fold the bias into the split:
521 // biased_sum = m + 254 (always nonnegative)
522 // biased_hi = biased_sum >> 1 = m/2 + 127
523 // biased_lo = biased_sum - biased_hi = (m - m/2) + 127
524 // so each <<23 directly yields the float bit pattern for 2^(m/2) and
525 // 2^(m - m/2).
526 const PacketI cst_double_bias = pset1<PacketI>(254);
527 const PacketI mi = pcast<Packet, PacketI>(m);
528 const PacketI biased_sum = padd(mi, cst_double_bias);
529 const PacketI biased_hi = plogical_shift_right<1>(biased_sum); // m/2 + 127
530 const PacketI biased_lo = psub(biased_sum, biased_hi); // (m - m/2) + 127
531 const Packet pow2_hi = preinterpret<Packet>(plogical_shift_left<23>(biased_hi));
532 const Packet pow2_lo = preinterpret<Packet>(plogical_shift_left<23>(biased_lo));
533 y = pmul(pmul(y, pow2_hi), pow2_lo);
534
535 EIGEN_IF_CONSTEXPR (!IsFinite) {
536 // Handle NaN: exp(nan) = nan. Use pmax to propagate NaN from input.
537 y = pmax(y, _x);
538 }
539 return y;
540}
541
542// exp(g) for g in [-ln(2)/2, ln(2)/2], by the rational interpolant exp(g) = 1 + 2 g P(g^2) / (Q(g^2) - g P(g^2)).
543template <typename Packet>
544EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pexp_double_reduced(const Packet& g) {
545 const Packet cst_1 = pset1<Packet>(1.0);
546 const Packet cst_2 = pset1<Packet>(2.0);
547 const Packet cst_cephes_exp_p0 = pset1<Packet>(1.26177193074810590878e-4);
548 const Packet cst_cephes_exp_p1 = pset1<Packet>(3.02994407707441961300e-2);
549 const Packet cst_cephes_exp_p2 = pset1<Packet>(9.99999999999999999910e-1);
550 const Packet cst_cephes_exp_q0 = pset1<Packet>(3.00198505138664455042e-6);
551 const Packet cst_cephes_exp_q1 = pset1<Packet>(2.52448340349684104192e-3);
552 const Packet cst_cephes_exp_q2 = pset1<Packet>(2.27265548208155028766e-1);
553 const Packet cst_cephes_exp_q3 = pset1<Packet>(2.00000000000000000009e0);
554
555 const Packet x2 = pmul(g, g);
556
557 // Evaluate the numerator polynomial of the rational interpolant.
558 Packet px = cst_cephes_exp_p0;
559 px = pmadd(px, x2, cst_cephes_exp_p1);
560 px = pmadd(px, x2, cst_cephes_exp_p2);
561 px = pmul(px, g);
562
563 // Evaluate the denominator polynomial of the rational interpolant.
564 Packet qx = cst_cephes_exp_q0;
565 qx = pmadd(qx, x2, cst_cephes_exp_q1);
566 qx = pmadd(qx, x2, cst_cephes_exp_q2);
567 qx = pmadd(qx, x2, cst_cephes_exp_q3);
568
569 // exp(g) = 1 + 2*px/(qx - px).
570 return pmadd(cst_2, pdiv(px, psub(qx, px)), cst_1);
571}
572
573// expm1(r) for r in [-ln(2)/2, ln(2)/2], as r + r^2 P(r). The 10th order polynomial P is Sollya's fpminimax for the
574// relative error of expm1(r), 2^-61. The rational interpolant of pexp_double_reduced without its leading 1 would lose
575// up to 5 ulp here, where no 1 hides the rounding errors of its quotient.
576template <typename Packet>
577EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet pexpm1_double_reduced(const Packet& r) {
578 const Packet cst_p2 = pset1<Packet>(0.5);
579 const Packet cst_p3 = pset1<Packet>(0.16666666666666677);
580 const Packet cst_p4 = pset1<Packet>(0.04166666666666661);
581 const Packet cst_p5 = pset1<Packet>(0.008333333333321436);
582 const Packet cst_p6 = pset1<Packet>(0.0013888888888923816);
583 const Packet cst_p7 = pset1<Packet>(0.00019841269886601229);
584 const Packet cst_p8 = pset1<Packet>(2.480158723753082e-05);
585 const Packet cst_p9 = pset1<Packet>(2.7557243734579987e-06);
586 const Packet cst_p10 = pset1<Packet>(2.7557352849435147e-07);
587 const Packet cst_p11 = pset1<Packet>(2.510920369520403e-08);
588 const Packet cst_p12 = pset1<Packet>(2.0889530999828774e-09);
589
590 // Even and odd powers in two chains for a shorter dependency chain.
591 const Packet r2 = pmul(r, r);
592 Packet p_even = pmadd(r2, cst_p12, cst_p10);
593 Packet p_odd = pmadd(r2, cst_p11, cst_p9);
594 p_even = pmadd(r2, p_even, cst_p8);
595 p_odd = pmadd(r2, p_odd, cst_p7);
596 p_even = pmadd(r2, p_even, cst_p6);
597 p_odd = pmadd(r2, p_odd, cst_p5);
598 p_even = pmadd(r2, p_even, cst_p4);
599 p_odd = pmadd(r2, p_odd, cst_p3);
600 p_even = pmadd(r2, p_even, cst_p2);
601 const Packet y = pmadd(r, p_odd, p_even);
602 return pmadd(r2, y, r);
603}
604
605template <typename Packet>
606EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp_double(const Packet _x) {
607 const Packet cst_exp_hi = pset1<Packet>(709.784);
608 const Packet cst_exp_lo = pset1<Packet>(-745.519);
609 const Packet cst_pldexp_threshold = pset1<Packet>(708.0);
610 const Packet cst_cephes_LOG2EF = pset1<Packet>(1.4426950408889634073599);
611 const Packet cst_cephes_exp_C1 = pset1<Packet>(0.693145751953125);
612 const Packet cst_cephes_exp_C2 = pset1<Packet>(1.42860682030941723212e-6);
613
614 // Clamp x.
615 Packet zero_mask = pcmp_lt(_x, cst_exp_lo);
616 Packet x = pmin(_x, cst_exp_hi);
617
618 // Express exp(x) as exp(g + n*log(2)).
619 // n = rint(x / ln(2)).
620 Packet fx = print(pmul(x, cst_cephes_LOG2EF));
621
622 // Get the remainder modulo log(2), i.e. the "g" described above. Subtract
623 // n*log(2) out in two steps, i.e. n*C1 + n*C2, C1+C2=log2 to get the last
624 // digits right.
625 x = pnmadd(fx, cst_cephes_exp_C1, x);
626 x = pnmadd(fx, cst_cephes_exp_C2, x);
627
628 x = pexp_double_reduced(x);
629
630 // Construct the result 2^n * exp(g) = e * x. The max is used to catch
631 // non-finite values in the input.
632 const Packet fast_pldexp_unsafe = pcmp_lt(cst_pldexp_threshold, pabs(_x));
633 if (!predux_any(fast_pldexp_unsafe)) {
634 // For |x| <= 708, we know the result is not zero or inf, and we can safely use
635 // the fast version of pldexp.
636 return pmax(pldexp_fast(x, fx), _x);
637 }
638 return pandnot(pmax(pldexp(x, fx), _x), zero_mask);
639}
640
641template <typename Packet, typename Scalar = typename unpacket_traits<Packet>::type>
642struct pexp_reduced_impl;
643template <typename Packet>
644struct pexp_reduced_impl<Packet, float> {
645 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& r) { return pexp_float_reduced(r); }
646};
647template <typename Packet>
648struct pexp_reduced_impl<Packet, double> {
649 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& r) { return pexp_double_reduced(r); }
650};
651
652// The reduced-argument kernel of expm1 and the two parts of ln(2) of its Cody-Waite reduction, those of pexp_float and
653// pexp_double.
654template <typename Packet, typename Scalar = typename unpacket_traits<Packet>::type>
655struct pexpm1_reduced_impl;
656template <typename Packet>
657struct pexpm1_reduced_impl<Packet, float> {
658 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& r) { return pexpm1_float_reduced(r); }
659 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float ln2_hi() { return 0.693359375f; }
660 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float ln2_lo() { return -2.12194440e-4f; }
661};
662template <typename Packet>
663struct pexpm1_reduced_impl<Packet, double> {
664 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run(const Packet& r) { return pexpm1_double_reduced(r); }
665 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double ln2_hi() { return 0.693145751953125; }
666 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double ln2_lo() { return 1.42860682030941723212e-6; }
667};
668
674template <typename Packet>
675EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_expm1(const Packet& _x) {
676 using Scalar = typename unpacket_traits<Packet>::type;
677 using Impl = pexpm1_reduced_impl<Packet>;
678 constexpr int max_exponent = std::numeric_limits<Scalar>::max_exponent;
679 constexpr int digits = std::numeric_limits<Scalar>::digits;
680 const Packet one = pset1<Packet>(Scalar(1));
681 // Above max_cap the result is inf, and below min_cap it rounds to -1.
682 const Scalar max_cap = Scalar(max_exponent + 1) * Scalar(EIGEN_LN2);
683 const Scalar min_cap = -Scalar(digits + 2) * Scalar(EIGEN_LN2);
684 const Packet x = pmax(pmin(_x, pset1<Packet>(max_cap)), pset1<Packet>(min_cap));
685
686 const Packet n = print(pmul(x, pset1<Packet>(Scalar(EIGEN_LOG2E))));
687 Packet r = pnmadd(n, pset1<Packet>(Impl::ln2_hi()), x);
688 r = pnmadd(n, pset1<Packet>(Impl::ln2_lo()), r);
689 const Packet em1 = Impl::run(r);
690
691 // c = 1 - 2^-n, exact while it has at most `digits` bits. Beyond that, 2^-n is below half an ulp of 1 for a
692 // positive n, hence the cap, and 1 is below half an ulp of 2^-n for a negative one.
693 const Packet c = psub(one, pldexp_fast(one, pnegate(pmin(n, pset1<Packet>(Scalar(digits + 2))))));
694 // 2^n (em1 + c) in two steps: 2^n itself overflows for the largest n. The barrier keeps -ffast-math from forming
695 // 2^(n - 1) 2, which overflows for n = max_exponent.
696 Packet y = pldexp_fast(padd(em1, c), psub(n, one));
697 pbarrier(y);
698 y = pmul(y, pset1<Packet>(Scalar(2)));
699 // Below epsilon / 4 in magnitude, x is the result to a quarter of an ulp; returning it keeps the sign of zero and
700 // the subnormals.
701 y = pselect(pcmp_lt(pabs(_x), pset1<Packet>(Scalar(0.25) * NumTraits<Scalar>::epsilon())), _x, y);
702 // A NaN input propagates through the clamp or this max, as in pexp, depending on the NaN convention of pmin and
703 // pmax. expm1(x) >= x, so otherwise the max only replaces a y that rounding left below x, with the closer x.
704 return pmax(y, _x);
705}
706
712template <typename Packet>
713EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_half_exp(const Packet& _x) {
714 using Scalar = typename unpacket_traits<Packet>::type;
715 using Impl = pexpm1_reduced_impl<Packet>;
716 constexpr int max_exponent = std::numeric_limits<Scalar>::max_exponent;
717 // Above max_cap the result is inf.
718 const Scalar max_cap = Scalar(max_exponent + 2) * Scalar(EIGEN_LN2);
719 const Packet x = pmin(_x, pset1<Packet>(max_cap));
720
721 const Packet n = print(pmul(x, pset1<Packet>(Scalar(EIGEN_LOG2E))));
722 Packet r = pnmadd(n, pset1<Packet>(Impl::ln2_hi()), x);
723 r = pnmadd(n, pset1<Packet>(Impl::ln2_lo()), r);
724 // 2^(n - 1) exp(r) in two steps: 2^(n - 1) itself overflows for the largest n. The barrier keeps -ffast-math from
725 // forming 2^(n - 2) 2.
726 Packet y = pldexp_fast(pexp_reduced_impl<Packet>::run(r), psub(n, pset1<Packet>(Scalar(2))));
727 pbarrier(y);
728 y = pmul(y, pset1<Packet>(Scalar(2)));
729 // A NaN input propagates through the clamp or this max, as in pexp, depending on the NaN convention of pmin and
730 // pmax. exp(x) / 2 > x, so otherwise the max changes nothing.
731 return pmax(y, _x);
732}
733
734// 2^f for |f| <= 1/2: f ln(2) = r_hi + r_lo with ln(2) = ln2_hi + ln2_lo, and exp(r) = exp(r_hi) (1 + r_lo) +
735// O(r_lo^2).
736template <typename Packet>
737EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_exp2_reduced(const Packet& f) {
738 using Scalar = typename unpacket_traits<Packet>::type;
739 // ln(2) - ln2_hi, from (double(ln(2)) - ln2_hi) exact in double plus ln(2) - double(ln(2)).
740 const Scalar ln2_hi = Scalar(EIGEN_LN2);
741 const Scalar ln2_lo = Scalar((static_cast<double>(EIGEN_LN2) - double(ln2_hi)) + 2.3190468138462996e-17);
742 Packet r_hi, r_lo;
743 twoprod(pset1<Packet>(ln2_hi), f, r_hi, r_lo);
744 r_lo = pmadd(pset1<Packet>(ln2_lo), f, r_lo);
745 const Packet e = pexp_reduced_impl<Packet>::run(r_hi);
746 return pmadd(e, r_lo, e);
747}
748
749// exp2(x) = 2^n 2^f with n = rint(x) and f = x - n, which is exact with |f| <= 1/2. So exp2(n) = 2^n exactly, and
750// neither the reduction nor the scaling adds an error that grows with |x|.
751template <typename Packet>
752EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet generic_exp2(const Packet& _x) {
753 using Scalar = typename unpacket_traits<Packet>::type;
754 constexpr int max_exponent = std::numeric_limits<Scalar>::max_exponent;
755 constexpr int digits = std::numeric_limits<Scalar>::digits;
756 // Beyond these, the result is inf or rounds to 0; within them, n / 2 and n - n / 2 are normal exponents.
757 constexpr Scalar max_cap = Scalar(max_exponent + 1);
758 constexpr Scalar min_cap = -Scalar(max_exponent + digits - 1);
759 const Packet x = pmax(pmin(_x, pset1<Packet>(max_cap)), pset1<Packet>(min_cap));
760 const Packet n = print(x);
761 const Packet y = generic_exp2_reduced(psub(x, n));
762 // One pldexp_fast while 2^n is normal. Near the ends of the range, y 2^n = (y 2^h) 2^(n - h) with h = rint(n / 2):
763 // both powers are normal and the first product is exact, so only the last one rounds, also into the subnormal range.
764 // The barrier keeps -ffast-math from forming 2^h 2^(n - h), which overflows for n = max_exponent.
765 if (!predux_any(pcmp_lt(pset1<Packet>(Scalar(max_exponent - 2)), pabs(n)))) return pldexp_fast(y, n);
766 const Packet h = print(pmul(n, pset1<Packet>(Scalar(0.5))));
767 Packet out = pldexp_fast(y, h);
768 pbarrier(out);
769 return pldexp_fast(out, psub(n, h));
770}
771
777template <typename Packet>
778EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog10_float(const Packet& x) {
779 // log10(e) in higher precision, split into hi+lo so log_x*(hi+lo) is reconstructed via FMA.
780 // hi = round-to-nearest-float of log10(e); lo = float(log10(e) - hi).
781 const Packet cst_log10e_hi = pset1<Packet>(0.4342944920063018f);
782 const Packet cst_log10e_lo = pset1<Packet>(-1.0103049952192578e-08f);
783 const Packet cst_inf = pinf<Packet>();
784 const Packet cst_zero = pzero(x);
785
786 const Packet log_x = plog(x);
787 const Packet finite_mask = pcmp_lt(pabs(log_x), cst_inf);
788 const Packet finite_log_x = pselect(finite_mask, log_x, cst_zero);
789 const Packet split_log10_x = pmadd(finite_log_x, cst_log10e_hi, pmul(finite_log_x, cst_log10e_lo));
790 return pselect(finite_mask, split_log10_x, log_x);
791}
792
796template <typename Packet>
797EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog10_double(const Packet& x) {
798 const Packet cst_log10e = pset1<Packet>(0.43429448190325182);
799 return pmul(plog(x), cst_log10e);
800}
801
802} // end namespace internal
803} // end namespace Eigen
804
805// Include the split-out sections. Order matters: Pow depends on exp/log and FrexpLdexp,
806// Trig depends on exp (for ptanh_float), Complex depends on Trig (for psincos_selector).
807#include "GenericPacketMathPow.h"
808#include "GenericPacketMathTrig.h"
809#include "GenericPacketMathComplex.h"
810
811namespace Eigen {
812namespace internal {
813
814//----------------------------------------------------------------------
815// Sign Function
816//----------------------------------------------------------------------
817
818template <typename Packet, typename Scalar = typename unpacket_traits<Packet>::type>
819struct psign_uses_binary_encoding
820 : bool_constant<!is_scalar<Packet>::value && packet_has_integer_packet<Packet>::value &&
821 (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value)> {};
822
823template <typename Packet>
824struct psign_impl<Packet, std::enable_if_t<psign_uses_binary_encoding<Packet>::value>> {
825 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) {
826 using Scalar = typename unpacket_traits<Packet>::type;
827 using PacketI = typename unpacket_traits<Packet>::integer_packet;
828 using IntScalar = typename unpacket_traits<PacketI>::type;
829 using Binary = binary_floating_point_traits<Scalar>;
830
831 // Magnitude is a nonnegative integer. Comparing it with 1 avoids Clang
832 // folding an integer == 0 test into a DAZ-sensitive FP comparison.
833 const PacketI magnitude = pand(preinterpret<PacketI>(a), pset1<PacketI>(IntScalar(~Binary::kSignBit)));
834 const Packet is_zero = preinterpret<Packet>(pcmp_lt(magnitude, pset1<PacketI>(IntScalar(1))));
835 const Packet is_nan = preinterpret<Packet>(pcmp_lt(pset1<PacketI>(IntScalar(Binary::kExponentMask)), magnitude));
836 const Packet signed_one = por(pandnot(a, pabs(a)), pset1<Packet>(Scalar(1)));
837 return pselect(is_nan, a, pandnot(signed_one, is_zero));
838 }
839};
840
841template <typename Packet>
842struct psign_impl<Packet, std::enable_if_t<!psign_uses_binary_encoding<Packet>::value && !is_scalar<Packet>::value &&
843 !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex &&
844 !NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>> {
845 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) {
846 using Scalar = typename unpacket_traits<Packet>::type;
847 const Packet cst_one = pset1<Packet>(Scalar(1));
848
849 const Packet abs_a = pabs(a);
850 const Packet sign_mask = pandnot(a, abs_a);
851 const Packet zero_mask = pcmp_eq(abs_a, pzero(abs_a));
852
853 return pselect(pisnan(a), a, pselect(zero_mask, abs_a, por(sign_mask, cst_one)));
854 }
855};
856
857template <typename Packet>
858struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value &&
859 !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex &&
860 NumTraits<typename unpacket_traits<Packet>::type>::IsSigned &&
861 NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>> {
862 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) {
863 using Scalar = typename unpacket_traits<Packet>::type;
864 const Packet cst_one = pset1<Packet>(Scalar(1));
865 const Packet cst_minus_one = pset1<Packet>(Scalar(-1));
866 const Packet cst_zero = pzero(a);
867
868 const Packet positive_mask = pcmp_lt(cst_zero, a);
869 const Packet positive = pand(positive_mask, cst_one);
870 const Packet negative_mask = pcmp_lt(a, cst_zero);
871 const Packet negative = pand(negative_mask, cst_minus_one);
872
873 return por(positive, negative);
874 }
875};
876
877template <typename Packet>
878struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value &&
879 !NumTraits<typename unpacket_traits<Packet>::type>::IsComplex &&
880 !NumTraits<typename unpacket_traits<Packet>::type>::IsSigned &&
881 NumTraits<typename unpacket_traits<Packet>::type>::IsInteger>> {
882 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) {
883 using Scalar = typename unpacket_traits<Packet>::type;
884 const Packet cst_one = pset1<Packet>(Scalar(1));
885 const Packet cst_zero = pzero(a);
886
887 const Packet zero_mask = pcmp_eq(cst_zero, a);
888 return pandnot(cst_one, zero_mask);
889 }
890};
891
892// \internal \returns the sign of a complex number z, defined as z / abs(z).
893template <typename Packet>
894struct psign_impl<Packet, std::enable_if_t<!is_scalar<Packet>::value &&
895 NumTraits<typename unpacket_traits<Packet>::type>::IsComplex &&
896 unpacket_traits<Packet>::vectorizable>> {
897 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) {
898 using Scalar = typename unpacket_traits<Packet>::type;
899 using RealScalar = typename Scalar::value_type;
900 using RealPacket = typename unpacket_traits<Packet>::as_real;
901
902 // Step 1. Compute (for each element z = x + i*y in a)
903 // l = abs(z) = sqrt(x^2 + y^2).
904 // To avoid over- and underflow, we use the stable formula for each hypotenuse
905 // l = (zmin == 0 ? zmax : zmax * sqrt(1 + (zmin/zmax)**2)),
906 // where zmax = max(|x|, |y|), zmin = min(|x|, |y|),
907 RealPacket a_abs = pabs(a.v);
908 RealPacket a_abs_flip = pcplxflip(Packet(a_abs)).v;
909 RealPacket a_max = pmax(a_abs, a_abs_flip);
910 RealPacket a_min = pmin(a_abs, a_abs_flip);
911 RealPacket a_min_zero_mask = pcmp_eq(a_min, pzero(a_min));
912 RealPacket a_max_zero_mask = pcmp_eq(a_max, pzero(a_max));
913 RealPacket r = pdiv(a_min, a_max);
914 const RealPacket cst_one = pset1<RealPacket>(RealScalar(1));
915 RealPacket l = pmul(a_max, psqrt(pmadd(r, r, cst_one))); // [l0, l0, l1, l1]
916 // Set l to a_max if a_min is zero, since the roundtrip sqrt(a_max^2) may be
917 // lossy.
918 l = pselect(a_min_zero_mask, a_max, l);
919 // Step 2 compute a / abs(a).
920 RealPacket sign_as_real = pandnot(pdiv(a.v, l), a_max_zero_mask);
921 Packet sign;
922 sign.v = sign_as_real;
923 return sign;
924 }
925};
926
927//----------------------------------------------------------------------
928// Rounding Functions
929//----------------------------------------------------------------------
930
931template <typename Packet>
932EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet generic_rint(const Packet& a) {
933 using Scalar = typename unpacket_traits<Packet>::type;
934 using IntType = typename numext::get_integer_by_size<sizeof(Scalar)>::signed_type;
935 // Adds and subtracts signum(a) * 2^kMantissaBits to force rounding.
936 const IntType kLimit = IntType(1) << (NumTraits<Scalar>::digits() - 1);
937 const Packet cst_limit = pset1<Packet>(static_cast<Scalar>(kLimit));
938 Packet abs_a = pabs(a);
939 Packet sign_a = pandnot(a, abs_a);
940 Packet rint_a = padd(abs_a, cst_limit);
941 // Don't compile-away addition and subtraction.
942 pbarrier(rint_a);
943 rint_a = psub(rint_a, cst_limit);
944 rint_a = por(rint_a, sign_a);
945 // If greater than limit (or NaN), simply return a.
946 Packet mask = pcmp_lt(abs_a, cst_limit);
947 Packet result = pselect(mask, rint_a, a);
948 return result;
949}
950
951template <typename Packet>
952EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet generic_floor(const Packet& a) {
953 using Scalar = typename unpacket_traits<Packet>::type;
954 const Packet cst_1 = pset1<Packet>(Scalar(1));
955 Packet rint_a = generic_rint(a);
956 // if a < rint(a), then rint(a) == ceil(a)
957 Packet mask = pcmp_lt(a, rint_a);
958 Packet offset = pand(cst_1, mask);
959 Packet result = psub(rint_a, offset);
960 return result;
961}
962
963template <typename Packet>
964EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet generic_ceil(const Packet& a) {
965 using Scalar = typename unpacket_traits<Packet>::type;
966 const Packet cst_1 = pset1<Packet>(Scalar(1));
967 const Packet sign_mask = psignmask<Packet>();
968 Packet rint_a = generic_rint(a);
969 // if rint(a) < a, then rint(a) == floor(a)
970 Packet mask = pcmp_lt(rint_a, a);
971 Packet offset = pand(cst_1, mask);
972 Packet result = padd(rint_a, offset);
973 // Signed zero must remain signed (e.g. ceil(-0.02) == -0).
974 result = por(result, pand(sign_mask, a));
975 return result;
976}
977
978template <typename Packet>
979EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet generic_trunc(const Packet& a) {
980 Packet abs_a = pabs(a);
981 Packet sign_a = pandnot(a, abs_a);
982 Packet floor_abs_a = generic_floor(abs_a);
983 Packet result = por(floor_abs_a, sign_a);
984 return result;
985}
986
987template <typename Packet>
988EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet generic_round(const Packet& a) {
989 using Scalar = typename unpacket_traits<Packet>::type;
990 const Packet cst_half = pset1<Packet>(Scalar(0.5));
991 const Packet cst_1 = pset1<Packet>(Scalar(1));
992 Packet abs_a = pabs(a);
993 Packet sign_a = pandnot(a, abs_a);
994 Packet floor_abs_a = generic_floor(abs_a);
995 Packet diff = psub(abs_a, floor_abs_a);
996 Packet mask = pcmp_le(cst_half, diff);
997 Packet offset = pand(cst_1, mask);
998 Packet result = padd(floor_abs_a, offset);
999 result = por(result, sign_a);
1000 return result;
1001}
1002
1003template <typename Packet>
1004struct nearest_integer_packetop_impl<Packet, /*IsScalar*/ false, /*IsInteger*/ false> {
1005 using Scalar = typename unpacket_traits<Packet>::type;
1006 static_assert(packet_traits<Scalar>::HasRound, "Generic nearest integer functions are disabled for this type.");
1007 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_floor(const Packet& x) { return generic_floor(x); }
1008 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_ceil(const Packet& x) { return generic_ceil(x); }
1009 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_rint(const Packet& x) { return generic_rint(x); }
1010 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_round(const Packet& x) { return generic_round(x); }
1011 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_trunc(const Packet& x) { return generic_trunc(x); }
1012};
1013
1014template <typename Packet>
1015struct nearest_integer_packetop_impl<Packet, /*IsScalar*/ false, /*IsInteger*/ true> {
1016 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_floor(const Packet& x) { return x; }
1017 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_ceil(const Packet& x) { return x; }
1018 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_rint(const Packet& x) { return x; }
1019 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_round(const Packet& x) { return x; }
1020 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet run_trunc(const Packet& x) { return x; }
1021};
1022
1023} // end namespace internal
1024} // end namespace Eigen
1025
1026#endif // EIGEN_ARCH_GENERIC_PACKET_MATH_FUNCTIONS_H