Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
MathFunctions.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2006-2010 Benoit Jacob <jacob.benoit.1@gmail.com>
5// Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_MATHFUNCTIONS_H
13#define EIGEN_MATHFUNCTIONS_H
14
15// TODO: consider moving these constants to NumTraits.
16// Source: WolframAlpha
17#define EIGEN_PI 3.141592653589793238462643383279502884197169399375105820974944592307816406L
18#define EIGEN_LOG2E 1.442695040888963407359924681001892137426645954152985934135449406931109219L
19#define EIGEN_LN2 0.693147180559945309417232121458176568075500134360255254120680009493393621L
20
21// IWYU pragma: private
22#include "./InternalHeaderCheck.h"
23
24namespace Eigen {
25
26namespace internal {
27
48
49template <typename T, typename dummy = void>
50struct global_math_functions_filtering_base {
51 using type = T;
52};
53
54template <typename T>
55struct global_math_functions_filtering_base<T,
56 void_t<typename T::Eigen_BaseClassForSpecializationOfGlobalMathFuncImpl>> {
57 using type = typename T::Eigen_BaseClassForSpecializationOfGlobalMathFuncImpl;
58};
59
60#define EIGEN_MATHFUNC_IMPL(func, scalar) \
61 Eigen::internal::func##_impl<typename Eigen::internal::global_math_functions_filtering_base<scalar>::type>
62
63/****************************************************************************
64 * Implementation of real *
65 ****************************************************************************/
66
67template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex>
68struct real_default_impl {
69 using RealScalar = typename NumTraits<Scalar>::Real;
70 EIGEN_DEVICE_FUNC static constexpr RealScalar run(const Scalar& x) { return x; }
71};
72
73template <typename Scalar>
74struct real_default_impl<Scalar, true> {
75 using RealScalar = typename NumTraits<Scalar>::Real;
76 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
77 using std::real;
78 return real(x);
79 }
80};
81
82template <typename Scalar>
83struct real_impl : real_default_impl<Scalar> {};
84
85#if defined(EIGEN_GPU_COMPILE_PHASE)
86template <typename T>
87struct real_impl<std::complex<T>> {
88 using RealScalar = T;
89 EIGEN_DEVICE_FUNC static inline T run(const std::complex<T>& x) { return x.real(); }
90};
91#endif
92
93/****************************************************************************
94 * Implementation of imag *
95 ****************************************************************************/
96
97template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex>
98struct imag_default_impl {
99 using RealScalar = typename NumTraits<Scalar>::Real;
100 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar&) { return RealScalar(0); }
101};
102
103template <typename Scalar>
104struct imag_default_impl<Scalar, true> {
105 using RealScalar = typename NumTraits<Scalar>::Real;
106 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
107 using std::imag;
108 return imag(x);
109 }
110};
111
112template <typename Scalar>
113struct imag_impl : imag_default_impl<Scalar> {};
114
115#if defined(EIGEN_GPU_COMPILE_PHASE)
116template <typename T>
117struct imag_impl<std::complex<T>> {
118 using RealScalar = T;
119 EIGEN_DEVICE_FUNC static inline T run(const std::complex<T>& x) { return x.imag(); }
120};
121#endif
122
123/****************************************************************************
124 * Implementation of real_ref *
125 ****************************************************************************/
126
127template <typename Scalar>
128struct real_ref_impl {
129 using RealScalar = typename NumTraits<Scalar>::Real;
130 EIGEN_DEVICE_FUNC static inline RealScalar& run(Scalar& x) { return reinterpret_cast<RealScalar*>(&x)[0]; }
131 EIGEN_DEVICE_FUNC static inline const RealScalar& run(const Scalar& x) {
132 return reinterpret_cast<const RealScalar*>(&x)[0];
133 }
134};
135
136/****************************************************************************
137 * Implementation of imag_ref *
138 ****************************************************************************/
139
140template <typename Scalar, bool IsComplex>
141struct imag_ref_default_impl {
142 using RealScalar = typename NumTraits<Scalar>::Real;
143 EIGEN_DEVICE_FUNC static inline RealScalar& run(Scalar& x) { return reinterpret_cast<RealScalar*>(&x)[1]; }
144 EIGEN_DEVICE_FUNC static inline const RealScalar& run(const Scalar& x) {
145 return reinterpret_cast<const RealScalar*>(&x)[1];
146 }
147};
148
149template <typename Scalar>
150struct imag_ref_default_impl<Scalar, false> {
151 using RealScalar = typename NumTraits<Scalar>::Real;
152 EIGEN_DEVICE_FUNC constexpr static inline RealScalar run(Scalar&) { return RealScalar(0); }
153 EIGEN_DEVICE_FUNC constexpr static inline RealScalar run(const Scalar&) { return RealScalar(0); }
154};
155
156template <typename Scalar>
157struct imag_ref_impl : imag_ref_default_impl<Scalar, NumTraits<Scalar>::IsComplex> {};
158
159} // namespace internal
160
161namespace numext {
162
163template <typename Scalar>
164EIGEN_DEVICE_FUNC inline decltype(auto) real(const Scalar& x) {
165 return EIGEN_MATHFUNC_IMPL(real, Scalar)::run(x);
166}
167
168template <typename Scalar>
169EIGEN_DEVICE_FUNC inline auto real_ref(const Scalar& x)
170 -> internal::add_const_on_value_type_t<decltype(internal::real_ref_impl<Scalar>::run(x))> {
171 return internal::real_ref_impl<Scalar>::run(x);
172}
173
174template <typename Scalar>
175EIGEN_DEVICE_FUNC inline auto real_ref(Scalar& x)
176 -> decltype(EIGEN_MATHFUNC_IMPL(real_ref, std::remove_const_t<Scalar>)::run(x)) {
177 return EIGEN_MATHFUNC_IMPL(real_ref, std::remove_const_t<Scalar>)::run(x);
178}
179
180template <typename Scalar>
181EIGEN_DEVICE_FUNC inline decltype(auto) imag(const Scalar& x) {
182 return EIGEN_MATHFUNC_IMPL(imag, Scalar)::run(x);
183}
184
185template <typename Scalar>
186EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar select(const Scalar& mask, const Scalar& a, const Scalar& b) {
187 return numext::is_exactly_zero(mask) ? b : a;
188}
189
190} // namespace numext
191
192namespace internal {
193
194/****************************************************************************
195 * Implementation of conj *
196 ****************************************************************************/
197
198template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex>
199struct conj_default_impl {
200 EIGEN_DEVICE_FUNC static constexpr Scalar run(const Scalar& x) { return x; }
201};
202
203template <typename Scalar>
204struct conj_default_impl<Scalar, true> {
205 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) {
206 using std::conj;
207 return conj(x);
208 }
209};
210
211template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex>
212struct conj_impl : conj_default_impl<Scalar, IsComplex> {};
213
214/****************************************************************************
215 * Implementation of mul *
216 ****************************************************************************/
217
218// Unsigned operands narrower than int promote to int, where a product whose wrapped value is representable
219// can still overflow: (unsigned short)0xffff squared is 0xfffe0001 > INT_MAX. Multiplying in unsigned int,
220// wider than every such operand, wraps instead. Mixed operand types, signed types, and bool keep operator*.
221template <typename Lhs, typename Rhs,
222 bool NarrowUnsigned = std::is_same<Lhs, Rhs>::value && std::is_unsigned<Lhs>::value &&
223 !std::is_same<Lhs, bool>::value && (sizeof(Lhs) < sizeof(int))>
224struct mul_impl {
225 EIGEN_DEVICE_FUNC static constexpr EIGEN_ALWAYS_INLINE auto run(const Lhs& a, const Rhs& b) { return a * b; }
226};
227
228template <typename Scalar>
229struct mul_impl<Scalar, Scalar, true> {
230 EIGEN_DEVICE_FUNC static constexpr EIGEN_ALWAYS_INLINE Scalar run(const Scalar& a, const Scalar& b) {
231 return static_cast<Scalar>(static_cast<unsigned int>(a) * static_cast<unsigned int>(b));
232 }
233};
234
235// MSVC's STL evaluates T * complex<T> as complex<T>(a) * b, whose 0 * inf terms turn 2 * (inf, 2) into
236// (inf, NaN). Scale component-wise, as complex<T> * T, libstdc++, libc++, and the GPU overloads do.
237template <typename RealScalar>
238struct mul_impl<RealScalar, std::complex<RealScalar>, false> {
239 EIGEN_DEVICE_FUNC static constexpr EIGEN_ALWAYS_INLINE std::complex<RealScalar> run(
240 const RealScalar& a, const std::complex<RealScalar>& b) {
241 return std::complex<RealScalar>(a * b.real(), a * b.imag());
242 }
243};
244
248template <typename Lhs, typename Rhs>
249EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE auto mul(const Lhs& a, const Rhs& b) {
250 return mul_impl<Lhs, Rhs>::run(a, b);
251}
252
253/****************************************************************************
254 * Implementation of abs2 *
255 ****************************************************************************/
256
257template <typename Scalar, bool IsComplex>
258struct abs2_impl_default {
259 using RealScalar = typename NumTraits<Scalar>::Real;
260 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) { return internal::mul(x, x); }
261};
262
263template <typename Scalar>
264struct abs2_impl_default<Scalar, true> // IsComplex
265{
266 using RealScalar = typename NumTraits<Scalar>::Real;
267 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
268 return numext::real(x) * numext::real(x) + numext::imag(x) * numext::imag(x);
269 }
270};
271
272template <typename Scalar>
273struct abs2_impl {
274 using RealScalar = typename NumTraits<Scalar>::Real;
275 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
276 return abs2_impl_default<Scalar, NumTraits<Scalar>::IsComplex>::run(x);
277 }
278};
279
280/****************************************************************************
281 * Implementation of sqrt/rsqrt *
282 ****************************************************************************/
283
284template <typename Scalar>
285struct sqrt_impl {
286 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE Scalar run(const Scalar& x) {
287 EIGEN_USING_STD(sqrt);
288 return sqrt(x);
289 }
290};
291
292// Complex sqrt defined in MathFunctionsImpl.h.
293template <typename ComplexT>
294EIGEN_DEVICE_FUNC constexpr ComplexT complex_sqrt(const ComplexT& a_x);
295
296// Custom implementation is faster than `std::sqrt`, works on
297// GPU, and correctly handles special cases (unlike MSVC).
298template <typename T>
299struct sqrt_impl<std::complex<T>> {
300 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE std::complex<T> run(const std::complex<T>& x) { return complex_sqrt(x); }
301};
302
303// Default implementation relies on numext::sqrt, at bottom of file.
304template <typename T>
305struct rsqrt_impl;
306
307// Complex rsqrt defined in MathFunctionsImpl.h.
308template <typename ComplexT>
309EIGEN_DEVICE_FUNC constexpr ComplexT complex_rsqrt(const ComplexT& a_x);
310
311template <typename T>
312struct rsqrt_impl<std::complex<T>> {
313 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE std::complex<T> run(const std::complex<T>& x) {
314 return complex_rsqrt(x);
315 }
316};
317
318/****************************************************************************
319 * Implementation of norm1 *
320 ****************************************************************************/
321
322template <typename Scalar, bool IsComplex>
323struct norm1_default_impl;
324
325template <typename Scalar>
326struct norm1_default_impl<Scalar, true> {
327 using RealScalar = typename NumTraits<Scalar>::Real;
328 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
329 EIGEN_USING_STD(abs);
330 return abs(numext::real(x)) + abs(numext::imag(x));
331 }
332};
333
334template <typename Scalar>
335struct norm1_default_impl<Scalar, false> {
336 using RealScalar = typename NumTraits<Scalar>::Real;
337 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
338 EIGEN_USING_STD(abs);
339 return abs(x);
340 }
341};
342
343template <typename Scalar>
344struct norm1_impl : norm1_default_impl<Scalar, NumTraits<Scalar>::IsComplex> {};
345
346/****************************************************************************
347 * Implementation of hypot *
348 ****************************************************************************/
349
350template <typename Scalar>
351struct hypot_impl;
352
353/****************************************************************************
354 * Implementation of cast *
355 ****************************************************************************/
356
357template <typename OldType, typename NewType, typename EnableIf = void>
358struct cast_impl {
359 EIGEN_DEVICE_FUNC static inline NewType run(const OldType& x) { return static_cast<NewType>(x); }
360};
361
362template <typename OldType>
363struct cast_impl<OldType, bool> {
364 EIGEN_DEVICE_FUNC static inline bool run(const OldType& x) { return x != OldType(0); }
365};
366
367// Casting from S -> Complex<T> leads to an implicit conversion from S to T,
368// generating warnings on clang. Here we explicitly cast the real component.
369template <typename OldType, typename NewType>
370struct cast_impl<OldType, NewType, std::enable_if_t<!NumTraits<OldType>::IsComplex && NumTraits<NewType>::IsComplex>> {
371 EIGEN_DEVICE_FUNC static inline NewType run(const OldType& x) {
372 using NewReal = typename NumTraits<NewType>::Real;
373 return static_cast<NewType>(cast_impl<OldType, NewReal>::run(x));
374 }
375};
376
377#if EIGEN_ARCH_ARM64 && EIGEN_COMP_CLANG && !EIGEN_CLANG_STRICT_AT_LEAST(21, 0, 0)
378// Clang before 21, and the Clang forks whose version does not say, vectorize a loop of 64-bit integer to float
379// conversions through double on AArch64, which rounds twice. Rounding x to odd on the multiples of 2^11 where
380// |x| >= 2^53 makes the conversion to double exact and leaves a direct conversion unchanged.
381template <typename OldType>
382struct cast_impl<OldType, float, std::enable_if_t<std::is_integral<OldType>::value && sizeof(OldType) == 8>> {
383 EIGEN_DEVICE_FUNC static inline float run(const OldType& x) {
384 using U = std::make_unsigned_t<OldType>;
385 const U u = static_cast<U>(x);
386 // In two's complement, clearing the low bits rounds toward -inf for either sign; setting bit 11 then picks the odd
387 // neighbor.
388 const U low = 0x7ff;
389 const U odd = (u & ~low) | ((u & low) != 0 ? U(0x800) : U(0));
390 const U magnitude = std::is_signed<OldType>::value && (u >> 63) != 0 ? U(0) - u : u;
391 return static_cast<float>(magnitude >= (U(1) << 53) ? numext::bit_cast<OldType>(odd) : x);
392 }
393};
394#endif
395
396// Returns NewType directly to avoid unintended intermediate conversions.
397
398template <typename OldType, typename NewType>
399EIGEN_DEVICE_FUNC inline NewType cast(const OldType& x) {
400 return cast_impl<OldType, NewType>::run(x);
401}
402
403/****************************************************************************
404 * Implementation of arg *
405 ****************************************************************************/
406
407// Visual Studio 2017 has a bug where arg(float) returns 0 for negative inputs.
408// This seems to be fixed in VS 2019.
409#if (!EIGEN_COMP_MSVC || EIGEN_COMP_MSVC >= 1920)
410// std::arg is only defined for types of std::complex, or integer types or float/double/long double
411template <typename Scalar, bool HasStdImpl = NumTraits<Scalar>::IsComplex || std::is_integral<Scalar>::value ||
412 std::is_same<Scalar, float>::value ||
413 std::is_same<Scalar, double>::value ||
414 std::is_same<Scalar, long double>::value>
415struct arg_default_impl;
416
417template <typename Scalar>
418struct arg_default_impl<Scalar, true> {
419 using RealScalar = typename NumTraits<Scalar>::Real;
420 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
421 // There is no official ::arg on device in CUDA/HIP, so we always need to use std::arg.
422 using std::arg;
423 return static_cast<RealScalar>(arg(x));
424 }
425};
426
427// Must be non-complex floating-point type (e.g. half/bfloat16).
428template <typename Scalar>
429struct arg_default_impl<Scalar, false> {
430 using RealScalar = typename NumTraits<Scalar>::Real;
431 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
432 return (x < Scalar(0)) ? RealScalar(EIGEN_PI) : RealScalar(0);
433 }
434};
435#else
436template <typename Scalar, bool IsComplex = NumTraits<Scalar>::IsComplex>
437struct arg_default_impl {
438 using RealScalar = typename NumTraits<Scalar>::Real;
439 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
440 return (x < RealScalar(0)) ? RealScalar(EIGEN_PI) : RealScalar(0);
441 }
442};
443
444template <typename Scalar>
445struct arg_default_impl<Scalar, true> {
446 using RealScalar = typename NumTraits<Scalar>::Real;
447 EIGEN_DEVICE_FUNC static inline RealScalar run(const Scalar& x) {
448 EIGEN_USING_STD(arg);
449 return arg(x);
450 }
451};
452#endif
453template <typename Scalar>
454struct arg_impl : arg_default_impl<Scalar> {};
455
456/****************************************************************************
457 * Implementation of expm1 *
458 ****************************************************************************/
459
460// This implementation is based on GSL Math's expm1.
461namespace std_fallback {
462// fallback expm1 implementation in case there is no expm1(Scalar) function in namespace of Scalar,
463// or that there is no suitable std::expm1 function available. Implementation
464// attributed to Kahan. See: http://www.plunk.org/~hatch/rightway.php.
465template <typename Scalar>
466EIGEN_DEVICE_FUNC inline Scalar expm1(const Scalar& x) {
467 EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar)
468 using RealScalar = typename NumTraits<Scalar>::Real;
469
470 EIGEN_USING_STD(exp);
471 Scalar u = exp(x);
472 if (numext::equal_strict(u, Scalar(1))) {
473 return x;
474 }
475 Scalar um1 = u - RealScalar(1);
476 if (numext::equal_strict(um1, Scalar(-1))) {
477 return RealScalar(-1);
478 }
479
480 EIGEN_USING_STD(log);
481 Scalar logu = log(u);
482 return numext::equal_strict(u, logu) ? u : (u - RealScalar(1)) * x / logu;
483}
484} // namespace std_fallback
485
486template <typename Scalar>
487struct expm1_impl {
488 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) {
489 EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar)
490 EIGEN_USING_STD(expm1);
491 return expm1(x);
492 }
493};
494
495/****************************************************************************
496 * Implementation of log *
497 ****************************************************************************/
498
499// Complex log defined in MathFunctionsImpl.h.
500template <typename ComplexT>
501EIGEN_DEVICE_FUNC constexpr ComplexT complex_log(const ComplexT& z);
502
503template <typename Scalar>
504struct log_impl {
505 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) {
506 EIGEN_USING_STD(log);
507 return static_cast<Scalar>(log(x));
508 }
509};
510
511template <typename Scalar>
512struct log_impl<std::complex<Scalar>> {
513 EIGEN_DEVICE_FUNC static inline std::complex<Scalar> run(const std::complex<Scalar>& z) { return complex_log(z); }
514};
515
516/****************************************************************************
517 * Implementation of log1p *
518 ****************************************************************************/
519
520namespace std_fallback {
521// fallback log1p implementation in case there is no log1p(Scalar) function in namespace of Scalar,
522// or that there is no suitable std::log1p function available
523template <typename Scalar>
524EIGEN_DEVICE_FUNC inline Scalar log1p(const Scalar& x) {
525 EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar)
526 using RealScalar = typename NumTraits<Scalar>::Real;
527 EIGEN_USING_STD(log);
528 Scalar x1p = RealScalar(1) + x;
529 Scalar log_1p = log_impl<Scalar>::run(x1p);
530 const bool is_small = numext::equal_strict(x1p, Scalar(1));
531 // The correction below is 0 * inf or inf / inf once log(1 + x) is not finite; that log is the answer then.
532 EIGEN_USING_STD(isfinite);
533 if (!((isfinite)(numext::real(log_1p)) && (isfinite)(numext::imag(log_1p)))) return log_1p;
534 return is_small ? x : x * (log_1p / (x1p - RealScalar(1)));
535}
536} // namespace std_fallback
537
538template <typename Scalar>
539struct log1p_impl {
540 EIGEN_STATIC_ASSERT_NON_INTEGER(Scalar)
541
542 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& x) {
543 EIGEN_USING_STD(log1p);
544 return log1p(x);
545 }
546};
547
548// Specialization for complex types that are not supported by std::log1p; defined after the numext functions it is
549// built from.
550template <typename RealScalar>
551struct log1p_impl<std::complex<RealScalar>>;
552
553// numext::exp2 of a complex argument; defined after the numext functions it is built from.
554template <typename RealScalar>
555struct complex_exp2_impl;
556
557/****************************************************************************
558 * Implementation of pow *
559 ****************************************************************************/
560
561template <typename ScalarX, typename ScalarY,
562 bool IsInteger = NumTraits<ScalarX>::IsInteger && NumTraits<ScalarY>::IsInteger>
563struct pow_impl {
564 using result_type =
565 typename ScalarBinaryOpTraits<ScalarX, ScalarY, internal::scalar_pow_op<ScalarX, ScalarY>>::ReturnType;
566 static EIGEN_DEVICE_FUNC inline result_type run(const ScalarX& x, const ScalarY& y) {
567 EIGEN_USING_STD(pow);
568 return pow(x, y);
569 }
570};
571
572template <typename ScalarX, typename ScalarY>
573struct pow_impl<ScalarX, ScalarY, true> {
574 using result_type = ScalarX;
575 static EIGEN_DEVICE_FUNC inline ScalarX run(ScalarX x, ScalarY y) {
576 ScalarX res(1);
577 eigen_assert(!NumTraits<ScalarY>::IsSigned || y >= 0);
578 if (y & 1) res = internal::mul(res, x);
579 y >>= 1;
580 while (y) {
581 x = internal::mul(x, x);
582 if (y & 1) res = internal::mul(res, x);
583 y >>= 1;
584 }
585 return res;
586 }
587};
588
589constexpr int floor_log2(unsigned int value) {
590 int result = 0;
591 while (value > 1) {
592 value >>= 1;
593 ++result;
594 }
595 return result;
596}
597
598template <typename BitsType, typename EnableIf = void>
599struct count_bits_impl {
600 static_assert(std::is_integral<BitsType>::value && std::is_unsigned<BitsType>::value,
601 "BitsType must be an unsigned integer");
602 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
603 int n = CHAR_BIT * sizeof(BitsType);
604 int shift = n / 2;
605 while (bits > 0 && shift > 0) {
606 BitsType y = bits >> shift;
607 if (y > 0) {
608 n -= shift;
609 bits = y;
610 }
611 shift /= 2;
612 }
613 if (shift == 0) {
614 --n;
615 }
616 return n;
617 }
618
619 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
620 int n = CHAR_BIT * sizeof(BitsType);
621 int shift = n / 2;
622 while (bits > 0 && shift > 0) {
623 BitsType y = bits << shift;
624 if (y > 0) {
625 n -= shift;
626 bits = y;
627 }
628 shift /= 2;
629 }
630 if (shift == 0) {
631 --n;
632 }
633 return n;
634 }
635
636 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
637 int n = 0;
638 while (bits) {
639 bits &= bits - 1;
640 ++n;
641 }
642 return n;
643 }
644};
645
646// Count leading zeros.
647template <typename BitsType>
648EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
649 return count_bits_impl<BitsType>::clz(bits);
650}
651
652// Count trailing zeros.
653template <typename BitsType>
654EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
655 return count_bits_impl<BitsType>::ctz(bits);
656}
657
658// Count set bits (population count).
659template <typename BitsType>
660EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
661 return count_bits_impl<BitsType>::popcount(bits);
662}
663
664#if defined(EIGEN_CUDA_ARCH)
665
666// nvcc's device code has only __builtin_clz, __builtin_clzll, __builtin_popcount and __builtin_popcountll. The other
667// builtins are host functions there, and with warning 20014 silenced, a call from here compiles and returns garbage:
668// ctz at every width, and clz and popcount where long has 64 bits. __clz and __clzll return the full width for zero.
669// From CUDA 13.2, their parameter and return types are unsigned on aarch64 with GCC 11.4 or later as the host compiler
670// and signed elsewhere and in earlier toolkits; the conversions below keep the bits under either declaration.
671template <typename BitsType>
672struct count_bits_impl<
673 BitsType, std::enable_if_t<std::is_integral<BitsType>::value && sizeof(BitsType) <= sizeof(unsigned int)>> {
674 static constexpr int kNumBits = static_cast<int>(sizeof(BitsType) * CHAR_BIT);
675 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
676 static constexpr int kLeadingBitsOffset = (sizeof(unsigned int) - sizeof(BitsType)) * CHAR_BIT;
677 return static_cast<int>(__clz(static_cast<unsigned int>(bits))) - kLeadingBitsOffset;
678 }
679
680 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
681 return bits == 0 ? kNumBits : __ffs(static_cast<int>(static_cast<unsigned int>(bits))) - 1;
682 }
683
684 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) { return __popc(static_cast<unsigned int>(bits)); }
685};
686
687template <typename BitsType>
688struct count_bits_impl<BitsType,
689 std::enable_if_t<std::is_integral<BitsType>::value && sizeof(unsigned int) < sizeof(BitsType) &&
690 sizeof(BitsType) <= sizeof(unsigned long long)>> {
691 static constexpr int kNumBits = static_cast<int>(sizeof(BitsType) * CHAR_BIT);
692 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
693 static constexpr int kLeadingBitsOffset = (sizeof(unsigned long long) - sizeof(BitsType)) * CHAR_BIT;
694 return static_cast<int>(__clzll(static_cast<unsigned long long>(bits))) - kLeadingBitsOffset;
695 }
696
697 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
698 return bits == 0 ? kNumBits : __ffsll(static_cast<long long>(static_cast<unsigned long long>(bits))) - 1;
699 }
700
701 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
702 return __popcll(static_cast<unsigned long long>(bits));
703 }
704};
705
706#elif EIGEN_COMP_GNUC || EIGEN_COMP_CLANG
707
708template <typename BitsType>
709struct count_bits_impl<
710 BitsType, std::enable_if_t<std::is_integral<BitsType>::value && sizeof(BitsType) <= sizeof(unsigned int)>> {
711 static constexpr int kNumBits = static_cast<int>(sizeof(BitsType) * CHAR_BIT);
712 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
713 static constexpr int kLeadingBitsOffset = (sizeof(unsigned int) - sizeof(BitsType)) * CHAR_BIT;
714 return bits == 0 ? kNumBits : __builtin_clz(static_cast<unsigned int>(bits)) - kLeadingBitsOffset;
715 }
716
717 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
718 return bits == 0 ? kNumBits : __builtin_ctz(static_cast<unsigned int>(bits));
719 }
720
721 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
722 return __builtin_popcount(static_cast<unsigned int>(bits));
723 }
724};
725
726template <typename BitsType>
727struct count_bits_impl<BitsType,
728 std::enable_if_t<std::is_integral<BitsType>::value && sizeof(unsigned int) < sizeof(BitsType) &&
729 sizeof(BitsType) <= sizeof(unsigned long)>> {
730 static constexpr int kNumBits = static_cast<int>(sizeof(BitsType) * CHAR_BIT);
731 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
732 static constexpr int kLeadingBitsOffset = (sizeof(unsigned long) - sizeof(BitsType)) * CHAR_BIT;
733 return bits == 0 ? kNumBits : __builtin_clzl(static_cast<unsigned long>(bits)) - kLeadingBitsOffset;
734 }
735
736 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
737 return bits == 0 ? kNumBits : __builtin_ctzl(static_cast<unsigned long>(bits));
738 }
739
740 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
741 return __builtin_popcountl(static_cast<unsigned long>(bits));
742 }
743};
744
745template <typename BitsType>
746struct count_bits_impl<BitsType,
747 std::enable_if_t<std::is_integral<BitsType>::value && sizeof(unsigned long) < sizeof(BitsType) &&
748 sizeof(BitsType) <= sizeof(unsigned long long)>> {
749 static constexpr int kNumBits = static_cast<int>(sizeof(BitsType) * CHAR_BIT);
750 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
751 static constexpr int kLeadingBitsOffset = (sizeof(unsigned long long) - sizeof(BitsType)) * CHAR_BIT;
752 return bits == 0 ? kNumBits : __builtin_clzll(static_cast<unsigned long long>(bits)) - kLeadingBitsOffset;
753 }
754
755 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
756 return bits == 0 ? kNumBits : __builtin_ctzll(static_cast<unsigned long long>(bits));
757 }
758
759 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
760 return __builtin_popcountll(static_cast<unsigned long long>(bits));
761 }
762};
763
764#elif EIGEN_COMP_MSVC
765
766// `__popcnt`/`__popcnt64` require `POPCNT` hardware support, which MSVC cannot guarantee
767// at its default baseline (unlike `_BitScanReverse`/`_BitScanForward`, which lower to
768// baseline `bsr`/`bsf`). Fall back to a portable count when building without SSE4.2
769// enabled.
770template <typename BitsType>
771EIGEN_DEVICE_FUNC inline int popcount_fallback(BitsType bits) {
772 int n = 0;
773 while (bits) {
774 bits &= bits - 1;
775 ++n;
776 }
777 return n;
778}
779
780template <typename BitsType>
781struct count_bits_impl<
782 BitsType, std::enable_if_t<std::is_integral<BitsType>::value && sizeof(BitsType) <= sizeof(unsigned long)>> {
783 static constexpr int kNumBits = static_cast<int>(sizeof(BitsType) * CHAR_BIT);
784 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
785 unsigned long out;
786 _BitScanReverse(&out, static_cast<unsigned long>(bits));
787 return bits == 0 ? kNumBits : (kNumBits - 1) - static_cast<int>(out);
788 }
789
790 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
791 unsigned long out;
792 _BitScanForward(&out, static_cast<unsigned long>(bits));
793 return bits == 0 ? kNumBits : static_cast<int>(out);
794 }
795
796 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
797#if defined(EIGEN_VECTORIZE_SSE4_2)
798 return static_cast<int>(__popcnt(static_cast<unsigned int>(bits)));
799#else
800 return popcount_fallback(bits);
801#endif
802 }
803};
804
805#ifdef _WIN64
806
807template <typename BitsType>
808struct count_bits_impl<BitsType,
809 std::enable_if_t<std::is_integral<BitsType>::value && sizeof(unsigned long) < sizeof(BitsType) &&
810 sizeof(BitsType) <= sizeof(__int64)>> {
811 static constexpr int kNumBits = static_cast<int>(sizeof(BitsType) * CHAR_BIT);
812 static EIGEN_DEVICE_FUNC inline int clz(BitsType bits) {
813 unsigned long out;
814 _BitScanReverse64(&out, static_cast<unsigned __int64>(bits));
815 return bits == 0 ? kNumBits : (kNumBits - 1) - static_cast<int>(out);
816 }
817
818 static EIGEN_DEVICE_FUNC inline int ctz(BitsType bits) {
819 unsigned long out;
820 _BitScanForward64(&out, static_cast<unsigned __int64>(bits));
821 return bits == 0 ? kNumBits : static_cast<int>(out);
822 }
823
824 static EIGEN_DEVICE_FUNC inline int popcount(BitsType bits) {
825#if defined(EIGEN_VECTORIZE_SSE4_2)
826 return static_cast<int>(__popcnt64(static_cast<unsigned __int64>(bits)));
827#else
828 return popcount_fallback(bits);
829#endif
830 }
831};
832
833#endif // _WIN64
834
835#endif // EIGEN_CUDA_ARCH
836
837template <typename BitsType>
838struct log_2_impl {
839 static constexpr int kTotalBits = sizeof(BitsType) * CHAR_BIT;
840 static EIGEN_DEVICE_FUNC inline int run_ceil(const BitsType& x) {
841 const int n = kTotalBits - clz(x);
842 bool power_of_two = (x & (x - 1)) == 0;
843 return x == 0 ? 0 : power_of_two ? (n - 1) : n;
844 }
845 static EIGEN_DEVICE_FUNC inline int run_floor(const BitsType& x) {
846 const int n = kTotalBits - clz(x);
847 return x == 0 ? 0 : n - 1;
848 }
849};
850
851template <typename BitsType>
852int log2_ceil(const BitsType& x) {
853 return log_2_impl<BitsType>::run_ceil(x);
854}
855
856template <typename BitsType>
857int log2_floor(const BitsType& x) {
858 return log_2_impl<BitsType>::run_floor(x);
859}
860
861// Implementation of is* functions
862
863template <typename T>
864EIGEN_DEVICE_FUNC std::enable_if_t<!(std::numeric_limits<T>::has_infinity || std::numeric_limits<T>::has_quiet_NaN ||
865 std::numeric_limits<T>::has_signaling_NaN),
866 bool>
867isfinite_impl(const T&) {
868 return true;
869}
870
871template <typename T>
872EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits<T>::has_infinity || std::numeric_limits<T>::has_quiet_NaN ||
873 std::numeric_limits<T>::has_signaling_NaN) &&
874 (!NumTraits<T>::IsComplex),
875 bool>
876isfinite_impl(const T& x) {
877 EIGEN_USING_STD(isfinite);
878 return isfinite EIGEN_NOT_A_MACRO(x);
879}
880
881template <typename T>
882EIGEN_DEVICE_FUNC std::enable_if_t<!std::numeric_limits<T>::has_infinity, bool> isinf_impl(const T&) {
883 return false;
884}
885
886template <typename T>
887EIGEN_DEVICE_FUNC std::enable_if_t<(std::numeric_limits<T>::has_infinity && !NumTraits<T>::IsComplex), bool> isinf_impl(
888 const T& x) {
889 EIGEN_USING_STD(isinf);
890 return isinf EIGEN_NOT_A_MACRO(x);
891}
892
893template <typename T>
894EIGEN_DEVICE_FUNC
895 std::enable_if_t<!(std::numeric_limits<T>::has_quiet_NaN || std::numeric_limits<T>::has_signaling_NaN), bool>
896 isnan_impl(const T&) {
897 return false;
898}
899
900template <typename T>
901EIGEN_DEVICE_FUNC std::enable_if_t<
902 (std::numeric_limits<T>::has_quiet_NaN || std::numeric_limits<T>::has_signaling_NaN) && (!NumTraits<T>::IsComplex),
903 bool>
904isnan_impl(const T& x) {
905 EIGEN_USING_STD(isnan);
906 return isnan EIGEN_NOT_A_MACRO(x);
907}
908
909// The following overload are defined at the end of this file
910template <typename T>
911EIGEN_DEVICE_FUNC bool isfinite_impl(const std::complex<T>& x);
912template <typename T>
913EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex<T>& x);
914template <typename T>
915EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex<T>& x);
916template <typename T>
917EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS T ptanh_float(const T& a_x);
918
919/****************************************************************************
920 * Implementation of sign *
921 ****************************************************************************/
922template <typename Scalar, bool IsComplex = (NumTraits<Scalar>::IsComplex != 0),
923 bool IsInteger = (NumTraits<Scalar>::IsInteger != 0)>
924struct sign_impl {
925 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a) { return Scalar((a > Scalar(0)) - (a < Scalar(0))); }
926};
927
928template <typename Scalar>
929struct sign_impl<Scalar, false, false> {
930 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a) {
931 return (isnan_impl<Scalar>)(a) ? a : Scalar((a > Scalar(0)) - (a < Scalar(0)));
932 }
933};
934
935template <typename Scalar, bool IsInteger>
936struct sign_impl<Scalar, true, IsInteger> {
937 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a) {
938 using real_type = typename NumTraits<Scalar>::Real;
939 EIGEN_USING_STD(abs);
940 real_type aa = abs(a);
941 if (aa == real_type(0)) return Scalar(0);
942 // Divide rather than multiply by 1/aa: the reciprocal overflows for subnormal aa and is itself
943 // subnormal, hence inexact, for aa near the top of the range. psign_impl for complex packets
944 // divides for the same reason.
945 return Scalar(numext::real(a) / aa, numext::imag(a) / aa);
946 }
947};
948
949// The sign function for bool is the identity.
950template <>
951struct sign_impl<bool, false, true> {
952 EIGEN_DEVICE_FUNC static inline bool run(const bool& a) { return a; }
953};
954
955// Extra namespace to prevent leaking std::copysign into Eigen::internal.
956namespace has_copysign_detail {
957
958template <typename T, typename EnableIf = void>
959struct has_copysign_impl : public std::false_type {};
960
961EIGEN_USING_STD(copysign)
962
963template <typename T>
964struct has_copysign_impl<T, std::enable_if_t<std::is_constructible<
965 T, decltype(copysign(std::declval<const T&>(), std::declval<const T&>()))>::value>>
966 : public std::true_type {};
967
968} // namespace has_copysign_detail
969
970// Detects whether copysign(T, T) is available via std:: or ADL. When unavailable,
971// copysign_impl falls back to `< Scalar(0)` comparisons, which cannot inspect the
972// sign bit of negative zero or NaN (e.g., copysign(-0, 1) returns -0, copysign(1, -0)
973// returns +1, and complex_sqrt(-1 - 0i) returns +i rather than -i).
974template <typename T>
975struct has_copysign : public has_copysign_detail::has_copysign_impl<T> {};
976
977template <typename Scalar, bool IsComplex = (NumTraits<Scalar>::IsComplex != 0),
978 bool IsInteger = (NumTraits<Scalar>::IsInteger != 0),
979 bool HasCopysign = (!IsComplex && !IsInteger && has_copysign<Scalar>::value)>
980struct copysign_impl {
981 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a, const Scalar& b) {
982 EIGEN_USING_STD(copysign);
983 return Scalar(copysign(a, b));
984 }
985};
986
987template <typename Scalar, bool IsInteger>
988struct copysign_impl<Scalar, true, IsInteger, false> {
989 using RealScalar = typename NumTraits<Scalar>::Real;
990 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a, const Scalar& b) {
991 return Scalar(copysign_impl<RealScalar>::run(numext::real(a), numext::real(b)),
992 copysign_impl<RealScalar>::run(numext::imag(a), numext::imag(b)));
993 }
994};
995
996template <typename Scalar, bool IsInteger>
997struct copysign_impl<Scalar, false, IsInteger, false> {
998 EIGEN_DEVICE_FUNC static inline Scalar run(const Scalar& a, const Scalar& b) {
999 EIGEN_IF_CONSTEXPR (IsInteger && !NumTraits<Scalar>::IsSigned) return a;
1000 const Scalar abs_a = a < Scalar(0) ? -a : a;
1001 return b < Scalar(0) ? -abs_a : abs_a;
1002 }
1003};
1004
1005template <>
1006struct copysign_impl<bool, false, true, false> {
1007 EIGEN_DEVICE_FUNC static inline bool run(const bool& a, const bool&) { return a; }
1008};
1009
1010// suppress "unary minus operator applied to unsigned type, result still unsigned" warnings on MSVC
1011// note: `0 - a` is distinct from `-a` when Scalar is a floating point type and `a` is zero
1012
1013template <typename Scalar, bool IsInteger = NumTraits<Scalar>::IsInteger>
1014struct negate_impl {
1015 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run(const Scalar& a) { return -a; }
1016};
1017
1018template <typename Scalar>
1019struct negate_impl<Scalar, true> {
1020 EIGEN_STATIC_ASSERT((!std::is_same<Scalar, bool>::value), NEGATE IS NOT DEFINED FOR BOOLEAN TYPES)
1021 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar run(const Scalar& a) { return Scalar(0) - a; }
1022};
1023
1024template <typename Scalar, bool IsInteger = NumTraits<typename unpacket_traits<Scalar>::type>::IsInteger>
1025struct nearest_integer_impl {
1026 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_floor(const Scalar& x) {
1027 EIGEN_USING_STD(floor) return floor(x);
1028 }
1029 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_ceil(const Scalar& x) {
1030 EIGEN_USING_STD(ceil) return ceil(x);
1031 }
1032 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_rint(const Scalar& x) {
1033 EIGEN_USING_STD(rint) return rint(x);
1034 }
1035 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_round(const Scalar& x) {
1036 EIGEN_USING_STD(round) return round(x);
1037 }
1038 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_trunc(const Scalar& x) {
1039 EIGEN_USING_STD(trunc) return trunc(x);
1040 }
1041};
1042template <typename Scalar>
1043struct nearest_integer_impl<Scalar, true> {
1044 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_floor(const Scalar& x) { return x; }
1045 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_ceil(const Scalar& x) { return x; }
1046 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_rint(const Scalar& x) { return x; }
1047 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_round(const Scalar& x) { return x; }
1048 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run_trunc(const Scalar& x) { return x; }
1049};
1050
1051// Extra namespace to prevent leaking std::fma into Eigen::internal.
1052namespace has_fma_detail {
1053
1054template <typename T, typename EnableIf = void>
1055struct has_fma_impl : public std::false_type {};
1056
1057using std::fma;
1058
1059template <typename T>
1060struct has_fma_impl<
1061 T, std::enable_if_t<std::is_same<T, decltype(fma(std::declval<T>(), std::declval<T>(), std::declval<T>()))>::value>>
1062 : public std::true_type {};
1063
1064} // namespace has_fma_detail
1065
1066template <typename T>
1067struct has_fma : public has_fma_detail::has_fma_impl<T> {};
1068
1069// Default implementation.
1070template <typename T, typename Enable = void>
1071struct fma_impl {
1072 static_assert(has_fma<T>::value, "No function fma(...) for type. Please provide an implementation.");
1073};
1074
1075// STD or ADL version if it exists.
1076template <typename T>
1077struct fma_impl<T, std::enable_if_t<has_fma<T>::value>> {
1078 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T run(const T& a, const T& b, const T& c) {
1079 using std::fma;
1080 return fma(a, b, c);
1081 }
1082};
1083
1084#if defined(EIGEN_GPUCC)
1085template <>
1086struct has_fma<float> : public std::true_type {};
1087
1088template <>
1089struct fma_impl<float, void> {
1090 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE float run(const float& a, const float& b, const float& c) {
1091 return ::fmaf(a, b, c);
1092 }
1093};
1094
1095template <>
1096struct has_fma<double> : public std::true_type {};
1097
1098template <>
1099struct fma_impl<double, void> {
1100 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE double run(const double& a, const double& b, const double& c) {
1101 return ::fma(a, b, c);
1102 }
1103};
1104#endif
1105
1106// Basic multiply-add.
1107template <typename Scalar, typename EnableIf = void>
1108struct madd_impl {
1109 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Scalar& x, const Scalar& y, const Scalar& z) {
1110 // Separate statements prevent FP_CONTRACT ON from fusing the unfused fallback.
1111 const Scalar xy = x * y;
1112 return xy + z;
1113 }
1114};
1115
1116// Whether fma on Scalar is an instruction rather than a libm routine. x87 extended and IBM double-double long double
1117// have no fma instruction, nor does binary128 on most targets.
1118template <typename Scalar>
1119struct has_fast_fma : std::false_type {};
1120#if defined(EIGEN_VECTORIZE_FMA) || defined(FP_FAST_FMAF) || defined(__FP_FAST_FMAF)
1121template <>
1122struct has_fast_fma<float> : std::true_type {};
1123#endif
1124#if defined(EIGEN_VECTORIZE_FMA) || defined(FP_FAST_FMA) || defined(__FP_FAST_FMA)
1125template <>
1126struct has_fast_fma<double> : std::true_type {};
1127#endif
1128#if defined(FP_FAST_FMAL) || defined(__FP_FAST_FMAL)
1129template <>
1130struct has_fast_fma<long double> : std::true_type {};
1131#else
1132// A long double with double's format is as fast as double.
1133template <>
1134struct has_fast_fma<long double>
1135 : bool_constant<std::numeric_limits<long double>::digits == std::numeric_limits<double>::digits &&
1136 has_fast_fma<double>::value> {};
1137#endif
1138
1139// Whether numext::madd fuses Scalar: as EIGEN_SCALAR_MADD_USE_FMA says, except that long double is fused only where
1140// its fma is an instruction, because scalar GEBP through libm's fmal is far slower than the unfused product and sum.
1141// has_fma<Scalar> is passed as a type, not a value, so it is instantiated only when selected, never when
1142// EIGEN_SCALAR_MADD_USE_FMA is 0.
1143template <typename Scalar>
1144struct scalar_madd_uses_fma
1145 : std::conditional_t<bool(EIGEN_SCALAR_MADD_USE_FMA) &&
1146 (!std::is_same<Scalar, long double>::value || has_fast_fma<Scalar>::value),
1147 has_fma<Scalar>, std::false_type> {};
1148
1149template <typename Scalar>
1150struct madd_impl<Scalar, std::enable_if_t<scalar_madd_uses_fma<Scalar>::value>> {
1151 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Scalar& x, const Scalar& y, const Scalar& z) {
1152 return fma_impl<Scalar>::run(x, y, z);
1153 }
1154};
1155
1156// Representation properties shared by operations that inspect an IEEE-style binary floating-point value.
1157template <typename Scalar>
1158struct binary_floating_point_traits {
1159 using Bits = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type;
1160 static constexpr int kFractionBits = std::numeric_limits<Scalar>::digits - 1;
1161 static constexpr int kExponentBits = int(sizeof(Scalar) * CHAR_BIT) - std::numeric_limits<Scalar>::digits;
1162 static constexpr int kExponentBias = std::numeric_limits<Scalar>::max_exponent - 1;
1163 static constexpr Bits kSignBit = Bits(1) << (sizeof(Scalar) * CHAR_BIT - 1);
1164 static constexpr Bits kExponentUnit = Bits(1) << kFractionBits;
1165 static constexpr Bits kFractionMask = kExponentUnit - 1;
1166 static constexpr Bits kExponentMask = ((Bits(1) << kExponentBits) - 1) << kFractionBits;
1167 static constexpr Bits kMaxFiniteExponentBits = kExponentMask - kExponentUnit;
1168
1169 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Bits bits(const Scalar& value) { return numext::bit_cast<Bits>(value); }
1170
1171 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Bits magnitude(const Scalar& value) { return bits(value) & ~kSignBit; }
1172};
1173
1174// Redundant out-of-class definitions are required pre-C++17 but deprecated since.
1175#if EIGEN_COMP_CXXVER < 17
1176template <typename Scalar>
1177constexpr int binary_floating_point_traits<Scalar>::kFractionBits;
1178template <typename Scalar>
1179constexpr int binary_floating_point_traits<Scalar>::kExponentBits;
1180template <typename Scalar>
1181constexpr int binary_floating_point_traits<Scalar>::kExponentBias;
1182template <typename Scalar>
1183constexpr typename binary_floating_point_traits<Scalar>::Bits binary_floating_point_traits<Scalar>::kSignBit;
1184template <typename Scalar>
1185constexpr typename binary_floating_point_traits<Scalar>::Bits binary_floating_point_traits<Scalar>::kExponentUnit;
1186template <typename Scalar>
1187constexpr typename binary_floating_point_traits<Scalar>::Bits binary_floating_point_traits<Scalar>::kFractionMask;
1188template <typename Scalar>
1189constexpr typename binary_floating_point_traits<Scalar>::Bits binary_floating_point_traits<Scalar>::kExponentMask;
1190template <typename Scalar>
1191constexpr
1192 typename binary_floating_point_traits<Scalar>::Bits binary_floating_point_traits<Scalar>::kMaxFiniteExponentBits;
1193#endif
1194
1195template <typename Scalar>
1196EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_exactly_zero_no_flush_impl(const Scalar& value, false_type) {
1197 return numext::is_exactly_zero(value);
1198}
1199
1200template <typename Scalar>
1201EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE bool is_zero_magnitude_bits(
1202 typename binary_floating_point_traits<Scalar>::Bits magnitude) {
1203 // Keep the integer comparison out of line: Clang can otherwise fold it into a
1204 // floating-point comparison that treats subnormals as zero under DAZ/FZ.
1205 return magnitude == 0;
1206}
1207
1208// Floating-point comparisons can treat subnormals as zero under DAZ/FTZ.
1209template <typename Scalar>
1210struct sign_impl_binary {
1211 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Scalar& a) {
1212 using Binary = binary_floating_point_traits<Scalar>;
1213 using Bits = typename Binary::Bits;
1214 const Bits bits = Binary::bits(a);
1215 const Bits magnitude = bits & ~Binary::kSignBit;
1216 // magnitude | -magnitude has its high bit set iff magnitude is nonzero.
1217 const Bits nonzero = (magnitude | (Bits(0) - magnitude)) >> (sizeof(Bits) * CHAR_BIT - 1);
1218 const Bits nonzero_mask = Bits(0) - nonzero;
1219 const Bits nan_mask = Bits(0) - Bits(magnitude > Binary::kExponentMask);
1220 const Bits signed_one = (bits & Binary::kSignBit) | Binary::bits(Scalar(1));
1221 return numext::bit_cast<Scalar>((bits & nan_mask) | (signed_one & nonzero_mask & ~nan_mask));
1222 }
1223};
1224
1225template <>
1226struct sign_impl<float, false, false> : sign_impl_binary<float> {};
1227
1228template <>
1229struct sign_impl<double, false, false> : sign_impl_binary<double> {};
1230
1231template <typename Scalar>
1232EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_exactly_zero_no_flush_impl(const Scalar& value, true_type) {
1233 return is_zero_magnitude_bits<Scalar>(binary_floating_point_traits<Scalar>::magnitude(value));
1234}
1235
1236// |x - y|, as the packet path computes it: abs keeps the +0 of (+0, -0) and the NaN of (inf, inf).
1237template <typename T, bool IsInteger = NumTraits<T>::IsInteger>
1238struct absdiff_impl {
1239 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE T run(const T& x, const T& y) {
1240 EIGEN_USING_STD(abs);
1241 return abs(x - y);
1242 }
1243};
1244
1245// Unsigned x - y wraps, so integers order the operands.
1246template <typename T>
1247struct absdiff_impl<T, true> {
1248 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE T run(const T& x, const T& y) { return x > y ? x - y : y - x; }
1249};
1250
1251} // end namespace internal
1252
1253/****************************************************************************
1254 * Generic math functions *
1255 ****************************************************************************/
1256
1257namespace numext {
1258
1259template <typename X>
1260EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool is_exactly_zero_no_flush(const X& x) {
1261 constexpr bool HasBinaryEncoding = (std::is_same<X, float>::value || std::is_same<X, double>::value) &&
1262 std::numeric_limits<X>::is_iec559 && std::numeric_limits<X>::radix == 2 &&
1263 (sizeof(X) == sizeof(uint32_t) || sizeof(X) == sizeof(uint64_t));
1264 return internal::is_exactly_zero_no_flush_impl(x, internal::bool_constant<HasBinaryEncoding>());
1265}
1266
1267#if !defined(EIGEN_GPUCC)
1268template <typename T>
1269EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T mini(const T& x, const T& y) {
1270 EIGEN_USING_STD(min)
1271 return min EIGEN_NOT_A_MACRO(x, y);
1272}
1273
1274template <typename T>
1275EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T maxi(const T& x, const T& y) {
1276 EIGEN_USING_STD(max)
1277 return max EIGEN_NOT_A_MACRO(x, y);
1278}
1279#else
1280template <typename T>
1281EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T mini(const T& x, const T& y) {
1282 return y < x ? y : x;
1283}
1284#if !defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)
1285// Without relaxed constexpr, numeric GPU scalars keep fmin/fmax's number-preferring
1286// NaN behavior. With relaxed constexpr, they use the constexpr ternary overloads,
1287// matching std::min/std::max behavior and supporting custom less-comparable scalars.
1288template <>
1289EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float mini(const float& x, const float& y) {
1290 return fminf(x, y);
1291}
1292template <>
1293EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double mini(const double& x, const double& y) {
1294 return fmin(x, y);
1295}
1296
1297#ifndef EIGEN_GPU_COMPILE_PHASE
1298template <>
1299EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE long double mini(const long double& x, const long double& y) {
1300#if defined(EIGEN_HIPCC)
1301 // no "fminl" on HIP yet
1302 return (x < y) ? x : y;
1303#else
1304 return fminl(x, y);
1305#endif
1306}
1307#endif
1308#endif
1309
1310template <typename T>
1311EIGEN_DEVICE_FUNC constexpr EIGEN_ALWAYS_INLINE T maxi(const T& x, const T& y) {
1312 return x < y ? y : x;
1313}
1314#if !defined(EIGEN_CONSTEXPR_ARE_DEVICE_FUNC)
1315template <>
1316EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float maxi(const float& x, const float& y) {
1317 return fmaxf(x, y);
1318}
1319template <>
1320EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double maxi(const double& x, const double& y) {
1321 return fmax(x, y);
1322}
1323#ifndef EIGEN_GPU_COMPILE_PHASE
1324template <>
1325EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE long double maxi(const long double& x, const long double& y) {
1326#if defined(EIGEN_HIPCC)
1327 // no "fmaxl" on HIP yet
1328 return (x > y) ? x : y;
1329#else
1330 return fmaxl(x, y);
1331#endif
1332}
1333#endif
1334#endif
1335#endif
1336
1337#if defined(EIGEN_SYCL_DEVICE_ONLY)
1338
1339#define SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_BINARY(NAME, FUNC) \
1340 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_char) \
1341 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_short) \
1342 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_int) \
1343 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_long)
1344#define SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_UNARY(NAME, FUNC) \
1345 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_char) \
1346 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_short) \
1347 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_int) \
1348 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_long)
1349#define SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_BINARY(NAME, FUNC) \
1350 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_uchar) \
1351 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_ushort) \
1352 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_uint) \
1353 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_ulong)
1354#define SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_UNARY(NAME, FUNC) \
1355 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_uchar) \
1356 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_ushort) \
1357 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_uint) \
1358 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_ulong)
1359#define SYCL_SPECIALIZE_INTEGER_TYPES_BINARY(NAME, FUNC) \
1360 SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_BINARY(NAME, FUNC) \
1361 SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_BINARY(NAME, FUNC)
1362#define SYCL_SPECIALIZE_INTEGER_TYPES_UNARY(NAME, FUNC) \
1363 SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_UNARY(NAME, FUNC) \
1364 SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_UNARY(NAME, FUNC)
1365#define SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(NAME, FUNC) \
1366 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_float) \
1367 SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, cl::sycl::cl_double)
1368#define SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(NAME, FUNC) \
1369 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_float) \
1370 SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, cl::sycl::cl_double)
1371#define SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(NAME, FUNC, RET_TYPE) \
1372 SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, RET_TYPE, cl::sycl::cl_float) \
1373 SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, RET_TYPE, cl::sycl::cl_double)
1374
1375#define SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE) \
1376 template <> \
1377 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE RET_TYPE NAME(const ARG_TYPE& x) { \
1378 return cl::sycl::FUNC(x); \
1379 }
1380
1381#define SYCL_SPECIALIZE_UNARY_FUNC(NAME, FUNC, TYPE) SYCL_SPECIALIZE_GEN_UNARY_FUNC(NAME, FUNC, TYPE, TYPE)
1382
1383#define SYCL_SPECIALIZE_GEN1_BINARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE1, ARG_TYPE2) \
1384 template <> \
1385 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE RET_TYPE NAME(const ARG_TYPE1& x, const ARG_TYPE2& y) { \
1386 return cl::sycl::FUNC(x, y); \
1387 }
1388
1389#define SYCL_SPECIALIZE_GEN2_BINARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE) \
1390 SYCL_SPECIALIZE_GEN1_BINARY_FUNC(NAME, FUNC, RET_TYPE, ARG_TYPE, ARG_TYPE)
1391
1392#define SYCL_SPECIALIZE_BINARY_FUNC(NAME, FUNC, TYPE) SYCL_SPECIALIZE_GEN2_BINARY_FUNC(NAME, FUNC, TYPE, TYPE)
1393
1394SYCL_SPECIALIZE_INTEGER_TYPES_BINARY(mini, min)
1395SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(mini, fmin)
1396SYCL_SPECIALIZE_INTEGER_TYPES_BINARY(maxi, max)
1397SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(maxi, fmax)
1398
1399#endif
1400
1401template <typename Scalar>
1402EIGEN_DEVICE_FUNC inline typename NumTraits<Scalar>::Real arg(const Scalar& x) {
1403 return EIGEN_MATHFUNC_IMPL(arg, Scalar)::run(x);
1404}
1405
1406template <typename Scalar>
1407EIGEN_DEVICE_FUNC inline auto imag_ref(const Scalar& x)
1408 -> internal::add_const_on_value_type_t<decltype(internal::imag_ref_impl<Scalar>::run(x))> {
1409 return internal::imag_ref_impl<Scalar>::run(x);
1410}
1411
1412template <typename Scalar>
1413EIGEN_DEVICE_FUNC inline auto imag_ref(Scalar& x)
1414 -> decltype(EIGEN_MATHFUNC_IMPL(imag_ref, std::remove_const_t<Scalar>)::run(x)) {
1415 return EIGEN_MATHFUNC_IMPL(imag_ref, std::remove_const_t<Scalar>)::run(x);
1416}
1417
1418template <typename Scalar>
1419EIGEN_DEVICE_FUNC inline Scalar conj(const Scalar& x) {
1420 return EIGEN_MATHFUNC_IMPL(conj, Scalar)::run(x);
1421}
1422
1423template <typename Scalar>
1424EIGEN_DEVICE_FUNC inline Scalar sign(const Scalar& x) {
1425 return EIGEN_MATHFUNC_IMPL(sign, Scalar)::run(x);
1426}
1427
1428template <typename Scalar>
1429EIGEN_DEVICE_FUNC inline Scalar copysign(const Scalar& x, const Scalar& y) {
1430 return EIGEN_MATHFUNC_IMPL(copysign, Scalar)::run(x, y);
1431}
1432
1433#if defined(EIGEN_SYCL_DEVICE_ONLY)
1434SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(copysign, copysign)
1435#endif
1436
1437template <typename Scalar>
1438EIGEN_DEVICE_FUNC inline Scalar negate(const Scalar& x) {
1439 return EIGEN_MATHFUNC_IMPL(negate, Scalar)::run(x);
1440}
1441
1442template <typename Scalar>
1443EIGEN_DEVICE_FUNC inline decltype(auto) abs2(const Scalar& x) {
1444 return EIGEN_MATHFUNC_IMPL(abs2, Scalar)::run(x);
1445}
1446
1447EIGEN_DEVICE_FUNC inline bool abs2(bool x) { return x; }
1448
1449template <typename T>
1450EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T absdiff(const T& x, const T& y) {
1451 return internal::absdiff_impl<T>::run(x, y);
1452}
1453template <>
1454EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float absdiff(const float& x, const float& y) {
1455 return fabsf(x - y);
1456}
1457template <>
1458EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double absdiff(const double& x, const double& y) {
1459 return fabs(x - y);
1460}
1461
1462// HIP and CUDA do not support long double.
1463#ifndef EIGEN_GPU_COMPILE_PHASE
1464template <>
1465EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE long double absdiff(const long double& x, const long double& y) {
1466 return fabsl(x - y);
1467}
1468#endif
1469
1470template <typename Scalar>
1471EIGEN_DEVICE_FUNC inline typename NumTraits<Scalar>::Real norm1(const Scalar& x) {
1472 return EIGEN_MATHFUNC_IMPL(norm1, Scalar)::run(x);
1473}
1474
1475template <typename Scalar>
1476EIGEN_DEVICE_FUNC inline typename NumTraits<Scalar>::Real hypot(const Scalar& x, const Scalar& y) {
1477 return EIGEN_MATHFUNC_IMPL(hypot, Scalar)::run(x, y);
1478}
1479
1480#if defined(EIGEN_SYCL_DEVICE_ONLY)
1481SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(hypot, hypot)
1482#endif
1483
1484template <typename Scalar>
1485EIGEN_DEVICE_FUNC inline Scalar log1p(const Scalar& x) {
1486 return EIGEN_MATHFUNC_IMPL(log1p, Scalar)::run(x);
1487}
1488
1489#if defined(EIGEN_SYCL_DEVICE_ONLY)
1490SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(log1p, log1p)
1491#endif
1492
1493#if defined(EIGEN_GPUCC)
1494template <>
1495EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float log1p(const float& x) {
1496 return ::log1pf(x);
1497}
1498
1499template <>
1500EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double log1p(const double& x) {
1501 return ::log1p(x);
1502}
1503#endif
1504
1505template <typename ScalarX, typename ScalarY>
1506EIGEN_DEVICE_FUNC inline typename internal::pow_impl<ScalarX, ScalarY>::result_type pow(const ScalarX& x,
1507 const ScalarY& y) {
1508 return internal::pow_impl<ScalarX, ScalarY>::run(x, y);
1509}
1510
1511#if defined(EIGEN_SYCL_DEVICE_ONLY)
1512SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(pow, pow)
1513#endif
1514
1515template <typename T>
1516EIGEN_DEVICE_FUNC bool(isnan)(const T& x) {
1517 return internal::isnan_impl(x);
1518}
1519template <typename T>
1520EIGEN_DEVICE_FUNC bool(isinf)(const T& x) {
1521 return internal::isinf_impl(x);
1522}
1523template <typename T>
1524EIGEN_DEVICE_FUNC bool(isfinite)(const T& x) {
1525 return internal::isfinite_impl(x);
1526}
1527
1528#if defined(EIGEN_SYCL_DEVICE_ONLY)
1529SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(isnan, isnan, bool)
1530SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(isinf, isinf, bool)
1531SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE(isfinite, isfinite, bool)
1532#endif
1533
1534template <typename Scalar>
1535EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar rint(const Scalar& x) {
1536 return internal::nearest_integer_impl<Scalar>::run_rint(x);
1537}
1538
1539template <typename Scalar>
1540EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar round(const Scalar& x) {
1541 return internal::nearest_integer_impl<Scalar>::run_round(x);
1542}
1543
1544template <typename Scalar>
1545EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar(floor)(const Scalar& x) {
1546 return internal::nearest_integer_impl<Scalar>::run_floor(x);
1547}
1548
1549template <typename Scalar>
1550EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar(ceil)(const Scalar& x) {
1551 return internal::nearest_integer_impl<Scalar>::run_ceil(x);
1552}
1553
1554template <typename Scalar>
1555EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar(trunc)(const Scalar& x) {
1556 return internal::nearest_integer_impl<Scalar>::run_trunc(x);
1557}
1558
1559#if defined(EIGEN_SYCL_DEVICE_ONLY)
1560SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(rint, rint)
1561SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(round, round)
1562SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(floor, floor)
1563SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(ceil, ceil)
1564SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(trunc, trunc)
1565#endif
1566
1567#if defined(EIGEN_GPUCC)
1568template <>
1569EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float floor(const float& x) {
1570 return ::floorf(x);
1571}
1572template <>
1573EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double floor(const double& x) {
1574 return ::floor(x);
1575}
1576template <>
1577EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float ceil(const float& x) {
1578 return ::ceilf(x);
1579}
1580template <>
1581EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double ceil(const double& x) {
1582 return ::ceil(x);
1583}
1584template <>
1585EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float trunc(const float& x) {
1586 return ::truncf(x);
1587}
1588template <>
1589EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double trunc(const double& x) {
1590 return ::trunc(x);
1591}
1592#endif
1593
1594// Integer division with rounding up.
1595// T is assumed to be an integer type with a>=0, and b>0
1596template <typename T>
1597EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr T div_ceil(T a, T b) {
1598 using UnsignedT = std::make_unsigned_t<T>;
1599 EIGEN_STATIC_ASSERT((NumTraits<T>::IsInteger), THIS FUNCTION IS FOR INTEGER TYPES)
1600 // Note: explicitly declaring a and b as non-negative values allows the compiler to use better optimizations
1601 const UnsignedT ua = UnsignedT(a);
1602 const UnsignedT ub = UnsignedT(b);
1603 // Note: This form is used because it cannot overflow.
1604 return ua == 0 ? 0 : (ua - 1) / ub + 1;
1605}
1606
1607// Integer round down to nearest power of b
1608// T is assumed to be an integer type with a>=0, and b>0
1609template <typename T, typename U>
1610EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE constexpr T round_down(T a, U b) {
1611 using UnsignedT = std::make_unsigned_t<T>;
1612 using UnsignedU = std::make_unsigned_t<U>;
1613 EIGEN_STATIC_ASSERT((NumTraits<T>::IsInteger), THIS FUNCTION IS FOR INTEGER TYPES)
1614 EIGEN_STATIC_ASSERT((NumTraits<U>::IsInteger), THIS FUNCTION IS FOR INTEGER TYPES)
1615 // Note: explicitly declaring a and b as non-negative values allows the compiler to use better optimizations
1616 const UnsignedT ua = UnsignedT(a);
1617 const UnsignedU ub = UnsignedU(b);
1618 return ub * (ua / ub);
1619}
1620
1621template <typename T>
1622EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T log10(T x) {
1623 EIGEN_USING_STD(log10);
1624 return log10(x);
1625}
1626
1627template <typename T>
1628EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T log2(T x) {
1629 EIGEN_USING_STD(log2);
1630 return log2(x);
1631}
1632
1633#if defined(EIGEN_SYCL_DEVICE_ONLY)
1634template <>
1635EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float log10(float x) {
1636 return cl::sycl::log10(x);
1637}
1638template <>
1639EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double log10(double x) {
1640 return cl::sycl::log10(x);
1641}
1642template <>
1643EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float log2(float x) {
1644 return cl::sycl::log2(x);
1645}
1646template <>
1647EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double log2(double x) {
1648 return cl::sycl::log2(x);
1649}
1650#endif
1651
1654constexpr int log2(int x) {
1655 unsigned int v(x);
1656 constexpr int table[32] = {0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30,
1657 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31};
1658 v |= v >> 1;
1659 v |= v >> 2;
1660 v |= v >> 4;
1661 v |= v >> 8;
1662 v |= v >> 16;
1663 return table[(v * 0x07C4ACDDU) >> 27];
1664}
1665
1675template <typename Scalar>
1676EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Scalar sqrt(const Scalar& x) {
1677 return EIGEN_MATHFUNC_IMPL(sqrt, Scalar)::run(x);
1678}
1679
1680// Boolean specialization, avoids implicit float to bool conversion (-Wimplicit-conversion-floating-point-to-bool).
1681template <>
1682EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS EIGEN_DEVICE_FUNC bool sqrt<bool>(const bool& x) {
1683 return x;
1684}
1685
1686#if defined(EIGEN_SYCL_DEVICE_ONLY)
1687SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(sqrt, sqrt)
1688#endif
1689
1690#if defined(EIGEN_VECTORIZE_SSE)
1691// Half/BFloat16 scalar wrappers precede the SSE definitions and must not instantiate the primary template.
1692template <>
1693EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float sqrt(const float& x);
1694template <>
1695EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double sqrt(const double& x);
1696#endif
1697
1699template <typename T>
1700EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<!NumTraits<T>::IsComplex, T> cbrt(const T& x) {
1701 EIGEN_USING_STD(cbrt);
1702 return static_cast<T>(cbrt(x));
1703}
1704
1705template <typename T>
1706EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<NumTraits<T>::IsComplex, T> cbrt(const T& x) {
1707 EIGEN_USING_STD(pow);
1708 return pow(x, typename NumTraits<T>::Real(1.0 / 3.0));
1709}
1710
1711#if defined(EIGEN_SYCL_DEVICE_ONLY)
1712SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(cbrt, cbrt)
1713#endif
1714
1716template <typename T>
1717EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T rsqrt(const T& x) {
1718 return internal::rsqrt_impl<T>::run(x);
1719}
1720
1721template <typename T>
1722EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T log(const T& x) {
1723 return internal::log_impl<T>::run(x);
1724}
1725
1726#if defined(EIGEN_SYCL_DEVICE_ONLY)
1727SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(log, log)
1728#endif
1729
1730#if defined(EIGEN_GPUCC)
1731template <>
1732EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float log(const float& x) {
1733 return ::logf(x);
1734}
1735
1736template <>
1737EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double log(const double& x) {
1738 return ::log(x);
1739}
1740#endif
1741
1742template <typename T>
1743EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<
1744 (NumTraits<T>::IsSigned || !NumTraits<T>::IsInteger) && !NumTraits<T>::IsComplex, typename NumTraits<T>::Real>
1745abs(const T& x) {
1746 EIGEN_USING_STD(abs);
1747 return abs(x);
1748}
1749
1750template <typename T>
1751EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<NumTraits<T>::IsComplex, typename NumTraits<T>::Real> abs(
1752 const T& x) {
1753 return numext::hypot(numext::real(x), numext::imag(x));
1754}
1755
1756template <typename T>
1757EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::enable_if_t<
1758 !NumTraits<T>::IsSigned && NumTraits<T>::IsInteger && !NumTraits<T>::IsComplex, typename NumTraits<T>::Real>
1759abs(const T& x) {
1760 return x;
1761}
1762
1763#if defined(EIGEN_SYCL_DEVICE_ONLY)
1764SYCL_SPECIALIZE_INTEGER_TYPES_UNARY(abs, abs)
1765SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(abs, fabs)
1766#endif
1767
1768#if defined(EIGEN_GPUCC)
1769template <>
1770EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float abs(const float& x) {
1771 return ::fabsf(x);
1772}
1773
1774template <>
1775EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double abs(const double& x) {
1776 return ::fabs(x);
1777}
1778
1779template <>
1780EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float abs(const std::complex<float>& x) {
1781 return ::hypotf(x.real(), x.imag());
1782}
1783
1784template <>
1785EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double abs(const std::complex<double>& x) {
1786 return ::hypot(x.real(), x.imag());
1787}
1788#endif
1789
1790template <typename Scalar, bool IsInteger = NumTraits<Scalar>::IsInteger, bool IsSigned = NumTraits<Scalar>::IsSigned>
1791struct signbit_impl;
1792template <typename Scalar>
1793struct signbit_impl<Scalar, false, true> {
1794 static constexpr size_t Size = sizeof(Scalar);
1795 static constexpr size_t Shift = (CHAR_BIT * Size) - 1;
1796 using intSize_t = typename get_integer_by_size<Size>::signed_type;
1797 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static Scalar run(const Scalar& x) {
1798 intSize_t a = bit_cast<intSize_t, Scalar>(x);
1799 a = a >> Shift;
1800 Scalar result = bit_cast<Scalar, intSize_t>(a);
1801 return result;
1802 }
1803};
1804template <typename Scalar>
1805struct signbit_impl<Scalar, true, true> {
1806 static constexpr size_t Size = sizeof(Scalar);
1807 static constexpr size_t Shift = (CHAR_BIT * Size) - 1;
1808 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Scalar run(const Scalar& x) { return x >> Shift; }
1809};
1810template <typename Scalar>
1811struct signbit_impl<Scalar, true, false> {
1812 EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Scalar run(const Scalar&) { return Scalar(0); }
1813};
1814template <typename Scalar>
1815EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static constexpr Scalar signbit(const Scalar& x) {
1816 return signbit_impl<Scalar>::run(x);
1817}
1818
1819template <typename T>
1820EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T exp(const T& x) {
1821 EIGEN_USING_STD(exp);
1822 return exp(x);
1823}
1824
1825// MSVC through at least 19.51 mishandles some finite-real, non-finite-imaginary inputs to std::exp(complex).
1826#if EIGEN_COMP_MSVC
1827template <typename RealScalar>
1828EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::complex<RealScalar> exp(const std::complex<RealScalar>& x) {
1829 // Evaluate eagerly to preserve the floating-point exceptions raised by std::exp before correcting its result.
1830 const std::complex<RealScalar> result = std::exp(x);
1831 // If z is (x,±∞) (for any finite x), the result is (NaN,NaN) and FE_INVALID is raised.
1832 // If z is (x,NaN) (for any finite x), the result is (NaN,NaN) and FE_INVALID may be raised.
1833 if ((isfinite)(real_ref(x)) && !(isfinite)(imag_ref(x))) {
1834 return std::complex<RealScalar>(NumTraits<RealScalar>::quiet_NaN(), NumTraits<RealScalar>::quiet_NaN());
1835 }
1836#if EIGEN_COMP_MSVC < 1931
1837 // Before 19.31, MSVC also mishandles a positive-infinite real part with a non-finite imaginary part.
1838 // If z is (+∞,±∞), the result is (±∞,NaN) and FE_INVALID is raised (the sign of the real part is unspecified)
1839 // If z is (+∞,NaN), the result is (±∞,NaN) (the sign of the real part is unspecified)
1840 if ((real_ref(x) == NumTraits<RealScalar>::infinity() && !(isfinite)(imag_ref(x)))) {
1841 return std::complex<RealScalar>(NumTraits<RealScalar>::infinity(), NumTraits<RealScalar>::quiet_NaN());
1842 }
1843#endif
1844 return result;
1845}
1846#endif
1847
1848#if defined(EIGEN_SYCL_DEVICE_ONLY)
1849SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(exp, exp)
1850#endif
1851
1852#if defined(EIGEN_GPUCC)
1853template <>
1854EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float exp(const float& x) {
1855 return ::expf(x);
1856}
1857
1858template <>
1859EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double exp(const double& x) {
1860 return ::exp(x);
1861}
1862
1863// exp(x + 0i) = exp(x) + 0i even where exp(x) overflows, so a zero imaginary part is kept rather than computed as
1864// inf * 0.
1865template <>
1866EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::complex<float> exp(const std::complex<float>& x) {
1867 float com = ::expf(x.real());
1868 float res_real = com * ::cosf(x.imag());
1869 float res_imag = x.imag() == 0.f ? x.imag() : com * ::sinf(x.imag());
1870 return std::complex<float>(res_real, res_imag);
1871}
1872
1873template <>
1874EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::complex<double> exp(const std::complex<double>& x) {
1875 double com = ::exp(x.real());
1876 double res_real = com * ::cos(x.imag());
1877 double res_imag = x.imag() == 0.0 ? x.imag() : com * ::sin(x.imag());
1878 return std::complex<double>(res_real, res_imag);
1879}
1880#endif
1881
1882template <typename T>
1883EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T exp2(const T& x) {
1884 EIGEN_USING_STD(exp2);
1885 return exp2(x);
1886}
1887
1888// There is no std::exp2 for complex arguments.
1889template <typename RealScalar>
1890EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::complex<RealScalar> exp2(const std::complex<RealScalar>& x) {
1891 return internal::complex_exp2_impl<RealScalar>::run(x);
1892}
1893
1894#if defined(EIGEN_SYCL_DEVICE_ONLY)
1895SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(exp2, exp2)
1896#endif
1897
1898#if defined(EIGEN_GPUCC)
1899template <>
1900EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float exp2(const float& x) {
1901 return ::exp2f(x);
1902}
1903
1904template <>
1905EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double exp2(const double& x) {
1906 return ::exp2(x);
1907}
1908#endif
1909
1910// Exact scaling by 2^exponent, including denormals and unrepresentable scale factors.
1911// Results outside the finite range saturate to zero or infinity.
1912template <typename T>
1913EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T ldexp(const T& x, int exponent) {
1914 EIGEN_USING_STD(ldexp);
1915 return static_cast<T>(ldexp(x, exponent));
1916}
1917
1918#if defined(EIGEN_SYCL_DEVICE_ONLY)
1919template <>
1920EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_float ldexp(const cl::sycl::cl_float& x, int exponent) {
1921 return cl::sycl::ldexp(x, exponent);
1922}
1923template <>
1924EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE cl::sycl::cl_double ldexp(const cl::sycl::cl_double& x, int exponent) {
1925 return cl::sycl::ldexp(x, exponent);
1926}
1927#endif
1928
1929#if defined(EIGEN_GPUCC)
1930template <>
1931EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float ldexp(const float& x, int exponent) {
1932 return ::ldexpf(x, exponent);
1933}
1934
1935template <>
1936EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double ldexp(const double& x, int exponent) {
1937 return ::ldexp(x, exponent);
1938}
1939#endif
1940
1941template <typename Scalar>
1942EIGEN_DEVICE_FUNC inline Scalar expm1(const Scalar& x) {
1943 return EIGEN_MATHFUNC_IMPL(expm1, Scalar)::run(x);
1944}
1945
1946#if defined(EIGEN_SYCL_DEVICE_ONLY)
1947SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(expm1, expm1)
1948#endif
1949
1950#if defined(EIGEN_GPUCC)
1951template <>
1952EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float expm1(const float& x) {
1953 return ::expm1f(x);
1954}
1955
1956template <>
1957EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double expm1(const double& x) {
1958 return ::expm1(x);
1959}
1960#endif
1961
1962template <typename T>
1963EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T cos(const T& x) {
1964 EIGEN_USING_STD(cos);
1965 return cos(x);
1966}
1967
1968#if defined(EIGEN_SYCL_DEVICE_ONLY)
1969SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(cos, cos)
1970#endif
1971
1972#if defined(EIGEN_GPUCC)
1973template <>
1974EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float cos(const float& x) {
1975 return ::cosf(x);
1976}
1977
1978template <>
1979EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double cos(const double& x) {
1980 return ::cos(x);
1981}
1982#endif
1983
1984template <typename T>
1985EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T sin(const T& x) {
1986 EIGEN_USING_STD(sin);
1987 return sin(x);
1988}
1989
1990#if defined(EIGEN_SYCL_DEVICE_ONLY)
1991SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(sin, sin)
1992#endif
1993
1994#if defined(EIGEN_GPUCC)
1995template <>
1996EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float sin(const float& x) {
1997 return ::sinf(x);
1998}
1999
2000template <>
2001EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double sin(const double& x) {
2002 return ::sin(x);
2003}
2004#endif
2005
2006template <typename T>
2007EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T tan(const T& x) {
2008 EIGEN_USING_STD(tan);
2009 return tan(x);
2010}
2011
2012#if defined(EIGEN_SYCL_DEVICE_ONLY)
2013SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(tan, tan)
2014#endif
2015
2016#if defined(EIGEN_GPUCC)
2017template <>
2018EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float tan(const float& x) {
2019 return ::tanf(x);
2020}
2021
2022template <>
2023EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double tan(const double& x) {
2024 return ::tan(x);
2025}
2026#endif
2027
2028template <typename T>
2029EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T acos(const T& x) {
2030 EIGEN_USING_STD(acos);
2031 return acos(x);
2032}
2033
2034template <typename T>
2035EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T acosh(const T& x) {
2036 EIGEN_USING_STD(acosh);
2037 return static_cast<T>(acosh(x));
2038}
2039
2040#if defined(EIGEN_SYCL_DEVICE_ONLY)
2041SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(acos, acos)
2042SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(acosh, acosh)
2043#endif
2044
2045#if defined(EIGEN_GPUCC)
2046template <>
2047EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float acos(const float& x) {
2048 return ::acosf(x);
2049}
2050
2051template <>
2052EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double acos(const double& x) {
2053 return ::acos(x);
2054}
2055#endif
2056
2057template <typename T>
2058EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T asin(const T& x) {
2059 EIGEN_USING_STD(asin);
2060 return asin(x);
2061}
2062
2063template <typename T>
2064EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T asinh(const T& x) {
2065 EIGEN_USING_STD(asinh);
2066 return static_cast<T>(asinh(x));
2067}
2068
2069#if defined(EIGEN_SYCL_DEVICE_ONLY)
2070SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(asin, asin)
2071SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(asinh, asinh)
2072#endif
2073
2074#if defined(EIGEN_GPUCC)
2075template <>
2076EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float asin(const float& x) {
2077 return ::asinf(x);
2078}
2079
2080template <>
2081EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double asin(const double& x) {
2082 return ::asin(x);
2083}
2084#endif
2085
2086template <typename T>
2087EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T atan(const T& x) {
2088 EIGEN_USING_STD(atan);
2089 return static_cast<T>(atan(x));
2090}
2091
2092template <typename T, std::enable_if_t<!NumTraits<T>::IsComplex, int> = 0>
2093EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T atan2(const T& y, const T& x) {
2094 EIGEN_USING_STD(atan2);
2095 return static_cast<T>(atan2(y, x));
2096}
2097
2098template <typename T>
2099EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T atanh(const T& x) {
2100 EIGEN_USING_STD(atanh);
2101 return static_cast<T>(atanh(x));
2102}
2103
2104#if defined(EIGEN_SYCL_DEVICE_ONLY)
2105SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(atan2, atan2)
2106SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(atan, atan)
2107SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(atanh, atanh)
2108#endif
2109
2110#if defined(EIGEN_GPUCC)
2111template <>
2112EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float atan(const float& x) {
2113 return ::atanf(x);
2114}
2115
2116template <>
2117EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double atan(const double& x) {
2118 return ::atan(x);
2119}
2120#endif
2121
2122template <typename T>
2123EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T cosh(const T& x) {
2124 EIGEN_USING_STD(cosh);
2125 return static_cast<T>(cosh(x));
2126}
2127
2128#if defined(EIGEN_SYCL_DEVICE_ONLY)
2129SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(cosh, cosh)
2130#endif
2131
2132#if defined(EIGEN_GPUCC)
2133template <>
2134EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float cosh(const float& x) {
2135 return ::coshf(x);
2136}
2137
2138template <>
2139EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double cosh(const double& x) {
2140 return ::cosh(x);
2141}
2142#endif
2143
2144template <typename T>
2145EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T sinh(const T& x) {
2146 EIGEN_USING_STD(sinh);
2147 return static_cast<T>(sinh(x));
2148}
2149
2150#if defined(EIGEN_SYCL_DEVICE_ONLY)
2151SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(sinh, sinh)
2152#endif
2153
2154#if defined(EIGEN_GPUCC)
2155template <>
2156EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float sinh(const float& x) {
2157 return ::sinhf(x);
2158}
2159
2160template <>
2161EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double sinh(const double& x) {
2162 return ::sinh(x);
2163}
2164#endif
2165
2166template <typename T>
2167EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T tanh(const T& x) {
2168 EIGEN_USING_STD(tanh);
2169 return tanh(x);
2170}
2171
2172#if (!defined(EIGEN_GPUCC)) && EIGEN_FAST_MATH && !defined(EIGEN_SYCL_DEVICE_ONLY)
2173EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float tanh(float x) { return internal::ptanh_float(x); }
2174#endif
2175
2176#if defined(EIGEN_SYCL_DEVICE_ONLY)
2177SYCL_SPECIALIZE_FLOATING_TYPES_UNARY(tanh, tanh)
2178#endif
2179
2180#if defined(EIGEN_GPUCC)
2181template <>
2182EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float tanh(const float& x) {
2183 return ::tanhf(x);
2184}
2185
2186template <>
2187EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double tanh(const double& x) {
2188 return ::tanh(x);
2189}
2190#endif
2191
2192template <typename T>
2193EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T fmod(const T& a, const T& b) {
2194 EIGEN_USING_STD(fmod);
2195 return fmod(a, b);
2196}
2197
2198#if defined(EIGEN_SYCL_DEVICE_ONLY)
2199SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(fmod, fmod)
2200#endif
2201
2202#if defined(EIGEN_GPUCC)
2203template <>
2204EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float fmod(const float& a, const float& b) {
2205 return ::fmodf(a, b);
2206}
2207
2208template <>
2209EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double fmod(const double& a, const double& b) {
2210 return ::fmod(a, b);
2211}
2212#endif
2213
2214template <typename T>
2215EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T nextafter(const T& from, const T& to) {
2216 EIGEN_USING_STD(nextafter);
2217 return nextafter(from, to);
2218}
2219
2225template <typename Scalar,
2226 std::enable_if_t<!std::is_same<Scalar, long double>::value && std::numeric_limits<Scalar>::is_iec559 &&
2227 std::numeric_limits<Scalar>::radix == 2 &&
2228 (sizeof(Scalar) == sizeof(uint16_t) || sizeof(Scalar) == sizeof(uint32_t) ||
2229 sizeof(Scalar) == sizeof(uint64_t)),
2230 int> = 0>
2231EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar ceil_power_of_two(const Scalar& value) {
2232 using Binary = internal::binary_floating_point_traits<Scalar>;
2233 using Bits = typename Binary::Bits;
2234 const Bits valueBits = Binary::bits(value);
2235 const Bits resultBits = Bits((valueBits + Binary::kFractionMask) & Binary::kExponentMask);
2236 return bit_cast<Scalar>(resultBits);
2237}
2238
2239// long double has no portable object representation: depending on the ABI it can be binary64, padded x87 extended
2240// precision, IEEE binary128, or IBM double-double. Construct the result arithmetically instead of inspecting its bits.
2241#if !defined(EIGEN_GPU_COMPILE_PHASE)
2242EIGEN_STRONG_INLINE long double ceil_power_of_two(const long double& value) {
2243 EIGEN_USING_STD(frexp);
2244 EIGEN_USING_STD(ldexp);
2245 int exponent = 0;
2246 const long double fraction = frexp(value, &exponent);
2247 if (numext::is_exactly_zero(fraction)) return 0.0L;
2248 if (numext::equal_strict(fraction, 0.5L)) --exponent;
2249 if (exponent < std::numeric_limits<long double>::min_exponent - 1)
2250 exponent = std::numeric_limits<long double>::min_exponent - 1;
2251 return ldexp(1.0L, exponent);
2252}
2253#endif
2254
2255#if defined(EIGEN_SYCL_DEVICE_ONLY)
2256SYCL_SPECIALIZE_FLOATING_TYPES_BINARY(nextafter, nextafter)
2257#endif
2258
2259#if defined(EIGEN_GPUCC)
2260template <>
2261EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE float nextafter(const float& from, const float& to) {
2262 return ::nextafterf(from, to);
2263}
2264
2265template <>
2266EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE double nextafter(const double& from, const double& to) {
2267 return ::nextafter(from, to);
2268}
2269#endif
2270
2271#if defined(EIGEN_SYCL_DEVICE_ONLY)
2272#undef SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_BINARY
2273#undef SYCL_SPECIALIZE_SIGNED_INTEGER_TYPES_UNARY
2274#undef SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_BINARY
2275#undef SYCL_SPECIALIZE_UNSIGNED_INTEGER_TYPES_UNARY
2276#undef SYCL_SPECIALIZE_INTEGER_TYPES_BINARY
2277#undef SYCL_SPECIALIZE_INTEGER_TYPES_UNARY
2278#undef SYCL_SPECIALIZE_FLOATING_TYPES_BINARY
2279#undef SYCL_SPECIALIZE_FLOATING_TYPES_UNARY
2280#undef SYCL_SPECIALIZE_FLOATING_TYPES_UNARY_FUNC_RET_TYPE
2281#undef SYCL_SPECIALIZE_GEN_UNARY_FUNC
2282#undef SYCL_SPECIALIZE_UNARY_FUNC
2283#undef SYCL_SPECIALIZE_GEN1_BINARY_FUNC
2284#undef SYCL_SPECIALIZE_GEN2_BINARY_FUNC
2285#undef SYCL_SPECIALIZE_BINARY_FUNC
2286#endif
2287
2288// A logical shift is a shift of the bit pattern, so it goes through the unsigned type of the same
2289// width whatever Scalar's signedness. A Scalar narrower than int promotes to int for the shift
2290// itself, which the explicit truncation undoes: for the left shift the promoted value can carry set
2291// bits above Scalar's width, and dropping them is the operation rather than an accident of the cast.
2292template <typename Scalar, typename Enable = std::enable_if_t<std::is_integral<Scalar>::value>>
2293EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar logical_shift_left(const Scalar& a, int n) {
2294 using UnsignedScalar = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type;
2295 return bit_cast<Scalar, UnsignedScalar>(static_cast<UnsignedScalar>(bit_cast<UnsignedScalar, Scalar>(a) << n));
2296}
2297
2298template <typename Scalar, typename Enable = std::enable_if_t<std::is_integral<Scalar>::value>>
2299EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar logical_shift_right(const Scalar& a, int n) {
2300 using UnsignedScalar = typename numext::get_integer_by_size<sizeof(Scalar)>::unsigned_type;
2301 return bit_cast<Scalar, UnsignedScalar>(static_cast<UnsignedScalar>(bit_cast<UnsignedScalar, Scalar>(a) >> n));
2302}
2303
2304// An arithmetic shift propagates the sign bit, so it coincides with the logical shift when Scalar is
2305// unsigned and has none. Shifting through the signed type unconditionally would sign-extend an
2306// ordinary value bit, which is what every backend's parithmetic_shift_right on unsigned packets
2307// avoids, leaving the scalar and vectorized paths of one expression disagreeing. Scalar's own
2308// operator>> already selects on its signedness, so no reinterpretation is needed; the cast only
2309// undoes the integral promotion a Scalar narrower than int is subject to.
2310template <typename Scalar, typename Enable = std::enable_if_t<std::is_integral<Scalar>::value>>
2311EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar arithmetic_shift_right(const Scalar& a, int n) {
2312 return static_cast<Scalar>(a >> n);
2313}
2314
2315template <typename Scalar>
2316EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar fma(const Scalar& x, const Scalar& y, const Scalar& z) {
2317 return internal::fma_impl<Scalar>::run(x, y, z);
2318}
2319
2320// Multiply-add.
2321template <typename Scalar>
2322EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar madd(const Scalar& x, const Scalar& y, const Scalar& z) {
2323 return internal::madd_impl<Scalar>::run(x, y, z);
2324}
2325
2326} // end namespace numext
2327
2328namespace internal {
2329
2330// Two-product residual using FMA, including software FMA when hardware FMA is unavailable.
2331template <typename Scalar>
2332EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar twoprod_low_fma(const Scalar& x, const Scalar& y, const Scalar& xy) {
2333 return numext::fma(x, y, Scalar(-xy));
2334}
2335
2336// p_hi = fl(x * y) and p_lo = fl(x * y - p_hi), for 1/2 <= |y| <= 2 and every x for which p_hi is normal. Unlike
2337// twoprod_low_fma, it takes Dekker's product on targets without an FMA instruction where that is exact for Scalar.
2338// Defined with the other error-free transformations in arch/Default/GenericPacketMathDoubleWord.h.
2339template <typename Scalar>
2340EIGEN_DEVICE_FUNC void twoprod_full_range(const Scalar& x, const Scalar& y, Scalar& p_hi, Scalar& p_lo);
2341
2342template <typename T>
2343EIGEN_DEVICE_FUNC bool isfinite_impl(const std::complex<T>& x) {
2344 EIGEN_IF_CONSTEXPR ((std::is_floating_point<T>::value)) {
2345 // Eager classification lets coefficient-wise loops remain branch-free.
2346 return static_cast<unsigned int>((numext::isfinite)(numext::real(x))) & (numext::isfinite)(numext::imag(x));
2347 } else {
2348 return (numext::isfinite)(numext::real(x)) && (numext::isfinite)(numext::imag(x));
2349 }
2350}
2351
2352template <typename T>
2353EIGEN_DEVICE_FUNC bool isnan_impl(const std::complex<T>& x) {
2354 EIGEN_IF_CONSTEXPR ((std::is_floating_point<T>::value)) {
2355 return static_cast<unsigned int>((numext::isnan)(numext::real(x))) | (numext::isnan)(numext::imag(x));
2356 } else {
2357 return (numext::isnan)(numext::real(x)) || (numext::isnan)(numext::imag(x));
2358 }
2359}
2360
2361template <typename T>
2362EIGEN_DEVICE_FUNC bool isinf_impl(const std::complex<T>& x) {
2363 EIGEN_IF_CONSTEXPR ((std::is_floating_point<T>::value)) {
2364 const bool has_inf = static_cast<unsigned int>((numext::isinf)(numext::real(x))) | (numext::isinf)(numext::imag(x));
2365 const bool has_nan = (numext::isnan)(x);
2366 return has_inf & !has_nan;
2367 } else {
2368 return ((numext::isinf)(numext::real(x)) || (numext::isinf)(numext::imag(x))) && (!(numext::isnan)(x));
2369 }
2370}
2371
2372/****************************************************************************
2373 * Implementation of fuzzy comparisons *
2374 ****************************************************************************/
2375
2376template <typename Scalar, bool IsComplex, bool IsInteger>
2377struct scalar_fuzzy_default_impl {};
2378
2379template <typename Scalar>
2380struct scalar_fuzzy_default_impl<Scalar, false, false> {
2381 using RealScalar = typename NumTraits<Scalar>::Real;
2382 template <typename OtherScalar>
2383 EIGEN_DEVICE_FUNC static inline bool isMuchSmallerThan(const Scalar& x, const OtherScalar& y,
2384 const RealScalar& prec) {
2385 return numext::abs(x) <= numext::abs(y) * prec;
2386 }
2387 EIGEN_DEVICE_FUNC static inline bool isApprox(const Scalar& x, const Scalar& y, const RealScalar& prec) {
2388 return numext::abs(x - y) <= numext::mini(numext::abs(x), numext::abs(y)) * prec;
2389 }
2390 EIGEN_DEVICE_FUNC static inline bool isApproxOrLessThan(const Scalar& x, const Scalar& y, const RealScalar& prec) {
2391 return x <= y || isApprox(x, y, prec);
2392 }
2393};
2394
2395template <typename Scalar>
2396struct scalar_fuzzy_default_impl<Scalar, false, true> {
2397 using RealScalar = typename NumTraits<Scalar>::Real;
2398 template <typename OtherScalar>
2399 EIGEN_DEVICE_FUNC static inline bool isMuchSmallerThan(const Scalar& x, const Scalar&, const RealScalar&) {
2400 return x == Scalar(0);
2401 }
2402 EIGEN_DEVICE_FUNC static inline bool isApprox(const Scalar& x, const Scalar& y, const RealScalar&) { return x == y; }
2403 EIGEN_DEVICE_FUNC static inline bool isApproxOrLessThan(const Scalar& x, const Scalar& y, const RealScalar&) {
2404 return x <= y;
2405 }
2406};
2407
2408// Compares moduli, not their squares: abs2 overflows above sqrt(highest) and underflows below sqrt(min), where unequal
2409// values then compared as approximately equal.
2410template <typename Scalar>
2411struct scalar_fuzzy_default_impl<Scalar, true, false> {
2412 using RealScalar = typename NumTraits<Scalar>::Real;
2413 template <typename OtherScalar>
2414 EIGEN_DEVICE_FUNC static inline bool isMuchSmallerThan(const Scalar& x, const OtherScalar& y,
2415 const RealScalar& prec) {
2416 return numext::abs(x) <= numext::abs(y) * prec;
2417 }
2418 EIGEN_DEVICE_FUNC static inline bool isApprox(const Scalar& x, const Scalar& y, const RealScalar& prec) {
2419 return numext::abs(x - y) <= numext::mini(numext::abs(x), numext::abs(y)) * prec;
2420 }
2421};
2422
2423template <typename Scalar>
2424struct scalar_fuzzy_impl
2425 : scalar_fuzzy_default_impl<Scalar, NumTraits<Scalar>::IsComplex, NumTraits<Scalar>::IsInteger> {};
2426
2427template <typename Scalar, typename OtherScalar>
2428EIGEN_DEVICE_FUNC inline bool isMuchSmallerThan(
2429 const Scalar& x, const OtherScalar& y,
2430 const typename NumTraits<Scalar>::Real& precision = NumTraits<Scalar>::dummy_precision()) {
2431 return scalar_fuzzy_impl<Scalar>::template isMuchSmallerThan<OtherScalar>(x, y, precision);
2432}
2433
2434template <typename Scalar>
2435EIGEN_DEVICE_FUNC inline bool isApprox(
2436 const Scalar& x, const Scalar& y,
2437 const typename NumTraits<Scalar>::Real& precision = NumTraits<Scalar>::dummy_precision()) {
2438 return scalar_fuzzy_impl<Scalar>::isApprox(x, y, precision);
2439}
2440
2441template <typename Scalar>
2442EIGEN_DEVICE_FUNC inline bool isApproxOrLessThan(
2443 const Scalar& x, const Scalar& y,
2444 const typename NumTraits<Scalar>::Real& precision = NumTraits<Scalar>::dummy_precision()) {
2445 return scalar_fuzzy_impl<Scalar>::isApproxOrLessThan(x, y, precision);
2446}
2447
2448/******************************************
2449*** The special case of the bool type ***
2450******************************************/
2451
2452template <>
2453struct scalar_fuzzy_impl<bool> {
2454 using RealScalar = bool;
2455
2456 template <typename OtherScalar>
2457 EIGEN_DEVICE_FUNC static inline bool isMuchSmallerThan(const bool& x, const bool&, const bool&) {
2458 return !x;
2459 }
2460
2461 EIGEN_DEVICE_FUNC static inline bool isApprox(bool x, bool y, bool) { return x == y; }
2462
2463 EIGEN_DEVICE_FUNC static inline bool isApproxOrLessThan(const bool& x, const bool& y, const bool&) {
2464 return (!x) || y;
2465 }
2466};
2467
2468} // end namespace internal
2469
2470// Default implementations that rely on other numext implementations
2471namespace internal {
2472
2473// Specialization for complex types that are not supported by std::expm1.
2474template <typename RealScalar>
2475struct expm1_impl<std::complex<RealScalar>> {
2476 EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar)
2477
2478 EIGEN_DEVICE_FUNC static inline std::complex<RealScalar> run(const std::complex<RealScalar>& x) {
2479 RealScalar xr = x.real();
2480 RealScalar xi = x.imag();
2481 // expm1(z) = exp(z) - 1
2482 // = exp(x + i * y) - 1
2483 // = exp(x) * (cos(y) + i * sin(y)) - 1
2484 // = exp(x) * cos(y) - 1 + i * exp(x) * sin(y)
2485 // Imag(expm1(z)) = exp(x) * sin(y)
2486 // Real(expm1(z)) = exp(x) * cos(y) - 1
2487 // = exp(x) * cos(y) - 1.
2488 // = expm1(x) + exp(x) * (cos(y) - 1)
2489 // = expm1(x) + exp(x) * (2 * sin(y / 2) ** 2)
2490 RealScalar erm1 = numext::expm1<RealScalar>(xr);
2491 RealScalar er = erm1 + RealScalar(1.);
2492 // C99 Annex G for exp, less one: (+inf, inf or NaN) -> (inf, NaN) and (-inf, inf or NaN) -> (-1, 0).
2493 // xi * 0 is that NaN, raising FE_INVALID for xi = inf as Annex G asks.
2494 if ((numext::isinf)(xr) && !(numext::isfinite)(xi)) {
2495 return xr > RealScalar(0) ? std::complex<RealScalar>(xr, xi * RealScalar(0))
2496 : std::complex<RealScalar>(RealScalar(-1), RealScalar(0));
2497 }
2498 const RealScalar s = numext::sin(xi);
2499 const RealScalar c = numext::cos(xi);
2500 // 1 - cos(y) = 2 sin(y / 2)^2, taken as sin(y)^2 / (1 + cos(y)) where the difference would cancel.
2501 const RealScalar one_minus_c = c > RealScalar(0) ? s * s / (RealScalar(1) + c) : RealScalar(1) - c;
2502 // exp(x) overflows where exp(x) cos(y) and exp(x) sin(y) need not; multiply exp(x / 2) in twice then, or
2503 // exp(x / 3) three times where that overflows too, and exp(x) sin(y) is finite only for denormal y.
2504 if ((numext::isinf)(er) && (numext::isfinite)(xr)) {
2505 const RealScalar h = numext::exp(xr / RealScalar(2));
2506 if (!(numext::isinf)(h)) {
2507 return std::complex<RealScalar>((h * c) * h - RealScalar(1), numext::is_exactly_zero(xi) ? xi : (h * s) * h);
2508 }
2509 const RealScalar h3 = numext::exp(xr / RealScalar(3));
2510 return std::complex<RealScalar>(((h3 * c) * h3) * h3, numext::is_exactly_zero(xi) ? xi : ((h3 * s) * h3) * h3);
2511 }
2512 // For x < 0 both terms of expm1(x) - exp(x) (1 - cos(y)) are <= 0, and the result is -1 exactly once exp(x) is
2513 // negligible. For x >= 0, expm1(x) cos(y) - (1 - cos(y)) has the smaller terms, and no inf - inf at x = inf.
2514 const RealScalar real_part = xr < RealScalar(0) ? erm1 - er * one_minus_c : erm1 * c - one_minus_c;
2515 // exp(x) * sin(0) is NaN for infinite exp(x); keep the exact zero instead.
2516 return std::complex<RealScalar>(real_part, numext::is_exactly_zero(xi) ? xi : er * s);
2517 }
2518};
2519
2520template <typename RealScalar>
2521struct log1p_impl<std::complex<RealScalar>> {
2522 EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar)
2523
2524 EIGEN_DEVICE_FUNC static inline std::complex<RealScalar> run(const std::complex<RealScalar>& z) {
2525 const RealScalar x = z.real();
2526 const RealScalar y = z.imag();
2527 // log|1 + z| = log1p(x (2 + x) + y^2) / 2. Rounding |1 + z| instead loses the real part to cancellation near
2528 // |1 + z| = 1, and with it the relative accuracy of log1p(z) for small z off the real axis.
2529 if (numext::abs(x) < RealScalar(0.5) && numext::abs(y) < RealScalar(0.5) && !numext::is_exactly_zero(y)) {
2530 return std::complex<RealScalar>(numext::log1p(x * (RealScalar(2) + x) + y * y) / RealScalar(2),
2531 numext::atan2(y, RealScalar(1) + x));
2532 }
2533 return std_fallback::log1p(z);
2534 }
2535};
2536
2537// 2^(a + ib) = 2^a (cos(t) + i sin(t)) with t = b ln(2). The modulus is the real exp2, so 2^n is exact. Rounding t
2538// would cost |t| eps in the phase, so b ln(2) = t_hi + t_lo is kept in double-word precision, ln(2) = ln2_hi + ln2_lo:
2539// cos(t) = cos(t_hi) + (cos(t_hi) (cos(t_lo) - 1) - sin(t_hi) sin(t_lo)),
2540// sin(t) = sin(t_hi) + (sin(t_hi) (cos(t_lo) - 1) + cos(t_hi) sin(t_lo)).
2541// Special values are those of exp(z) in C99 Annex G, since ln(2) > 0.
2542template <typename RealScalar>
2543struct complex_exp2_impl {
2544 EIGEN_STATIC_ASSERT_NON_INTEGER(RealScalar)
2545
2546 EIGEN_DEVICE_FUNC static inline std::complex<RealScalar> run(const std::complex<RealScalar>& x) {
2547 using ComplexScalar = std::complex<RealScalar>;
2548 const RealScalar a = x.real();
2549 const RealScalar b = x.imag();
2550 // (a, +-0) -> (2^a, +-0), also for a = +-inf and NaN.
2551 if (numext::is_exactly_zero(b)) return ComplexScalar(numext::exp2(a), b);
2552 // (+inf, inf or NaN) -> (inf, NaN) and (-inf, inf or NaN) -> (0, 0), where 2^a cos(t) would be NaN.
2553 // b * 0 is that NaN, raising FE_INVALID for b = inf as Annex G asks.
2554 if ((numext::isinf)(a) && !(numext::isfinite)(b)) {
2555 return a > RealScalar(0) ? ComplexScalar(a, b * RealScalar(0)) : ComplexScalar(RealScalar(0), RealScalar(0));
2556 }
2557 // ln2_lo = (double(ln(2)) - ln2_hi) + (ln(2) - double(ln(2))), the first difference exact in Wide.
2558 // nvcc 11.8 drops parentheses around a `>` in a template argument when it regenerates host code.
2559 constexpr bool kWiderThanDouble = NumTraits<RealScalar>::digits() > NumTraits<double>::digits();
2560 using Wide = std::conditional_t<kWiderThanDouble, RealScalar, double>;
2561 const RealScalar ln2_hi = RealScalar(EIGEN_LN2);
2562 const RealScalar ln2_lo = RealScalar((Wide(static_cast<double>(EIGEN_LN2)) - Wide(ln2_hi)) +
2563 Wide(2.3190468138462996154948554638754786504e-17L));
2564 // The phase is computed for |b|: a libm sin need not be exactly odd (IBM double-double's is not), and
2565 // exp2(conj(z)) = conj(exp2(z)) should hold exactly.
2566 const bool negative = b < RealScalar(0);
2567 const RealScalar abs_b = numext::abs(b);
2568 RealScalar t_hi, t_lo;
2569 twoprod_full_range(abs_b, ln2_hi, t_hi, t_lo);
2570 t_lo += abs_b * ln2_lo;
2571 // |t_lo| <= ulp(t_hi) / 2, so the Taylor terms below are exact to rounding unless |t| exceeds about 2 / sqrt(eps).
2572 // The phase is within an ulp of b ln(2) while |t| < 1 / eps.
2573 RealScalar sin_lo = t_lo;
2574 RealScalar cos_lo_m1 = -t_lo * t_lo / RealScalar(2);
2575 if (numext::abs(t_lo) > numext::sqrt(NumTraits<RealScalar>::epsilon())) {
2576 sin_lo = numext::sin(t_lo);
2577 const RealScalar sin_half = numext::sin(t_lo / RealScalar(2));
2578 cos_lo_m1 = RealScalar(-2) * sin_half * sin_half;
2579 }
2580 const RealScalar cos_hi = numext::cos(t_hi);
2581 const RealScalar sin_hi = numext::sin(t_hi);
2582 const RealScalar c = cos_hi + (cos_hi * cos_lo_m1 - sin_hi * sin_lo);
2583 const RealScalar s = sin_hi + (sin_hi * cos_lo_m1 + cos_hi * sin_lo);
2584 // 2^a overflows or is denormal where 2^a cos(t) and 2^a sin(t) need not be; apply 2^e after the products then.
2585 // Past 2 e_max, a part is finite only for denormal t.
2586 const int e_max = NumTraits<RealScalar>::max_exponent() - 1;
2587 const int e = a > RealScalar(e_max) ? (a > RealScalar(2 * e_max) ? 2 * e_max : e_max)
2588 : (a < RealScalar(NumTraits<RealScalar>::min_exponent()) ? -e_max : 0);
2589 const RealScalar m = numext::exp2(a - RealScalar(e));
2590 if (numext::abs(t_hi) < (numext::numeric_limits<RealScalar>::min)()) {
2591 // A denormal t has lost bits that 2^a can bring back. There cos(t) = 1 and sin(t) = t, taken from b 2^digits.
2592 const int p = NumTraits<RealScalar>::digits();
2593 const RealScalar b_p = numext::ldexp(abs_b, p);
2594 RealScalar u_hi, u_lo;
2595 twoprod_full_range(b_p, ln2_hi, u_hi, u_lo);
2596 const RealScalar im = numext::ldexp(m * (u_hi + (u_lo + b_p * ln2_lo)), e - p);
2597 return ComplexScalar(numext::ldexp(m, e), negative ? -im : im);
2598 }
2599 const RealScalar re = e == 0 ? m * c : numext::ldexp(m * c, e);
2600 const RealScalar im = e == 0 ? m * s : numext::ldexp(m * s, e);
2601 return ComplexScalar(re, negative ? -im : im);
2602 }
2603};
2604
2605template <typename T>
2606struct rsqrt_impl {
2607// C4804: unsafe use of type 'bool' in operation. Unavoidable when instantiated with T=bool.
2608#if EIGEN_COMP_MSVC
2609#pragma warning(push)
2610#pragma warning(disable : 4804)
2611#endif
2612 EIGEN_DEVICE_FUNC static EIGEN_ALWAYS_INLINE T run(const T& x) { return T(1) / numext::sqrt(x); }
2613#if EIGEN_COMP_MSVC
2614#pragma warning(pop)
2615#endif
2616};
2617
2618#if defined(EIGEN_GPU_COMPILE_PHASE)
2619template <typename T>
2620struct conj_impl<std::complex<T>, true> {
2621 EIGEN_DEVICE_FUNC static inline std::complex<T> run(const std::complex<T>& x) {
2622 return std::complex<T>(numext::real(x), -numext::imag(x));
2623 }
2624};
2625#endif
2626
2627// Complex multiply and division operators.
2628// Note that these do not handle the case if inf+NaNi, which is considered an infinity.
2629// This is for consistency with our standard pmul, pdiv implementations.
2630template <typename T>
2631EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> complex_multiply(const std::complex<T>& a,
2632 const std::complex<T>& b) {
2633 const T a_real = numext::real(a);
2634 const T a_imag = numext::imag(a);
2635 const T b_real = numext::real(b);
2636 const T b_imag = numext::imag(b);
2637 return std::complex<T>(a_real * b_real - a_imag * b_imag, a_imag * b_real + a_real * b_imag);
2638}
2639
2640template <typename T>
2641EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> complex_divide_fast(const std::complex<T>& a,
2642 const std::complex<T>& b) {
2643 const T a_real = numext::real(a);
2644 const T a_imag = numext::imag(a);
2645 const T b_real = numext::real(b);
2646 const T b_imag = numext::imag(b);
2647 const T norm = (b_real * b_real + b_imag * b_imag);
2648 return std::complex<T>((a_real * b_real + a_imag * b_imag) / norm, (a_imag * b_real - a_real * b_imag) / norm);
2649}
2650
2651template <typename T>
2652EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> complex_divide_smith(const std::complex<T>& a,
2653 const std::complex<T>& b) {
2654 const T a_real = numext::real(a);
2655 const T a_imag = numext::imag(a);
2656 T b_real = numext::real(b);
2657 T b_imag = numext::imag(b);
2658 // Smith's complex division (https://arxiv.org/pdf/1210.4539.pdf),
2659 // guards against over/under-flow.
2660 const bool scale_imag = numext::abs(b_imag) <= numext::abs(b_real);
2661 T rscale = T(1);
2662 T iscale = T(1);
2663 if (numext::abs(b_imag) == numext::abs(b_real)) {
2664 // d / c = +-1 for |c| = |d|, which the quotient misses as 0 / 0 or inf / inf when b = 0 or both parts of b are
2665 // infinite; a zero divisor then gives an infinity and an infinite one zero, as in pdiv_complex. The sign is not
2666 // taken from b_real * b_imag, which can overflow or underflow.
2667 iscale = numext::copysign(T(1), b_real) * numext::copysign(T(1), b_imag);
2668 } else if (scale_imag) {
2669 // Each quotient divides by the larger part of b, which is not zero here. The barrier keeps the quotient on its
2670 // branch: an if-converted form evaluates both quotients, and the untaken one divides by zero for a divisor on an
2671 // axis, raising FE_DIVBYZERO. The barrier is applied in place, so that the branch costs no register copy.
2672 volatile_barrier(b_real);
2673 iscale = b_imag / b_real;
2674 } else {
2675 volatile_barrier(b_imag);
2676 rscale = b_real / b_imag;
2677 }
2678 const T denominator = b_real * rscale + b_imag * iscale;
2679 return std::complex<T>((a_real * rscale + a_imag * iscale) / denominator,
2680 (a_imag * rscale - a_real * iscale) / denominator);
2681}
2682
2683template <typename T>
2684EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> complex_divide(const std::complex<T>& a,
2685 const std::complex<T>& b) {
2686#if EIGEN_FAST_MATH
2687 return complex_divide_fast(a, b);
2688#else
2689 return complex_divide_smith(a, b);
2690#endif
2691}
2692
2693} // end namespace internal
2694
2695namespace numext {
2696
2705template <typename LhsScalar, typename RhsScalar>
2706EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto divide(const LhsScalar& a, const RhsScalar& b) -> decltype(a / b) {
2707 return a / b;
2708}
2709
2710template <typename T>
2711EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> divide(const std::complex<T>& a, const std::complex<T>& b) {
2712 return internal::complex_divide_smith(a, b);
2713}
2714
2715template <typename T>
2716EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> divide(const T& a, const std::complex<T>& b) {
2717 return internal::complex_divide_smith(std::complex<T>(a, T(0)), b);
2718}
2719
2720template <typename T>
2721EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex<T> divide(const std::complex<T>& a, const T& b) {
2722 return a / b;
2723}
2724
2725} // end namespace numext
2726
2727} // end namespace Eigen
2728
2729#endif // EIGEN_MATHFUNCTIONS_H