Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
SpecialFunctionsImpl.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2015 Eugene Brevdo <ebrevdo@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_SPECIAL_FUNCTIONS_H
12#define EIGEN_SPECIAL_FUNCTIONS_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18namespace internal {
19
20// Parts of this code are based on the Cephes Math Library.
21//
22// Cephes Math Library Release 2.8: June, 2000
23// Copyright 1984, 1987, 1992, 2000 by Stephen L. Moshier
24//
25// Permission has been kindly provided by the original author
26// to incorporate the Cephes software into the Eigen codebase:
27//
28// From: Stephen Moshier
29// To: Eugene Brevdo
30// Subject: Re: Permission to wrap several cephes functions in Eigen
31//
32// Hello Eugene,
33//
34// Thank you for writing.
35//
36// If your licensing is similar to BSD, the formal way that has been
37// handled is simply to add a statement to the effect that you are incorporating
38// the Cephes software by permission of the author.
39//
40// Good luck with your project,
41// Steve
42
43/****************************************************************************
44 * Implementation of lgamma *
45 ****************************************************************************/
46
47template <typename Scalar>
48struct lgamma_impl {
49 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Scalar) {
50 EIGEN_STATIC_ASSERT((!std::is_same<Scalar, Scalar>::value), THIS_TYPE_IS_NOT_SUPPORTED)
51 return Scalar(0);
52 }
53};
54
55// Since glibc 2.19
56#if defined(__GLIBC__) && ((__GLIBC__ >= 2 && __GLIBC_MINOR__ >= 19) || __GLIBC__ > 2) && \
57 (defined(_DEFAULT_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE))
58#define EIGEN_HAS_LGAMMA_R
59#endif
60
61// Glibc versions before 2.19
62#if defined(__GLIBC__) && ((__GLIBC__ == 2 && __GLIBC_MINOR__ < 19) || __GLIBC__ < 2) && \
63 (defined(_BSD_SOURCE) || defined(_SVID_SOURCE))
64#define EIGEN_HAS_LGAMMA_R
65#endif
66
67template <>
68struct lgamma_impl<float> {
69 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float run(float x) {
70#if !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(__SYCL_DEVICE_ONLY__) && defined(EIGEN_HAS_LGAMMA_R) && \
71 !defined(__APPLE__)
72 int dummy;
73 return ::lgammaf_r(x, &dummy);
74#elif defined(EIGEN_SYCL_DEVICE_ONLY)
75 return cl::sycl::lgamma(x);
76#else
77 return ::lgammaf(x);
78#endif
79 }
80};
81
82template <>
83struct lgamma_impl<double> {
84 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double run(double x) {
85#if !defined(EIGEN_GPU_COMPILE_PHASE) && !defined(__SYCL_DEVICE_ONLY__) && defined(EIGEN_HAS_LGAMMA_R) && \
86 !defined(__APPLE__)
87 int dummy;
88 return ::lgamma_r(x, &dummy);
89#elif defined(EIGEN_SYCL_DEVICE_ONLY)
90 return cl::sycl::lgamma(x);
91#else
92 return ::lgamma(x);
93#endif
94 }
95};
96
97#undef EIGEN_HAS_LGAMMA_R
98
99/****************************************************************************
100 * Implementation of digamma (psi), based on Cephes *
101 ****************************************************************************/
102
103/*
104 *
105 * Polynomial evaluation helper for the Psi (digamma) function.
106 *
107 * digamma_impl_maybe_poly::run(s) evaluates the asymptotic Psi expansion for
108 * input Scalar s, assuming s is above 10.0.
109 *
110 * If s is above a certain threshold for the given Scalar type, zero
111 * is returned. Otherwise the polynomial is evaluated with enough
112 * coefficients for results matching Scalar machine precision.
113 *
114 *
115 */
116template <typename Scalar>
117struct digamma_impl_maybe_poly {
118 EIGEN_STATIC_ASSERT((std::is_same<Scalar, Scalar>::value == false), THIS_TYPE_IS_NOT_SUPPORTED)
119
120 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Scalar) { return Scalar(0); }
121};
122
123template <>
124struct digamma_impl_maybe_poly<float> {
125 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float run(const float s) {
126 constexpr float A[] = {-4.16666666666666666667E-3f, 3.96825396825396825397E-3f, -8.33333333333333333333E-3f,
127 8.33333333333333333333E-2f};
128
129 float z;
130 if (s < 1.0e8f) {
131 z = 1.0f / (s * s);
132 return z * internal::ppolevl<float, 3>::run(z, A);
133 } else
134 return 0.0f;
135 }
136};
137
138template <>
139struct digamma_impl_maybe_poly<double> {
140 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double run(const double s) {
141 constexpr double A[] = {8.33333333333333333333E-2, -2.10927960927960927961E-2, 7.57575757575757575758E-3,
142 -4.16666666666666666667E-3, 3.96825396825396825397E-3, -8.33333333333333333333E-3,
143 8.33333333333333333333E-2};
144
145 double z;
146 if (s < 1.0e17) {
147 z = 1.0 / (s * s);
148 return z * internal::ppolevl<double, 6>::run(z, A);
149 } else
150 return 0.0;
151 }
152};
153
154template <typename Scalar>
155struct digamma_impl {
156 EIGEN_DEVICE_FUNC static Scalar run(Scalar x) {
157 /*
158 *
159 * Psi (digamma) function (modified for Eigen)
160 *
161 *
162 * SYNOPSIS:
163 *
164 * double x, y, psi();
165 *
166 * y = psi( x );
167 *
168 *
169 * DESCRIPTION:
170 *
171 * d -
172 * psi(x) = -- ln | (x)
173 * dx
174 *
175 * is the logarithmic derivative of the gamma function.
176 * For integer x,
177 * n-1
178 * -
179 * psi(n) = -EUL + > 1/k.
180 * -
181 * k=1
182 *
183 * If x is negative, it is transformed to a positive argument by the
184 * reflection formula psi(1-x) = psi(x) + pi cot(pi x).
185 * For general positive x, the argument is made greater than 10
186 * using the recurrence psi(x+1) = psi(x) + 1/x.
187 * Then the following asymptotic expansion is applied:
188 *
189 * inf. B
190 * - 2k
191 * psi(x) = log(x) - 1/2x - > -------
192 * - 2k
193 * k=1 2k x
194 *
195 * where the B2k are Bernoulli numbers.
196 *
197 * ACCURACY (float):
198 * Relative error (except absolute when |psi| < 1):
199 * arithmetic domain # trials peak rms
200 * IEEE 0,30 30000 1.3e-15 1.4e-16
201 * IEEE -30,0 40000 1.5e-15 2.2e-16
202 *
203 * ACCURACY (double):
204 * Absolute error, relative when |psi| > 1 :
205 * arithmetic domain # trials peak rms
206 * IEEE -33,0 30000 8.2e-7 1.2e-7
207 * IEEE 0,33 100000 7.3e-7 7.7e-8
208 *
209 * ERROR MESSAGES:
210 * message condition value returned
211 * psi singularity x integer <=0 INFINITY
212 */
213
214 Scalar p, q, nz, s, w, y;
215 bool negative = false;
216
217 const Scalar nan = NumTraits<Scalar>::quiet_NaN();
218 const Scalar m_pi = Scalar(EIGEN_PI);
219
220 const Scalar zero = Scalar(0);
221 const Scalar one = Scalar(1);
222 const Scalar half = Scalar(0.5);
223 nz = zero;
224
225 // Near 0, psi(x) ~ -1/x - gamma.
226 // For x = +0.0: psi(+0.0) -> -infinity.
227 // For x = -0.0: psi(-0.0) -> +infinity.
228 if (x == zero) {
229 return (std::signbit(x)) ? NumTraits<Scalar>::infinity() : -NumTraits<Scalar>::infinity();
230 }
231 if (x < zero) {
232 negative = true;
233 q = x;
234 p = numext::floor(q);
235 if (p == q) {
236 return nan;
237 }
238 /* Remove the zeros of tan(m_pi x)
239 * by subtracting the nearest integer from x
240 */
241 nz = q - p;
242 if (nz != half) {
243 if (nz > half) {
244 p += one;
245 nz = q - p;
246 }
247 nz = m_pi / numext::tan(m_pi * nz);
248 } else {
249 nz = zero;
250 }
251 x = one - x;
252 }
253
254 /* use the recurrence psi(x+1) = psi(x) + 1/x. */
255 s = x;
256 w = zero;
257 while (s < Scalar(10)) {
258 w += one / s;
259 s += one;
260 }
261
262 y = digamma_impl_maybe_poly<Scalar>::run(s);
263
264 y = numext::log(s) - (half / s) - y - w;
265
266 return (negative) ? y - nz : y;
267 }
268};
269
270// Does unqualified lookup of erf/erfc succeed for T? The lookup below mirrors
271// the one at the call sites in erf_impl/erfc_impl, so it finds std::erf/std::erfc
272// and any overload visible through argument-dependent lookup.
273namespace unqualified_erf {
274EIGEN_USING_STD(erf)
275EIGEN_USING_STD(erfc)
276template <typename T>
277auto test_erf(int) -> decltype(void(erf(std::declval<const T&>())), std::true_type{});
278template <typename T>
279std::false_type test_erf(...);
280template <typename T>
281auto test_erfc(int) -> decltype(void(erfc(std::declval<const T&>())), std::true_type{});
282template <typename T>
283std::false_type test_erfc(...);
284} // namespace unqualified_erf
285
286template <typename T>
287struct has_erf : decltype(unqualified_erf::test_erf<T>(0)) {};
288template <typename T>
289struct has_erfc : decltype(unqualified_erf::test_erfc<T>(0)) {};
290
291/***************************************************************************
292 * Implementation of erfc.
293 ****************************************************************************/
294template <typename Scalar>
295struct generic_fast_erfc {
296 template <typename T>
297 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T run(const T& x_in);
298};
299
300template <>
301template <typename T>
302EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T generic_fast_erfc<float>::run(const T& x_in) {
303 constexpr float kClamp = 11.0f;
304 const T x = pmin<PropagateNaN>(pmax<PropagateNaN>(x_in, pset1<T>(-kClamp)), pset1<T>(kClamp));
305
306 // erfc(x) = 1 + x * S(x^2), |x| <= 1.
307 //
308 // Coefficients for S and T generated with Rminimax command:
309 // ./ratapprox --function="erfc(x)-1" --dom='[-1,1]' --type=[11,0] --num="odd"
310 // --numF="[SG]" --denF="[SG]" --log --dispCoeff="dec"
311 constexpr float alpha[] = {5.61802298761904239654541015625e-04, -4.91381669417023658752441406250e-03,
312 2.67075151205062866210937500000e-02, -1.12800106406211853027343750000e-01,
313 3.76122951507568359375000000000e-01, -1.12837910652160644531250000000e+00};
314 const T x2 = pmul(x, x);
315 const T one = pset1<T>(1.0f);
316 const T erfc_small = pmadd(x, ppolevl<T, 5>::run(x2, alpha), one);
317
318 // Return early if we don't need the more expensive approximation for any
319 // entry in a.
320 const T x_abs_gt_one_mask = pcmp_lt(one, x2);
321 if (!predux_any(x_abs_gt_one_mask)) return erfc_small;
322
323 // erfc(x) = exp(-x^2) * 1/x * P(1/x^2) / Q(1/x^2), 1 < x < 9.
324 //
325 // Coefficients for P and Q generated with Rminimax command:
326 // ./ratapprox --function="erfc(1/sqrt(x))*exp(1/x)/sqrt(x)"
327 // --dom='[0.01,1]' --type=[3,4] --numF="[SG]" --denF="[SG]" --log
328 // --dispCoeff="dec"
329 constexpr float gamma[] = {1.0208116471767425537109375e-01f, 4.2920666933059692382812500e-01f,
330 3.2379078865051269531250000e-01f, 5.3971976041793823242187500e-02f};
331 constexpr float delta[] = {1.7251677811145782470703125e-02f, 3.9137163758277893066406250e-01f,
332 1.0000000000000000000000000e+00f, 6.2173241376876831054687500e-01f,
333 9.5662862062454223632812500e-02f};
334 const T x2_lo = twoprod_low(x, x, x2);
335 // Here we use that
336 // exp(-x^2) = exp(-(x2+x2_lo)^2) ~= exp(-x2)*exp(-x2_lo) ~= exp(-x2)*(1-x2_lo)
337 // since x2_lo < kClamp * eps << 1 in the region we care about. This trick reduces the max error
338 // from 34 ulps to below 5 ulps.
339 const T exp2_hi = pexp(pnegate(x2));
340 const T z = pnmadd(exp2_hi, x2_lo, exp2_hi);
341 const T q2 = preciprocal(x2);
342 const T num = ppolevl<T, 3>::run(q2, gamma);
343 const T denom = pmul(x, ppolevl<T, 4>::run(q2, delta));
344 const T r = pdiv(num, denom);
345 const T maybe_two = pselect(pcmp_lt(x, pset1<T>(0.0f)), pset1<T>(2.0f), pset1<T>(0.0f));
346 const T erfc_large = pmadd(z, r, maybe_two);
347 return pselect(x_abs_gt_one_mask, erfc_large, erfc_small);
348}
349
350// Computes erf(x)/x for |x| <= 1. Used by both erf and erfc implementations.
351// Takes x2 = x^2 as input.
352//
353// PRECONDITION: x2 <= 1.
354template <typename T>
355EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T erf_over_x_double_small(const T& x2) {
356 // erf(x)/x = S(x^2) / T(x^2), x^2 <= 1.
357 //
358 // Coefficients for S and T generated with Rminimax command:
359 // ./ratapprox --function="erf(x)" --dom='[-1,1]' --type=[9,10]
360 // --num="odd" --numF="[D]" --den="even" --denF="[D]" --log --dispCoeff="dec"
361 constexpr double alpha[] = {1.9493725660006057018823477644531294572516344487667083740234375e-04,
362 1.8272566210022942682217328425053892715368419885635375976562500e-03,
363 4.5303363351690106863856044583371840417385101318359375000000000e-02,
364 1.4215015503619179981775744181504705920815467834472656250000000e-01,
365 1.1283791670955125585606992899556644260883331298828125000000000e+00};
366 constexpr double beta[] = {2.0294484101083099089526257108317963684385176748037338256835938e-05,
367 6.8117805899186819641732970609382391558028757572174072265625000e-04,
368 1.0582026056098614921752165685120417037978768348693847656250000e-02,
369 9.3252603143757495374188692949246615171432495117187500000000000e-02,
370 4.5931062818368939559832142549566924571990966796875000000000000e-01,
371 1.0};
372 const T num_small = ppolevl<T, 4>::run(x2, alpha);
373 const T denom_small = ppolevl<T, 5>::run(x2, beta);
374 return pdiv(num_small, denom_small);
375}
376
377// erfc(x) = exp(-x^2) * 1/x * P(1/x^2) / Q(1/x^2), 1 < x < 28.
378//
379// Coefficients for P and Q generated with Rminimax command:
380// ./ratapprox --function="erfc(1/sqrt(x))*exp(1/x)/sqrt(x)" --dom='[0.0013717,1]' --type=[9,9] --numF="[D]"
381// --denF="[D]" --log --dispCoeff="dec"
382//
383// PRECONDITION: 1 < x < 28.
384template <typename T>
385EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T erfc_double_large(const T& x, const T& x2) {
386 constexpr double gamma[] = {1.5252844933226974316088642158462107545346952974796295166015625e-04,
387 1.0909912393738931124520519233556115068495273590087890625000000e-02,
388 1.0628604636755033252537572252549580298364162445068359375000000e-01,
389 3.3492472973137982217295416376146022230386734008789062500000000e-01,
390 4.5065776215933289750026347064704168587923049926757812500000000e-01,
391 2.9433039130294824659017649537418037652969360351562500000000000e-01,
392 9.8792676360600226170838311645638896152377128601074218750000000e-02,
393 1.7095935395503719655962981960328761488199234008789062500000000e-02,
394 1.4249109729504577659398023570247460156679153442382812500000000e-03,
395 4.4567378313647954771875570045835956989321857690811157226562500e-05};
396 constexpr double delta[] = {2.041985103115789845773520028160419315099716186523437500000000e-03,
397 5.316030659946043707142493417450168635696172714233398437500000e-02,
398 3.426242193784684864077405563875799998641014099121093750000000e-01,
399 8.565637124308049799026321124983951449394226074218750000000000e-01,
400 1.000000000000000000000000000000000000000000000000000000000000e+00,
401 5.968805280570776972126623149961233139038085937500000000000000e-01,
402 1.890922854723317836356244470152887515723705291748046875000000e-01,
403 3.152505418656005586885981983868987299501895904541015625000000e-02,
404 2.565085751861882583380047861965067568235099315643310546875000e-03,
405 7.899362131678837697403017248376499992446042597293853759765625e-05};
406 // Compute exp(-x^2).
407 const T x2_lo = twoprod_low(x, x, x2);
408 // Here we use that
409 // exp(-x^2) = exp(-(x2+x2_lo)^2) ~= exp(-x2)*exp(-x2_lo) ~= exp(-x2)*(1-x2_lo)
410 // since x2_lo < kClamp * eps << 1 in the region we care about. This trick reduces the max error
411 // from 258 ulps to below 7 ulps.
412 const T exp2_hi = pexp(pnegate(x2));
413 const T z = pnmadd(exp2_hi, x2_lo, exp2_hi);
414 // Compute r = P / Q.
415 const T q2 = preciprocal(x2);
416 const T num_large = ppolevl<T, 9>::run(q2, gamma);
417 const T denom_large = pmul(x, ppolevl<T, 9>::run(q2, delta));
418 const T r = pdiv(num_large, denom_large);
419 const T maybe_two = pselect(pcmp_lt(x, pset1<T>(0.0)), pset1<T>(2.0), pset1<T>(0.0));
420 return pmadd(z, r, maybe_two);
421}
422
423template <>
424template <typename T>
425EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T generic_fast_erfc<double>::run(const T& x_in) {
426 // Clamp x to [-28:28] beyond which erfc(x) is either two or zero (below the underflow threshold).
427 // This avoids having to deal with twoprod(x,x) producing NaN for sufficiently large x.
428 constexpr double kClamp = 28.0;
429 const T x = pmin<PropagateNaN>(pmax<PropagateNaN>(x_in, pset1<T>(-kClamp)), pset1<T>(kClamp));
430
431 // For |x| < 1, we use erfc(x) = 1 - erf(x).
432 const T x2 = pmul(x, x);
433 const T one = pset1<T>(1.0);
434 const T erfc_small = pnmadd(x, erf_over_x_double_small(x2), one);
435
436 // Return early if we don't need the more expensive approximation for any
437 // entry in a.
438 const T x_abs_gt_one_mask = pcmp_lt(one, x2);
439 if (!predux_any(x_abs_gt_one_mask)) return erfc_small;
440
441 const T erfc_large = erfc_double_large(x, x2);
442 return pselect(x_abs_gt_one_mask, erfc_large, erfc_small);
443}
444
445template <typename T>
446struct erfc_impl {
447 using Scalar = typename unpacket_traits<T>::type;
448 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T& x) { return run_impl(x, std::is_same<T, Scalar>()); }
449
450 private:
451 // Packets of float/double: vectorized rational approximation.
452 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_impl(const T& x, std::false_type) {
453 return generic_fast_erfc<Scalar>::run(x);
454 }
455 // Any other scalar type: defer to an erfc found by argument-dependent lookup
456 // (or std::erfc), keeping custom scalars on their own implementation instead
457 // of the float/double-tuned polynomials.
458 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_impl(const T& x, std::true_type) {
459 EIGEN_STATIC_ASSERT_NON_INTEGER(T)
460 return run_scalar(x, has_erfc<T>());
461 }
462 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_scalar(const T& x, std::true_type) {
463 EIGEN_USING_STD(erfc);
464 return erfc(x);
465 }
466 // Reject the type here instead of letting overload resolution fail inside the
467 // call above: the approximations in this file are tuned for float and double
468 // and are not valid for an arbitrary scalar, so a scalar type that wants erfc
469 // has to supply it.
470 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_scalar(const T& x, std::false_type) {
471 EIGEN_STATIC_ASSERT(has_erfc<T>::value, SCALAR_TYPE_MUST_PROVIDE_AN_ERFC_OVERLOAD_FOUND_BY_ADL_OR_IN_NAMESPACE_STD)
472 return x;
473 }
474};
475
476template <>
477struct erfc_impl<float> {
478 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float run(const float x) {
479#if defined(EIGEN_SYCL_DEVICE_ONLY)
480 return cl::sycl::erfc(x);
481#else
482 return generic_fast_erfc<float>::run(x);
483#endif
484 }
485};
486
487template <>
488struct erfc_impl<double> {
489 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double run(const double x) {
490#if defined(EIGEN_SYCL_DEVICE_ONLY)
491 return cl::sycl::erfc(x);
492#else
493 return generic_fast_erfc<double>::run(x);
494#endif
495 }
496};
497
498/****************************************************************************
499 * Implementation of erf.
500 ****************************************************************************/
501
502template <typename Scalar>
503struct generic_fast_erf {
504 template <typename T>
505 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T run(const T& x_in);
506};
507
514template <>
515template <typename T>
516EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T generic_fast_erf<float>::run(const T& x) {
517 // The monomial coefficients of the numerator polynomial (odd).
518 constexpr float alpha[] = {2.123732201653183437883853912353515625e-06f, 2.861979592125862836837768554687500000e-04f,
519 3.658048342913389205932617187500000000e-03f, 5.243302136659622192382812500000000000e-02f,
520 1.874160766601562500000000000000000000e-01f, 1.128379106521606445312500000000000000e+00f};
521
522 // The monomial coefficients of the denominator polynomial (even).
523 constexpr float beta[] = {3.89185734093189239501953125000e-05f, 1.14329601638019084930419921875e-03f,
524 1.47520881146192550659179687500e-02f, 1.12945675849914550781250000000e-01f,
525 4.99425798654556274414062500000e-01f, 1.0f};
526
527 // Since the polynomials are odd/even, we need x^2.
528 // Since erf(4) == 1 in float, we clamp x^2 to 16 to avoid computing Inf/Inf below.
529 // NaN need not survive this clamp: multiplying by x below restores it.
530 const T x2 = pmin(pset1<T>(16.0f), pmul(x, x));
531
532 // Evaluate the numerator polynomial p.
533 T p = ppolevl<T, 5>::run(x2, alpha);
534 p = pmul(x, p);
535
536 // Evaluate the denominator polynomial q.
537 T q = ppolevl<T, 5>::run(x2, beta);
538 const T r = pdiv(p, q);
539
540 // Clamp to [-1:1].
541 return pmax<PropagateNaN>(pmin<PropagateNaN>(r, pset1<T>(1.0f)), pset1<T>(-1.0f));
542}
543
544template <>
545template <typename T>
546EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T generic_fast_erf<double>::run(const T& x_in) {
547 // Clamp x to [-28:28] beyond which erf(x) is ±1 within double precision.
548 // This avoids NaN from twoprod and exp operations for infinite inputs.
549 constexpr double kClamp = 28.0;
550 const T x = pmin<PropagateNaN>(pmax<PropagateNaN>(x_in, pset1<T>(-kClamp)), pset1<T>(kClamp));
551 T x2 = pmul(x, x);
552 T erf_small = pmul(x, erf_over_x_double_small(x2));
553
554 // Return early if we don't need the more expensive approximation for any
555 // entry in a.
556 const T one = pset1<T>(1.0);
557 const T x_abs_gt_one_mask = pcmp_lt(one, x2);
558 if (!predux_any(x_abs_gt_one_mask)) return erf_small;
559
560 // For |x| > 1, use erf(x) = 1 - erfc(x).
561 const T erf_large = psub(one, erfc_double_large(x, x2));
562 return pselect(x_abs_gt_one_mask, erf_large, erf_small);
563}
564
565template <typename T>
566struct erf_impl {
567 using Scalar = typename unpacket_traits<T>::type;
568 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run(const T& x) { return run_impl(x, std::is_same<T, Scalar>()); }
569
570 private:
571 // Packets of float/double: vectorized rational approximation.
572 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_impl(const T& x, std::false_type) {
573 return generic_fast_erf<Scalar>::run(x);
574 }
575 // Any other scalar type: defer to an erf found by argument-dependent lookup
576 // (or std::erf), keeping custom scalars on their own implementation instead
577 // of the float/double-tuned polynomials.
578 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_impl(const T& x, std::true_type) {
579 EIGEN_STATIC_ASSERT_NON_INTEGER(T)
580 return run_scalar(x, has_erf<T>());
581 }
582 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_scalar(const T& x, std::true_type) {
583 EIGEN_USING_STD(erf);
584 return erf(x);
585 }
586 // Reject the type here instead of letting overload resolution fail inside the
587 // call above: the approximations in this file are tuned for float and double
588 // and are not valid for an arbitrary scalar, so a scalar type that wants erf
589 // has to supply it.
590 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE T run_scalar(const T& x, std::false_type) {
591 EIGEN_STATIC_ASSERT(has_erf<T>::value, SCALAR_TYPE_MUST_PROVIDE_AN_ERF_OVERLOAD_FOUND_BY_ADL_OR_IN_NAMESPACE_STD)
592 return x;
593 }
594};
595
596template <>
597struct erf_impl<float> {
598 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float run(const float x) {
599#if defined(EIGEN_SYCL_DEVICE_ONLY)
600 return cl::sycl::erf(x);
601#else
602 return generic_fast_erf<float>::run(x);
603#endif
604 }
605};
606
607template <>
608struct erf_impl<double> {
609 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double run(const double x) {
610#if defined(EIGEN_SYCL_DEVICE_ONLY)
611 return cl::sycl::erf(x);
612#else
613 return generic_fast_erf<double>::run(x);
614#endif
615 }
616};
617
618/***************************************************************************
619 * Implementation of ndtri. *
620 ****************************************************************************/
621
622/* Inverse of Normal distribution function (modified for Eigen).
623 *
624 *
625 * SYNOPSIS:
626 *
627 * double x, y, ndtri();
628 *
629 * x = ndtri( y );
630 *
631 *
632 *
633 * DESCRIPTION:
634 *
635 * Returns the argument, x, for which the area under the
636 * Gaussian probability density function (integrated from
637 * minus infinity to x) is equal to y.
638 *
639 *
640 * For small arguments 0 < y < exp(-2), the program computes
641 * z = sqrt( -2.0 * log(y) ); then the approximation is
642 * x = z - log(z)/z - (1/z) P(1/z) / Q(1/z).
643 * There are two rational functions P/Q, one for 0 < y < exp(-32)
644 * and the other for y up to exp(-2). For larger arguments,
645 * w = y - 0.5, and x/sqrt(2pi) = w + w**3 R(w**2)/S(w**2)).
646 *
647 *
648 * ACCURACY:
649 *
650 * Relative error:
651 * arithmetic domain # trials peak rms
652 * DEC 0.125, 1 5500 9.5e-17 2.1e-17
653 * DEC 6e-39, 0.135 3500 5.7e-17 1.3e-17
654 * IEEE 0.125, 1 20000 7.2e-16 1.3e-16
655 * IEEE 3e-308, 0.135 50000 4.6e-16 9.8e-17
656 *
657 *
658 * ERROR MESSAGES:
659 *
660 * message condition value returned
661 * ndtri domain x == 0 -INF
662 * ndtri domain x == 1 INF
663 * ndtri domain x < 0, x > 1 NAN
664 */
665/*
666 Cephes Math Library Release 2.2: June, 1992
667 Copyright 1985, 1987, 1992 by Stephen L. Moshier
668 Direct inquiries to 30 Frost Street, Cambridge, MA 02140
669*/
670
671// TODO: Add a cheaper approximation for float.
672
673template <typename T, bool IsScalar = is_scalar<T>::value>
674struct flipsign_impl;
675
676template <typename T>
677struct flipsign_impl<T, false> {
678 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T run(const T& should_flipsign, const T& x) {
679 const T sign_mask = psignmask<T>();
680 const T sign_bit = pand<T>(should_flipsign, sign_mask);
681 return pxor<T>(sign_bit, x);
682 }
683};
684
685template <typename T>
686struct flipsign_impl<T, true> {
687 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T run(const T& should_flipsign, const T& x) {
688 return should_flipsign == T(0) ? x : -x;
689 }
690};
691
692template <typename T>
693EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T flipsign(const T& should_flipsign, const T& x) {
694 return flipsign_impl<T>::run(should_flipsign, x);
695}
696
697template <typename T, bool IsScalar = is_scalar<T>::value>
698struct ndtri_negative_infinity_impl;
699
700template <typename T>
701struct ndtri_negative_infinity_impl<T, false> {
702 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T run(const T& positive_infinity) {
703 return por(psignmask<T>(), positive_infinity);
704 }
705};
706
707template <typename T>
708struct ndtri_negative_infinity_impl<T, true> {
709 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T run(const T& positive_infinity) { return -positive_infinity; }
710};
711
712// We split this computation in to two so that in the scalar path
713// only one branch is evaluated (due to our template specialization of pselect
714// being an if statement.)
715
716template <typename T, typename ScalarType>
717EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T generic_ndtri_gt_exp_neg_two(const T& b) {
718 const ScalarType p0[] = {ScalarType(-5.99633501014107895267e1), ScalarType(9.80010754185999661536e1),
719 ScalarType(-5.66762857469070293439e1), ScalarType(1.39312609387279679503e1),
720 ScalarType(-1.23916583867381258016e0)};
721 const ScalarType q0[] = {ScalarType(1.0),
722 ScalarType(1.95448858338141759834e0),
723 ScalarType(4.67627912898881538453e0),
724 ScalarType(8.63602421390890590575e1),
725 ScalarType(-2.25462687854119370527e2),
726 ScalarType(2.00260212380060660359e2),
727 ScalarType(-8.20372256168333339912e1),
728 ScalarType(1.59056225126211695515e1),
729 ScalarType(-1.18331621121330003142e0)};
730 const T sqrt2pi = pset1<T>(ScalarType(2.50662827463100050242e0));
731 const T half = pset1<T>(ScalarType(0.5));
732 T c, c2, ndtri_gt_exp_neg_two;
733
734 c = psub(b, half);
735 c2 = pmul(c, c);
736 ndtri_gt_exp_neg_two =
737 pmadd(c, pmul(c2, pdiv(internal::ppolevl<T, 4>::run(c2, p0), internal::ppolevl<T, 8>::run(c2, q0))), c);
738 return pmul(ndtri_gt_exp_neg_two, sqrt2pi);
739}
740
741template <typename T, typename ScalarType>
742EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T generic_ndtri_lt_exp_neg_two(const T& b, const T& should_flipsign) {
743 /* Approximation for interval z = sqrt(-2 log a ) between 2 and 8
744 * i.e., a between exp(-2) = .135 and exp(-32) = 1.27e-14.
745 */
746 const ScalarType p1[] = {ScalarType(4.05544892305962419923e0), ScalarType(3.15251094599893866154e1),
747 ScalarType(5.71628192246421288162e1), ScalarType(4.40805073893200834700e1),
748 ScalarType(1.46849561928858024014e1), ScalarType(2.18663306850790267539e0),
749 ScalarType(-1.40256079171354495875e-1), ScalarType(-3.50424626827848203418e-2),
750 ScalarType(-8.57456785154685413611e-4)};
751 const ScalarType q1[] = {ScalarType(1.0),
752 ScalarType(1.57799883256466749731e1),
753 ScalarType(4.53907635128879210584e1),
754 ScalarType(4.13172038254672030440e1),
755 ScalarType(1.50425385692907503408e1),
756 ScalarType(2.50464946208309415979e0),
757 ScalarType(-1.42182922854787788574e-1),
758 ScalarType(-3.80806407691578277194e-2),
759 ScalarType(-9.33259480895457427372e-4)};
760 /* Approximation for interval z = sqrt(-2 log a ) between 8 and 64
761 * i.e., a between exp(-32) = 1.27e-14 and exp(-2048) = 3.67e-890.
762 */
763 const ScalarType p2[] = {ScalarType(3.23774891776946035970e0), ScalarType(6.91522889068984211695e0),
764 ScalarType(3.93881025292474443415e0), ScalarType(1.33303460815807542389e0),
765 ScalarType(2.01485389549179081538e-1), ScalarType(1.23716634817820021358e-2),
766 ScalarType(3.01581553508235416007e-4), ScalarType(2.65806974686737550832e-6),
767 ScalarType(6.23974539184983293730e-9)};
768 const ScalarType q2[] = {ScalarType(1.0),
769 ScalarType(6.02427039364742014255e0),
770 ScalarType(3.67983563856160859403e0),
771 ScalarType(1.37702099489081330271e0),
772 ScalarType(2.16236993594496635890e-1),
773 ScalarType(1.34204006088543189037e-2),
774 ScalarType(3.28014464682127739104e-4),
775 ScalarType(2.89247864745380683936e-6),
776 ScalarType(6.79019408009981274425e-9)};
777 const T eight = pset1<T>(ScalarType(8.0));
778 const T neg_two = pset1<T>(ScalarType(-2));
779 T x, x0, correction, z;
780
781 x = psqrt(pmul(neg_two, plog(b)));
782 x0 = psub(x, pdiv(plog(x), x));
783 z = preciprocal(x);
784 correction =
785 pselect(pcmp_lt(x, eight), pdiv(internal::ppolevl<T, 8>::run(z, p1), internal::ppolevl<T, 8>::run(z, q1)),
786 pdiv(internal::ppolevl<T, 8>::run(z, p2), internal::ppolevl<T, 8>::run(z, q2)));
787 return flipsign(should_flipsign, pnmadd(z, correction, x0));
788}
789
790template <typename T, typename ScalarType>
791EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T generic_ndtri(const T& a) {
792 const T maxnum = pinf<T>();
793 const T neg_maxnum = ndtri_negative_infinity_impl<T>::run(maxnum);
794
795 const T zero = pset1<T>(ScalarType(0));
796 const T one = pset1<T>(ScalarType(1));
797 // exp(-2)
798 const T exp_neg_two = pset1<T>(ScalarType(0.13533528323661269189));
799 T b, ndtri, should_flipsign;
800
801 should_flipsign = pcmp_le(a, psub(one, exp_neg_two));
802 b = pselect(should_flipsign, a, psub(one, a));
803
804 ndtri = pselect(pcmp_lt(exp_neg_two, b), generic_ndtri_gt_exp_neg_two<T, ScalarType>(b),
805 generic_ndtri_lt_exp_neg_two<T, ScalarType>(b, should_flipsign));
806
807 return pselect(pcmp_eq(a, zero), neg_maxnum, pselect(pcmp_eq(one, a), maxnum, ndtri));
808}
809
810template <typename Scalar>
811struct ndtri_impl {
812 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Scalar x) { return generic_ndtri<Scalar, Scalar>(x); }
813};
814
815/**************************************************************************************************************
816 * Implementation of igammac (complemented incomplete gamma integral), based on Cephes *
817 **************************************************************************************************************/
818
819// NOTE: cephes_helper is also used to implement zeta
820template <typename Scalar>
821struct cephes_helper {
822 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar machep() {
823 eigen_assert(false && "machep not supported for this type");
824 return 0.0;
825 }
826 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar big() {
827 eigen_assert(false && "big not supported for this type");
828 return 0.0;
829 }
830 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar biginv() {
831 eigen_assert(false && "biginv not supported for this type");
832 return 0.0;
833 }
834};
835
836template <>
837struct cephes_helper<float> {
838 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float machep() {
839 return NumTraits<float>::epsilon() / 2; // 1.0 - machep == 1.0
840 }
841 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float big() {
842 // use epsneg (1.0 - epsneg == 1.0)
843 return 1.0f / (NumTraits<float>::epsilon() / 2);
844 }
845 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float biginv() {
846 // epsneg
847 return machep();
848 }
849};
850
851template <>
852struct cephes_helper<double> {
853 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double machep() {
854 return NumTraits<double>::epsilon() / 2; // 1.0 - machep == 1.0
855 }
856 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double big() { return 1.0 / NumTraits<double>::epsilon(); }
857 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double biginv() {
858 // inverse of eps
859 return NumTraits<double>::epsilon();
860 }
861};
862
863enum IgammaComputationMode { VALUE, DERIVATIVE, SAMPLE_DERIVATIVE };
864
865template <typename Scalar>
866EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar main_igamma_term(Scalar a, Scalar x) {
867 /* Compute x**a * exp(-x) / gamma(a) */
868 Scalar logax = a * numext::log(x) - x - lgamma_impl<Scalar>::run(a);
869 if (logax < -numext::log(NumTraits<Scalar>::highest()) ||
870 // Assuming x and a aren't Nan.
871 (numext::isnan)(logax)) {
872 return Scalar(0);
873 }
874 return numext::exp(logax);
875}
876
877template <typename Scalar, IgammaComputationMode mode>
878EIGEN_DEVICE_FUNC constexpr int igamma_num_iterations() {
879 /* Returns the maximum number of internal iterations for igamma computation.
880 */
881 return mode == VALUE ? 2000
882 : std::is_same<Scalar, float>::value ? 200
883 : std::is_same<Scalar, double>::value ? 500
884 : 2000;
885}
886
887template <typename Scalar, IgammaComputationMode mode>
888struct igammac_cf_impl {
889 /* Computes igamc(a, x) or derivative (depending on the mode)
890 * using the continued fraction expansion of the complementary
891 * incomplete Gamma function.
892 *
893 * Preconditions:
894 * a > 0
895 * x >= 1
896 * x >= a
897 */
898 EIGEN_DEVICE_FUNC static Scalar run(Scalar a, Scalar x) {
899 const Scalar zero = 0;
900 const Scalar one = 1;
901 const Scalar two = 2;
902 const Scalar machep = cephes_helper<Scalar>::machep();
903 const Scalar big = cephes_helper<Scalar>::big();
904 const Scalar biginv = cephes_helper<Scalar>::biginv();
905
906 if ((numext::isinf)(x)) {
907 return zero;
908 }
909
910 Scalar ax = main_igamma_term<Scalar>(a, x);
911 // This is independent of mode. If this value is zero,
912 // then the function value is zero. If the function value is zero,
913 // then we are in a neighborhood where the function value evaluates to zero,
914 // so the derivative is zero.
915 if (ax == zero) {
916 return zero;
917 }
918
919 // continued fraction
920 Scalar y = one - a;
921 Scalar z = x + y + one;
922 Scalar c = zero;
923 Scalar pkm2 = one;
924 Scalar qkm2 = x;
925 Scalar pkm1 = x + one;
926 Scalar qkm1 = z * x;
927 Scalar ans = pkm1 / qkm1;
928
929 Scalar dpkm2_da = zero;
930 Scalar dqkm2_da = zero;
931 Scalar dpkm1_da = zero;
932 Scalar dqkm1_da = -x;
933 Scalar dans_da = (dpkm1_da - ans * dqkm1_da) / qkm1;
934
935 for (int i = 0; i < igamma_num_iterations<Scalar, mode>(); i++) {
936 c += one;
937 y += one;
938 z += two;
939
940 Scalar yc = y * c;
941 Scalar pk = pkm1 * z - pkm2 * yc;
942 Scalar qk = qkm1 * z - qkm2 * yc;
943
944 Scalar dpk_da = dpkm1_da * z - pkm1 - dpkm2_da * yc + pkm2 * c;
945 Scalar dqk_da = dqkm1_da * z - qkm1 - dqkm2_da * yc + qkm2 * c;
946
947 if (qk != zero) {
948 Scalar ans_prev = ans;
949 ans = pk / qk;
950
951 Scalar dans_da_prev = dans_da;
952 dans_da = (dpk_da - ans * dqk_da) / qk;
953
954 if (mode == VALUE) {
955 if (numext::abs(ans_prev - ans) <= machep * numext::abs(ans)) {
956 break;
957 }
958 } else {
959 if (numext::abs(dans_da - dans_da_prev) <= machep) {
960 break;
961 }
962 }
963 }
964
965 pkm2 = pkm1;
966 pkm1 = pk;
967 qkm2 = qkm1;
968 qkm1 = qk;
969
970 dpkm2_da = dpkm1_da;
971 dpkm1_da = dpk_da;
972 dqkm2_da = dqkm1_da;
973 dqkm1_da = dqk_da;
974
975 if (numext::abs(pk) > big) {
976 pkm2 *= biginv;
977 pkm1 *= biginv;
978 qkm2 *= biginv;
979 qkm1 *= biginv;
980
981 dpkm2_da *= biginv;
982 dpkm1_da *= biginv;
983 dqkm2_da *= biginv;
984 dqkm1_da *= biginv;
985 }
986 }
987
988 /* Compute x**a * exp(-x) / gamma(a) */
989 Scalar dlogax_da = numext::log(x) - digamma_impl<Scalar>::run(a);
990 Scalar dax_da = ax * dlogax_da;
991
992 switch (mode) {
993 case VALUE:
994 return ans * ax;
995 case DERIVATIVE:
996 return ans * dax_da + dans_da * ax;
997 case SAMPLE_DERIVATIVE:
998 default: // this is needed to suppress clang warning
999 return -(dans_da + ans * dlogax_da) * x;
1000 }
1001 }
1002};
1003
1004template <typename Scalar, IgammaComputationMode mode>
1005struct igamma_series_impl {
1006 /* Computes igam(a, x) or its derivative (depending on the mode)
1007 * using the series expansion of the incomplete Gamma function.
1008 *
1009 * Preconditions:
1010 * x > 0
1011 * a > 0
1012 * !(x > 1 && x > a)
1013 */
1014 EIGEN_DEVICE_FUNC static Scalar run(Scalar a, Scalar x) {
1015 const Scalar zero = 0;
1016 const Scalar one = 1;
1017 const Scalar machep = cephes_helper<Scalar>::machep();
1018
1019 Scalar ax = main_igamma_term<Scalar>(a, x);
1020
1021 // This is independent of mode. If this value is zero,
1022 // then the function value is zero. If the function value is zero,
1023 // then we are in a neighborhood where the function value evaluates to zero,
1024 // so the derivative is zero.
1025 if (ax == zero) {
1026 return zero;
1027 }
1028
1029 ax /= a;
1030
1031 /* power series */
1032 Scalar r = a;
1033 Scalar c = one;
1034 Scalar ans = one;
1035
1036 Scalar dc_da = zero;
1037 Scalar dans_da = zero;
1038
1039 for (int i = 0; i < igamma_num_iterations<Scalar, mode>(); i++) {
1040 r += one;
1041 Scalar term = x / r;
1042 Scalar dterm_da = -x / (r * r);
1043 dc_da = term * dc_da + dterm_da * c;
1044 dans_da += dc_da;
1045 c *= term;
1046 ans += c;
1047
1048 if (mode == VALUE) {
1049 if (c <= machep * ans) {
1050 break;
1051 }
1052 } else {
1053 if (numext::abs(dc_da) <= machep * numext::abs(dans_da)) {
1054 break;
1055 }
1056 }
1057 }
1058
1059 Scalar dlogax_da = numext::log(x) - digamma_impl<Scalar>::run(a + one);
1060 Scalar dax_da = ax * dlogax_da;
1061
1062 switch (mode) {
1063 case VALUE:
1064 return ans * ax;
1065 case DERIVATIVE:
1066 return ans * dax_da + dans_da * ax;
1067 case SAMPLE_DERIVATIVE:
1068 default: // this is needed to suppress clang warning
1069 return -(dans_da + ans * dlogax_da) * x / a;
1070 }
1071 }
1072};
1073
1074template <typename Scalar>
1075struct igammac_impl {
1076 EIGEN_DEVICE_FUNC static Scalar run(Scalar a, Scalar x) {
1077 /* igamc()
1078 *
1079 * Incomplete gamma integral (modified for Eigen)
1080 *
1081 *
1082 *
1083 * SYNOPSIS:
1084 *
1085 * double a, x, y, igamc();
1086 *
1087 * y = igamc( a, x );
1088 *
1089 * DESCRIPTION:
1090 *
1091 * The function is defined by
1092 *
1093 *
1094 * igamc(a,x) = 1 - igam(a,x)
1095 *
1096 * inf.
1097 * -
1098 * 1 | | -t a-1
1099 * = ----- | e t dt.
1100 * - | |
1101 * | (a) -
1102 * x
1103 *
1104 *
1105 * In this implementation both arguments must be positive.
1106 * The integral is evaluated by either a power series or
1107 * continued fraction expansion, depending on the relative
1108 * values of a and x.
1109 *
1110 * ACCURACY (float):
1111 *
1112 * Relative error:
1113 * arithmetic domain # trials peak rms
1114 * IEEE 0,30 30000 7.8e-6 5.9e-7
1115 *
1116 *
1117 * ACCURACY (double):
1118 *
1119 * Tested at random a, x.
1120 * a x Relative error:
1121 * arithmetic domain domain # trials peak rms
1122 * IEEE 0.5,100 0,100 200000 1.9e-14 1.7e-15
1123 * IEEE 0.01,0.5 0,100 200000 1.4e-13 1.6e-15
1124 *
1125 */
1126 /*
1127 Cephes Math Library Release 2.2: June, 1992
1128 Copyright 1985, 1987, 1992 by Stephen L. Moshier
1129 Direct inquiries to 30 Frost Street, Cambridge, MA 02140
1130 */
1131 const Scalar zero = 0;
1132 const Scalar one = 1;
1133 const Scalar nan = NumTraits<Scalar>::quiet_NaN();
1134
1135 if ((x < zero) || (a <= zero)) {
1136 // domain error
1137 return nan;
1138 }
1139
1140 if ((numext::isnan)(a) || (numext::isnan)(x)) { // propagate nans
1141 return nan;
1142 }
1143
1144 if ((x < one) || (x < a)) {
1145 // Clamp to [0,1] since 1-igamma can produce tiny negative values
1146 // due to floating-point cancellation for extreme arguments.
1147 return numext::mini(one, numext::maxi(zero, one - igamma_series_impl<Scalar, VALUE>::run(a, x)));
1148 }
1149
1150 return igammac_cf_impl<Scalar, VALUE>::run(a, x);
1151 }
1152};
1153
1154/************************************************************************************************
1155 * Implementation of igamma (incomplete gamma integral), based on Cephes *
1156 ************************************************************************************************/
1157
1158template <typename Scalar, IgammaComputationMode mode>
1159struct igamma_generic_impl {
1160 EIGEN_DEVICE_FUNC static Scalar run(Scalar a, Scalar x) {
1161 /* Depending on the mode, returns
1162 * - VALUE: incomplete Gamma function igamma(a, x)
1163 * - DERIVATIVE: derivative of incomplete Gamma function d/da igamma(a, x)
1164 * - SAMPLE_DERIVATIVE: implicit derivative of a Gamma random variable
1165 * x ~ Gamma(x | a, 1), dx/da = -1 / Gamma(x | a, 1) * d igamma(a, x) / dx
1166 *
1167 * Derivatives are implemented by forward-mode differentiation.
1168 */
1169 const Scalar zero = 0;
1170 const Scalar one = 1;
1171 const Scalar nan = NumTraits<Scalar>::quiet_NaN();
1172
1173 if ((numext::isnan)(a) || (numext::isnan)(x)) { // propagate nans
1174 return nan;
1175 }
1176
1177 if (x == zero) return zero;
1178
1179 if ((x < zero) || (a <= zero)) { // domain error
1180 return nan;
1181 }
1182
1183 if ((x > one) && (x > a)) {
1184 Scalar ret = igammac_cf_impl<Scalar, mode>::run(a, x);
1185 if (mode == VALUE) {
1186 // Clamp to [0,1] since 1-igammac can produce tiny negative values
1187 // due to floating-point cancellation for extreme arguments.
1188 return numext::mini(one, numext::maxi(zero, one - ret));
1189 } else {
1190 return -ret;
1191 }
1192 }
1193
1194 Scalar ret = igamma_series_impl<Scalar, mode>::run(a, x);
1195 if (mode == VALUE) {
1196 // Clamp to [0,1] since accumulated series terms can slightly exceed 1.0
1197 // due to floating-point rounding for extreme arguments.
1198 return numext::mini(one, numext::maxi(zero, ret));
1199 }
1200 return ret;
1201 }
1202};
1203
1204template <typename Scalar>
1205struct igamma_impl : igamma_generic_impl<Scalar, VALUE> {
1206 /* igam()
1207 * Incomplete gamma integral.
1208 *
1209 * The CDF of Gamma(a, 1) random variable at the point x.
1210 *
1211 * Accuracy estimation. For each a in [10^-2, 10^-1...10^3] we sample
1212 * 50 Gamma random variables x ~ Gamma(x | a, 1), a total of 300 points.
1213 * The ground truth is computed by mpmath. Mean absolute error:
1214 * float: 1.26713e-05
1215 * double: 2.33606e-12
1216 *
1217 * Cephes documentation below.
1218 *
1219 * SYNOPSIS:
1220 *
1221 * double a, x, y, igam();
1222 *
1223 * y = igam( a, x );
1224 *
1225 * DESCRIPTION:
1226 *
1227 * The function is defined by
1228 *
1229 * x
1230 * -
1231 * 1 | | -t a-1
1232 * igam(a,x) = ----- | e t dt.
1233 * - | |
1234 * | (a) -
1235 * 0
1236 *
1237 *
1238 * In this implementation both arguments must be positive.
1239 * The integral is evaluated by either a power series or
1240 * continued fraction expansion, depending on the relative
1241 * values of a and x.
1242 *
1243 * ACCURACY (double):
1244 *
1245 * Relative error:
1246 * arithmetic domain # trials peak rms
1247 * IEEE 0,30 200000 3.6e-14 2.9e-15
1248 * IEEE 0,100 300000 9.9e-14 1.5e-14
1249 *
1250 *
1251 * ACCURACY (float):
1252 *
1253 * Relative error:
1254 * arithmetic domain # trials peak rms
1255 * IEEE 0,30 20000 7.8e-6 5.9e-7
1256 *
1257 */
1258 /*
1259 Cephes Math Library Release 2.2: June, 1992
1260 Copyright 1985, 1987, 1992 by Stephen L. Moshier
1261 Direct inquiries to 30 Frost Street, Cambridge, MA 02140
1262 */
1263
1264 /* left tail of incomplete gamma function:
1265 *
1266 * inf. k
1267 * a -x - x
1268 * x e > ----------
1269 * - -
1270 * k=0 | (a+k+1)
1271 *
1272 */
1273};
1274
1275template <typename Scalar>
1276struct igamma_der_a_impl : igamma_generic_impl<Scalar, DERIVATIVE> {
1277 /* Derivative of the incomplete Gamma function with respect to a.
1278 *
1279 * Computes d/da igamma(a, x) by forward differentiation of the igamma code.
1280 *
1281 * Accuracy estimation. For each a in [10^-2, 10^-1...10^3] we sample
1282 * 50 Gamma random variables x ~ Gamma(x | a, 1), a total of 300 points.
1283 * The ground truth is computed by mpmath. Mean absolute error:
1284 * float: 6.17992e-07
1285 * double: 4.60453e-12
1286 *
1287 * Reference:
1288 * R. Moore. "Algorithm AS 187: Derivatives of the incomplete gamma
1289 * integral". Journal of the Royal Statistical Society. 1982
1290 */
1291};
1292
1293template <typename Scalar>
1294struct gamma_sample_der_alpha_impl : igamma_generic_impl<Scalar, SAMPLE_DERIVATIVE> {
1295 /* Derivative of a Gamma random variable sample with respect to alpha.
1296 *
1297 * Consider a sample of a Gamma random variable with the concentration
1298 * parameter alpha: sample ~ Gamma(alpha, 1). The reparameterization
1299 * derivative that we want to compute is dsample / dalpha =
1300 * d igammainv(alpha, u) / dalpha, where u = igamma(alpha, sample).
1301 * However, this formula is numerically unstable and expensive, so instead
1302 * we use implicit differentiation:
1303 *
1304 * igamma(alpha, sample) = u, where u ~ Uniform(0, 1).
1305 * Apply d / dalpha to both sides:
1306 * d igamma(alpha, sample) / dalpha
1307 * + d igamma(alpha, sample) / dsample * dsample/dalpha = 0
1308 * d igamma(alpha, sample) / dalpha
1309 * + Gamma(sample | alpha, 1) dsample / dalpha = 0
1310 * dsample/dalpha = - (d igamma(alpha, sample) / dalpha)
1311 * / Gamma(sample | alpha, 1)
1312 *
1313 * Here Gamma(sample | alpha, 1) is the PDF of the Gamma distribution
1314 * (note that the derivative of the CDF w.r.t. sample is the PDF).
1315 * See the reference below for more details.
1316 *
1317 * The derivative of igamma(alpha, sample) is computed by forward
1318 * differentiation of the igamma code. Division by the Gamma PDF is performed
1319 * in the same code, increasing the accuracy and speed due to cancellation
1320 * of some terms.
1321 *
1322 * Accuracy estimation. For each alpha in [10^-2, 10^-1...10^3] we sample
1323 * 50 Gamma random variables sample ~ Gamma(sample | alpha, 1), a total of 300
1324 * points. The ground truth is computed by mpmath. Mean absolute error:
1325 * float: 2.1686e-06
1326 * double: 1.4774e-12
1327 *
1328 * Reference:
1329 * M. Figurnov, S. Mohamed, A. Mnih "Implicit Reparameterization Gradients".
1330 * 2018
1331 */
1332};
1333
1334/*****************************************************************************
1335 * Implementation of Riemann zeta function of two arguments, based on Cephes *
1336 *****************************************************************************/
1337
1338template <typename Scalar>
1339struct zeta_impl_series {
1340 EIGEN_STATIC_ASSERT((std::is_same<Scalar, Scalar>::value == false), THIS_TYPE_IS_NOT_SUPPORTED)
1341
1342 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(const Scalar) { return Scalar(0); }
1343};
1344
1345template <>
1346struct zeta_impl_series<float> {
1347 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE bool run(float& a, float& b, float& s, const float x,
1348 const float machep) {
1349 int i = 0;
1350 while ((i < 9) || (a <= 9.0f)) {
1351 i += 1;
1352 a += 1.0f;
1353 b = numext::pow(a, -x);
1354 s += b;
1355 if (numext::abs(b / s) < machep) return true;
1356 }
1357
1358 // Return whether we are done
1359 return false;
1360 }
1361};
1362
1363template <>
1364struct zeta_impl_series<double> {
1365 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE bool run(double& a, double& b, double& s, const double x,
1366 const double machep) {
1367 int i = 0;
1368 while ((i < 9) || (a <= 9.0)) {
1369 i += 1;
1370 a += 1.0;
1371 b = numext::pow(a, -x);
1372 s += b;
1373 if (numext::abs(b / s) < machep) return true;
1374 }
1375
1376 // Return whether we are done
1377 return false;
1378 }
1379};
1380
1381template <typename Scalar>
1382struct zeta_impl {
1383 EIGEN_DEVICE_FUNC static Scalar run(Scalar x, Scalar q) {
1384 /* zeta.c
1385 *
1386 * Riemann zeta function of two arguments
1387 *
1388 *
1389 *
1390 * SYNOPSIS:
1391 *
1392 * double x, q, y, zeta();
1393 *
1394 * y = zeta( x, q );
1395 *
1396 *
1397 *
1398 * DESCRIPTION:
1399 *
1400 *
1401 *
1402 * inf.
1403 * - -x
1404 * zeta(x,q) = > (k+q)
1405 * -
1406 * k=0
1407 *
1408 * where x > 1 and q is not a negative integer or zero.
1409 * The Euler-Maclaurin summation formula is used to obtain
1410 * the expansion
1411 *
1412 * n
1413 * - -x
1414 * zeta(x,q) = > (k+q)
1415 * -
1416 * k=1
1417 *
1418 * 1-x inf. B x(x+1)...(x+2j)
1419 * (n+q) 1 - 2j
1420 * + --------- - ------- + > --------------------
1421 * x-1 x - x+2j+1
1422 * 2(n+q) j=1 (2j)! (n+q)
1423 *
1424 * where the B2j are Bernoulli numbers. Note that (see zetac.c)
1425 * zeta(x,1) = zetac(x) + 1.
1426 *
1427 *
1428 *
1429 * ACCURACY:
1430 *
1431 * Relative error for single precision:
1432 * arithmetic domain # trials peak rms
1433 * IEEE 0,25 10000 6.9e-7 1.0e-7
1434 *
1435 * Large arguments may produce underflow in powf(), in which
1436 * case the results are inaccurate.
1437 *
1438 * REFERENCE:
1439 *
1440 * Gradshteyn, I. S., and I. M. Ryzhik, Tables of Integrals,
1441 * Series, and Products, p. 1073; Academic Press, 1980.
1442 *
1443 */
1444
1445 int i;
1446 Scalar p, r, a, b, k, s, t, w;
1447
1448 const Scalar A[] = {
1449 Scalar(12.0),
1450 Scalar(-720.0),
1451 Scalar(30240.0),
1452 Scalar(-1209600.0),
1453 Scalar(47900160.0),
1454 Scalar(-1.8924375803183791606e9), /*1.307674368e12/691*/
1455 Scalar(7.47242496e10),
1456 Scalar(-2.950130727918164224e12), /*1.067062284288e16/3617*/
1457 Scalar(1.1646782814350067249e14), /*5.109094217170944e18/43867*/
1458 Scalar(-4.5979787224074726105e15), /*8.028576626982912e20/174611*/
1459 Scalar(1.8152105401943546773e17), /*1.5511210043330985984e23/854513*/
1460 Scalar(-7.1661652561756670113e18) /*1.6938241367317436694528e27/236364091*/
1461 };
1462
1463 const Scalar maxnum = NumTraits<Scalar>::infinity();
1464 const Scalar zero = Scalar(0.0), half = Scalar(0.5), one = Scalar(1.0);
1465 const Scalar machep = cephes_helper<Scalar>::machep();
1466 const Scalar nan = NumTraits<Scalar>::quiet_NaN();
1467
1468 if (x == one) return maxnum;
1469
1470 if (x < one) {
1471 return nan;
1472 }
1473
1474 if (q <= zero) {
1475 if (q == numext::floor(q)) {
1476 if (numext::rint(Scalar(0.5) * x) == Scalar(0.5) * x) {
1477 return maxnum;
1478 } else {
1479 return nan;
1480 }
1481 }
1482 p = x;
1483 r = numext::floor(p);
1484 if (p != r) return nan;
1485 }
1486
1487 /* Permit negative q but continue sum until n+q > +9 .
1488 * This case should be handled by a reflection formula.
1489 * If q<0 and x is an integer, there is a relation to
1490 * the polygamma function.
1491 */
1492 s = numext::pow(q, -x);
1493 a = q;
1494 b = zero;
1495 // Run the summation in a helper function that is specific to the floating precision
1496 if (zeta_impl_series<Scalar>::run(a, b, s, x, machep)) {
1497 return s;
1498 }
1499
1500 // If b is zero, then the tail sum will also end up being zero.
1501 // Exiting early here can prevent NaNs for some large inputs, where
1502 // the tail sum computed below has term `a` which can overflow to `inf`.
1503 if (numext::equal_strict(b, zero)) {
1504 return s;
1505 }
1506
1507 w = a;
1508 s += b * w / (x - one);
1509 s -= half * b;
1510 a = one;
1511 k = zero;
1512
1513 for (i = 0; i < 12; i++) {
1514 a *= x + k;
1515 b /= w;
1516 t = a * b / A[i];
1517 s = s + t;
1518 t = numext::abs(t / s);
1519 if (t < machep) {
1520 break;
1521 }
1522 k += one;
1523 a *= x + k;
1524 b /= w;
1525 k += one;
1526 }
1527 return s;
1528 }
1529};
1530
1531/****************************************************************************
1532 * Implementation of polygamma function *
1533 ****************************************************************************/
1534
1535template <typename Scalar>
1536struct polygamma_impl {
1537 EIGEN_DEVICE_FUNC static Scalar run(Scalar n, Scalar x) {
1538 Scalar zero = 0.0, one = 1.0;
1539 Scalar nplus = n + one;
1540 const Scalar nan = NumTraits<Scalar>::quiet_NaN();
1541
1542 // Check that n is a non-negative integer
1543 if (numext::floor(n) != n || n < zero) {
1544 return nan;
1545 }
1546 // Just return the digamma function for n = 0
1547 else if (n == zero) {
1548 return digamma_impl<Scalar>::run(x);
1549 }
1550 // Use the same implementation as scipy
1551 else {
1552 Scalar factorial = numext::exp(lgamma_impl<Scalar>::run(nplus));
1553 return numext::pow(-one, nplus) * factorial * zeta_impl<Scalar>::run(nplus, x);
1554 }
1555 }
1556};
1557
1558/************************************************************************************************
1559 * Implementation of betainc (incomplete beta integral), based on Cephes *
1560 ************************************************************************************************/
1561
1562template <typename Scalar>
1563struct betainc_impl {
1564 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(Scalar, Scalar, Scalar) {
1565 EIGEN_STATIC_ASSERT((!std::is_same<Scalar, Scalar>::value), THIS_TYPE_IS_NOT_SUPPORTED)
1566 /* betaincf.c
1567 *
1568 * Incomplete beta integral
1569 *
1570 *
1571 * SYNOPSIS:
1572 *
1573 * float a, b, x, y, betaincf();
1574 *
1575 * y = betaincf( a, b, x );
1576 *
1577 *
1578 * DESCRIPTION:
1579 *
1580 * Returns incomplete beta integral of the arguments, evaluated
1581 * from zero to x. The function is defined as
1582 *
1583 * x
1584 * - -
1585 * | (a+b) | | a-1 b-1
1586 * ----------- | t (1-t) dt.
1587 * - - | |
1588 * | (a) | (b) -
1589 * 0
1590 *
1591 * The domain of definition is 0 <= x <= 1. In this
1592 * implementation a and b are restricted to positive values.
1593 * The integral from x to 1 may be obtained by the symmetry
1594 * relation
1595 *
1596 * 1 - betainc( a, b, x ) = betainc( b, a, 1-x ).
1597 *
1598 * The integral is evaluated by a continued fraction expansion.
1599 * If a < 1, the function calls itself recursively after a
1600 * transformation to increase a to a+1.
1601 *
1602 * ACCURACY (float):
1603 *
1604 * Tested at random points (a,b,x) with a and b in the indicated
1605 * interval and x between 0 and 1.
1606 *
1607 * arithmetic domain # trials peak rms
1608 * Relative error:
1609 * IEEE 0,30 10000 3.7e-5 5.1e-6
1610 * IEEE 0,100 10000 1.7e-4 2.5e-5
1611 * The useful domain for relative error is limited by underflow
1612 * of the single precision exponential function.
1613 * Absolute error:
1614 * IEEE 0,30 100000 2.2e-5 9.6e-7
1615 * IEEE 0,100 10000 6.5e-5 3.7e-6
1616 *
1617 * Larger errors may occur for extreme ratios of a and b.
1618 *
1619 * ACCURACY (double):
1620 * arithmetic domain # trials peak rms
1621 * IEEE 0,5 10000 6.9e-15 4.5e-16
1622 * IEEE 0,85 250000 2.2e-13 1.7e-14
1623 * IEEE 0,1000 30000 5.3e-12 6.3e-13
1624 * IEEE 0,10000 250000 9.3e-11 7.1e-12
1625 * IEEE 0,100000 10000 8.7e-10 4.8e-11
1626 * Outputs smaller than the IEEE gradual underflow threshold
1627 * were excluded from these statistics.
1628 *
1629 * ERROR MESSAGES:
1630 * message condition value returned
1631 * incbet domain x<0, x>1 nan
1632 * incbet underflow nan
1633 */
1634 return Scalar(0);
1635 }
1636};
1637
1638/* Continued fraction expansion #1 for incomplete beta integral (small_branch = True)
1639 * Continued fraction expansion #2 for incomplete beta integral (small_branch = False)
1640 */
1641template <typename Scalar>
1642struct incbeta_cfe {
1643 EIGEN_STATIC_ASSERT((std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value),
1644 THIS_TYPE_IS_NOT_SUPPORTED)
1645
1646 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Scalar run(Scalar a, Scalar b, Scalar x, bool small_branch) {
1647 const Scalar big = cephes_helper<Scalar>::big();
1648 const Scalar machep = cephes_helper<Scalar>::machep();
1649 const Scalar biginv = cephes_helper<Scalar>::biginv();
1650
1651 const Scalar zero = 0;
1652 const Scalar one = 1;
1653 const Scalar two = 2;
1654
1655 Scalar xk, pk, pkm1, pkm2, qk, qkm1, qkm2;
1656 Scalar k1, k2, k3, k4, k5, k6, k7, k8, k26update;
1657 Scalar ans;
1658 int n;
1659
1660 constexpr int num_iters = (std::is_same<Scalar, float>::value) ? 100 : 300;
1661 const Scalar thresh = (std::is_same<Scalar, float>::value) ? machep : Scalar(3) * machep;
1662 Scalar r = (std::is_same<Scalar, float>::value) ? zero : one;
1663
1664 if (small_branch) {
1665 k1 = a;
1666 k2 = a + b;
1667 k3 = a;
1668 k4 = a + one;
1669 k5 = one;
1670 k6 = b - one;
1671 k7 = k4;
1672 k8 = a + two;
1673 k26update = one;
1674 } else {
1675 k1 = a;
1676 k2 = b - one;
1677 k3 = a;
1678 k4 = a + one;
1679 k5 = one;
1680 k6 = a + b;
1681 k7 = a + one;
1682 k8 = a + two;
1683 k26update = -one;
1684 x = x / (one - x);
1685 }
1686
1687 pkm2 = zero;
1688 qkm2 = one;
1689 pkm1 = one;
1690 qkm1 = one;
1691 ans = one;
1692 n = 0;
1693
1694 do {
1695 xk = -(x * k1 * k2) / (k3 * k4);
1696 pk = pkm1 + pkm2 * xk;
1697 qk = qkm1 + qkm2 * xk;
1698 pkm2 = pkm1;
1699 pkm1 = pk;
1700 qkm2 = qkm1;
1701 qkm1 = qk;
1702
1703 xk = (x * k5 * k6) / (k7 * k8);
1704 pk = pkm1 + pkm2 * xk;
1705 qk = qkm1 + qkm2 * xk;
1706 pkm2 = pkm1;
1707 pkm1 = pk;
1708 qkm2 = qkm1;
1709 qkm1 = qk;
1710
1711 if (qk != zero) {
1712 r = pk / qk;
1713 if (numext::abs(ans - r) < numext::abs(r) * thresh) {
1714 return r;
1715 }
1716 ans = r;
1717 }
1718
1719 k1 += one;
1720 k2 += k26update;
1721 k3 += two;
1722 k4 += two;
1723 k5 += one;
1724 k6 -= k26update;
1725 k7 += two;
1726 k8 += two;
1727
1728 if ((numext::abs(qk) + numext::abs(pk)) > big) {
1729 pkm2 *= biginv;
1730 pkm1 *= biginv;
1731 qkm2 *= biginv;
1732 qkm1 *= biginv;
1733 }
1734 if ((numext::abs(qk) < biginv) || (numext::abs(pk) < biginv)) {
1735 pkm2 *= big;
1736 pkm1 *= big;
1737 qkm2 *= big;
1738 qkm1 *= big;
1739 }
1740 } while (++n < num_iters);
1741
1742 return ans;
1743 }
1744};
1745
1746/* Helper functions depending on the Scalar type */
1747template <typename Scalar>
1748struct betainc_helper {};
1749
1750template <>
1751struct betainc_helper<float> {
1752 /* Core implementation, assumes a large (> 1.0) */
1753 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float incbsa(float aa, float bb, float xx) {
1754 float ans, a, b, t, x, onemx;
1755 bool reversed_a_b = false;
1756
1757 onemx = 1.0f - xx;
1758
1759 /* see if x is greater than the mean */
1760 if (xx > (aa / (aa + bb))) {
1761 reversed_a_b = true;
1762 a = bb;
1763 b = aa;
1764 t = xx;
1765 x = onemx;
1766 } else {
1767 a = aa;
1768 b = bb;
1769 t = onemx;
1770 x = xx;
1771 }
1772
1773 /* Choose expansion for optimal convergence */
1774 if (b > 10.0f) {
1775 if (numext::abs(b * x / a) < 0.3f) {
1776 t = betainc_helper<float>::incbps(a, b, x);
1777 if (reversed_a_b) t = 1.0f - t;
1778 return t;
1779 }
1780 }
1781
1782 ans = x * (a + b - 2.0f) / (a - 1.0f);
1783 if (ans < 1.0f) {
1784 ans = incbeta_cfe<float>::run(a, b, x, true /* small_branch */);
1785 t = b * numext::log(t);
1786 } else {
1787 ans = incbeta_cfe<float>::run(a, b, x, false /* small_branch */);
1788 t = (b - 1.0f) * numext::log(t);
1789 }
1790
1791 t += a * numext::log(x) + lgamma_impl<float>::run(a + b) - lgamma_impl<float>::run(a) - lgamma_impl<float>::run(b);
1792 t += numext::log(ans / a);
1793 t = numext::exp(t);
1794
1795 if (reversed_a_b) t = 1.0f - t;
1796 return t;
1797 }
1798
1799 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE float incbps(float a, float b, float x) {
1800 float t, u, y, s;
1801 const float machep = cephes_helper<float>::machep();
1802
1803 y = a * numext::log(x) + (b - 1.0f) * numext::log1p(-x) - numext::log(a);
1804 y -= lgamma_impl<float>::run(a) + lgamma_impl<float>::run(b);
1805 y += lgamma_impl<float>::run(a + b);
1806
1807 t = x / (1.0f - x);
1808 s = 0.0f;
1809 u = 1.0f;
1810 do {
1811 b -= 1.0f;
1812 if (b == 0.0f) {
1813 break;
1814 }
1815 a += 1.0f;
1816 u *= t * b / a;
1817 s += u;
1818 } while (numext::abs(u) > machep);
1819
1820 return numext::exp(y) * (1.0f + s);
1821 }
1822};
1823
1824template <>
1825struct betainc_impl<float> {
1826 EIGEN_DEVICE_FUNC static float run(float a, float b, float x) {
1827 const float nan = NumTraits<float>::quiet_NaN();
1828 float ans, t;
1829
1830 if ((numext::isnan)(a) || (numext::isnan)(b) || (numext::isnan)(x)) return nan;
1831 if (a == 0.0f && b == 0.0f) return nan;
1832 if (x < 0.0f || x > 1.0f) return nan;
1833 if (a < 0.0f) return nan;
1834 if (b < 0.0f) return nan;
1835 if (a == 0.0f) return 1.0f;
1836 if (b == 0.0f) return 0.0f;
1837 if (x == 0.0f) return 0.0f;
1838 if (x == 1.0f) return 1.0f;
1839 // mtherr("betaincf", DOMAIN);
1840
1841 /* transformation for small aa */
1842 if (a <= 1.0f) {
1843 ans = betainc_helper<float>::incbsa(a + 1.0f, b, x);
1844 t = a * numext::log(x) + b * numext::log1p(-x) + lgamma_impl<float>::run(a + b) -
1845 lgamma_impl<float>::run(a + 1.0f) - lgamma_impl<float>::run(b);
1846 return (ans + numext::exp(t));
1847 } else {
1848 return betainc_helper<float>::incbsa(a, b, x);
1849 }
1850 }
1851};
1852
1853template <>
1854struct betainc_helper<double> {
1855 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE double incbps(double a, double b, double x) {
1856 const double machep = cephes_helper<double>::machep();
1857
1858 double s, t, u, v, n, t1, z, ai;
1859
1860 ai = 1.0 / a;
1861 u = (1.0 - b) * x;
1862 v = u / (a + 1.0);
1863 t1 = v;
1864 t = u;
1865 n = 2.0;
1866 s = 0.0;
1867 z = machep * ai;
1868 while (numext::abs(v) > z) {
1869 u = (n - b) * x / n;
1870 t *= u;
1871 v = t / (a + n);
1872 s += v;
1873 n += 1.0;
1874 }
1875 s += t1;
1876 s += ai;
1877
1878 u = a * numext::log(x);
1879 // TODO: gamma() is not directly implemented in Eigen.
1880 /*
1881 if ((a + b) < maxgam && numext::abs(u) < maxlog) {
1882 t = gamma(a + b) / (gamma(a) * gamma(b));
1883 s = s * t * pow(x, a);
1884 }
1885 */
1886 t = lgamma_impl<double>::run(a + b) - lgamma_impl<double>::run(a) - lgamma_impl<double>::run(b) + u +
1887 numext::log(s);
1888 return numext::exp(t);
1889 }
1890};
1891
1892template <>
1893struct betainc_impl<double> {
1894 EIGEN_DEVICE_FUNC static double run(double aa, double bb, double xx) {
1895 const double nan = NumTraits<double>::quiet_NaN();
1896 const double machep = cephes_helper<double>::machep();
1897 double a, b, t, x, xc, w, y;
1898 bool reversed_a_b = false;
1899
1900 if ((numext::isnan)(aa) || (numext::isnan)(bb) || (numext::isnan)(xx)) return nan;
1901 if (aa == 0.0 && bb == 0.0) return nan;
1902 if (xx < 0.0 || xx > 1.0) return nan;
1903 if (aa < 0.0) return nan;
1904 if (bb < 0.0) return nan;
1905 if (aa == 0.0) return 1.0;
1906 if (bb == 0.0) return 0.0;
1907 if (xx == 0.0) return 0.0;
1908 if (xx == 1.0) return 1.0;
1909 // mtherr("incbet", DOMAIN);
1910
1911 if ((bb * xx) <= 1.0 && xx <= 0.95) {
1912 return betainc_helper<double>::incbps(aa, bb, xx);
1913 }
1914
1915 w = 1.0 - xx;
1916
1917 /* Reverse a and b if x is greater than the mean. */
1918 if (xx > (aa / (aa + bb))) {
1919 reversed_a_b = true;
1920 a = bb;
1921 b = aa;
1922 xc = xx;
1923 x = w;
1924 } else {
1925 a = aa;
1926 b = bb;
1927 xc = w;
1928 x = xx;
1929 }
1930
1931 if (reversed_a_b && (b * x) <= 1.0 && x <= 0.95) {
1932 t = betainc_helper<double>::incbps(a, b, x);
1933 if (t <= machep) {
1934 t = 1.0 - machep;
1935 } else {
1936 t = 1.0 - t;
1937 }
1938 return t;
1939 }
1940
1941 /* Choose expansion for better convergence. */
1942 y = x * (a + b - 2.0) - (a - 1.0);
1943 if (y < 0.0) {
1944 w = incbeta_cfe<double>::run(a, b, x, true /* small_branch */);
1945 } else {
1946 w = incbeta_cfe<double>::run(a, b, x, false /* small_branch */) / xc;
1947 }
1948
1949 /* Multiply w by the factor
1950 a b _ _ _
1951 x (1-x) | (a+b) / ( a | (a) | (b) ) . */
1952
1953 y = a * numext::log(x);
1954 t = b * numext::log(xc);
1955 // TODO: gamma is not directly implemented in Eigen.
1956 /*
1957 if ((a + b) < maxgam && numext::abs(y) < maxlog && numext::abs(t) < maxlog)
1958 {
1959 t = pow(xc, b);
1960 t *= pow(x, a);
1961 t /= a;
1962 t *= w;
1963 t *= gamma(a + b) / (gamma(a) * gamma(b));
1964 } else {
1965 */
1966 /* Resort to logarithms. */
1967 y += t + lgamma_impl<double>::run(a + b) - lgamma_impl<double>::run(a) - lgamma_impl<double>::run(b);
1968 y += numext::log(w / a);
1969 t = numext::exp(y);
1970
1971 /* } */
1972 // done:
1973
1974 if (reversed_a_b) {
1975 if (t <= machep) {
1976 t = 1.0 - machep;
1977 } else {
1978 t = 1.0 - t;
1979 }
1980 }
1981 return t;
1982 }
1983};
1984
1985} // end namespace internal
1986
1987namespace numext {
1988
1989template <typename Scalar>
1990EIGEN_DEVICE_FUNC inline auto lgamma(const Scalar& x) -> decltype(EIGEN_MATHFUNC_IMPL(lgamma, Scalar)::run(x)) {
1991 return EIGEN_MATHFUNC_IMPL(lgamma, Scalar)::run(x);
1992}
1993
1994template <typename Scalar>
1995EIGEN_DEVICE_FUNC inline auto digamma(const Scalar& x) -> decltype(EIGEN_MATHFUNC_IMPL(digamma, Scalar)::run(x)) {
1996 return EIGEN_MATHFUNC_IMPL(digamma, Scalar)::run(x);
1997}
1998
1999template <typename Scalar>
2000EIGEN_DEVICE_FUNC inline auto zeta(const Scalar& x, const Scalar& q)
2001 -> decltype(EIGEN_MATHFUNC_IMPL(zeta, Scalar)::run(x, q)) {
2002 return EIGEN_MATHFUNC_IMPL(zeta, Scalar)::run(x, q);
2003}
2004
2005template <typename Scalar>
2006EIGEN_DEVICE_FUNC inline auto polygamma(const Scalar& n, const Scalar& x)
2007 -> decltype(EIGEN_MATHFUNC_IMPL(polygamma, Scalar)::run(n, x)) {
2008 return EIGEN_MATHFUNC_IMPL(polygamma, Scalar)::run(n, x);
2009}
2010
2011template <typename Scalar>
2012EIGEN_DEVICE_FUNC inline auto erf(const Scalar& x) -> decltype(EIGEN_MATHFUNC_IMPL(erf, Scalar)::run(x)) {
2013 return EIGEN_MATHFUNC_IMPL(erf, Scalar)::run(x);
2014}
2015
2016template <typename Scalar>
2017EIGEN_DEVICE_FUNC inline auto erfc(const Scalar& x) -> decltype(EIGEN_MATHFUNC_IMPL(erfc, Scalar)::run(x)) {
2018 return EIGEN_MATHFUNC_IMPL(erfc, Scalar)::run(x);
2019}
2020
2021template <typename Scalar>
2022EIGEN_DEVICE_FUNC inline auto ndtri(const Scalar& x) -> decltype(EIGEN_MATHFUNC_IMPL(ndtri, Scalar)::run(x)) {
2023 return EIGEN_MATHFUNC_IMPL(ndtri, Scalar)::run(x);
2024}
2025
2026template <typename Scalar>
2027EIGEN_DEVICE_FUNC inline auto igamma(const Scalar& a, const Scalar& x)
2028 -> decltype(EIGEN_MATHFUNC_IMPL(igamma, Scalar)::run(a, x)) {
2029 return EIGEN_MATHFUNC_IMPL(igamma, Scalar)::run(a, x);
2030}
2031
2032template <typename Scalar>
2033EIGEN_DEVICE_FUNC inline auto igamma_der_a(const Scalar& a, const Scalar& x)
2034 -> decltype(EIGEN_MATHFUNC_IMPL(igamma_der_a, Scalar)::run(a, x)) {
2035 return EIGEN_MATHFUNC_IMPL(igamma_der_a, Scalar)::run(a, x);
2036}
2037
2038template <typename Scalar>
2039EIGEN_DEVICE_FUNC inline auto gamma_sample_der_alpha(const Scalar& a, const Scalar& x)
2040 -> decltype(EIGEN_MATHFUNC_IMPL(gamma_sample_der_alpha, Scalar)::run(a, x)) {
2041 return EIGEN_MATHFUNC_IMPL(gamma_sample_der_alpha, Scalar)::run(a, x);
2042}
2043
2044template <typename Scalar>
2045EIGEN_DEVICE_FUNC inline auto igammac(const Scalar& a, const Scalar& x)
2046 -> decltype(EIGEN_MATHFUNC_IMPL(igammac, Scalar)::run(a, x)) {
2047 return EIGEN_MATHFUNC_IMPL(igammac, Scalar)::run(a, x);
2048}
2049
2050template <typename Scalar>
2051EIGEN_DEVICE_FUNC inline auto betainc(const Scalar& a, const Scalar& b, const Scalar& x)
2052 -> decltype(EIGEN_MATHFUNC_IMPL(betainc, Scalar)::run(a, b, x)) {
2053 return EIGEN_MATHFUNC_IMPL(betainc, Scalar)::run(a, b, x);
2054}
2055
2056} // end namespace numext
2057} // end namespace Eigen
2058
2059#endif // EIGEN_SPECIAL_FUNCTIONS_H
Namespace containing all symbols from the Eigen library.