12#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_COMPLEX_H
13#define EIGEN_ARCH_GENERIC_PACKET_MATH_COMPLEX_H
16#include "../../InternalHeaderCheck.h"
21EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_PUSH
27template <
typename Packet>
28EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pdiv_complex(
const Packet& x,
const Packet& y) {
29 using RealPacket =
typename unpacket_traits<Packet>::as_real;
30 using RealScalar =
typename unpacket_traits<RealPacket>::type;
34 const RealPacket one = pset1<RealPacket>(RealScalar(1));
35 const RealPacket abs_y = pabs(y.v);
36 const RealPacket abs_y_flip = pcplxflip(Packet(abs_y)).v;
38 const RealPacket mask = pcmp_lt(abs_y, abs_y_flip);
42 const RealPacket ratio =
43 pdiv(pmin(abs_y, abs_y_flip),
44 pselect(pcmp_eq(abs_y, abs_y_flip), pset1<RealPacket>(RealScalar(2)), pmax(abs_y, abs_y_flip)));
45 RealPacket y_scaled = pselect(mask, ratio, one);
46 y_scaled = por(y_scaled, pandnot(y.v, abs_y));
47 RealPacket denom = pmul(y.v, y_scaled);
48 denom = padd(denom, pcplxflip(Packet(denom)).v);
49 Packet num = pmul(x, pconj(Packet(y_scaled)));
50 RealPacket result = pdiv(num.v, denom);
56 const RealPacket quarter_highest = pset1<RealPacket>(NumTraits<RealScalar>::highest() / RealScalar(4));
57 if (EIGEN_PREDICT_FALSE(predux_any(pcmp_lt(quarter_highest, pmax(pabs(x.v), abs_y))))) {
58 const RealPacket cst_quarter = pset1<RealPacket>(RealScalar(0.25));
59 const RealPacket abs_x = pabs(x.v);
60 const RealPacket large_x = pcmp_lt(quarter_highest, pmax(abs_x, pcplxflip(Packet(abs_x)).v));
61 const RealPacket large_y = pcmp_lt(quarter_highest, pmax(abs_y, abs_y_flip));
62 const Packet x_in(pselect(large_x, pmul(x.v, cst_quarter), x.v));
63 const RealPacket y_in = pselect(large_y, pmul(y.v, cst_quarter), y.v);
64 const RealPacket unscale =
65 pmul(pselect(large_x, pset1<RealPacket>(RealScalar(4)), one), pselect(large_y, cst_quarter, one));
66 RealPacket denom_in = pmul(y_in, y_scaled);
67 denom_in = padd(denom_in, pcplxflip(Packet(denom_in)).v);
68 const Packet num_in = pmul(x_in, pconj(Packet(y_scaled)));
69 const RealPacket inf = pset1<RealPacket>(NumTraits<RealScalar>::infinity());
70 RealPacket overflow = por(pcmp_eq(pabs(num.v), inf), pcmp_eq(pabs(denom), inf));
71 overflow = por(overflow, pcplxflip(Packet(overflow)).v);
72 result = pselect(overflow, pmul(pdiv(num_in.v, denom_in), unscale), result);
74 return Packet(result);
77template <
typename Packet>
78EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pmul_complex(
const Packet& x,
const Packet& y) {
82 Packet x_re = pdupreal(x);
83 Packet x_im = pdupimag(x);
84 Packet tmp_re = Packet(pmul(x_re.v, y.v));
85 Packet tmp_im = Packet(pmul(x_im.v, y.v));
86 tmp_im = pcplxflip(pconj(tmp_im));
87 return padd(tmp_im, tmp_re);
92template <
typename Packet,
bool Base2,
bool Scaled>
93EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_complex_impl(
const Packet& x) {
94 using RealPacket =
typename unpacket_traits<Packet>::as_real;
95 using RealScalar =
typename unpacket_traits<RealPacket>::type;
97 RealPacket x_flip = pcplxflip(x).v;
104 RealPacket x_abs = pabs(x.v);
105 const RealPacket x_max = pmax(x_abs, pcplxflip(Packet(x_abs)).v);
106 const RealScalar lo = numext::sqrt((numext::numeric_limits<RealScalar>::min)() / NumTraits<RealScalar>::epsilon());
107 const RealScalar hi = numext::sqrt(NumTraits<RealScalar>::highest()) / RealScalar(2);
108 const RealPacket is_large = pcmp_lt(pset1<RealPacket>(hi), x_max);
109 const RealPacket is_small = pcmp_lt(x_max, pset1<RealPacket>(lo));
111 constexpr int kLarge = NumTraits<RealScalar>::max_exponent() / 2 + 2;
112 constexpr int kSmall = -((3 * NumTraits<RealScalar>::digits() - NumTraits<RealScalar>::min_exponent() - 1) / 2);
113 const RealPacket k_large = pset1<RealPacket>(RealScalar(kLarge));
114 const RealPacket k_small = pset1<RealPacket>(RealScalar(kSmall));
115 const RealPacket cst_one = pset1<RealPacket>(RealScalar(1));
116 const RealPacket k = pselect(is_large, k_large, pand(is_small, k_small));
117 const RealPacket scale = pselect(is_large, pldexp_fast(cst_one, pnegate(k_large)),
118 pselect(is_small, pldexp_fast(cst_one, pnegate(k_small)), cst_one));
121 const RealPacket r = phypot_complex(Packet(pmul(x.v, scale))).v;
122 xlogr = Base2 ? padd(k, plog2(r)) : pmadd(k, pset1<RealPacket>(RealScalar(EIGEN_LN2)), plog(r));
124 const RealPacket cst_pos_inf = pinf<RealPacket>();
125 RealPacket is_x_pos_inf = pcmp_eq(x_abs, cst_pos_inf);
126 RealPacket is_y_pos_inf = pcplxflip(Packet(is_x_pos_inf)).v;
127 xlogr = pselect(por(is_x_pos_inf, is_y_pos_inf), cst_pos_inf, xlogr);
129 const RealPacket r = phypot_complex(x).v;
130 xlogr = Base2 ? plog2(r) : plog(r);
134 RealPacket ximg = patan2(x.v, x_flip);
135 if (Base2) ximg = pmul(pset1<RealPacket>(RealScalar(EIGEN_LOG2E)), ximg);
137 return Packet(pselect(peven_mask(xlogr), xlogr, ximg));
141template <
typename Packet,
bool Base2>
142EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE Packet plog_complex_scaled(
const Packet& x) {
143 return plog_complex_impl<Packet, Base2, true>(x);
146template <
typename Packet,
bool Base2>
147EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_complex(
const Packet& x) {
148 using RealPacket =
typename unpacket_traits<Packet>::as_real;
149 using RealScalar =
typename unpacket_traits<RealPacket>::type;
150 const RealPacket x_abs = pabs(x.v);
151 const RealPacket x_max = pmax(x_abs, pcplxflip(Packet(x_abs)).v);
152 const RealScalar lo = numext::sqrt((numext::numeric_limits<RealScalar>::min)() / NumTraits<RealScalar>::epsilon());
153 const RealScalar hi = numext::sqrt(NumTraits<RealScalar>::highest()) / RealScalar(2);
156 const RealPacket is_small = pand(pcmp_lt(pzero(x_max), x_max), pcmp_lt(x_max, pset1<RealPacket>(lo)));
157 if (predux_any(por(pcmp_lt(pset1<RealPacket>(hi), x_abs), is_small))) {
158 return plog_complex_scaled<Packet, Base2>(x);
160 return plog_complex_impl<Packet, Base2, false>(x);
163template <
typename Packet>
164struct plog2_impl<Packet, true> {
165 static EIGEN_DEVICE_FUNC
inline Packet run(
const Packet& a) {
return plog_complex<Packet, true>(a); }
168template <
typename Packet>
169EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp_complex(
const Packet& a) {
170 using RealPacket =
typename unpacket_traits<Packet>::as_real;
171 using Scalar =
typename unpacket_traits<Packet>::type;
172 using RealScalar =
typename Scalar::value_type;
173 const RealPacket even_mask = peven_mask(a.v);
174 const RealPacket odd_mask = pcplxflip(Packet(even_mask)).v;
180 RealPacket x = pand(a.v, even_mask);
181 x = por(x, pcplxflip(Packet(x)).v);
182 RealPacket expx = pexp(x);
185 RealPacket y = pand(odd_mask, a.v);
186 y = por(y, pcplxflip(Packet(y)).v);
187 RealPacket cisy = psincos_selector<RealPacket>(y);
188 cisy = pcplxflip(Packet(cisy)).v;
192 const RealScalar large_x = RealScalar(NumTraits<RealScalar>::max_exponent() - 1) * RealScalar(EIGEN_LN2);
193 RealPacket expx_factor = expx;
194 if (predux_any(pcmp_le(pset1<RealPacket>(large_x), pabs(x)))) {
195 const RealPacket cst_pos_inf = pinf<RealPacket>();
196 const RealPacket is_x_pos_inf = pcmp_eq(x, cst_pos_inf);
197 const RealPacket split = pandnot(pcmp_eq(expx, cst_pos_inf), is_x_pos_inf);
198 const RealPacket cst_neg_inf = por(psignmask<RealPacket>(), pinf<RealPacket>());
203 RealPacket cisy_sign = por(pandnot(cisy, pabs(cisy)), pset1<RealPacket>(RealScalar(1)));
204 cisy = pselect(pcmp_eq(x, cst_neg_inf), cisy_sign, cisy);
208 RealPacket y_sign = por(pandnot(y, pabs(y)), pset1<RealPacket>(RealScalar(1)));
209 cisy = pselect(pand(is_x_pos_inf, pisnan(cisy)), pand(y_sign, even_mask), cisy);
215 RealPacket cisy_sign_one = por(pand(cisy, psignmask<RealPacket>()), pset1<RealPacket>(RealScalar(1)));
216 RealPacket x_inf_y_finite = pand(is_x_pos_inf, pcmp_lt(pabs(y), cst_pos_inf));
217 cisy = pselect(x_inf_y_finite, cisy_sign_one, cisy);
226 const int digits = NumTraits<RealScalar>::digits();
227 const RealScalar x_max =
228 RealScalar(NumTraits<RealScalar>::max_exponent() - NumTraits<RealScalar>::min_exponent() + digits + 1) *
229 RealScalar(EIGEN_LN2);
230 constexpr bool is_float = std::is_same<RealScalar, float>::value;
231 const RealScalar ln2_hi = is_float ? RealScalar(0.693359375f) : RealScalar(0.693145751953125);
232 const RealScalar ln2_lo = is_float ? RealScalar(-2.12194440e-4f) : RealScalar(1.42860682030941723212e-6);
233 const RealPacket x_split = pselect(split, pmin(x, pset1<RealPacket>(x_max)), pzero(x));
234 const RealPacket n = print(pmul(x_split, pset1<RealPacket>(RealScalar(EIGEN_LOG2E))));
235 const RealPacket r = pnmadd(n, pset1<RealPacket>(ln2_lo), pnmadd(n, pset1<RealPacket>(ln2_hi), x_split));
236 const RealPacket scaled_cisy = pmul(cisy, pset1<RealPacket>(RealScalar(numext::uint64_t(1) << digits)));
237 const RealPacket split_result = pldexp(pmul(pexp_reduced_impl<RealPacket>::run(r), scaled_cisy),
238 psub(n, pset1<RealPacket>(RealScalar(digits))));
239 cisy = pselect(split, split_result, cisy);
240 expx_factor = pselect(split, pset1<RealPacket>(RealScalar(1)), expx);
243 Packet result = Packet(pmul(expx_factor, cisy));
246 result = pselect(Packet(pcmp_eq(y, pzero(y))), Packet(por(pand(expx, even_mask), pand(y, odd_mask))), result);
253template <
typename Packet>
254EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psqrt_complex_impl(
const Packet& z,
bool scaled) {
255 using Scalar =
typename unpacket_traits<Packet>::type;
256 using RealScalar =
typename Scalar::value_type;
257 using RealPacket =
typename unpacket_traits<Packet>::as_real;
263 RealPacket a_abs = pabs(z.v);
264 RealPacket a_abs_flip = pcplxflip(Packet(a_abs)).v;
265 RealPacket a_max = pmax(a_abs, a_abs_flip);
266 const RealPacket cst_one = pset1<RealPacket>(RealScalar(1));
268 RealPacket unscale = cst_one;
270 const RealPacket is_large = pcmp_lt(pset1<RealPacket>(NumTraits<RealScalar>::highest() / RealScalar(4)), a_max);
271 const RealPacket is_small =
272 pcmp_lt(a_max, pset1<RealPacket>(RealScalar(2) * (numext::numeric_limits<RealScalar>::min)()));
273 const int m = (NumTraits<RealScalar>::digits() + 1) / 2;
274 const RealScalar two_m = RealScalar(numext::uint64_t(1) << m);
275 const RealPacket scale = pselect(is_large, pset1<RealPacket>(RealScalar(0.25)),
276 pselect(is_small, pset1<RealPacket>(two_m * two_m), cst_one));
277 unscale = pselect(is_large, pset1<RealPacket>(RealScalar(2)),
278 pselect(is_small, pset1<RealPacket>(RealScalar(1) / two_m), cst_one));
279 a.v = pmul(z.v, scale);
282 a_abs_flip = pcplxflip(Packet(a_abs)).v;
283 a_max = pmax(a_abs, a_abs_flip);
322 RealPacket a_min = pmin(a_abs, a_abs_flip);
323 RealPacket a_min_zero_mask = pcmp_eq(a_min, pzero(a_min));
324 RealPacket a_max_zero_mask = pcmp_eq(a_max, pzero(a_max));
325 RealPacket r = pdiv(a_min, a_max);
326 RealPacket l = pmul(a_max, psqrt(pmadd(r, r, cst_one)));
328 l = pselect(a_min_zero_mask, a_max, l);
333 const RealPacket cst_half = pset1<RealPacket>(RealScalar(0.5));
335 rho.v = psqrt(pmul(cst_half, padd(a_abs, l)));
336 if (scaled) rho.v = pmul(rho.v, unscale);
341 const RealPacket rho_flip = pcplxflip(rho).v;
342 RealPacket eta = pselect(a_max_zero_mask, a.v, pdiv(z.v, padd(rho_flip, rho_flip)));
343 RealPacket real_mask = peven_mask(a.v);
344 Packet positive_real_result;
346 positive_real_result.v = pselect(real_mask, rho.v, eta);
351 const RealPacket cst_imag_sign_mask = pandnot(psignmask<RealPacket>(), real_mask);
352 RealPacket imag_signs = pand(a.v, cst_imag_sign_mask);
353 Packet negative_real_result;
355 negative_real_result.v = por(pabs(pcplxflip(positive_real_result).v), imag_signs);
358 Packet negative_real_mask;
359 negative_real_mask.v = pcmp_lt(pand(real_mask, a.v), pzero(a.v));
360 negative_real_mask.v = por(negative_real_mask.v, pcplxflip(negative_real_mask).v);
361 Packet result = pselect(negative_real_mask, negative_real_result, positive_real_result);
363 Packet result_is_nan = pisnan(result);
364 result = por(result_is_nan, result);
365 if (!scaled)
return result;
372 const RealPacket cst_pos_inf = pinf<RealPacket>();
374 is_inf.v = pcmp_eq(a_abs, cst_pos_inf);
376 is_real_inf.v = pand(is_inf.v, real_mask);
377 is_real_inf = por(is_real_inf, pcplxflip(is_real_inf));
379 Packet real_inf_result;
380 real_inf_result.v = pmul(a_abs, pset1<Packet>(Scalar(RealScalar(1.0), RealScalar(0.0))).v);
381 real_inf_result.v = pselect(negative_real_mask.v, pcplxflip(real_inf_result).v, real_inf_result.v);
383 real_inf_result.v = por(real_inf_result.v, imag_signs);
386 is_imag_inf.v = pandnot(is_inf.v, real_mask);
387 is_imag_inf = por(is_imag_inf, pcplxflip(is_imag_inf));
388 Packet imag_inf_result;
389 imag_inf_result.v = por(pand(cst_pos_inf, real_mask), pandnot(a.v, real_mask));
390 return pselect(is_imag_inf, imag_inf_result, pselect(is_real_inf, real_inf_result, result));
393template <
typename Packet>
394EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psqrt_complex(
const Packet& z) {
395 using RealScalar =
typename unpacket_traits<Packet>::type::value_type;
396 using RealPacket =
typename unpacket_traits<Packet>::as_real;
397 const RealPacket z_abs = pabs(z.v);
398 const RealPacket z_max = pmax(z_abs, pcplxflip(Packet(z_abs)).v);
399 const RealPacket hi = pset1<RealPacket>(NumTraits<RealScalar>::highest() / RealScalar(4));
400 const RealPacket lo = pset1<RealPacket>(RealScalar(2) * (numext::numeric_limits<RealScalar>::min)());
403 const RealPacket is_small = pand(pcmp_lt(pzero(z_max), z_max), pcmp_lt(z_max, lo));
404 if (predux_any(por(pcmp_lt(hi, z_abs), is_small))) {
407 pbarrier(z_scaled.v);
408 return psqrt_complex_impl(z_scaled,
true);
410 return psqrt_complex_impl(z,
false);
422template <
typename Packet>
423EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet phypot_complex(
const Packet& a) {
424 using Scalar =
typename unpacket_traits<Packet>::type;
425 using RealScalar =
typename Scalar::value_type;
426 using RealPacket =
typename unpacket_traits<Packet>::as_real;
428 const RealPacket cst_zero_rp = pset1<RealPacket>(
static_cast<RealScalar
>(0.0));
429 const RealPacket cst_half_rp = pset1<RealPacket>(
static_cast<RealScalar
>(0.5));
431 RealPacket a_abs = pabs(a.v);
432 RealPacket a_flip = pcplxflip(Packet(a_abs)).v;
433 RealPacket a_all = pmax(a_abs, a_flip);
434 RealPacket b_all = pmin(a_abs, a_flip);
436 RealPacket a2 = pmul(a.v, a.v);
437 RealPacket a2_flip = pcplxflip(Packet(a2)).v;
438 RealPacket h = psqrt(padd(a2, a2_flip));
439 RealPacket h_sq = pmul(h, h);
440 RealPacket a_sq = pmax(a2, a2_flip);
441 RealPacket b_sq = pmin(a2, a2_flip);
442 RealPacket h_lo = twoprod_low(h, h, h_sq);
443 RealPacket a_lo = twoprod_low(a_all, a_all, a_sq);
444 RealPacket b_lo = twoprod_low(b_all, b_all, b_sq);
445 RealPacket x = padd(psub(psub(h_sq, a_sq), b_sq), psub(h_lo, padd(a_lo, b_lo)));
446 h = pnmadd(pdiv(x, h), cst_half_rp, h);
449 RealPacket iszero = pcmp_eq(por(a_abs, a_flip), cst_zero_rp);
451 h = pandnot(h, iszero);
455EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_POP