Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GenericPacketMathComplex.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2009-2019 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Copyright (C) 2018-2025 Rasmus Munk Larsen <rmlarsen@gmail.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_ARCH_GENERIC_PACKET_MATH_COMPLEX_H
13#define EIGEN_ARCH_GENERIC_PACKET_MATH_COMPLEX_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19namespace internal {
20
21EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_PUSH
22
23//----------------------------------------------------------------------
24// Complex Arithmetic and Functions
25//----------------------------------------------------------------------
26
27template <typename Packet>
28EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pdiv_complex(const Packet& x, const Packet& y) {
29 using RealPacket = typename unpacket_traits<Packet>::as_real;
30 using RealScalar = typename unpacket_traits<RealPacket>::type;
31 // In the following we annotate the code for the case where the inputs
32 // are a pair length-2 SIMD vectors representing a single pair of complex
33 // numbers x = a + i*b, y = c + i*d.
34 const RealPacket one = pset1<RealPacket>(RealScalar(1));
35 const RealPacket abs_y = pabs(y.v);
36 const RealPacket abs_y_flip = pcplxflip(Packet(abs_y)).v;
37
38 const RealPacket mask = pcmp_lt(abs_y, abs_y_flip); // |c| < |d|
39 // min(|c|, |d|) / max(|c|, |d|) never divides by zero for a nonzero denominator, even in unused elements. Elements
40 // with |c| == |d|, which include 0 / 0 and inf / inf, divide by 2 instead; they select 1 below. A divisor of 1 would
41 // not do: Clang rewrites x / (z ? 1 : m) as z ? x : x / m.
42 const RealPacket ratio =
43 pdiv(pmin(abs_y, abs_y_flip),
44 pselect(pcmp_eq(abs_y, abs_y_flip), pset1<RealPacket>(RealScalar(2)), pmax(abs_y, abs_y_flip)));
45 RealPacket y_scaled = pselect(mask, ratio, one);
46 y_scaled = por(y_scaled, pandnot(y.v, abs_y)); // copy signs in case |c| == |d|
47 RealPacket denom = pmul(y.v, y_scaled);
48 denom = padd(denom, pcplxflip(Packet(denom)).v); // c * c' + d * d'
49 Packet num = pmul(x, pconj(Packet(y_scaled))); // a * c' + b * d', -a * d + b * c
50 RealPacket result = pdiv(num.v, denom);
51 // y_scaled = y / max(|c|, |d|), so the denominator lies in [m, 2 m] for m = max(|c|, |d|) and the numerator's
52 // components are at most |a| + |b|. Either can overflow where x / y is finite, which needs a component above
53 // highest / 4. Elements whose numerator or denominator overflowed are recomputed with x (y) scaled by 1/4 where it
54 // has such a component, and the quotient by the inverse scale. The scaling can lose a subnormal component, so
55 // elements that did not overflow keep the unscaled quotient.
56 const RealPacket quarter_highest = pset1<RealPacket>(NumTraits<RealScalar>::highest() / RealScalar(4));
57 if (EIGEN_PREDICT_FALSE(predux_any(pcmp_lt(quarter_highest, pmax(pabs(x.v), abs_y))))) {
58 const RealPacket cst_quarter = pset1<RealPacket>(RealScalar(0.25));
59 const RealPacket abs_x = pabs(x.v);
60 const RealPacket large_x = pcmp_lt(quarter_highest, pmax(abs_x, pcplxflip(Packet(abs_x)).v));
61 const RealPacket large_y = pcmp_lt(quarter_highest, pmax(abs_y, abs_y_flip));
62 const Packet x_in(pselect(large_x, pmul(x.v, cst_quarter), x.v));
63 const RealPacket y_in = pselect(large_y, pmul(y.v, cst_quarter), y.v);
64 const RealPacket unscale =
65 pmul(pselect(large_x, pset1<RealPacket>(RealScalar(4)), one), pselect(large_y, cst_quarter, one));
66 RealPacket denom_in = pmul(y_in, y_scaled);
67 denom_in = padd(denom_in, pcplxflip(Packet(denom_in)).v);
68 const Packet num_in = pmul(x_in, pconj(Packet(y_scaled)));
69 const RealPacket inf = pset1<RealPacket>(NumTraits<RealScalar>::infinity());
70 RealPacket overflow = por(pcmp_eq(pabs(num.v), inf), pcmp_eq(pabs(denom), inf));
71 overflow = por(overflow, pcplxflip(Packet(overflow)).v);
72 result = pselect(overflow, pmul(pdiv(num_in.v, denom_in), unscale), result);
73 }
74 return Packet(result);
75}
76
77template <typename Packet>
78EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pmul_complex(const Packet& x, const Packet& y) {
79 // In the following we annotate the code for the case where the inputs
80 // are a pair length-2 SIMD vectors representing a single pair of complex
81 // numbers x = a + i*b, y = c + i*d.
82 Packet x_re = pdupreal(x); // a, a
83 Packet x_im = pdupimag(x); // b, b
84 Packet tmp_re = Packet(pmul(x_re.v, y.v)); // a*c, a*d
85 Packet tmp_im = Packet(pmul(x_im.v, y.v)); // b*c, b*d
86 tmp_im = pcplxflip(pconj(tmp_im)); // -b*d, d*c
87 return padd(tmp_im, tmp_re); // a*c - b*d, a*d + b*c
88}
89
90// \internal plog_complex. Only when `Scaled` does it scale the elements outside [lo, hi] and handle infinities, which
91// plog_complex guarantees are absent otherwise.
92template <typename Packet, bool Base2, bool Scaled>
93EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_complex_impl(const Packet& x) {
94 using RealPacket = typename unpacket_traits<Packet>::as_real;
95 using RealScalar = typename unpacket_traits<RealPacket>::type;
96
97 RealPacket x_flip = pcplxflip(x).v; // b, a
98 RealPacket xlogr;
99 if (Scaled) {
100 // Real part. phypot_complex squares the components, which neither overflows nor loses accuracy to underflow for
101 // M = max(|a|, |b|) in [lo, hi] = [sqrt(min / eps), sqrt(highest) / 2]. Elements with M > hi are scaled by 2^-k
102 // with k = max_exponent / 2 + 2, and those with M < lo by 2^-k with k = -(3 digits - min_exponent - 1) / 2; both
103 // are exact and land M in [lo, hi]. Then log|x| = log|x * 2^-k| + k log(2).
104 RealPacket x_abs = pabs(x.v);
105 const RealPacket x_max = pmax(x_abs, pcplxflip(Packet(x_abs)).v);
106 const RealScalar lo = numext::sqrt((numext::numeric_limits<RealScalar>::min)() / NumTraits<RealScalar>::epsilon());
107 const RealScalar hi = numext::sqrt(NumTraits<RealScalar>::highest()) / RealScalar(2);
108 const RealPacket is_large = pcmp_lt(pset1<RealPacket>(hi), x_max);
109 const RealPacket is_small = pcmp_lt(x_max, pset1<RealPacket>(lo));
110 // Both divisions round down: for double, k_small = -589 maps denorm_min = 2^-1074 exactly onto lo = 2^-485.
111 constexpr int kLarge = NumTraits<RealScalar>::max_exponent() / 2 + 2;
112 constexpr int kSmall = -((3 * NumTraits<RealScalar>::digits() - NumTraits<RealScalar>::min_exponent() - 1) / 2);
113 const RealPacket k_large = pset1<RealPacket>(RealScalar(kLarge));
114 const RealPacket k_small = pset1<RealPacket>(RealScalar(kSmall));
115 const RealPacket cst_one = pset1<RealPacket>(RealScalar(1));
116 const RealPacket k = pselect(is_large, k_large, pand(is_small, k_small));
117 const RealPacket scale = pselect(is_large, pldexp_fast(cst_one, pnegate(k_large)),
118 pselect(is_small, pldexp_fast(cst_one, pnegate(k_small)), cst_one));
119 // log(sqrt(a^2 + b^2)), log(sqrt(a^2 + b^2)); in base 2 the real part comes from plog2, which is exact where |x| is
120 // a power of two, rather than from log2(e) log|x|.
121 const RealPacket r = phypot_complex(Packet(pmul(x.v, scale))).v;
122 xlogr = Base2 ? padd(k, plog2(r)) : pmadd(k, pset1<RealPacket>(RealScalar(EIGEN_LN2)), plog(r));
123
124 const RealPacket cst_pos_inf = pinf<RealPacket>();
125 RealPacket is_x_pos_inf = pcmp_eq(x_abs, cst_pos_inf);
126 RealPacket is_y_pos_inf = pcplxflip(Packet(is_x_pos_inf)).v;
127 xlogr = pselect(por(is_x_pos_inf, is_y_pos_inf), cst_pos_inf, xlogr);
128 } else {
129 const RealPacket r = phypot_complex(x).v;
130 xlogr = Base2 ? plog2(r) : plog(r);
131 }
132
133 // Imag part
134 RealPacket ximg = patan2(x.v, x_flip); // atan2(a, b), atan2(b, a)
135 if (Base2) ximg = pmul(pset1<RealPacket>(RealScalar(EIGEN_LOG2E)), ximg);
136
137 return Packet(pselect(peven_mask(xlogr), xlogr, ximg)); // log(sqrt(a^2 + b^2)), atan2(b, a)
138}
139
140// Out of line, so that the common path stays small enough to inline into the caller.
141template <typename Packet, bool Base2>
142EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE Packet plog_complex_scaled(const Packet& x) {
143 return plog_complex_impl<Packet, Base2, true>(x);
144}
145
146template <typename Packet, bool Base2>
147EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet plog_complex(const Packet& x) {
148 using RealPacket = typename unpacket_traits<Packet>::as_real;
149 using RealScalar = typename unpacket_traits<RealPacket>::type;
150 const RealPacket x_abs = pabs(x.v);
151 const RealPacket x_max = pmax(x_abs, pcplxflip(Packet(x_abs)).v);
152 const RealScalar lo = numext::sqrt((numext::numeric_limits<RealScalar>::min)() / NumTraits<RealScalar>::epsilon());
153 const RealScalar hi = numext::sqrt(NumTraits<RealScalar>::highest()) / RealScalar(2);
154 // Infinite components exceed hi. |a| and |b| are compared separately, because pmax may drop the inf of (NaN, inf).
155 // Zero, which is common, needs no scaling: log|0| = -inf either way.
156 const RealPacket is_small = pand(pcmp_lt(pzero(x_max), x_max), pcmp_lt(x_max, pset1<RealPacket>(lo)));
157 if (predux_any(por(pcmp_lt(pset1<RealPacket>(hi), x_abs), is_small))) {
158 return plog_complex_scaled<Packet, Base2>(x);
159 }
160 return plog_complex_impl<Packet, Base2, false>(x);
161}
162
163template <typename Packet>
164struct plog2_impl<Packet, true> {
165 static EIGEN_DEVICE_FUNC inline Packet run(const Packet& a) { return plog_complex<Packet, true>(a); }
166};
167
168template <typename Packet>
169EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet pexp_complex(const Packet& a) {
170 using RealPacket = typename unpacket_traits<Packet>::as_real;
171 using Scalar = typename unpacket_traits<Packet>::type;
172 using RealScalar = typename Scalar::value_type;
173 const RealPacket even_mask = peven_mask(a.v);
174 const RealPacket odd_mask = pcplxflip(Packet(even_mask)).v;
175
176 // Let a = x + iy.
177 // exp(a) = exp(x) * cis(y), plus some special edge-case handling.
178
179 // exp(x):
180 RealPacket x = pand(a.v, even_mask);
181 x = por(x, pcplxflip(Packet(x)).v);
182 RealPacket expx = pexp(x); // exp(x);
183
184 // cis(y):
185 RealPacket y = pand(odd_mask, a.v);
186 y = por(y, pcplxflip(Packet(y)).v);
187 RealPacket cisy = psincos_selector<RealPacket>(y);
188 cisy = pcplxflip(Packet(cisy)).v; // cos(y) + i * sin(y)
189
190 // The fixups below need x = +-inf or an overflowing exp(x). Such inputs are rare, so packets without
191 // |x| >= (max_exponent - 1) ln(2), just below the overflow threshold, skip them.
192 const RealScalar large_x = RealScalar(NumTraits<RealScalar>::max_exponent() - 1) * RealScalar(EIGEN_LN2);
193 RealPacket expx_factor = expx;
194 if (predux_any(pcmp_le(pset1<RealPacket>(large_x), pabs(x)))) {
195 const RealPacket cst_pos_inf = pinf<RealPacket>();
196 const RealPacket is_x_pos_inf = pcmp_eq(x, cst_pos_inf);
197 const RealPacket split = pandnot(pcmp_eq(expx, cst_pos_inf), is_x_pos_inf);
198 const RealPacket cst_neg_inf = por(psignmask<RealPacket>(), pinf<RealPacket>());
199
200 // If x is -inf, we know that cossin(y) is bounded,
201 // so the result is (0, +/-0), where the sign of the imaginary part comes
202 // from the sign of cossin(y).
203 RealPacket cisy_sign = por(pandnot(cisy, pabs(cisy)), pset1<RealPacket>(RealScalar(1)));
204 cisy = pselect(pcmp_eq(x, cst_neg_inf), cisy_sign, cisy);
205
206 // If x is inf, and cos(y) has unknown sign (y is inf or NaN), the result
207 // is (+/-inf, NaN), where the signs are undetermined (take the sign of y).
208 RealPacket y_sign = por(pandnot(y, pabs(y)), pset1<RealPacket>(RealScalar(1)));
209 cisy = pselect(pand(is_x_pos_inf, pisnan(cisy)), pand(y_sign, even_mask), cisy);
210
211 // If x is +inf and y is finite, replace cisy with copysign(1, cisy) to
212 // prevent inf * 0 = NaN. The vectorized sincos may compute exact zero
213 // for near-zero values like cos(pi/2), and inf * +-1 = +-inf is correct.
214 // The y=0 case is handled separately below.
215 RealPacket cisy_sign_one = por(pand(cisy, psignmask<RealPacket>()), pset1<RealPacket>(RealScalar(1)));
216 RealPacket x_inf_y_finite = pand(is_x_pos_inf, pcmp_lt(pabs(y), cst_pos_inf));
217 cisy = pselect(x_inf_y_finite, cisy_sign_one, cisy);
218
219 // exp(x) cis(y) can have a finite part where exp(x) overflows, up to (max_exponent - min_exponent + digits) ln(2),
220 // where exp(x) denorm_min overflows. Those elements (finite x) use x = n ln(2) + r, |r| <= ln(2)/2:
221 // exp(x) cis(y) = (exp(r) (2^digits cis(y))) 2^(n - digits),
222 // where 2^digits cis(y) is exact and normal and the result is normal or overflows, so only exp(r) and one product
223 // round. x is clamped to x_max, one ln(2) further so that its rounding stays past the overflow point, and
224 // r = (x - n ln2_hi) - n ln2_lo as in pexp_float/pexp_double, the first difference exact for x <= x_max. Other
225 // elements use x = 0, which raises no exception, and keep exp(x) cis(y), whatever shares their packet.
226 const int digits = NumTraits<RealScalar>::digits();
227 const RealScalar x_max =
228 RealScalar(NumTraits<RealScalar>::max_exponent() - NumTraits<RealScalar>::min_exponent() + digits + 1) *
229 RealScalar(EIGEN_LN2);
230 constexpr bool is_float = std::is_same<RealScalar, float>::value;
231 const RealScalar ln2_hi = is_float ? RealScalar(0.693359375f) : RealScalar(0.693145751953125);
232 const RealScalar ln2_lo = is_float ? RealScalar(-2.12194440e-4f) : RealScalar(1.42860682030941723212e-6);
233 const RealPacket x_split = pselect(split, pmin(x, pset1<RealPacket>(x_max)), pzero(x));
234 const RealPacket n = print(pmul(x_split, pset1<RealPacket>(RealScalar(EIGEN_LOG2E))));
235 const RealPacket r = pnmadd(n, pset1<RealPacket>(ln2_lo), pnmadd(n, pset1<RealPacket>(ln2_hi), x_split));
236 const RealPacket scaled_cisy = pmul(cisy, pset1<RealPacket>(RealScalar(numext::uint64_t(1) << digits)));
237 const RealPacket split_result = pldexp(pmul(pexp_reduced_impl<RealPacket>::run(r), scaled_cisy),
238 psub(n, pset1<RealPacket>(RealScalar(digits))));
239 cisy = pselect(split, split_result, cisy);
240 expx_factor = pselect(split, pset1<RealPacket>(RealScalar(1)), expx);
241 }
242
243 Packet result = Packet(pmul(expx_factor, cisy));
244
245 // If y is +/- 0, the input is real, so take the real result for consistency.
246 result = pselect(Packet(pcmp_eq(y, pzero(y))), Packet(por(pand(expx, even_mask), pand(y, odd_mask))), result);
247
248 return result;
249}
250
251// \internal psqrt_complex. Only when `scaled` does it scale the elements outside [2 * min, highest / 4] and handle
252// infinities (step 6), which psqrt_complex guarantees are absent otherwise.
253template <typename Packet>
254EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psqrt_complex_impl(const Packet& z, bool scaled) {
255 using Scalar = typename unpacket_traits<Packet>::type;
256 using RealScalar = typename Scalar::value_type;
257 using RealPacket = typename unpacket_traits<Packet>::as_real;
258
259 // Let z = x + i*y, l = |z| and M = max(|x|, |y|). For M in [2 * min, highest / 4], |x| + l cannot overflow and
260 // 0.5 * l is normal. A packet with M outside that range for some element runs steps 1 and 2 on a = s * z, where
261 // s = 1/4 for M > highest / 4, s = 4^m with m = ceil(digits / 2) for M < 2 * min, and s = 1 otherwise; then
262 // rho(z) = rho(a) / sqrt(s) exactly. Step 3 divides the unscaled y, so that a subnormal eta is rounded once.
263 RealPacket a_abs = pabs(z.v); // [|x0|, |y0|, |x1|, |y1|]
264 RealPacket a_abs_flip = pcplxflip(Packet(a_abs)).v; // [|y0|, |x0|, |y1|, |x1|]
265 RealPacket a_max = pmax(a_abs, a_abs_flip);
266 const RealPacket cst_one = pset1<RealPacket>(RealScalar(1));
267 Packet a = z;
268 RealPacket unscale = cst_one;
269 if (scaled) {
270 const RealPacket is_large = pcmp_lt(pset1<RealPacket>(NumTraits<RealScalar>::highest() / RealScalar(4)), a_max);
271 const RealPacket is_small =
272 pcmp_lt(a_max, pset1<RealPacket>(RealScalar(2) * (numext::numeric_limits<RealScalar>::min)()));
273 const int m = (NumTraits<RealScalar>::digits() + 1) / 2;
274 const RealScalar two_m = RealScalar(numext::uint64_t(1) << m);
275 const RealPacket scale = pselect(is_large, pset1<RealPacket>(RealScalar(0.25)),
276 pselect(is_small, pset1<RealPacket>(two_m * two_m), cst_one));
277 unscale = pselect(is_large, pset1<RealPacket>(RealScalar(2)),
278 pselect(is_small, pset1<RealPacket>(RealScalar(1) / two_m), cst_one));
279 a.v = pmul(z.v, scale);
280 // Recomputed from a rather than scaled: on x86 every product with a subnormal operand takes a microcode assist.
281 a_abs = pabs(a.v);
282 a_abs_flip = pcplxflip(Packet(a_abs)).v;
283 a_max = pmax(a_abs, a_abs_flip);
284 }
285
286 // Computes the principal sqrt of the complex numbers in the input.
287 //
288 // For example, for packets containing 2 complex numbers stored in interleaved format
289 // a = [a0, a1] = [x0, y0, x1, y1],
290 // where x0 = real(a0), y0 = imag(a0) etc., this function returns
291 // b = [b0, b1] = [u0, v0, u1, v1],
292 // such that b0^2 = a0, b1^2 = a1.
293 //
294 // To derive the formula for the complex square roots, let's consider the equation for
295 // a single complex square root of the number x + i*y. We want to find real numbers
296 // u and v such that
297 // (u + i*v)^2 = x + i*y <=>
298 // u^2 - v^2 + i*2*u*v = x + i*y.
299 // By equating the real and imaginary parts we get:
300 // u^2 - v^2 = x
301 // 2*u*v = y.
302 //
303 // For x >= 0, this has the numerically stable solution
304 // u = sqrt(0.5 * (x + sqrt(x^2 + y^2)))
305 // v = 0.5 * (y / u)
306 // and for x < 0,
307 // v = sign(y) * sqrt(0.5 * (-x + sqrt(x^2 + y^2)))
308 // u = 0.5 * (y / v)
309 //
310 // To avoid unnecessary over- and underflow, we compute sqrt(x^2 + y^2) as
311 // l = max(|x|, |y|) * sqrt(1 + (min(|x|, |y|) / max(|x|, |y|))^2) ,
312
313 // In the following, without loss of generality, we have annotated the code, assuming
314 // that the input is a packet of 2 complex numbers.
315 //
316 // Step 1. Compute l = [l0, l0, l1, l1], where
317 // l0 = sqrt(x0^2 + y0^2), l1 = sqrt(x1^2 + y1^2)
318 // To avoid over- and underflow, we use the stable formula for each hypotenuse
319 // l0 = (min0 == 0 ? max0 : max0 * sqrt(1 + (min0/max0)**2)),
320 // where max0 = max(|x0|, |y0|), min0 = min(|x0|, |y0|), and similarly for l1.
321
322 RealPacket a_min = pmin(a_abs, a_abs_flip);
323 RealPacket a_min_zero_mask = pcmp_eq(a_min, pzero(a_min));
324 RealPacket a_max_zero_mask = pcmp_eq(a_max, pzero(a_max));
325 RealPacket r = pdiv(a_min, a_max);
326 RealPacket l = pmul(a_max, psqrt(pmadd(r, r, cst_one))); // [l0, l0, l1, l1]
327 // Set l to a_max if a_min is zero.
328 l = pselect(a_min_zero_mask, a_max, l);
329
330 // Step 2. Compute [rho0, *, rho1, *], where
331 // rho0 = sqrt(0.5 * (l0 + |x0|)), rho1 = sqrt(0.5 * (l1 + |x1|))
332 // We don't care about the imaginary parts computed here. They will be overwritten later.
333 const RealPacket cst_half = pset1<RealPacket>(RealScalar(0.5));
334 Packet rho;
335 rho.v = psqrt(pmul(cst_half, padd(a_abs, l)));
336 if (scaled) rho.v = pmul(rho.v, unscale);
337
338 // Step 3. Compute [rho0, eta0, rho1, eta1], where
339 // eta0 = y0 / (2 * rho0), and eta1 = y1 / (2 * rho1).
340 // For z = 0, eta is y itself, so that sqrt(+-0, -0) = (+0, -0).
341 const RealPacket rho_flip = pcplxflip(rho).v;
342 RealPacket eta = pselect(a_max_zero_mask, a.v, pdiv(z.v, padd(rho_flip, rho_flip)));
343 RealPacket real_mask = peven_mask(a.v);
344 Packet positive_real_result;
345 // Compute result for inputs with positive real part.
346 positive_real_result.v = pselect(real_mask, rho.v, eta);
347
348 // Step 4. Compute solution for inputs with negative real part:
349 // [|eta0|, sign(y0)*rho0, |eta1|, sign(y1)*rho1]
350 // [+0.0, -0.0, ...]: the sign bit of the imaginary (odd) lanes only.
351 const RealPacket cst_imag_sign_mask = pandnot(psignmask<RealPacket>(), real_mask);
352 RealPacket imag_signs = pand(a.v, cst_imag_sign_mask);
353 Packet negative_real_result;
354 // Notice that rho is positive, so taking its absolute value is a noop.
355 negative_real_result.v = por(pabs(pcplxflip(positive_real_result).v), imag_signs);
356
357 // Step 5. Select solution branch based on the sign of the real parts.
358 Packet negative_real_mask;
359 negative_real_mask.v = pcmp_lt(pand(real_mask, a.v), pzero(a.v));
360 negative_real_mask.v = por(negative_real_mask.v, pcplxflip(negative_real_mask).v);
361 Packet result = pselect(negative_real_mask, negative_real_result, positive_real_result);
362 // unless otherwise specified, if either the real or imaginary component is nan, the entire result is nan
363 Packet result_is_nan = pisnan(result);
364 result = por(result_is_nan, result);
365 if (!scaled) return result;
366
367 // Step 6. Handle special cases for infinities:
368 // * If z is (x,+∞), the result is (+∞,+∞) even if x is NaN
369 // * If z is (x,-∞), the result is (+∞,-∞) even if x is NaN
370 // * If z is (-∞,y), the result is (0*|y|,+∞) for finite or NaN y
371 // * If z is (+∞,y), the result is (+∞,0*|y|) for finite or NaN y
372 const RealPacket cst_pos_inf = pinf<RealPacket>();
373 Packet is_inf;
374 is_inf.v = pcmp_eq(a_abs, cst_pos_inf);
375 Packet is_real_inf;
376 is_real_inf.v = pand(is_inf.v, real_mask);
377 is_real_inf = por(is_real_inf, pcplxflip(is_real_inf));
378 // prepare packet of (+∞,0*|y|) or (0*|y|,+∞), depending on the sign of the infinite real part.
379 Packet real_inf_result;
380 real_inf_result.v = pmul(a_abs, pset1<Packet>(Scalar(RealScalar(1.0), RealScalar(0.0))).v);
381 real_inf_result.v = pselect(negative_real_mask.v, pcplxflip(real_inf_result).v, real_inf_result.v);
382 // The imaginary part takes the sign of y: (0, -inf) for (-inf, -y) and (+inf, -0) for (+inf, -y).
383 real_inf_result.v = por(real_inf_result.v, imag_signs);
384 // prepare packet of (+∞,+∞) or (+∞,-∞), depending on the sign of the infinite imaginary part.
385 Packet is_imag_inf;
386 is_imag_inf.v = pandnot(is_inf.v, real_mask);
387 is_imag_inf = por(is_imag_inf, pcplxflip(is_imag_inf));
388 Packet imag_inf_result;
389 imag_inf_result.v = por(pand(cst_pos_inf, real_mask), pandnot(a.v, real_mask));
390 return pselect(is_imag_inf, imag_inf_result, pselect(is_real_inf, real_inf_result, result));
391}
392
393template <typename Packet>
394EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet psqrt_complex(const Packet& z) {
395 using RealScalar = typename unpacket_traits<Packet>::type::value_type;
396 using RealPacket = typename unpacket_traits<Packet>::as_real;
397 const RealPacket z_abs = pabs(z.v);
398 const RealPacket z_max = pmax(z_abs, pcplxflip(Packet(z_abs)).v);
399 const RealPacket hi = pset1<RealPacket>(NumTraits<RealScalar>::highest() / RealScalar(4));
400 const RealPacket lo = pset1<RealPacket>(RealScalar(2) * (numext::numeric_limits<RealScalar>::min)());
401 // Infinite components exceed hi. |x| and |y| are compared separately, because pmax may drop the inf of (NaN, inf).
402 // Zero, which is common, needs no scaling: step 3 handles it.
403 const RealPacket is_small = pand(pcmp_lt(pzero(z_max), z_max), pcmp_lt(z_max, lo));
404 if (predux_any(por(pcmp_lt(hi, z_abs), is_small))) {
405 Packet z_scaled = z;
406 // Keeps compilers from executing this branch speculatively on the common path.
407 pbarrier(z_scaled.v);
408 return psqrt_complex_impl(z_scaled, true);
409 }
410 return psqrt_complex_impl(z, false);
411}
412
413// \internal \returns the norm of a complex number z = x + i*y, defined as sqrt(x^2 + y^2), in both lanes of each
414// element. This is the corrected hypot of Borges (2019), "An improved algorithm for hypot(a, b)",
415// https://doi.org/10.48550/arXiv.1904.09481: h = fl(sqrt(fl(a^2 + b^2))) is refined by one Newton step with the
416// residual x = h^2 - a^2 - b^2 formed from the exact squares h^2 = h_sq + h_lo, a^2 = a_sq + a_lo and b^2 = b_sq +
417// b_lo. With a the larger component, a_sq <= h_sq <= 2 a_sq and h_sq - a_sq is exact; the remaining terms are of order
418// u h^2, so their rounding errors are of order u^2 h^2 and the result is within (1/2 + O(u)) ulp of sqrt(a^2 + b^2).
419// It is correctly rounded unless the exact value lies within O(u) ulp of a rounding midpoint: for a = b^2 with b odd,
420// sqrt(a^2 + a) = a + 1/2 - 1/(8a) + ..., and the result rounds up to a + 1. Both the order and the exact residuals
421// matter: with the components unordered, or with residuals from an unfused pmadd, the error exceeds 0.8 and 1.3 ulps.
422template <typename Packet>
423EIGEN_DEFINE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS Packet phypot_complex(const Packet& a) {
424 using Scalar = typename unpacket_traits<Packet>::type;
425 using RealScalar = typename Scalar::value_type;
426 using RealPacket = typename unpacket_traits<Packet>::as_real;
427
428 const RealPacket cst_zero_rp = pset1<RealPacket>(static_cast<RealScalar>(0.0));
429 const RealPacket cst_half_rp = pset1<RealPacket>(static_cast<RealScalar>(0.5));
430
431 RealPacket a_abs = pabs(a.v);
432 RealPacket a_flip = pcplxflip(Packet(a_abs)).v; // |b|, |a|
433 RealPacket a_all = pmax(a_abs, a_flip); // the larger component, in both lanes
434 RealPacket b_all = pmin(a_abs, a_flip); // the smaller component, in both lanes
435
436 RealPacket a2 = pmul(a.v, a.v); // |a^2, b^2|
437 RealPacket a2_flip = pcplxflip(Packet(a2)).v; // |b^2, a^2|
438 RealPacket h = psqrt(padd(a2, a2_flip)); // |sqrt(a^2 + b^2), sqrt(a^2 + b^2)|
439 RealPacket h_sq = pmul(h, h);
440 RealPacket a_sq = pmax(a2, a2_flip);
441 RealPacket b_sq = pmin(a2, a2_flip);
442 RealPacket h_lo = twoprod_low(h, h, h_sq);
443 RealPacket a_lo = twoprod_low(a_all, a_all, a_sq);
444 RealPacket b_lo = twoprod_low(b_all, b_all, b_sq);
445 RealPacket x = padd(psub(psub(h_sq, a_sq), b_sq), psub(h_lo, padd(a_lo, b_lo)));
446 h = pnmadd(pdiv(x, h), cst_half_rp, h); // |h - x/(2*h), h - x/(2*h)|
447
448 // handle zero-case
449 RealPacket iszero = pcmp_eq(por(a_abs, a_flip), cst_zero_rp);
450
451 h = pandnot(h, iszero); // |sqrt(a^2+b^2), sqrt(a^2+b^2)|
452 return Packet(h); // |sqrt(a^2+b^2), sqrt(a^2+b^2)|
453}
454
455EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_POP
456
457} // end namespace internal
458} // end namespace Eigen
459
460#endif // EIGEN_ARCH_GENERIC_PACKET_MATH_COMPLEX_H