Eigen  5.0.1
 
Loading...
Searching...
No Matches
Complex.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// copyright (c) 2023 zang ruochen <zangruochen@loongson.cn>
5// copyright (c) 2024 XiWei Gu <guxiwei-hf@loongson.cn>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-FileCopyrightText: The Eigen Authors
11// SPDX-License-Identifier: MPL-2.0
12
13#ifndef EIGEN_COMPLEX_LSX_H
14#define EIGEN_COMPLEX_LSX_H
15
16// IWYU pragma: private
17#include "../../InternalHeaderCheck.h"
18
19namespace Eigen {
20
21namespace internal {
22
23EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_PUSH
24
25//---------- float ----------
26struct Packet2cf {
27 EIGEN_STRONG_INLINE Packet2cf() {}
28 EIGEN_STRONG_INLINE explicit Packet2cf(const __m128& a) : v(a) {}
29 Packet4f v;
30};
31
32template <>
33struct packet_traits<std::complex<float> > : default_packet_traits {
34 typedef Packet2cf type;
35 typedef Packet2cf half;
36 enum {
37 Vectorizable = 1,
38 AlignedOnScalar = 1,
39 size = 2,
40
41 HasAdd = 1,
42 HasSub = 1,
43 HasMul = 1,
44 HasDiv = 1,
45 HasNegate = 1,
46 HasSqrt = 1,
47 HasExp = 1,
48 HasAbs = 0,
49 HasLog = 1,
50 HasAbs2 = 0,
51 HasMin = 0,
52 HasMax = 0,
53 HasSetLinear = 0
54 };
55};
56
57template <>
58struct unpacket_traits<Packet2cf> {
59 typedef std::complex<float> type;
60 typedef Packet2cf half;
61 typedef Packet4f as_real;
62 enum {
63 size = 2,
64 alignment = Aligned16,
65 vectorizable = true,
66 masked_load_available = false,
67 masked_store_available = false
68 };
69};
70
71template <>
72EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
73 return Packet2cf(__lsx_vfadd_s(a.v, b.v));
74}
75template <>
76EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
77 return Packet2cf(__lsx_vfsub_s(a.v, b.v));
78}
79
80template <>
81EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) {
82 const uint32_t b[4] = {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u};
83 Packet4i mask = (Packet4i)__lsx_vld(b, 0);
84 Packet2cf res;
85 res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, mask);
86 return res;
87}
88
89template <>
90EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) {
91 const uint32_t b[4] = {0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u};
92 Packet4i mask = (__m128i)__lsx_vld(b, 0);
93 Packet2cf res;
94 res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, mask);
95 return res;
96}
97
98template <>
99EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
100 Packet4f part0_tmp = (Packet4f)__lsx_vfmul_s(a.v, b.v);
101 Packet4f part0 = __lsx_vfsub_s(part0_tmp, (__m128)__lsx_vshuf4i_w(part0_tmp, 0x31));
102 Packet4f part1_tmp = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(a.v, 0xb1), b.v);
103 Packet4f part1 = __lsx_vfadd_s(part1_tmp, (__m128)__lsx_vshuf4i_w(part1_tmp, 0x31));
104 Packet2cf res;
105 res.v = (Packet4f)__lsx_vpackev_w((__m128i)part1, (__m128i)part0);
106 return res;
107}
108
109template <>
110EIGEN_STRONG_INLINE Packet2cf ptrue<Packet2cf>(const Packet2cf& a) {
111 return Packet2cf(ptrue(Packet4f(a.v)));
112}
113
114template <>
115EIGEN_STRONG_INLINE Packet2cf pand<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
116 Packet2cf res;
117 res.v = (Packet4f)__lsx_vand_v((__m128i)a.v, (__m128i)b.v);
118 return res;
119}
120
121template <>
122EIGEN_STRONG_INLINE Packet2cf por<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
123 Packet2cf res;
124 res.v = (Packet4f)__lsx_vor_v((__m128i)a.v, (__m128i)b.v);
125 return res;
126}
127
128template <>
129EIGEN_STRONG_INLINE Packet2cf pxor<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
130 Packet2cf res;
131 res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, (__m128i)b.v);
132 return res;
133}
134
135template <>
136EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
137 Packet2cf res;
138 res.v = (Packet4f)__lsx_vandn_v((__m128i)b.v, (__m128i)a.v);
139 return res;
140}
141
142template <>
143EIGEN_STRONG_INLINE Packet2cf pload<Packet2cf>(const std::complex<float>* from) {
144 EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>(&numext::real_ref(*from)));
145}
146
147template <>
148EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) {
149 EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>(&numext::real_ref(*from)));
150}
151
152template <>
153EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) {
154 float f0 = from.real(), f1 = from.imag();
155 Packet4f re = {f0, f0, f0, f0};
156 Packet4f im = {f1, f1, f1, f1};
157 return Packet2cf((Packet4f)__lsx_vilvl_w((__m128i)im, (__m128i)re));
158}
159
160template <>
161EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) {
162 return pset1<Packet2cf>(*from);
163}
164
165template <>
166EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet2cf& from) {
167 EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), Packet4f(from.v));
168}
169
170template <>
171EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet2cf& from) {
172 EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), Packet4f(from.v));
173}
174
175template <>
176EIGEN_DEVICE_FUNC inline Packet2cf pgather<std::complex<float>, Packet2cf>(const std::complex<float>* from,
177 Index stride) {
178 Packet2cf res;
179 __m128i tmp = __lsx_vldrepl_d(from, 0);
180 __m128i tmp1 = __lsx_vldrepl_d(from + stride, 0);
181 tmp = __lsx_vilvl_d(tmp1, tmp);
182 res.v = (__m128)tmp;
183 return res;
184}
185
186template <>
187EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet2cf>(std::complex<float>* to, const Packet2cf& from,
188 Index stride) {
189 __lsx_vstelm_d((__m128i)from.v, to, 0, 0);
190 __lsx_vstelm_d((__m128i)from.v, to + stride, 0, 1);
191}
192
193template <>
194EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float>* addr) {
195 __builtin_prefetch(addr);
196}
197
198template <>
199EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) {
200 EIGEN_ALIGN16 std::complex<float> res[2];
201 __lsx_vst(a.v, res, 0);
202 return res[0];
203}
204
205template <>
206EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) {
207 Packet2cf res;
208 res.v = (Packet4f)__lsx_vshuf4i_w(a.v, 0x4e);
209 return res;
210}
211
212template <>
213EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a) {
214 return pfirst(Packet2cf(__lsx_vfadd_s(a.v, vec4f_movehl(a.v, a.v))));
215}
216
217template <>
218EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet2cf>(const Packet2cf& a) {
219 return pfirst(pmul(a, Packet2cf(vec4f_movehl(a.v, a.v))));
220}
221
222EIGEN_STRONG_INLINE Packet2cf pcplxflip /* <Packet2cf> */ (const Packet2cf& x) {
223 return Packet2cf(vec4f_swizzle1(x.v, 1, 0, 3, 2));
224}
225
226EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf, Packet4f)
227
228template <>
229EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
230 return pdiv_complex(a, b);
231}
232
233template <>
234EIGEN_STRONG_INLINE Packet2cf pzero(const Packet2cf& /* a */) {
235 __m128 v = {0.0f, 0.0f, 0.0f, 0.0f};
236 return (Packet2cf)v;
237}
238
239template <>
240EIGEN_STRONG_INLINE Packet2cf pmadd<Packet2cf>(const Packet2cf& a, const Packet2cf& b, const Packet2cf& c) {
241 Packet2cf result, t0, t1, t2;
242 t1 = pzero(t1);
243 t0.v = (__m128)__lsx_vpackev_w((__m128i)a.v, (__m128i)a.v);
244 t2.v = __lsx_vfmadd_s(t0.v, b.v, c.v);
245 result.v = __lsx_vfadd_s(t2.v, t1.v);
246 t1.v = __lsx_vfsub_s(t1.v, a.v);
247 t1.v = (__m128)__lsx_vpackod_w((__m128i)a.v, (__m128i)t1.v);
248 t2.v = (__m128)__lsx_vshuf4i_w((__m128i)b.v, 0xb1);
249 result.v = __lsx_vfmadd_s(t1.v, t2.v, result.v);
250 return result;
251}
252
253//---------- double ----------
254struct Packet1cd {
255 EIGEN_STRONG_INLINE Packet1cd() {}
256 EIGEN_STRONG_INLINE explicit Packet1cd(const __m128d& a) : v(a) {}
257 Packet2d v;
258};
259
260template <>
261struct packet_traits<std::complex<double> > : default_packet_traits {
262 typedef Packet1cd type;
263 typedef Packet1cd half;
264 enum {
265 Vectorizable = 1,
266 AlignedOnScalar = 0,
267 size = 1,
268
269 HasAdd = 1,
270 HasSub = 1,
271 HasMul = 1,
272 HasDiv = 1,
273 HasNegate = 1,
274 HasSqrt = 1,
275 HasAbs = 0,
276 HasLog = 1,
277 HasAbs2 = 0,
278 HasMin = 0,
279 HasMax = 0,
280 HasSetLinear = 0
281 };
282};
283
284template <>
285struct unpacket_traits<Packet1cd> {
286 typedef std::complex<double> type;
287 typedef Packet1cd half;
288 typedef Packet2d as_real;
289 enum {
290 size = 1,
291 alignment = Aligned16,
292 vectorizable = true,
293 masked_load_available = false,
294 masked_store_available = false
295 };
296};
297
298template <>
299EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
300 return Packet1cd(__lsx_vfadd_d(a.v, b.v));
301}
302template <>
303EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
304 return Packet1cd(__lsx_vfsub_d(a.v, b.v));
305}
306template <>
307EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) {
308 return Packet1cd(pnegate(Packet2d(a.v)));
309}
310
311template <>
312EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) {
313 const uint64_t tmp[2] = {0x0000000000000000u, 0x8000000000000000u};
314 __m128i mask = __lsx_vld(tmp, 0);
315 Packet1cd res;
316 res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, mask);
317 return res;
318}
319
320template <>
321EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
322 Packet2d tmp_real = __lsx_vfmul_d(a.v, b.v);
323 Packet2d real = __lsx_vfsub_d(tmp_real, preverse(tmp_real));
324
325 Packet2d tmp_imag = __lsx_vfmul_d(preverse(a.v), b.v);
326 Packet2d imag = (__m128d)__lsx_vfadd_d((__m128d)tmp_imag, preverse(tmp_imag));
327 Packet1cd res;
328 res.v = (__m128d)__lsx_vilvl_d((__m128i)imag, (__m128i)real);
329 return res;
330}
331
332template <>
333EIGEN_STRONG_INLINE Packet1cd ptrue<Packet1cd>(const Packet1cd& a) {
334 return Packet1cd(ptrue(Packet2d(a.v)));
335}
336template <>
337EIGEN_STRONG_INLINE Packet1cd pand<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
338 Packet1cd res;
339 res.v = (Packet2d)__lsx_vand_v((__m128i)a.v, (__m128i)b.v);
340 return res;
341}
342template <>
343EIGEN_STRONG_INLINE Packet1cd por<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
344 Packet1cd res;
345 res.v = (Packet2d)__lsx_vor_v((__m128i)a.v, (__m128i)b.v);
346 return res;
347}
348template <>
349EIGEN_STRONG_INLINE Packet1cd pxor<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
350 Packet1cd res;
351 res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, (__m128i)b.v);
352 return res;
353}
354template <>
355EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
356 Packet1cd res;
357 res.v = (Packet2d)__lsx_vandn_v((__m128i)b.v, (__m128i)a.v);
358 return res;
359}
360
361template <>
362EIGEN_STRONG_INLINE Packet1cd pload<Packet1cd>(const std::complex<double>* from) {
363 EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from));
364}
365template <>
366EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) {
367 EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from));
368}
369template <>
370EIGEN_STRONG_INLINE Packet1cd
371pset1<Packet1cd>(const std::complex<double>& from) { /* here we really have to use unaligned loads :( */
372 return ploadu<Packet1cd>(&from);
373}
374
375// The generic ploaddup broadcasts a copy made by pload1_scalar, which GCC spills and reloads
376// around __lsx_vld instead of loading *from once; binding *from directly does not.
377template <>
378EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) {
379 return pset1<Packet1cd>(*from);
380}
381
382template <>
383EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, const Packet1cd& from) {
384 EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, Packet2d(from.v));
385}
386template <>
387EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double>* to, const Packet1cd& from) {
388 EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, Packet2d(from.v));
389}
390
391template <>
392EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double>* addr) {
393 __builtin_prefetch(addr);
394}
395
396template <>
397EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) {
398 EIGEN_ALIGN16 double res[2];
399 __lsx_vst(a.v, res, 0);
400 return std::complex<double>(res[0], res[1]);
401}
402
403EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd, Packet2d)
404
405template <>
406EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
407 return pdiv_complex(a, b);
408}
409
410EIGEN_STRONG_INLINE Packet1cd pcplxflip /* <Packet1cd> */ (const Packet1cd& x) {
411 return Packet1cd(preverse(Packet2d(x.v)));
412}
413
414EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2cf, 2>& kernel) {
415 Packet4f tmp1 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
416 Packet4f tmp2 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
417 kernel.packet[0].v = (Packet4f)__lsx_vshuf4i_w(tmp1, 0xd8);
418 kernel.packet[1].v = (Packet4f)__lsx_vshuf4i_w(tmp2, 0xd8);
419}
420
421template <>
422EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) {
423 Packet4f eq = (Packet4f)__lsx_vfcmp_ceq_s(a.v, b.v);
424 return Packet2cf(pand<Packet4f>(eq, vec4f_swizzle1(eq, 1, 0, 3, 2)));
425}
426
427template <>
428EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) {
429 Packet2d eq = (Packet2d)__lsx_vfcmp_ceq_d(a.v, b.v);
430 return Packet1cd(pand<Packet2d>(eq, preverse(eq)));
431}
432
433template <>
434EIGEN_DEVICE_FUNC inline Packet2cf pselect(const Packet2cf& mask, const Packet2cf& a, const Packet2cf& b) {
435 Packet2cf res;
436 res.v = (Packet4f)__lsx_vbitsel_v((__m128i)b.v, (__m128i)a.v, (__m128i)mask.v);
437 return res;
438}
439
440EIGEN_INSTANTIATE_COMPLEX_MATH_FUNCS(Packet2cf)
441EIGEN_INSTANTIATE_COMPLEX_MATH_FUNCS_NO_EXP(Packet1cd)
442
443template <>
444EIGEN_STRONG_INLINE Packet1cd pzero<Packet1cd>(const Packet1cd& /* a */) {
445 __m128d v = {0.0, 0.0};
446 return (Packet1cd)v;
447}
448
449template <>
450EIGEN_STRONG_INLINE Packet1cd pmadd<Packet1cd>(const Packet1cd& a, const Packet1cd& b, const Packet1cd& c) {
451 Packet1cd result, t0, t1, t2;
452 t1 = pzero(t1);
453 t0.v = (__m128d)__lsx_vpackev_d((__m128i)a.v, (__m128i)a.v);
454 t2.v = __lsx_vfmadd_d(t0.v, b.v, c.v);
455 result.v = __lsx_vfadd_d(t2.v, t1.v);
456 t1.v = __lsx_vfsub_d(t1.v, a.v);
457 t1.v = (__m128d)__lsx_vpackod_d((__m128i)a.v, (__m128i)t1.v);
458 t2.v = (__m128d)__lsx_vshuf4i_d((__m128i)t2.v, (__m128i)b.v, 0xb);
459 result.v = __lsx_vfmadd_d(t1.v, t2.v, result.v);
460 return result;
461}
462
463template <>
464EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from,
465 Index /* stride */) {
466 Packet1cd res;
467 __m128i tmp = __lsx_vld((void*)from, 0);
468 res.v = (__m128d)tmp;
469 return res;
470}
471
472template <>
473EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to, const Packet1cd& from,
474 Index /* stride */) {
475 __lsx_vst((__m128i)from.v, (void*)to, 0);
476}
477
478EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet1cd, 2>& kernel) {
479 Packet2d tmp = (__m128d)__lsx_vilvl_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
480 kernel.packet[1].v = (__m128d)__lsx_vilvh_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
481 kernel.packet[0].v = tmp;
482}
483
484EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_POP
485
486} // end namespace internal
487} // end namespace Eigen
488
489#endif // EIGEN_COMPLEX_LSX_H
Eigen::Aligned16
@ Aligned16
Definition Constants.h:238