Eigen  5.0.1-dev+284dcc12
 
Loading...
Searching...
No Matches
Complex.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// copyright (c) 2023 zang ruochen <zangruochen@loongson.cn>
5// copyright (c) 2024 XiWei Gu <guxiwei-hf@loongson.cn>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_COMPLEX_LSX_H
12#define EIGEN_COMPLEX_LSX_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21//---------- float ----------
22struct Packet2cf {
23 EIGEN_STRONG_INLINE Packet2cf() {}
24 EIGEN_STRONG_INLINE explicit Packet2cf(const __m128& a) : v(a) {}
25 Packet4f v;
26};
27
28template <>
29struct packet_traits<std::complex<float> > : default_packet_traits {
30 typedef Packet2cf type;
31 typedef Packet2cf half;
32 enum {
33 Vectorizable = 1,
34 AlignedOnScalar = 1,
35 size = 2,
36
37 HasAdd = 1,
38 HasSub = 1,
39 HasMul = 1,
40 HasDiv = 1,
41 HasNegate = 1,
42 HasSqrt = 1,
43 HasExp = 1,
44 HasAbs = 0,
45 HasLog = 1,
46 HasAbs2 = 0,
47 HasMin = 0,
48 HasMax = 0,
49 HasSetLinear = 0
50 };
51};
52
53template <>
54struct unpacket_traits<Packet2cf> {
55 typedef std::complex<float> type;
56 typedef Packet2cf half;
57 typedef Packet4f as_real;
58 enum {
59 size = 2,
60 alignment = Aligned16,
61 vectorizable = true,
62 masked_load_available = false,
63 masked_store_available = false
64 };
65};
66
67template <>
68EIGEN_STRONG_INLINE Packet2cf padd<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
69 return Packet2cf(__lsx_vfadd_s(a.v, b.v));
70}
71template <>
72EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
73 return Packet2cf(__lsx_vfsub_s(a.v, b.v));
74}
75
76template <>
77EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) {
78 const uint32_t b[4] = {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u};
79 Packet4i mask = (Packet4i)__lsx_vld(b, 0);
80 Packet2cf res;
81 res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, mask);
82 return res;
83}
84
85template <>
86EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) {
87 const uint32_t b[4] = {0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u};
88 Packet4i mask = (__m128i)__lsx_vld(b, 0);
89 Packet2cf res;
90 res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, mask);
91 return res;
92}
93
94template <>
95EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
96 Packet4f part0_tmp = (Packet4f)__lsx_vfmul_s(a.v, b.v);
97 Packet4f part0 = __lsx_vfsub_s(part0_tmp, (__m128)__lsx_vshuf4i_w(part0_tmp, 0x31));
98 Packet4f part1_tmp = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(a.v, 0xb1), b.v);
99 Packet4f part1 = __lsx_vfadd_s(part1_tmp, (__m128)__lsx_vshuf4i_w(part1_tmp, 0x31));
100 Packet2cf res;
101 res.v = (Packet4f)__lsx_vpackev_w((__m128i)part1, (__m128i)part0);
102 return res;
103}
104
105template <>
106EIGEN_STRONG_INLINE Packet2cf ptrue<Packet2cf>(const Packet2cf& a) {
107 return Packet2cf(ptrue(Packet4f(a.v)));
108}
109
110template <>
111EIGEN_STRONG_INLINE Packet2cf pand<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
112 Packet2cf res;
113 res.v = (Packet4f)__lsx_vand_v((__m128i)a.v, (__m128i)b.v);
114 return res;
115}
116
117template <>
118EIGEN_STRONG_INLINE Packet2cf por<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
119 Packet2cf res;
120 res.v = (Packet4f)__lsx_vor_v((__m128i)a.v, (__m128i)b.v);
121 return res;
122}
123
124template <>
125EIGEN_STRONG_INLINE Packet2cf pxor<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
126 Packet2cf res;
127 res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, (__m128i)b.v);
128 return res;
129}
130
131template <>
132EIGEN_STRONG_INLINE Packet2cf pandnot<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
133 Packet2cf res;
134 res.v = (Packet4f)__lsx_vandn_v((__m128i)b.v, (__m128i)a.v);
135 return res;
136}
137
138template <>
139EIGEN_STRONG_INLINE Packet2cf pload<Packet2cf>(const std::complex<float>* from) {
140 EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload<Packet4f>(&numext::real_ref(*from)));
141}
142
143template <>
144EIGEN_STRONG_INLINE Packet2cf ploadu<Packet2cf>(const std::complex<float>* from) {
145 EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu<Packet4f>(&numext::real_ref(*from)));
146}
147
148template <>
149EIGEN_STRONG_INLINE Packet2cf pset1<Packet2cf>(const std::complex<float>& from) {
150 float f0 = from.real(), f1 = from.imag();
151 Packet4f re = {f0, f0, f0, f0};
152 Packet4f im = {f1, f1, f1, f1};
153 return Packet2cf((Packet4f)__lsx_vilvl_w((__m128i)im, (__m128i)re));
154}
155
156template <>
157EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<float>* from) {
158 return pset1<Packet2cf>(*from);
159}
160
161template <>
162EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet2cf& from) {
163 EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), Packet4f(from.v));
164}
165
166template <>
167EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet2cf& from) {
168 EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), Packet4f(from.v));
169}
170
171template <>
172EIGEN_DEVICE_FUNC inline Packet2cf pgather<std::complex<float>, Packet2cf>(const std::complex<float>* from,
173 Index stride) {
174 Packet2cf res;
175 __m128i tmp = __lsx_vldrepl_d(from, 0);
176 __m128i tmp1 = __lsx_vldrepl_d(from + stride, 0);
177 tmp = __lsx_vilvl_d(tmp1, tmp);
178 res.v = (__m128)tmp;
179 return res;
180}
181
182template <>
183EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet2cf>(std::complex<float>* to, const Packet2cf& from,
184 Index stride) {
185 __lsx_vstelm_d((__m128i)from.v, to, 0, 0);
186 __lsx_vstelm_d((__m128i)from.v, to + stride, 0, 1);
187}
188
189template <>
190EIGEN_STRONG_INLINE void prefetch<std::complex<float> >(const std::complex<float>* addr) {
191 __builtin_prefetch(addr);
192}
193
194template <>
195EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2cf>(const Packet2cf& a) {
196 EIGEN_ALIGN16 std::complex<float> res[2];
197 __lsx_vst(a.v, res, 0);
198 return res[0];
199}
200
201template <>
202EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) {
203 Packet2cf res;
204 res.v = (Packet4f)__lsx_vshuf4i_w(a.v, 0x4e);
205 return res;
206}
207
208template <>
209EIGEN_STRONG_INLINE std::complex<float> predux<Packet2cf>(const Packet2cf& a) {
210 return pfirst(Packet2cf(__lsx_vfadd_s(a.v, vec4f_movehl(a.v, a.v))));
211}
212
213template <>
214EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet2cf>(const Packet2cf& a) {
215 return pfirst(pmul(a, Packet2cf(vec4f_movehl(a.v, a.v))));
216}
217
218EIGEN_STRONG_INLINE Packet2cf pcplxflip /* <Packet2cf> */ (const Packet2cf& x) {
219 return Packet2cf(vec4f_swizzle1(x.v, 1, 0, 3, 2));
220}
221
222EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf, Packet4f)
223
224template <>
225EIGEN_STRONG_INLINE Packet2cf pdiv<Packet2cf>(const Packet2cf& a, const Packet2cf& b) {
226 return pdiv_complex(a, b);
227}
228
229template <>
230EIGEN_STRONG_INLINE Packet2cf plog<Packet2cf>(const Packet2cf& a) {
231 return plog_complex(a);
232}
233
234template <>
235EIGEN_STRONG_INLINE Packet2cf pzero(const Packet2cf& /* a */) {
236 __m128 v = {0.0f, 0.0f, 0.0f, 0.0f};
237 return (Packet2cf)v;
238}
239
240template <>
241EIGEN_STRONG_INLINE Packet2cf pmadd<Packet2cf>(const Packet2cf& a, const Packet2cf& b, const Packet2cf& c) {
242 Packet2cf result, t0, t1, t2;
243 t1 = pzero(t1);
244 t0.v = (__m128)__lsx_vpackev_w((__m128i)a.v, (__m128i)a.v);
245 t2.v = __lsx_vfmadd_s(t0.v, b.v, c.v);
246 result.v = __lsx_vfadd_s(t2.v, t1.v);
247 t1.v = __lsx_vfsub_s(t1.v, a.v);
248 t1.v = (__m128)__lsx_vpackod_w((__m128i)a.v, (__m128i)t1.v);
249 t2.v = (__m128)__lsx_vshuf4i_w((__m128i)b.v, 0xb1);
250 result.v = __lsx_vfmadd_s(t1.v, t2.v, result.v);
251 return result;
252}
253
254template <>
255EIGEN_STRONG_INLINE Packet2cf pexp<Packet2cf>(const Packet2cf& a) {
256 return pexp_complex(a);
257}
258
259//---------- double ----------
260struct Packet1cd {
261 EIGEN_STRONG_INLINE Packet1cd() {}
262 EIGEN_STRONG_INLINE explicit Packet1cd(const __m128d& a) : v(a) {}
263 Packet2d v;
264};
265
266template <>
267struct packet_traits<std::complex<double> > : default_packet_traits {
268 typedef Packet1cd type;
269 typedef Packet1cd half;
270 enum {
271 Vectorizable = 1,
272 AlignedOnScalar = 0,
273 size = 1,
274
275 HasAdd = 1,
276 HasSub = 1,
277 HasMul = 1,
278 HasDiv = 1,
279 HasNegate = 1,
280 HasSqrt = 1,
281 HasAbs = 0,
282 HasLog = 1,
283 HasAbs2 = 0,
284 HasMin = 0,
285 HasMax = 0,
286 HasSetLinear = 0
287 };
288};
289
290template <>
291struct unpacket_traits<Packet1cd> {
292 typedef std::complex<double> type;
293 typedef Packet1cd half;
294 typedef Packet2d as_real;
295 enum {
296 size = 1,
297 alignment = Aligned16,
298 vectorizable = true,
299 masked_load_available = false,
300 masked_store_available = false
301 };
302};
303
304template <>
305EIGEN_STRONG_INLINE Packet1cd padd<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
306 return Packet1cd(__lsx_vfadd_d(a.v, b.v));
307}
308template <>
309EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
310 return Packet1cd(__lsx_vfsub_d(a.v, b.v));
311}
312template <>
313EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) {
314 return Packet1cd(pnegate(Packet2d(a.v)));
315}
316
317template <>
318EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) {
319 const uint64_t tmp[2] = {0x0000000000000000u, 0x8000000000000000u};
320 __m128i mask = __lsx_vld(tmp, 0);
321 Packet1cd res;
322 res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, mask);
323 return res;
324}
325
326template <>
327EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
328 Packet2d tmp_real = __lsx_vfmul_d(a.v, b.v);
329 Packet2d real = __lsx_vfsub_d(tmp_real, preverse(tmp_real));
330
331 Packet2d tmp_imag = __lsx_vfmul_d(preverse(a.v), b.v);
332 Packet2d imag = (__m128d)__lsx_vfadd_d((__m128d)tmp_imag, preverse(tmp_imag));
333 Packet1cd res;
334 res.v = (__m128d)__lsx_vilvl_d((__m128i)imag, (__m128i)real);
335 return res;
336}
337
338template <>
339EIGEN_STRONG_INLINE Packet1cd ptrue<Packet1cd>(const Packet1cd& a) {
340 return Packet1cd(ptrue(Packet2d(a.v)));
341}
342template <>
343EIGEN_STRONG_INLINE Packet1cd pand<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
344 Packet1cd res;
345 res.v = (Packet2d)__lsx_vand_v((__m128i)a.v, (__m128i)b.v);
346 return res;
347}
348template <>
349EIGEN_STRONG_INLINE Packet1cd por<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
350 Packet1cd res;
351 res.v = (Packet2d)__lsx_vor_v((__m128i)a.v, (__m128i)b.v);
352 return res;
353}
354template <>
355EIGEN_STRONG_INLINE Packet1cd pxor<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
356 Packet1cd res;
357 res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, (__m128i)b.v);
358 return res;
359}
360template <>
361EIGEN_STRONG_INLINE Packet1cd pandnot<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
362 Packet1cd res;
363 res.v = (Packet2d)__lsx_vandn_v((__m128i)b.v, (__m128i)a.v);
364 return res;
365}
366
367// FIXME force unaligned load, this is a temporary fix
368template <>
369EIGEN_STRONG_INLINE Packet1cd pload<Packet1cd>(const std::complex<double>* from) {
370 EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload<Packet2d>((const double*)from));
371}
372template <>
373EIGEN_STRONG_INLINE Packet1cd ploadu<Packet1cd>(const std::complex<double>* from) {
374 EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu<Packet2d>((const double*)from));
375}
376template <>
377EIGEN_STRONG_INLINE Packet1cd
378pset1<Packet1cd>(const std::complex<double>& from) { /* here we really have to use unaligned loads :( */
379 return ploadu<Packet1cd>(&from);
380}
381
382template <>
383EIGEN_STRONG_INLINE Packet1cd ploaddup<Packet1cd>(const std::complex<double>* from) {
384 return pset1<Packet1cd>(*from);
385}
386
387// FIXME force unaligned store, this is a temporary fix
388template <>
389EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, const Packet1cd& from) {
390 EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, Packet2d(from.v));
391}
392template <>
393EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double>* to, const Packet1cd& from) {
394 EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, Packet2d(from.v));
395}
396
397template <>
398EIGEN_STRONG_INLINE void prefetch<std::complex<double> >(const std::complex<double>* addr) {
399 __builtin_prefetch(addr);
400}
401
402template <>
403EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet1cd>(const Packet1cd& a) {
404 EIGEN_ALIGN16 double res[2];
405 __lsx_vst(a.v, res, 0);
406 return std::complex<double>(res[0], res[1]);
407}
408
409template <>
410EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) {
411 return a;
412}
413
414template <>
415EIGEN_STRONG_INLINE std::complex<double> predux<Packet1cd>(const Packet1cd& a) {
416 return pfirst(a);
417}
418
419template <>
420EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet1cd>(const Packet1cd& a) {
421 return pfirst(a);
422}
423
424EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd, Packet2d)
425
426template <>
427EIGEN_STRONG_INLINE Packet1cd pdiv<Packet1cd>(const Packet1cd& a, const Packet1cd& b) {
428 return pdiv_complex(a, b);
429}
430
431EIGEN_STRONG_INLINE Packet1cd pcplxflip /* <Packet1cd> */ (const Packet1cd& x) {
432 return Packet1cd(preverse(Packet2d(x.v)));
433}
434
435EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet2cf, 2>& kernel) {
436 Packet4f tmp1 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
437 Packet4f tmp2 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
438 kernel.packet[0].v = (Packet4f)__lsx_vshuf4i_w(tmp1, 0xd8);
439 kernel.packet[1].v = (Packet4f)__lsx_vshuf4i_w(tmp2, 0xd8);
440}
441
442template <>
443EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) {
444 Packet4f eq = (Packet4f)__lsx_vfcmp_ceq_s(a.v, b.v);
445 return Packet2cf(pand<Packet4f>(eq, vec4f_swizzle1(eq, 1, 0, 3, 2)));
446}
447
448template <>
449EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) {
450 Packet2d eq = (Packet2d)__lsx_vfcmp_ceq_d(a.v, b.v);
451 return Packet1cd(pand<Packet2d>(eq, preverse(eq)));
452}
453
454template <>
455EIGEN_DEVICE_FUNC inline Packet2cf pselect(const Packet2cf& mask, const Packet2cf& a, const Packet2cf& b) {
456 Packet2cf res;
457 res.v = (Packet4f)__lsx_vbitsel_v((__m128i)b.v, (__m128i)a.v, (__m128i)mask.v);
458 return res;
459}
460
461template <>
462EIGEN_STRONG_INLINE Packet1cd psqrt<Packet1cd>(const Packet1cd& a) {
463 return psqrt_complex<Packet1cd>(a);
464}
465
466template <>
467EIGEN_STRONG_INLINE Packet2cf psqrt<Packet2cf>(const Packet2cf& a) {
468 return psqrt_complex<Packet2cf>(a);
469}
470
471template <>
472EIGEN_STRONG_INLINE Packet1cd plog<Packet1cd>(const Packet1cd& a) {
473 return plog_complex(a);
474}
475
476template <>
477EIGEN_STRONG_INLINE Packet1cd pzero<Packet1cd>(const Packet1cd& /* a */) {
478 __m128d v = {0.0, 0.0};
479 return (Packet1cd)v;
480}
481
482template <>
483EIGEN_STRONG_INLINE Packet1cd pmadd<Packet1cd>(const Packet1cd& a, const Packet1cd& b, const Packet1cd& c) {
484 Packet1cd result, t0, t1, t2;
485 t1 = pzero(t1);
486 t0.v = (__m128d)__lsx_vpackev_d((__m128i)a.v, (__m128i)a.v);
487 t2.v = __lsx_vfmadd_d(t0.v, b.v, c.v);
488 result.v = __lsx_vfadd_d(t2.v, t1.v);
489 t1.v = __lsx_vfsub_d(t1.v, a.v);
490 t1.v = (__m128d)__lsx_vpackod_d((__m128i)a.v, (__m128i)t1.v);
491 t2.v = (__m128d)__lsx_vshuf4i_d((__m128i)t2.v, (__m128i)b.v, 0xb);
492 result.v = __lsx_vfmadd_d(t1.v, t2.v, result.v);
493 return result;
494}
495
496template <>
497EIGEN_DEVICE_FUNC inline Packet1cd pgather<std::complex<double>, Packet1cd>(const std::complex<double>* from,
498 Index /* stride */) {
499 Packet1cd res;
500 __m128i tmp = __lsx_vld((void*)from, 0);
501 res.v = (__m128d)tmp;
502 return res;
503}
504
505template <>
506EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet1cd>(std::complex<double>* to, const Packet1cd& from,
507 Index /* stride */) {
508 __lsx_vst((__m128i)from.v, (void*)to, 0);
509}
510
511EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet1cd, 2>& kernel) {
512 Packet2d tmp = (__m128d)__lsx_vilvl_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
513 kernel.packet[1].v = (__m128d)__lsx_vilvh_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v);
514 kernel.packet[0].v = tmp;
515}
516
517} // end namespace internal
518} // end namespace Eigen
519
520#endif // EIGEN_COMPLEX_LSX_H
Eigen::Aligned16
@ Aligned16
Definition Constants.h:237
Eigen
Namespace containing all symbols from the Eigen library.
Definition B01_Experimental.dox:1
Eigen::real
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_real_op< typename Derived::Scalar >, const Derived > real(const Eigen::ArrayBase< Derived > &x)
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:82
Eigen::imag
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_imag_op< typename Derived::Scalar >, const Derived > imag(const Eigen::ArrayBase< Derived > &x)