Eigen  5.0.1
 
Loading...
Searching...
No Matches
Complex.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2025 Kseniya Zaytseva <kseniya.zaytseva@syntacore.com>
5// Copyright (C) 2026 Chip Kerchner <ckerchner@tenstorrent.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_COMPLEX_RVV10_H
13#define EIGEN_COMPLEX_RVV10_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_PUSH
23
24/********************************* float32 ************************************/
25
26#if 0
27#if EIGEN_RISCV64_DEFAULT_LMUL == 4
28#define USE_LMUL4_ONLY
29#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
30#define USE_LMUL2_ONLY
31#else
32#define USE_LMUL1_ONLY
33#endif
34#endif
35
36template <typename RealPacketT, int N>
37struct complex_packet_wrapper {
38 complex_packet_wrapper() = default;
39 EIGEN_STRONG_INLINE explicit complex_packet_wrapper(const RealPacketT& a) : v(a) {}
40
41 RealPacketT v;
42};
43
44typedef complex_packet_wrapper<Packet2Xf, 29> Packet2Xcf;
45typedef complex_packet_wrapper<Packet4Xf, 30> Packet4Xcf;
46typedef complex_packet_wrapper<Packet1Xf, 31> Packet1Xcf;
47
48#if EIGEN_RISCV64_DEFAULT_LMUL == 1
49typedef Packet1Xcf PacketXcf;
50
51template <>
52struct packet_traits<std::complex<float>> : default_packet_traits {
53 typedef Packet1Xcf type;
54 typedef Packet1Xcf half;
55 enum {
56 Vectorizable = 1,
57 AlignedOnScalar = 0,
58 size = rvv_packet_size_selector<std::complex<float>, EIGEN_RISCV64_RVV_VL, 1>::size,
59
60 HasAdd = 1,
61 HasSub = 1,
62 HasMul = 1,
63 HasDiv = 1,
64 HasNegate = 1,
65 HasSqrt = 1,
66 HasLog = 1,
67 HasExp = 1,
68 HasSign = 0,
69 HasAbs = 0,
70 HasAbs2 = 0,
71 HasMin = 0,
72 HasMax = 0,
73 HasConj = 1,
74 HasArg = 0,
75 HasSetLinear = 0
76 };
77};
78#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
79typedef Packet2Xcf PacketXcf;
80
81template <>
82struct packet_traits<std::complex<float>> : default_packet_traits {
83 typedef Packet2Xcf type;
84#ifndef USE_LMUL2_ONLY
85 typedef Packet1Xcf half;
86#else
87 typedef Packet2Xcf half;
88#endif
89 enum {
90 Vectorizable = 1,
91 AlignedOnScalar = 0,
92 size = rvv_packet_size_selector<std::complex<float>, EIGEN_RISCV64_RVV_VL, 2>::size,
93
94 HasAdd = 1,
95 HasSub = 1,
96 HasMul = 1,
97 HasDiv = 1,
98 HasNegate = 1,
99 HasSqrt = 1,
100 HasLog = 1,
101 HasExp = 1,
102 HasSign = 0,
103 HasAbs = 0,
104 HasAbs2 = 0,
105 HasMin = 0,
106 HasMax = 0,
107 HasConj = 1,
108 HasArg = 0,
109 HasSetLinear = 0
110 };
111};
112#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
113typedef Packet4Xcf PacketXcf;
114
115template <>
116struct packet_traits<std::complex<float>> : default_packet_traits {
117 typedef Packet4Xcf type;
118#ifndef USE_LMUL4_ONLY
119 typedef Packet2Xcf half;
120#else
121 typedef Packet4Xcf half;
122#endif
123 enum {
124 Vectorizable = 1,
125 AlignedOnScalar = 0,
126 size = rvv_packet_size_selector<std::complex<float>, EIGEN_RISCV64_RVV_VL, 4>::size,
127
128 HasAdd = 1,
129 HasSub = 1,
130 HasMul = 1,
131 HasDiv = 1,
132 HasNegate = 1,
133 HasSqrt = 1,
134 HasLog = 1,
135 HasExp = 1,
136 HasSign = 0,
137 HasAbs = 0,
138 HasAbs2 = 0,
139 HasMin = 0,
140 HasMax = 0,
141 HasConj = 1,
142 HasArg = 0,
143 HasSetLinear = 0
144 };
145};
146#endif
147
148template <>
149struct unpacket_traits<Packet2Xcf> : default_unpacket_traits {
150 typedef std::complex<float> type;
151#ifndef USE_LMUL2_ONLY
152 typedef Packet1Xcf half;
153#else
154 typedef Packet2Xcf half;
155#endif
156 typedef Packet2Xf as_real;
157 enum {
158 size = rvv_packet_size_selector<std::complex<float>, EIGEN_RISCV64_RVV_VL, 2>::size,
159 alignment = rvv_packet_alignment_selector<EIGEN_RISCV64_RVV_VL, 2>::alignment,
160 vectorizable = true,
161 masked_load_available = false,
162 masked_store_available = false
163 };
164};
165
166template <>
167struct unpacket_traits<Packet4Xcf> : default_unpacket_traits {
168 typedef std::complex<float> type;
169#ifndef USE_LMUL4_ONLY
170 typedef Packet2Xcf half;
171#else
172 typedef Packet4Xcf half;
173#endif
174 typedef Packet4Xf as_real;
175 enum {
176 size = rvv_packet_size_selector<std::complex<float>, EIGEN_RISCV64_RVV_VL, 4>::size,
177 alignment = rvv_packet_alignment_selector<EIGEN_RISCV64_RVV_VL, 4>::alignment,
178 vectorizable = true,
179 masked_load_available = false,
180 masked_store_available = false
181 };
182};
183
184template <>
185struct unpacket_traits<Packet1Xcf> : default_unpacket_traits {
186 typedef std::complex<float> type;
187 typedef Packet1Xcf half;
188 typedef Packet1Xf as_real;
189 enum {
190 size = rvv_packet_size_selector<std::complex<float>, EIGEN_RISCV64_RVV_VL, 1>::size,
191 alignment = rvv_packet_alignment_selector<EIGEN_RISCV64_RVV_VL, 1>::alignment,
192 vectorizable = true,
193 masked_load_available = false,
194 masked_store_available = false
195 };
196};
197
198template <>
199EIGEN_STRONG_INLINE Packet2Xcf pcast<Packet2Xf, Packet2Xcf>(const Packet2Xf& a) {
200 return Packet2Xcf(a);
201}
202
203template <>
204EIGEN_STRONG_INLINE Packet2Xf pcast<Packet2Xcf, Packet2Xf>(const Packet2Xcf& a) {
205 return a.v;
206}
207
208EIGEN_STRONG_INLINE Packet2Xul __riscv_vreinterpret_v_f32m2_u64m2(const Packet2Xf& a) {
209 return __riscv_vreinterpret_v_u32m2_u64m2(__riscv_vreinterpret_v_f32m2_u32m2(a));
210}
211
212EIGEN_STRONG_INLINE Packet2Xl __riscv_vreinterpret_v_f32m2_i64m2(const Packet2Xf& a) {
213 return __riscv_vreinterpret_v_u64m2_i64m2(__riscv_vreinterpret_v_u32m2_u64m2(__riscv_vreinterpret_v_f32m2_u32m2(a)));
214}
215
216EIGEN_STRONG_INLINE Packet2Xf __riscv_vreinterpret_v_i64m2_f32m2(const Packet2Xl& a) {
217 return __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vreinterpret_v_u64m2_u32m2(__riscv_vreinterpret_v_i64m2_u64m2(a)));
218}
219
220EIGEN_STRONG_INLINE void prealimag2(const Packet2Xcf& a, Packet2Xf& real, Packet2Xf& imag) {
221 const PacketMask16 mask =
222 __riscv_vreinterpret_v_i8m1_b16(__riscv_vmv_v_x_i8m1(static_cast<char>(0xaa), unpacket_traits<Packet1Xc>::size));
223 Packet2Xu res = __riscv_vreinterpret_v_f32m2_u32m2(a.v);
224 real = __riscv_vreinterpret_v_u32m2_f32m2(
225 __riscv_vslide1up_vx_u32m2_tumu(mask, res, res, 0, unpacket_traits<Packet2Xi>::size));
226 imag = __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vslide1down_vx_u32m2_tumu(
227 __riscv_vmnot_m_b16(mask, unpacket_traits<Packet1Xs>::size), res, res, 0, unpacket_traits<Packet2Xi>::size));
228}
229
230template <>
231EIGEN_STRONG_INLINE Packet2Xcf pset1<Packet2Xcf>(const std::complex<float>& from) {
232 const numext::int64_t from2 = *reinterpret_cast<const numext::int64_t*>(reinterpret_cast<const void*>(&from));
233 Packet2Xf res = __riscv_vreinterpret_v_i64m2_f32m2(pset1<Packet2Xl>(from2));
234 return Packet2Xcf(res);
235}
236
237template <>
238EIGEN_STRONG_INLINE Packet2Xcf padd<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
239 return Packet2Xcf(padd<Packet2Xf>(a.v, b.v));
240}
241
242template <>
243EIGEN_STRONG_INLINE Packet2Xcf psub<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
244 return Packet2Xcf(psub<Packet2Xf>(a.v, b.v));
245}
246
247template <>
248EIGEN_STRONG_INLINE Packet2Xcf pnegate(const Packet2Xcf& a) {
249 return Packet2Xcf(pnegate<Packet2Xf>(a.v));
250}
251
252template <>
253EIGEN_STRONG_INLINE Packet2Xcf pconj(const Packet2Xcf& a) {
254 return Packet2Xcf(__riscv_vreinterpret_v_u64m2_f32m2(__riscv_vxor_vx_u64m2(
255 __riscv_vreinterpret_v_f32m2_u64m2(a.v), 0x8000000000000000ull, unpacket_traits<Packet2Xl>::size)));
256}
257
258template <>
259EIGEN_STRONG_INLINE Packet2Xcf pcplxflip<Packet2Xcf>(const Packet2Xcf& a) {
260#ifndef __riscv_zvbb
261 Packet2Xu res = __riscv_vreinterpret_v_f32m2_u32m2(a.v);
262 const PacketMask16 mask =
263 __riscv_vreinterpret_v_i8m1_b16(__riscv_vmv_v_x_i8m1(static_cast<char>(0xaa), unpacket_traits<Packet1Xc>::size));
264 Packet2Xu data = __riscv_vslide1down_vx_u32m2(res, 0, unpacket_traits<Packet2Xi>::size);
265 Packet2Xf res2 = __riscv_vreinterpret_v_u32m2_f32m2(
266 __riscv_vslide1up_vx_u32m2_tumu(mask, data, res, 0, unpacket_traits<Packet2Xf>::size));
267 return Packet2Xcf(res2);
268#else
269 Packet2Xf res = __riscv_vreinterpret_v_u64m2_f32m2(
270 __riscv_vror_vx_u64m2(__riscv_vreinterpret_v_f32m2_u64m2(a.v), 32, unpacket_traits<Packet2Xl>::size));
271 return Packet2Xcf(res);
272#endif
273}
274
275template <>
276EIGEN_STRONG_INLINE Packet2Xcf pmul<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
277 Packet2Xf real, imag;
278 prealimag2(a, real, imag);
279 return Packet2Xcf(pmadd<Packet2Xf>(imag, pcplxflip<Packet2Xcf>(pconj<Packet2Xcf>(b)).v, pmul<Packet2Xf>(real, b.v)));
280}
281
282template <>
283EIGEN_STRONG_INLINE Packet2Xcf pmadd<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b, const Packet2Xcf& c) {
284 Packet2Xf real, imag;
285 prealimag2(a, real, imag);
286 return Packet2Xcf(
287 pmadd<Packet2Xf>(imag, pcplxflip<Packet2Xcf>(pconj<Packet2Xcf>(b)).v, pmadd<Packet2Xf>(real, b.v, c.v)));
288}
289
290template <>
291EIGEN_STRONG_INLINE Packet2Xcf pmsub<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b, const Packet2Xcf& c) {
292 Packet2Xf real, imag;
293 prealimag2(a, real, imag);
294 return Packet2Xcf(
295 pmadd<Packet2Xf>(imag, pcplxflip<Packet2Xcf>(pconj<Packet2Xcf>(b)).v, pmsub<Packet2Xf>(real, b.v, c.v)));
296}
297
298template <>
299EIGEN_STRONG_INLINE Packet2Xcf pcmp_eq(const Packet2Xcf& a, const Packet2Xcf& b) {
300 Packet2Xi c = __riscv_vundefined_i32m2();
301 PacketMask16 mask = __riscv_vmfeq_vv_f32m2_b16(a.v, b.v, unpacket_traits<Packet2Xf>::size);
302 Packet2Xl res = __riscv_vreinterpret_v_i32m2_i64m2(
303 __riscv_vmerge_vvm_i32m2(pzero<Packet2Xi>(c), ptrue<Packet2Xi>(c), mask, unpacket_traits<Packet2Xi>::size));
304 Packet2Xf res2 = __riscv_vreinterpret_v_i64m2_f32m2(
305 __riscv_vsra_vx_i64m2(__riscv_vand_vv_i64m2(__riscv_vsll_vx_i64m2(res, 32, unpacket_traits<Packet2Xl>::size), res,
306 unpacket_traits<Packet2Xl>::size),
307 32, unpacket_traits<Packet2Xl>::size));
308 return Packet2Xcf(res2);
309}
310
311template <>
312EIGEN_STRONG_INLINE Packet2Xcf pand<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
313 return Packet2Xcf(pand<Packet2Xf>(a.v, b.v));
314}
315
316template <>
317EIGEN_STRONG_INLINE Packet2Xcf por<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
318 return Packet2Xcf(por<Packet2Xf>(a.v, b.v));
319}
320
321template <>
322EIGEN_STRONG_INLINE Packet2Xcf pxor<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
323 return Packet2Xcf(pxor<Packet2Xf>(a.v, b.v));
324}
325
326template <>
327EIGEN_STRONG_INLINE Packet2Xcf pandnot<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
328 return Packet2Xcf(pandnot<Packet2Xf>(a.v, b.v));
329}
330
331template <>
332EIGEN_STRONG_INLINE Packet2Xcf pnot<Packet2Xcf>(const Packet2Xcf& a) {
333 return Packet2Xcf(pnot<Packet2Xf>(a.v));
334}
335
336template <>
337EIGEN_STRONG_INLINE Packet2Xcf pload<Packet2Xcf>(const std::complex<float>* from) {
338 Packet2Xf res = pload<Packet2Xf>(reinterpret_cast<const float*>(from));
339 EIGEN_DEBUG_ALIGNED_LOAD return Packet2Xcf(res);
340}
341
342template <>
343EIGEN_STRONG_INLINE Packet2Xcf ploadu<Packet2Xcf>(const std::complex<float>* from) {
344 Packet2Xf res = ploadu<Packet2Xf>(reinterpret_cast<const float*>(from));
345 EIGEN_DEBUG_UNALIGNED_LOAD return Packet2Xcf(res);
346}
347
348template <>
349EIGEN_STRONG_INLINE Packet2Xcf ploaddup<Packet2Xcf>(const std::complex<float>* from) {
350 Packet2Xl res = ploaddup<Packet2Xl>(reinterpret_cast<const numext::int64_t*>(reinterpret_cast<const void*>(from)));
351 return Packet2Xcf(__riscv_vreinterpret_v_i64m2_f32m2(res));
352}
353
354template <>
355EIGEN_STRONG_INLINE Packet2Xcf ploadquad<Packet2Xcf>(const std::complex<float>* from) {
356 Packet2Xl res = ploadquad<Packet2Xl>(reinterpret_cast<const numext::int64_t*>(reinterpret_cast<const void*>(from)));
357 return Packet2Xcf(__riscv_vreinterpret_v_i64m2_f32m2(res));
358}
359
360template <>
361EIGEN_STRONG_INLINE void pstore<std::complex<float>>(std::complex<float>* to, const Packet2Xcf& from) {
362 EIGEN_DEBUG_ALIGNED_STORE pstore<float>(reinterpret_cast<float*>(to), from.v);
363}
364
365template <>
366EIGEN_STRONG_INLINE void pstoreu<std::complex<float>>(std::complex<float>* to, const Packet2Xcf& from) {
367 EIGEN_DEBUG_UNALIGNED_STORE pstoreu<float>(reinterpret_cast<float*>(to), from.v);
368}
369
370template <>
371EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2Xcf
372pgather<std::complex<float>, Packet2Xcf>(const std::complex<float>* from, Index stride) {
373 return Packet2Xcf(__riscv_vreinterpret_v_i64m2_f32m2(pgather<int64_t, Packet2Xl>(
374 reinterpret_cast<const numext::int64_t*>(reinterpret_cast<const void*>(from)), stride)));
375}
376
377template <>
378EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<std::complex<float>, Packet2Xcf>(std::complex<float>* to,
379 const Packet2Xcf& from,
380 Index stride) {
381 pscatter<int64_t, Packet2Xl>(reinterpret_cast<numext::int64_t*>(reinterpret_cast<void*>(to)),
382 __riscv_vreinterpret_v_f32m2_i64m2(from.v), stride);
383}
384
385template <>
386EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet2Xcf>(const Packet2Xcf& a) {
387 numext::int64_t res = pfirst<Packet2Xl>(__riscv_vreinterpret_v_f32m2_i64m2(a.v));
388 return numext::bit_cast<std::complex<float>>(res);
389}
390
391template <>
392EIGEN_STRONG_INLINE Packet2Xcf preverse(const Packet2Xcf& a) {
393 return Packet2Xcf(__riscv_vreinterpret_v_i64m2_f32m2(preverse<Packet2Xl>(__riscv_vreinterpret_v_f32m2_i64m2(a.v))));
394}
395
396template <>
397EIGEN_STRONG_INLINE std::complex<float> predux<Packet2Xcf>(const Packet2Xcf& a) {
398 Packet2Xl res = __riscv_vreinterpret_v_f32m2_i64m2(a.v);
399 Packet2Xf real = __riscv_vreinterpret_v_i64m2_f32m2(
400 __riscv_vand_vx_i64m2(res, 0x00000000ffffffffull, unpacket_traits<Packet2Xl>::size));
401 Packet2Xf imag = __riscv_vreinterpret_v_i64m2_f32m2(
402 __riscv_vand_vx_i64m2(res, 0xffffffff00000000ull, unpacket_traits<Packet2Xl>::size));
403 return std::complex<float>(predux<Packet2Xf>(real), predux<Packet2Xf>(imag));
404}
405
406template <>
407EIGEN_STRONG_INLINE Packet2Xcf pdiv<Packet2Xcf>(const Packet2Xcf& a, const Packet2Xcf& b) {
408 return pdiv_complex(a, b);
409}
410
411template <int N>
412EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void ptranspose(PacketBlock<Packet2Xcf, N>& kernel) {
413 numext::int64_t buffer[unpacket_traits<Packet2Xl>::size * N] = {0};
414 int i = 0;
415
416 for (i = 0; i < N; i++) {
417 __riscv_vsse64(&buffer[i], N * sizeof(numext::int64_t), __riscv_vreinterpret_v_f32m2_i64m2(kernel.packet[i].v),
418 unpacket_traits<Packet2Xl>::size);
419 }
420 for (i = 0; i < N; i++) {
421 kernel.packet[i] = Packet2Xcf(__riscv_vreinterpret_v_i64m2_f32m2(
422 __riscv_vle64_v_i64m2(&buffer[i * unpacket_traits<Packet2Xl>::size], unpacket_traits<Packet2Xl>::size)));
423 }
424}
425
426template <>
427EIGEN_STRONG_INLINE Packet2Xcf psqrt<Packet2Xcf>(const Packet2Xcf& a) {
428 return psqrt_complex(a);
429}
430
431template <>
432EIGEN_STRONG_INLINE Packet2Xcf plog<Packet2Xcf>(const Packet2Xcf& a) {
433 return plog_complex(a);
434}
435
436template <>
437EIGEN_STRONG_INLINE Packet2Xcf pexp<Packet2Xcf>(const Packet2Xcf& a) {
438 return pexp_complex(a);
439}
440
441template <typename Packet = Packet2Xcf>
442EIGEN_STRONG_INLINE Packet1Xcf predux_half(const Packet2Xcf& a) {
443 return Packet1Xcf(__riscv_vfadd_vv_f32m1(__riscv_vget_v_f32m2_f32m1(a.v, 0), __riscv_vget_v_f32m2_f32m1(a.v, 1),
444 unpacket_traits<Packet1Xf>::size));
445}
446
447EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2Xcf, Packet2Xf)
448
449/********************************* double ************************************/
450
451typedef complex_packet_wrapper<Packet2Xd, 32> Packet2Xcd;
452typedef complex_packet_wrapper<Packet4Xd, 33> Packet4Xcd;
453typedef complex_packet_wrapper<Packet1Xd, 34> Packet1Xcd;
454
455#if EIGEN_RISCV64_DEFAULT_LMUL == 1
456typedef Packet1Xcd PacketXcd;
457
458template <>
459struct packet_traits<std::complex<double>> : default_packet_traits {
460 typedef Packet1Xcd type;
461 typedef Packet1Xcd half;
462 enum {
463 Vectorizable = 1,
464 AlignedOnScalar = 0,
465 size = rvv_packet_size_selector<std::complex<double>, EIGEN_RISCV64_RVV_VL, 1>::size,
466
467 HasAdd = 1,
468 HasSub = 1,
469 HasMul = 1,
470 HasDiv = 1,
471 HasNegate = 1,
472 HasSqrt = 1,
473 HasLog = 1,
474 HasExp = 1,
475 HasSign = 0,
476 HasAbs = 0,
477 HasAbs2 = 0,
478 HasMin = 0,
479 HasMax = 0,
480 HasConj = 1,
481 HasArg = 0,
482 HasSetLinear = 0
483 };
484};
485#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
486typedef Packet2Xcd PacketXcd;
487
488template <>
489struct packet_traits<std::complex<double>> : default_packet_traits {
490 typedef Packet2Xcd type;
491#ifndef USE_LMUL2_ONLY
492 typedef Packet1Xcd half;
493#else
494 typedef Packet2Xcd half;
495#endif
496 enum {
497 Vectorizable = 1,
498 AlignedOnScalar = 0,
499 size = rvv_packet_size_selector<std::complex<double>, EIGEN_RISCV64_RVV_VL, 2>::size,
500
501 HasAdd = 1,
502 HasSub = 1,
503 HasMul = 1,
504 HasDiv = 1,
505 HasNegate = 1,
506 HasSqrt = 1,
507 HasLog = 1,
508 HasExp = 1,
509 HasSign = 0,
510 HasAbs = 0,
511 HasAbs2 = 0,
512 HasMin = 0,
513 HasMax = 0,
514 HasConj = 1,
515 HasArg = 0,
516 HasSetLinear = 0
517 };
518};
519#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
520typedef Packet4Xcd PacketXcd;
521
522template <>
523struct packet_traits<std::complex<double>> : default_packet_traits {
524 typedef Packet4Xcd type;
525#ifndef USE_LMUL4_ONLY
526 typedef Packet2Xcd half;
527#else
528 typedef Packet4Xcd half;
529#endif
530 enum {
531 Vectorizable = 1,
532 AlignedOnScalar = 0,
533 size = rvv_packet_size_selector<std::complex<double>, EIGEN_RISCV64_RVV_VL, 4>::size,
534
535 HasAdd = 1,
536 HasSub = 1,
537 HasMul = 1,
538 HasDiv = 1,
539 HasNegate = 1,
540 HasSqrt = 1,
541 HasLog = 1,
542 HasExp = 1,
543 HasSign = 0,
544 HasAbs = 0,
545 HasAbs2 = 0,
546 HasMin = 0,
547 HasMax = 0,
548 HasConj = 1,
549 HasArg = 0,
550 HasSetLinear = 0
551 };
552};
553#endif
554
555template <>
556struct unpacket_traits<Packet2Xcd> : default_unpacket_traits {
557 typedef std::complex<double> type;
558#ifndef USE_LMUL2_ONLY
559 typedef Packet1Xcd half;
560#else
561 typedef Packet2Xcd half;
562#endif
563 typedef Packet2Xd as_real;
564 enum {
565 size = rvv_packet_size_selector<std::complex<double>, EIGEN_RISCV64_RVV_VL, 2>::size,
566 alignment = rvv_packet_alignment_selector<EIGEN_RISCV64_RVV_VL, 2>::alignment,
567 vectorizable = true,
568 masked_load_available = false,
569 masked_store_available = false
570 };
571};
572
573template <>
574struct unpacket_traits<Packet4Xcd> : default_unpacket_traits {
575 typedef std::complex<double> type;
576#ifndef USE_LMUL4_ONLY
577 typedef Packet2Xcd half;
578#else
579 typedef Packet4Xcd half;
580#endif
581 typedef Packet4Xd as_real;
582 enum {
583 size = rvv_packet_size_selector<std::complex<double>, EIGEN_RISCV64_RVV_VL, 4>::size,
584 alignment = rvv_packet_alignment_selector<EIGEN_RISCV64_RVV_VL, 4>::alignment,
585 vectorizable = true,
586 masked_load_available = false,
587 masked_store_available = false
588 };
589};
590
591template <>
592struct unpacket_traits<Packet1Xcd> : default_unpacket_traits {
593 typedef std::complex<double> type;
594 typedef Packet1Xcd half;
595 typedef Packet1Xd as_real;
596 enum {
597 size = rvv_packet_size_selector<std::complex<double>, EIGEN_RISCV64_RVV_VL, 1>::size,
598 alignment = rvv_packet_alignment_selector<EIGEN_RISCV64_RVV_VL, 1>::alignment,
599 vectorizable = true,
600 masked_load_available = false,
601 masked_store_available = false
602 };
603};
604
605template <>
606EIGEN_STRONG_INLINE Packet2Xcd pcast<Packet2Xd, Packet2Xcd>(const Packet2Xd& a) {
607 return Packet2Xcd(a);
608}
609
610template <>
611EIGEN_STRONG_INLINE Packet2Xd pcast<Packet2Xcd, Packet2Xd>(const Packet2Xcd& a) {
612 return a.v;
613}
614
615EIGEN_STRONG_INLINE void prealimag2(const Packet2Xcd& a, Packet2Xd& real, Packet2Xd& imag) {
616 const PacketMask32 mask =
617 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0xaa), unpacket_traits<Packet1Xc>::size));
618 real = __riscv_vfslide1up_vf_f64m2_tumu(mask, a.v, a.v, 0.0, unpacket_traits<Packet2Xd>::size);
619 imag = __riscv_vfslide1down_vf_f64m2_tumu(__riscv_vmnot_m_b32(mask, unpacket_traits<Packet1Xi>::size), a.v, a.v, 0.0,
620 unpacket_traits<Packet2Xd>::size);
621}
622
623template <>
624EIGEN_STRONG_INLINE Packet2Xcd pset1<Packet2Xcd>(const std::complex<double>& from) {
625 const PacketMask32 mask =
626 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0xaa), unpacket_traits<Packet1Xc>::size));
627 Packet2Xd res = __riscv_vmerge_vvm_f64m2(pset1<Packet2Xd>(from.real()), pset1<Packet2Xd>(from.imag()), mask,
628 unpacket_traits<Packet2Xd>::size);
629 return Packet2Xcd(res);
630}
631
632template <>
633EIGEN_STRONG_INLINE Packet2Xcd padd<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
634 return Packet2Xcd(padd<Packet2Xd>(a.v, b.v));
635}
636
637template <>
638EIGEN_STRONG_INLINE Packet2Xcd psub<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
639 return Packet2Xcd(psub<Packet2Xd>(a.v, b.v));
640}
641
642template <>
643EIGEN_STRONG_INLINE Packet2Xcd pnegate(const Packet2Xcd& a) {
644 return Packet2Xcd(pnegate<Packet2Xd>(a.v));
645}
646
647template <>
648EIGEN_STRONG_INLINE Packet2Xcd pconj(const Packet2Xcd& a) {
649 const PacketMask32 mask =
650 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0xaa), unpacket_traits<Packet1Xc>::size));
651 return Packet2Xcd(__riscv_vfsgnjn_vv_f64m2_tumu(mask, a.v, a.v, a.v, unpacket_traits<Packet2Xd>::size));
652}
653
654template <>
655EIGEN_STRONG_INLINE Packet2Xcd pcplxflip<Packet2Xcd>(const Packet2Xcd& a) {
656 Packet2Xul res = __riscv_vreinterpret_v_f64m2_u64m2(a.v);
657 const PacketMask32 mask =
658 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0xaa), unpacket_traits<Packet1Xc>::size));
659 Packet2Xul data = __riscv_vslide1down_vx_u64m2(res, 0, unpacket_traits<Packet2Xl>::size);
660 Packet2Xd res2 = __riscv_vreinterpret_v_u64m2_f64m2(
661 __riscv_vslide1up_vx_u64m2_tumu(mask, data, res, 0, unpacket_traits<Packet2Xl>::size));
662 return Packet2Xcd(res2);
663}
664
665template <>
666EIGEN_STRONG_INLINE Packet2Xcd pmul<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
667 Packet2Xd real, imag;
668 prealimag2(a, real, imag);
669 return Packet2Xcd(pmadd<Packet2Xd>(imag, pcplxflip<Packet2Xcd>(pconj<Packet2Xcd>(b)).v, pmul<Packet2Xd>(real, b.v)));
670}
671
672template <>
673EIGEN_STRONG_INLINE Packet2Xcd pmadd<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b, const Packet2Xcd& c) {
674 Packet2Xd real, imag;
675 prealimag2(a, real, imag);
676 return Packet2Xcd(
677 pmadd<Packet2Xd>(imag, pcplxflip<Packet2Xcd>(pconj<Packet2Xcd>(b)).v, pmadd<Packet2Xd>(real, b.v, c.v)));
678}
679
680template <>
681EIGEN_STRONG_INLINE Packet2Xcd pmsub<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b, const Packet2Xcd& c) {
682 Packet2Xd real, imag;
683 prealimag2(a, real, imag);
684 return Packet2Xcd(
685 pmadd<Packet2Xd>(imag, pcplxflip<Packet2Xcd>(pconj<Packet2Xcd>(b)).v, pmsub<Packet2Xd>(real, b.v, c.v)));
686}
687
688template <>
689EIGEN_STRONG_INLINE Packet2Xcd pcmp_eq(const Packet2Xcd& a, const Packet2Xcd& b) {
690 Packet2Xl c = __riscv_vundefined_i64m2();
691 Packet1Xu mask =
692 __riscv_vreinterpret_v_b32_u32m1(__riscv_vmfeq_vv_f64m2_b32(a.v, b.v, unpacket_traits<Packet2Xd>::size));
693 Packet1Xu mask_r = __riscv_vsrl_vx_u32m1(__riscv_vand_vx_u32m1(mask, 0xaaaaaaaa, unpacket_traits<Packet1Xi>::size), 1,
694 unpacket_traits<Packet1Xi>::size);
695 mask = __riscv_vand_vv_u32m1(mask, mask_r, unpacket_traits<Packet1Xi>::size);
696 mask = __riscv_vor_vv_u32m1(__riscv_vsll_vx_u32m1(mask, 1, unpacket_traits<Packet1Xi>::size), mask,
697 unpacket_traits<Packet1Xi>::size);
698 Packet2Xd res = __riscv_vreinterpret_v_i64m2_f64m2(__riscv_vmerge_vvm_i64m2(pzero<Packet2Xl>(c), ptrue<Packet2Xl>(c),
699 __riscv_vreinterpret_v_u32m1_b32(mask),
700 unpacket_traits<Packet2Xl>::size));
701 return Packet2Xcd(res);
702}
703
704template <>
705EIGEN_STRONG_INLINE Packet2Xcd pand<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
706 return Packet2Xcd(pand<Packet2Xd>(a.v, b.v));
707}
708
709template <>
710EIGEN_STRONG_INLINE Packet2Xcd por<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
711 return Packet2Xcd(por<Packet2Xd>(a.v, b.v));
712}
713
714template <>
715EIGEN_STRONG_INLINE Packet2Xcd pxor<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
716 return Packet2Xcd(pxor<Packet2Xd>(a.v, b.v));
717}
718
719template <>
720EIGEN_STRONG_INLINE Packet2Xcd pandnot<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
721 return Packet2Xcd(pandnot<Packet2Xd>(a.v, b.v));
722}
723
724template <>
725EIGEN_STRONG_INLINE Packet2Xcd pnot<Packet2Xcd>(const Packet2Xcd& a) {
726 return Packet2Xcd(pnot<Packet2Xd>(a.v));
727}
728
729template <>
730EIGEN_STRONG_INLINE Packet2Xcd pload<Packet2Xcd>(const std::complex<double>* from) {
731 Packet2Xd res = pload<Packet2Xd>(reinterpret_cast<const double*>(from));
732 EIGEN_DEBUG_ALIGNED_LOAD return Packet2Xcd(res);
733}
734
735template <>
736EIGEN_STRONG_INLINE Packet2Xcd ploadu<Packet2Xcd>(const std::complex<double>* from) {
737 Packet2Xd res = ploadu<Packet2Xd>(reinterpret_cast<const double*>(from));
738 EIGEN_DEBUG_UNALIGNED_LOAD return Packet2Xcd(res);
739}
740
741template <>
742EIGEN_STRONG_INLINE Packet2Xcd ploaddup<Packet2Xcd>(const std::complex<double>* from) {
743 const PacketMask32 mask =
744 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0x66), unpacket_traits<Packet1Xc>::size));
745 Packet2Xul idx1 =
746 __riscv_vsrl_vx_u64m2(__riscv_vid_v_u64m2(unpacket_traits<Packet2Xd>::size), 1, unpacket_traits<Packet2Xd>::size);
747 Packet2Xul idx2 = __riscv_vxor_vx_u64m2_tumu(mask, idx1, idx1, 1, unpacket_traits<Packet2Xl>::size);
748 return Packet2Xcd(__riscv_vrgather_vv_f64m2(
749 __riscv_vlmul_ext_v_f64m1_f64m2(pload<Packet1Xd>(reinterpret_cast<const double*>(from))), idx2,
750 unpacket_traits<Packet2Xd>::size));
751}
752
753template <>
754EIGEN_STRONG_INLINE Packet2Xcd ploadquad<Packet2Xcd>(const std::complex<double>* from) {
755 const PacketMask32 mask =
756 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0x5a), unpacket_traits<Packet1Xc>::size));
757 Packet2Xul idx1 =
758 __riscv_vsrl_vx_u64m2(__riscv_vid_v_u64m2(unpacket_traits<Packet2Xd>::size), 2, unpacket_traits<Packet2Xd>::size);
759 Packet2Xul idx2 = __riscv_vxor_vx_u64m2_tumu(mask, idx1, idx1, 1, unpacket_traits<Packet2Xl>::size);
760 return Packet2Xcd(__riscv_vrgather_vv_f64m2(
761 __riscv_vlmul_ext_v_f64m1_f64m2(pload<Packet1Xd>(reinterpret_cast<const double*>(from))), idx2,
762 unpacket_traits<Packet2Xd>::size));
763}
764
765template <>
766EIGEN_STRONG_INLINE void pstore<std::complex<double>>(std::complex<double>* to, const Packet2Xcd& from) {
767 EIGEN_DEBUG_ALIGNED_STORE pstore<double>(reinterpret_cast<double*>(to), from.v);
768}
769
770template <>
771EIGEN_STRONG_INLINE void pstoreu<std::complex<double>>(std::complex<double>* to, const Packet2Xcd& from) {
772 EIGEN_DEBUG_UNALIGNED_STORE pstoreu<double>(reinterpret_cast<double*>(to), from.v);
773}
774
775template <>
776EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet2Xcd
777pgather<std::complex<double>, Packet2Xcd>(const std::complex<double>* from, Index stride) {
778 const PacketMask32 mask =
779 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0x55), unpacket_traits<Packet1Xc>::size));
780 const double* from2 = reinterpret_cast<const double*>(from);
781 Packet2Xd res = __riscv_vundefined_f64m2();
782 res = __riscv_vlse64_v_f64m2_tumu(mask, res, &from2[0 - (0 * stride)], stride * sizeof(double),
783 unpacket_traits<Packet2Xd>::size);
784 res =
785 __riscv_vlse64_v_f64m2_tumu(__riscv_vmnot_m_b32(mask, unpacket_traits<Packet1Xi>::size), res,
786 &from2[1 - (1 * stride)], stride * sizeof(double), unpacket_traits<Packet2Xd>::size);
787 return Packet2Xcd(res);
788}
789
790template <>
791EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pscatter<std::complex<double>, Packet2Xcd>(std::complex<double>* to,
792 const Packet2Xcd& from,
793 Index stride) {
794 const PacketMask32 mask =
795 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0x55), unpacket_traits<Packet1Xc>::size));
796 double* to2 = reinterpret_cast<double*>(to);
797 __riscv_vsse64_v_f64m2_m(mask, &to2[0 - (0 * stride)], stride * sizeof(double), from.v,
798 unpacket_traits<Packet2Xd>::size);
799 __riscv_vsse64_v_f64m2_m(__riscv_vmnot_m_b32(mask, unpacket_traits<Packet1Xi>::size), &to2[1 - (1 * stride)],
800 stride * sizeof(double), from.v, unpacket_traits<Packet2Xd>::size);
801}
802
803template <>
804EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet2Xcd>(const Packet2Xcd& a) {
805 double real = pfirst<Packet2Xd>(a.v);
806 double imag = pfirst<Packet2Xd>(__riscv_vfslide1down_vf_f64m2(a.v, 0.0, unpacket_traits<Packet2Xd>::size));
807 return std::complex<double>(real, imag);
808}
809
810template <>
811EIGEN_STRONG_INLINE Packet2Xcd preverse(const Packet2Xcd& a) {
812 Packet2Xul idx = __riscv_vxor_vx_u64m2(__riscv_vid_v_u64m2(unpacket_traits<Packet2Xl>::size),
813 unpacket_traits<Packet2Xl>::size - 2, unpacket_traits<Packet2Xl>::size);
814 Packet2Xd res = __riscv_vrgather_vv_f64m2(a.v, idx, unpacket_traits<Packet2Xd>::size);
815 return Packet2Xcd(res);
816}
817
818template <>
819EIGEN_STRONG_INLINE std::complex<double> predux<Packet2Xcd>(const Packet2Xcd& a) {
820 const PacketMask32 mask =
821 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0xaa), unpacket_traits<Packet1Xc>::size));
822 Packet2Xl res = __riscv_vreinterpret_v_f64m2_i64m2(a.v);
823 Packet2Xd real = __riscv_vreinterpret_v_i64m2_f64m2(
824 __riscv_vand_vx_i64m2_tumu(mask, res, res, 0, unpacket_traits<Packet2Xl>::size));
825 Packet2Xd imag = __riscv_vreinterpret_v_i64m2_f64m2(__riscv_vand_vx_i64m2_tumu(
826 __riscv_vmnot_m_b32(mask, unpacket_traits<Packet1Xi>::size), res, res, 0, unpacket_traits<Packet2Xl>::size));
827 return std::complex<double>(predux<Packet2Xd>(real), predux<Packet2Xd>(imag));
828}
829
830template <>
831EIGEN_STRONG_INLINE Packet2Xcd pdiv<Packet2Xcd>(const Packet2Xcd& a, const Packet2Xcd& b) {
832 return pdiv_complex(a, b);
833}
834
835template <int N>
836EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void ptranspose(PacketBlock<Packet2Xcd, N>& kernel) {
837 double buffer[unpacket_traits<Packet2Xd>::size * N];
838 int i = 0;
839
840 const PacketMask32 mask =
841 __riscv_vreinterpret_v_i8m1_b32(__riscv_vmv_v_x_i8m1(static_cast<char>(0x55), unpacket_traits<Packet1Xc>::size));
842
843 for (i = 0; i < N; i++) {
844 __riscv_vsse64_v_f64m2_m(mask, &buffer[(i * 2) - (0 * N) + 0], N * sizeof(double), kernel.packet[i].v,
845 unpacket_traits<Packet2Xd>::size);
846 __riscv_vsse64_v_f64m2_m(__riscv_vmnot_m_b32(mask, unpacket_traits<Packet1Xi>::size),
847 &buffer[(i * 2) - (1 * N) + 1], N * sizeof(double), kernel.packet[i].v,
848 unpacket_traits<Packet2Xd>::size);
849 }
850
851 for (i = 0; i < N; i++) {
852 kernel.packet[i] = Packet2Xcd(
853 __riscv_vle64_v_f64m2(&buffer[i * unpacket_traits<Packet2Xd>::size], unpacket_traits<Packet2Xd>::size));
854 }
855}
856
857template <>
858EIGEN_STRONG_INLINE Packet2Xcd psqrt<Packet2Xcd>(const Packet2Xcd& a) {
859 return psqrt_complex(a);
860}
861
862template <>
863EIGEN_STRONG_INLINE Packet2Xcd plog<Packet2Xcd>(const Packet2Xcd& a) {
864 return plog_complex(a);
865}
866
867template <>
868EIGEN_STRONG_INLINE Packet2Xcd pexp<Packet2Xcd>(const Packet2Xcd& a) {
869 return pexp_complex(a);
870}
871
872template <typename Packet = Packet2Xcd>
873EIGEN_STRONG_INLINE Packet1Xcd predux_half(const Packet2Xcd& a) {
874 return Packet1Xcd(__riscv_vfadd_vv_f64m1(__riscv_vget_v_f64m2_f64m1(a.v, 0), __riscv_vget_v_f64m2_f64m1(a.v, 1),
875 unpacket_traits<Packet1Xd>::size));
876}
877
878EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2Xcd, Packet2Xd)
879
880EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_POP
881
882} // end namespace internal
883
884} // end namespace Eigen
885
886#endif // EIGEN_COMPLEX_RVV10_H