Eigen  5.0.1
 
Loading...
Searching...
No Matches
Complex.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018 Gael Guennebaud <gael.guennebaud@inria.fr>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_COMPLEX_AVX512_H
12#define EIGEN_COMPLEX_AVX512_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_PUSH
22
23//---------- float ----------
24struct Packet8cf {
25 EIGEN_STRONG_INLINE Packet8cf() {}
26 EIGEN_STRONG_INLINE explicit Packet8cf(const __m512& a) : v(a) {}
27 __m512 v;
28};
29
30template <>
31struct packet_traits<std::complex<float> > : default_packet_traits {
32 typedef Packet8cf type;
33 typedef Packet4cf half;
34 enum {
35 Vectorizable = 1,
36 AlignedOnScalar = 1,
37 size = 8,
38
39 HasAdd = 1,
40 HasSub = 1,
41 HasMul = 1,
42 HasDiv = 1,
43 HasNegate = 1,
44 HasSqrt = 1,
45 HasLog = 1,
46 HasExp = 1,
47 HasAbs = 0,
48 HasAbs2 = 0,
49 HasMin = 0,
50 HasMax = 0,
51 HasSetLinear = 0
52 };
53};
54
55template <>
56struct unpacket_traits<Packet8cf> {
57 typedef std::complex<float> type;
58 typedef Packet4cf half;
59 typedef Packet16f as_real;
60 enum {
61 size = 8,
62 alignment = unpacket_traits<Packet16f>::alignment,
63 vectorizable = true,
64 masked_load_available = false,
65 masked_store_available = false
66 };
67};
68
69template <>
70EIGEN_STRONG_INLINE Packet8cf ptrue<Packet8cf>(const Packet8cf& a) {
71 return Packet8cf(ptrue(Packet16f(a.v)));
72}
73template <>
74EIGEN_STRONG_INLINE Packet8cf padd<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
75 return Packet8cf(_mm512_add_ps(a.v, b.v));
76}
77template <>
78EIGEN_STRONG_INLINE Packet8cf psub<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
79 return Packet8cf(_mm512_sub_ps(a.v, b.v));
80}
81template <>
82EIGEN_STRONG_INLINE Packet8cf pnegate(const Packet8cf& a) {
83 return Packet8cf(pnegate(a.v));
84}
85template <>
86EIGEN_STRONG_INLINE Packet8cf pconj(const Packet8cf& a) {
87 const __m512 mask = _mm512_castsi512_ps(_mm512_setr_epi32(
88 0x00000000, SIGN_MASK_I32, 0x00000000, SIGN_MASK_I32, 0x00000000, SIGN_MASK_I32, 0x00000000, SIGN_MASK_I32,
89 0x00000000, SIGN_MASK_I32, 0x00000000, SIGN_MASK_I32, 0x00000000, SIGN_MASK_I32, 0x00000000, SIGN_MASK_I32));
90 return Packet8cf(pxor(a.v, mask));
91}
92
93template <>
94EIGEN_STRONG_INLINE Packet8cf pmul<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
95 __m512 tmp2 = _mm512_mul_ps(_mm512_movehdup_ps(a.v), _mm512_permute_ps(b.v, _MM_SHUFFLE(2, 3, 0, 1)));
96 return Packet8cf(_mm512_fmaddsub_ps(_mm512_moveldup_ps(a.v), b.v, tmp2));
97}
98
99template <>
100EIGEN_STRONG_INLINE Packet8cf pand<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
101 return Packet8cf(pand(a.v, b.v));
102}
103template <>
104EIGEN_STRONG_INLINE Packet8cf por<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
105 return Packet8cf(por(a.v, b.v));
106}
107template <>
108EIGEN_STRONG_INLINE Packet8cf pxor<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
109 return Packet8cf(pxor(a.v, b.v));
110}
111template <>
112EIGEN_STRONG_INLINE Packet8cf pandnot<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
113 return Packet8cf(pandnot(a.v, b.v));
114}
115
116template <>
117EIGEN_STRONG_INLINE Packet8cf pcmp_eq(const Packet8cf& a, const Packet8cf& b) {
118 __m512 eq = pcmp_eq<Packet16f>(a.v, b.v);
119 return Packet8cf(pand(eq, _mm512_permute_ps(eq, 0xB1)));
120}
121
122template <>
123EIGEN_STRONG_INLINE Packet8cf pload<Packet8cf>(const std::complex<float>* from) {
124 EIGEN_DEBUG_ALIGNED_LOAD return Packet8cf(pload<Packet16f>(&numext::real_ref(*from)));
125}
126template <>
127EIGEN_STRONG_INLINE Packet8cf ploadu<Packet8cf>(const std::complex<float>* from) {
128 EIGEN_DEBUG_UNALIGNED_LOAD return Packet8cf(ploadu<Packet16f>(&numext::real_ref(*from)));
129}
130
131template <>
132EIGEN_STRONG_INLINE Packet8cf pset1<Packet8cf>(const std::complex<float>& from) {
133 const float re = std::real(from);
134 const float im = std::imag(from);
135 return Packet8cf(_mm512_set_ps(im, re, im, re, im, re, im, re, im, re, im, re, im, re, im, re));
136}
137
138template <>
139EIGEN_STRONG_INLINE Packet8cf ploaddup<Packet8cf>(const std::complex<float>* from) {
140 return Packet8cf(_mm512_castpd_ps(ploaddup<Packet8d>((const double*)(const void*)from)));
141}
142template <>
143EIGEN_STRONG_INLINE Packet8cf ploadquad<Packet8cf>(const std::complex<float>* from) {
144 return Packet8cf(_mm512_castpd_ps(ploadquad<Packet8d>((const double*)(const void*)from)));
145}
146
147template <>
148EIGEN_STRONG_INLINE void pstore<std::complex<float> >(std::complex<float>* to, const Packet8cf& from) {
149 EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), from.v);
150}
151template <>
152EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float>* to, const Packet8cf& from) {
153 EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), from.v);
154}
155
156template <>
157EIGEN_DEVICE_FUNC inline Packet8cf pgather<std::complex<float>, Packet8cf>(const std::complex<float>* from,
158 Index stride) {
159 return Packet8cf(_mm512_castpd_ps(pgather<double, Packet8d>((const double*)(const void*)from, stride)));
160}
161
162template <>
163EIGEN_DEVICE_FUNC inline void pscatter<std::complex<float>, Packet8cf>(std::complex<float>* to, const Packet8cf& from,
164 Index stride) {
165 pscatter((double*)(void*)to, _mm512_castps_pd(from.v), stride);
166}
167
168template <>
169EIGEN_STRONG_INLINE std::complex<float> pfirst<Packet8cf>(const Packet8cf& a) {
170 return pfirst(Packet2cf(_mm512_castps512_ps128(a.v)));
171}
172
173template <>
174EIGEN_STRONG_INLINE Packet8cf preverse(const Packet8cf& a) {
175 return Packet8cf(_mm512_castsi512_ps(_mm512_permutexvar_epi64(
176 _mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), _mm512_castps_si512(a.v))));
177}
178
179template <>
180EIGEN_STRONG_INLINE std::complex<float> predux<Packet8cf>(const Packet8cf& a) {
181 return predux(padd(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v))));
182}
183
184template <>
185EIGEN_STRONG_INLINE std::complex<float> predux_mul<Packet8cf>(const Packet8cf& a) {
186 return predux_mul(pmul(Packet4cf(extract256<0>(a.v)), Packet4cf(extract256<1>(a.v))));
187}
188
189template <>
190EIGEN_STRONG_INLINE Packet4cf predux_half<Packet8cf>(const Packet8cf& a) {
191 __m256 lane0 = extract256<0>(a.v);
192 __m256 lane1 = extract256<1>(a.v);
193 __m256 res = _mm256_add_ps(lane0, lane1);
194 return Packet4cf(res);
195}
196
197EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet8cf, Packet16f)
198
199template <>
200EIGEN_STRONG_INLINE Packet8cf pdiv<Packet8cf>(const Packet8cf& a, const Packet8cf& b) {
201 return pdiv_complex(a, b);
202}
203
204template <>
205EIGEN_STRONG_INLINE Packet8cf pcplxflip<Packet8cf>(const Packet8cf& x) {
206 return Packet8cf(EIGEN_AVX512_SHUFFLE_PS(x.v, x.v, _MM_SHUFFLE(2, 3, 0, 1)));
207}
208
209//---------- double ----------
210struct Packet4cd {
211 EIGEN_STRONG_INLINE Packet4cd() {}
212 EIGEN_STRONG_INLINE explicit Packet4cd(const __m512d& a) : v(a) {}
213 __m512d v;
214};
215
216template <>
217struct packet_traits<std::complex<double> > : default_packet_traits {
218 typedef Packet4cd type;
219 typedef Packet2cd half;
220 enum {
221 Vectorizable = 1,
222 AlignedOnScalar = 0,
223 size = 4,
224
225 HasAdd = 1,
226 HasSub = 1,
227 HasMul = 1,
228 HasDiv = 1,
229 HasNegate = 1,
230 HasSqrt = 1,
231 HasLog = 1,
232 HasExp = 1,
233 HasAbs = 0,
234 HasAbs2 = 0,
235 HasMin = 0,
236 HasMax = 0,
237 HasSetLinear = 0
238 };
239};
240
241template <>
242struct unpacket_traits<Packet4cd> {
243 typedef std::complex<double> type;
244 typedef Packet2cd half;
245 typedef Packet8d as_real;
246 enum {
247 size = 4,
248 alignment = unpacket_traits<Packet8d>::alignment,
249 vectorizable = true,
250 masked_load_available = false,
251 masked_store_available = false
252 };
253};
254
255template <>
256EIGEN_STRONG_INLINE Packet4cd padd<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
257 return Packet4cd(_mm512_add_pd(a.v, b.v));
258}
259template <>
260EIGEN_STRONG_INLINE Packet4cd psub<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
261 return Packet4cd(_mm512_sub_pd(a.v, b.v));
262}
263template <>
264EIGEN_STRONG_INLINE Packet4cd pnegate(const Packet4cd& a) {
265 return Packet4cd(pnegate(a.v));
266}
267template <>
268EIGEN_STRONG_INLINE Packet4cd pconj(const Packet4cd& a) {
269 const __m512d mask =
270 _mm512_castsi512_pd(_mm512_set_epi32(SIGN_MASK_I32, 0x0, 0x0, 0x0, SIGN_MASK_I32, 0x0, 0x0, 0x0, SIGN_MASK_I32,
271 0x0, 0x0, 0x0, SIGN_MASK_I32, 0x0, 0x0, 0x0));
272 return Packet4cd(pxor(a.v, mask));
273}
274
275template <>
276EIGEN_STRONG_INLINE Packet4cd pmul<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
277 __m512d tmp1 = _mm512_shuffle_pd(a.v, a.v, 0x0);
278 __m512d tmp2 = _mm512_shuffle_pd(a.v, a.v, 0xFF);
279 __m512d tmp3 = _mm512_shuffle_pd(b.v, b.v, 0x55);
280 __m512d odd = _mm512_mul_pd(tmp2, tmp3);
281 return Packet4cd(_mm512_fmaddsub_pd(tmp1, b.v, odd));
282}
283
284template <>
285EIGEN_STRONG_INLINE Packet4cd ptrue<Packet4cd>(const Packet4cd& a) {
286 return Packet4cd(ptrue(Packet8d(a.v)));
287}
288template <>
289EIGEN_STRONG_INLINE Packet4cd pand<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
290 return Packet4cd(pand(a.v, b.v));
291}
292template <>
293EIGEN_STRONG_INLINE Packet4cd por<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
294 return Packet4cd(por(a.v, b.v));
295}
296template <>
297EIGEN_STRONG_INLINE Packet4cd pxor<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
298 return Packet4cd(pxor(a.v, b.v));
299}
300template <>
301EIGEN_STRONG_INLINE Packet4cd pandnot<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
302 return Packet4cd(pandnot(a.v, b.v));
303}
304
305template <>
306EIGEN_STRONG_INLINE Packet4cd pcmp_eq(const Packet4cd& a, const Packet4cd& b) {
307 __m512d eq = pcmp_eq<Packet8d>(a.v, b.v);
308 return Packet4cd(pand(eq, _mm512_permute_pd(eq, 0x55)));
309}
310
311template <>
312EIGEN_STRONG_INLINE Packet4cd pload<Packet4cd>(const std::complex<double>* from) {
313 EIGEN_DEBUG_ALIGNED_LOAD return Packet4cd(pload<Packet8d>((const double*)from));
314}
315template <>
316EIGEN_STRONG_INLINE Packet4cd ploadu<Packet4cd>(const std::complex<double>* from) {
317 EIGEN_DEBUG_UNALIGNED_LOAD return Packet4cd(ploadu<Packet8d>((const double*)from));
318}
319
320template <>
321EIGEN_STRONG_INLINE Packet4cd pset1<Packet4cd>(const std::complex<double>& from) {
322 return Packet4cd(_mm512_castps_pd(_mm512_broadcast_f32x4(_mm_castpd_ps(pset1<Packet1cd>(from).v))));
323}
324
325template <>
326EIGEN_STRONG_INLINE Packet4cd ploaddup<Packet4cd>(const std::complex<double>* from) {
327 return Packet4cd(
328 _mm512_insertf64x4(_mm512_castpd256_pd512(ploaddup<Packet2cd>(from).v), ploaddup<Packet2cd>(from + 1).v, 1));
329}
330
331template <>
332EIGEN_STRONG_INLINE void pstore<std::complex<double> >(std::complex<double>* to, const Packet4cd& from) {
333 EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, from.v);
334}
335template <>
336EIGEN_STRONG_INLINE void pstoreu<std::complex<double> >(std::complex<double>* to, const Packet4cd& from) {
337 EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, from.v);
338}
339
340template <>
341EIGEN_DEVICE_FUNC inline Packet4cd pgather<std::complex<double>, Packet4cd>(const std::complex<double>* from,
342 Index stride) {
343 return Packet4cd(_mm512_insertf64x4(
344 _mm512_castpd256_pd512(_mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from + 0 * stride).v),
345 ploadu<Packet1cd>(from + 1 * stride).v, 1)),
346 _mm256_insertf128_pd(_mm256_castpd128_pd256(ploadu<Packet1cd>(from + 2 * stride).v),
347 ploadu<Packet1cd>(from + 3 * stride).v, 1),
348 1));
349}
350
351template <>
352EIGEN_DEVICE_FUNC inline void pscatter<std::complex<double>, Packet4cd>(std::complex<double>* to, const Packet4cd& from,
353 Index stride) {
354 __m512i fromi = _mm512_castpd_si512(from.v);
355 double* tod = (double*)(void*)to;
356 _mm_storeu_pd(tod + 0 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 0)));
357 _mm_storeu_pd(tod + 2 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 1)));
358 _mm_storeu_pd(tod + 4 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 2)));
359 _mm_storeu_pd(tod + 6 * stride, _mm_castsi128_pd(_mm512_extracti32x4_epi32(fromi, 3)));
360}
361
362template <>
363EIGEN_STRONG_INLINE std::complex<double> pfirst<Packet4cd>(const Packet4cd& a) {
364 __m128d low = extract128<0>(a.v);
365 EIGEN_ALIGN16 double res[2];
366 _mm_store_pd(res, low);
367 return std::complex<double>(res[0], res[1]);
368}
369
370template <>
371EIGEN_STRONG_INLINE Packet4cd preverse(const Packet4cd& a) {
372 return Packet4cd(_mm512_shuffle_f64x2(a.v, a.v, (shuffle_mask<3, 2, 1, 0>::mask)));
373}
374
375template <>
376EIGEN_STRONG_INLINE std::complex<double> predux<Packet4cd>(const Packet4cd& a) {
377 return predux(padd(Packet2cd(_mm512_extractf64x4_pd(a.v, 0)), Packet2cd(_mm512_extractf64x4_pd(a.v, 1))));
378}
379
380template <>
381EIGEN_STRONG_INLINE std::complex<double> predux_mul<Packet4cd>(const Packet4cd& a) {
382 return predux_mul(pmul(Packet2cd(_mm512_extractf64x4_pd(a.v, 0)), Packet2cd(_mm512_extractf64x4_pd(a.v, 1))));
383}
384
385EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet4cd, Packet8d)
386
387template <>
388EIGEN_STRONG_INLINE Packet4cd pdiv<Packet4cd>(const Packet4cd& a, const Packet4cd& b) {
389 return pdiv_complex(a, b);
390}
391
392template <>
393EIGEN_STRONG_INLINE Packet4cd pcplxflip<Packet4cd>(const Packet4cd& x) {
394 return Packet4cd(_mm512_permute_pd(x.v, 0x55));
395}
396
397EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8cf, 4>& kernel) {
398 PacketBlock<Packet8d, 4> pb;
399
400 pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
401 pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
402 pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
403 pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
404 ptranspose(pb);
405 kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
406 kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
407 kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
408 kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
409}
410
411EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8cf, 8>& kernel) {
412 PacketBlock<Packet8d, 8> pb;
413
414 pb.packet[0] = _mm512_castps_pd(kernel.packet[0].v);
415 pb.packet[1] = _mm512_castps_pd(kernel.packet[1].v);
416 pb.packet[2] = _mm512_castps_pd(kernel.packet[2].v);
417 pb.packet[3] = _mm512_castps_pd(kernel.packet[3].v);
418 pb.packet[4] = _mm512_castps_pd(kernel.packet[4].v);
419 pb.packet[5] = _mm512_castps_pd(kernel.packet[5].v);
420 pb.packet[6] = _mm512_castps_pd(kernel.packet[6].v);
421 pb.packet[7] = _mm512_castps_pd(kernel.packet[7].v);
422 ptranspose(pb);
423 kernel.packet[0].v = _mm512_castpd_ps(pb.packet[0]);
424 kernel.packet[1].v = _mm512_castpd_ps(pb.packet[1]);
425 kernel.packet[2].v = _mm512_castpd_ps(pb.packet[2]);
426 kernel.packet[3].v = _mm512_castpd_ps(pb.packet[3]);
427 kernel.packet[4].v = _mm512_castpd_ps(pb.packet[4]);
428 kernel.packet[5].v = _mm512_castpd_ps(pb.packet[5]);
429 kernel.packet[6].v = _mm512_castpd_ps(pb.packet[6]);
430 kernel.packet[7].v = _mm512_castpd_ps(pb.packet[7]);
431}
432
433EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4cd, 4>& kernel) {
434 __m512d T0 =
435 _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<0, 1, 0, 1>::mask)); // [a0 a1 b0 b1]
436 __m512d T1 =
437 _mm512_shuffle_f64x2(kernel.packet[0].v, kernel.packet[1].v, (shuffle_mask<2, 3, 2, 3>::mask)); // [a2 a3 b2 b3]
438 __m512d T2 =
439 _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<0, 1, 0, 1>::mask)); // [c0 c1 d0 d1]
440 __m512d T3 =
441 _mm512_shuffle_f64x2(kernel.packet[2].v, kernel.packet[3].v, (shuffle_mask<2, 3, 2, 3>::mask)); // [c2 c3 d2 d3]
442
443 kernel.packet[3] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<1, 3, 1, 3>::mask))); // [a3 b3 c3 d3]
444 kernel.packet[2] = Packet4cd(_mm512_shuffle_f64x2(T1, T3, (shuffle_mask<0, 2, 0, 2>::mask))); // [a2 b2 c2 d2]
445 kernel.packet[1] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<1, 3, 1, 3>::mask))); // [a1 b1 c1 d1]
446 kernel.packet[0] = Packet4cd(_mm512_shuffle_f64x2(T0, T2, (shuffle_mask<0, 2, 0, 2>::mask))); // [a0 b0 c0 d0]
447}
448
449EIGEN_INSTANTIATE_COMPLEX_MATH_FUNCS(Packet4cd)
450EIGEN_INSTANTIATE_COMPLEX_MATH_FUNCS(Packet8cf)
451
452/*---------------- load/store segment support ----------------*/
453
454// A complex lane is two real lanes, so the k-mask is formed over 2 * begin and 2 * count.
455
456/*---------------- std::complex<float> ----------------*/
457
458template <>
459struct has_packet_segment<Packet8cf> : std::true_type {};
460
461template <>
462inline Packet8cf ploaduSegment<Packet8cf>(const std::complex<float>* from, Index begin, Index count) {
463 return Packet8cf(
464 _mm512_maskz_loadu_ps(static_cast<__mmask16>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from)));
465}
466
467template <>
468inline void pstoreuSegment<std::complex<float>, Packet8cf>(std::complex<float>* to, const Packet8cf& from, Index begin,
469 Index count) {
470 _mm512_mask_storeu_ps(&numext::real_ref(*to), static_cast<__mmask16>(segment_kmask(2 * begin, 2 * count)), from.v);
471}
472
473/*---------------- std::complex<double> ----------------*/
474
475template <>
476struct has_packet_segment<Packet4cd> : std::true_type {};
477
478template <>
479inline Packet4cd ploaduSegment<Packet4cd>(const std::complex<double>* from, Index begin, Index count) {
480 return Packet4cd(
481 _mm512_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from)));
482}
483
484template <>
485inline void pstoreuSegment<std::complex<double>, Packet4cd>(std::complex<double>* to, const Packet4cd& from,
486 Index begin, Index count) {
487 _mm512_mask_storeu_pd(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v);
488}
489
490#ifdef EIGEN_VECTORIZE_AVX512VL
491
492// The 256- and 128-bit packets declare has_packet_segment in AVX/Complex.h.
493
494template <>
495inline Packet4cf ploaduSegment<Packet4cf>(const std::complex<float>* from, Index begin, Index count) {
496 return Packet4cf(
497 _mm256_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from)));
498}
499
500template <>
501inline void pstoreuSegment<std::complex<float>, Packet4cf>(std::complex<float>* to, const Packet4cf& from, Index begin,
502 Index count) {
503 _mm256_mask_storeu_ps(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v);
504}
505
506template <>
507inline Packet2cf ploaduSegment<Packet2cf>(const std::complex<float>* from, Index begin, Index count) {
508 return Packet2cf(
509 _mm_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from)));
510}
511
512template <>
513inline void pstoreuSegment<std::complex<float>, Packet2cf>(std::complex<float>* to, const Packet2cf& from, Index begin,
514 Index count) {
515 _mm_mask_storeu_ps(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v);
516}
517
518template <>
519inline Packet2cd ploaduSegment<Packet2cd>(const std::complex<double>* from, Index begin, Index count) {
520 return Packet2cd(
521 _mm256_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), &numext::real_ref(*from)));
522}
523
524template <>
525inline void pstoreuSegment<std::complex<double>, Packet2cd>(std::complex<double>* to, const Packet2cd& from,
526 Index begin, Index count) {
527 _mm256_mask_storeu_pd(&numext::real_ref(*to), static_cast<__mmask8>(segment_kmask(2 * begin, 2 * count)), from.v);
528}
529
530#endif // EIGEN_VECTORIZE_AVX512VL
531
532/*---------------- end load/store segment support ----------------*/
533
534EIGEN_GCC_FAST_MATH_COMPLEX_VECTORIZE_WORKAROUND_POP
535
536} // end namespace internal
537} // end namespace Eigen
538
539#endif // EIGEN_COMPLEX_AVX512_H