Eigen  5.0.1-dev+284dcc12
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_PACKET_MATH_AVX512_H
11#define EIGEN_PACKET_MATH_AVX512_H
12
13// IWYU pragma: private
14#include "../../InternalHeaderCheck.h"
15
16namespace Eigen {
17
18namespace internal {
19
20#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
21#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
22#endif
23
24#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
25#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
26#endif
27
28#ifdef EIGEN_VECTORIZE_FMA
29#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
30#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
31#endif
32#endif
33
34typedef __m512 Packet16f;
35typedef __m512i Packet16i;
36typedef __m512d Packet8d;
37typedef eigen_packet_wrapper<__m512i, 1> Packet8l;
38#ifndef EIGEN_VECTORIZE_AVX512FP16
39typedef eigen_packet_wrapper<__m256i, 1> Packet16h;
40#endif
41typedef eigen_packet_wrapper<__m256i, 2> Packet16bf;
42
43typedef eigen_packet_wrapper<__m512i, 6> Packet32s;
44typedef eigen_packet_wrapper<__m256i, 6> Packet16s;
45typedef eigen_packet_wrapper<__m128i, 6> Packet8s;
46
47template <>
48struct is_arithmetic<__m512> {
49 enum { value = true };
50};
51template <>
52struct is_arithmetic<__m512i> {
53 enum { value = true };
54};
55template <>
56struct is_arithmetic<__m512d> {
57 enum { value = true };
58};
59template <>
60struct is_arithmetic<Packet8l> {
61 enum { value = true };
62};
63
64#ifndef EIGEN_VECTORIZE_AVX512FP16
65template <>
66struct is_arithmetic<Packet16h> {
67 enum { value = true };
68};
69
70template <>
71struct packet_traits<half> : default_packet_traits {
72 typedef Packet16h type;
73 // There is no half-size packet for Packet16h.
74 typedef Packet16h half;
75 enum {
76 Vectorizable = 1,
77 AlignedOnScalar = 1,
78 size = 16,
79
80 HasCmp = 1,
81 HasAdd = 1,
82 HasSub = 1,
83 HasMul = 1,
84 HasDiv = 1,
85 HasNegate = 1,
86 HasAbs = 1,
87 HasAbs2 = 0,
88 HasMin = 1,
89 HasMax = 1,
90 HasConj = 1,
91 HasSetLinear = 0,
92 HasSqrt = 1,
93 HasRsqrt = 1,
94 HasLog = 1,
95 HasLog1p = 1,
96 HasExp = 1,
97 HasExpm1 = 1,
98 HasBessel = 1,
99 HasNdtri = 1,
100 HasSin = EIGEN_FAST_MATH,
101 HasCos = EIGEN_FAST_MATH,
102 HasTanh = EIGEN_FAST_MATH,
103 HasErf = EIGEN_FAST_MATH,
104 HasBlend = 0
105 };
106};
107#endif
108
109template <>
110struct packet_traits<float> : default_packet_traits {
111 typedef Packet16f type;
112 typedef Packet8f half;
113 enum {
114 Vectorizable = 1,
115 AlignedOnScalar = 1,
116 size = 16,
117
118 HasAbs = 1,
119 HasMin = 1,
120 HasMax = 1,
121 HasConj = 1,
122 HasBlend = 1,
123 HasSin = EIGEN_FAST_MATH,
124 HasCos = EIGEN_FAST_MATH,
125 HasACos = 1,
126 HasASin = 1,
127 HasATan = 1,
128 HasATanh = 1,
129 HasSqrt = 1,
130 HasRsqrt = 1,
131 HasCbrt = 1,
132 HasLog = 1,
133 HasLog1p = 1,
134 HasExpm1 = 1,
135 HasNdtri = 1,
136 HasBessel = 1,
137 HasExp = 1,
138 HasPow = 1,
139 HasReciprocal = EIGEN_FAST_MATH,
140 HasTanh = EIGEN_FAST_MATH,
141 HasErf = EIGEN_FAST_MATH,
142 HasErfc = EIGEN_FAST_MATH,
143 HasCmp = 1,
144 HasDiv = 1
145 };
146};
147template <>
148struct packet_traits<double> : default_packet_traits {
149 typedef Packet8d type;
150 typedef Packet4d half;
151 enum {
152 Vectorizable = 1,
153 AlignedOnScalar = 1,
154 size = 8,
155 HasBlend = 1,
156 HasSqrt = 1,
157 HasRsqrt = 1,
158 HasCbrt = 1,
159 HasSin = EIGEN_FAST_MATH,
160 HasCos = EIGEN_FAST_MATH,
161 HasLog = 1,
162 HasExp = 1,
163 HasPow = 1,
164 HasATan = 1,
165 HasTanh = EIGEN_FAST_MATH,
166 HasErf = EIGEN_FAST_MATH,
167 HasErfc = EIGEN_FAST_MATH,
168 HasATanh = 1,
169 HasCmp = 1,
170 HasDiv = 1
171 };
172};
173
174template <>
175struct packet_traits<int> : default_packet_traits {
176 typedef Packet16i type;
177 typedef Packet8i half;
178 enum { Vectorizable = 1, AlignedOnScalar = 1, HasBlend = 0, HasCmp = 1, HasDiv = 1, size = 16 };
179};
180
181template <>
182struct packet_traits<int64_t> : default_packet_traits {
183 typedef Packet8l type;
184 typedef Packet4l half;
185 enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, size = 8 };
186};
187
188template <>
189struct unpacket_traits<Packet16f> {
190 typedef float type;
191 typedef Packet8f half;
192 typedef Packet16i integer_packet;
193 typedef uint16_t mask_t;
194 enum {
195 size = 16,
196 alignment = Aligned64,
197 vectorizable = true,
198 masked_load_available = true,
199 masked_store_available = true,
200 masked_fpops_available = true
201 };
202};
203template <>
204struct unpacket_traits<Packet8d> {
205 typedef double type;
206 typedef Packet4d half;
207 typedef Packet8l integer_packet;
208 typedef uint8_t mask_t;
209 enum {
210 size = 8,
211 alignment = Aligned64,
212 vectorizable = true,
213 masked_load_available = true,
214 masked_store_available = true,
215 masked_fpops_available = true
216 };
217};
218template <>
219struct unpacket_traits<Packet16i> {
220 typedef int type;
221 typedef Packet8i half;
222 enum {
223 size = 16,
224 alignment = Aligned64,
225 vectorizable = true,
226 masked_load_available = false,
227 masked_store_available = false
228 };
229};
230
231template <>
232struct unpacket_traits<Packet8l> {
233 typedef int64_t type;
234 typedef Packet4l half;
235 enum {
236 size = 8,
237 alignment = Aligned64,
238 vectorizable = true,
239 masked_load_available = false,
240 masked_store_available = false
241 };
242};
243
244#ifndef EIGEN_VECTORIZE_AVX512FP16
245template <>
246struct unpacket_traits<Packet16h> {
247 typedef Eigen::half type;
248 typedef Packet8h half;
249 enum {
250 size = 16,
251 alignment = Aligned32,
252 vectorizable = true,
253 masked_load_available = false,
254 masked_store_available = false
255 };
256};
257#endif
258
259template <>
260struct unpacket_traits<Packet32s> {
261 typedef numext::int16_t type;
262 typedef Packet16s half;
263 enum {
264 size = 32,
265 alignment = Aligned64,
266 vectorizable = false,
267 };
268};
269
270template <>
271struct unpacket_traits<Packet16s> {
272 typedef numext::int16_t type;
273 typedef Packet8s half;
274 enum {
275 size = 16,
276 alignment = Aligned32,
277 vectorizable = false,
278 };
279};
280
281template <>
282struct unpacket_traits<Packet8s> {
283 typedef numext::int16_t type;
284 typedef Packet8s half;
285 enum {
286 size = 8,
287 alignment = Aligned16,
288 vectorizable = false,
289 };
290};
291
292template <>
293EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
294 return _mm512_set1_ps(from);
295}
296template <>
297EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) {
298 return _mm512_set1_pd(from);
299}
300template <>
301EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) {
302 return _mm512_set1_epi32(from);
303}
304template <>
305EIGEN_STRONG_INLINE Packet8l pset1<Packet8l>(const int64_t& from) {
306 return _mm512_set1_epi64(from);
307}
308
309template <>
310EIGEN_STRONG_INLINE Packet16f pset1frombits<Packet16f>(unsigned int from) {
311 return _mm512_castsi512_ps(_mm512_set1_epi32(from));
312}
313
314template <>
315EIGEN_STRONG_INLINE Packet8d pset1frombits<Packet8d>(const numext::uint64_t from) {
316 return _mm512_castsi512_pd(_mm512_set1_epi64(from));
317}
318
319template <>
320EIGEN_STRONG_INLINE Packet16f pzero(const Packet16f& /*a*/) {
321 return _mm512_setzero_ps();
322}
323template <>
324EIGEN_STRONG_INLINE Packet8d pzero(const Packet8d& /*a*/) {
325 return _mm512_setzero_pd();
326}
327template <>
328EIGEN_STRONG_INLINE Packet16i pzero(const Packet16i& /*a*/) {
329 return _mm512_setzero_si512();
330}
331
332template <>
333EIGEN_STRONG_INLINE Packet8l pzero(const Packet8l& /*a*/) {
334 return _mm512_setzero_si512();
335}
336
337template <>
338EIGEN_STRONG_INLINE Packet16f peven_mask(const Packet16f& /*a*/) {
339 return _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1));
340}
341template <>
342EIGEN_STRONG_INLINE Packet16i peven_mask(const Packet16i& /*a*/) {
343 return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1);
344}
345template <>
346EIGEN_STRONG_INLINE Packet8d peven_mask(const Packet8d& /*a*/) {
347 return _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1));
348}
349template <>
350EIGEN_STRONG_INLINE Packet8l peven_mask(const Packet8l& /*a*/) {
351 return _mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1);
352}
353
354template <>
355EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) {
356#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
357 // Inline asm here helps reduce some register spilling in TRSM kernels.
358 // See note in unrolls::gemm::microKernel in TrsmKernel.h
359 Packet16f ret;
360 __asm__("vbroadcastss %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
361 return ret;
362#else
363 return _mm512_broadcastss_ps(_mm_load_ps1(from));
364#endif
365}
366template <>
367EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) {
368#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
369 Packet8d ret;
370 __asm__("vbroadcastsd %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
371 return ret;
372#else
373 return _mm512_set1_pd(*from);
374#endif
375}
376
377template <>
378EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) {
379 return _mm512_add_ps(_mm512_set1_ps(a), _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f,
380 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
381}
382template <>
383EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) {
384 return _mm512_add_pd(_mm512_set1_pd(a), _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
385}
386template <>
387EIGEN_STRONG_INLINE Packet16i plset<Packet16i>(const int& a) {
388 return _mm512_add_epi32(_mm512_set1_epi32(a), _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
389}
390template <>
391EIGEN_STRONG_INLINE Packet8l plset<Packet8l>(const int64_t& a) {
392 return _mm512_add_epi64(_mm512_set1_epi64(a), _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0));
393}
394
395template <>
396EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, const Packet16f& b) {
397 return _mm512_add_ps(a, b);
398}
399template <>
400EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, const Packet8d& b) {
401 return _mm512_add_pd(a, b);
402}
403template <>
404EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(const Packet16i& a, const Packet16i& b) {
405 return _mm512_add_epi32(a, b);
406}
407template <>
408EIGEN_STRONG_INLINE Packet8l padd<Packet8l>(const Packet8l& a, const Packet8l& b) {
409 return _mm512_add_epi64(a, b);
410}
411
412template <>
413EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, const Packet16f& b, uint16_t umask) {
414 __mmask16 mask = static_cast<__mmask16>(umask);
415 return _mm512_maskz_add_ps(mask, a, b);
416}
417template <>
418EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, const Packet8d& b, uint8_t umask) {
419 __mmask8 mask = static_cast<__mmask8>(umask);
420 return _mm512_maskz_add_pd(mask, a, b);
421}
422
423template <>
424EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a, const Packet16f& b) {
425 return _mm512_sub_ps(a, b);
426}
427template <>
428EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a, const Packet8d& b) {
429 return _mm512_sub_pd(a, b);
430}
431template <>
432EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(const Packet16i& a, const Packet16i& b) {
433 return _mm512_sub_epi32(a, b);
434}
435template <>
436EIGEN_STRONG_INLINE Packet8l psub<Packet8l>(const Packet8l& a, const Packet8l& b) {
437 return _mm512_sub_epi64(a, b);
438}
439
440template <>
441EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
442 // NOTE: MSVC seems to struggle with _mm512_set1_epi32, leading to random results.
443 // The intel docs give it a relatively high latency as well, so we're probably
444 // better off with using _mm512_set_epi32 directly anyways.
445 const __m512i mask =
446 _mm512_set_epi32(0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000,
447 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000);
448 return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a), mask));
449}
450template <>
451EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) {
452 const __m512i mask =
453 _mm512_set_epi64(0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL,
454 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL);
455 return _mm512_castsi512_pd(_mm512_xor_epi64(_mm512_castpd_si512(a), mask));
456}
457template <>
458EIGEN_STRONG_INLINE Packet16i pnegate(const Packet16i& a) {
459 return _mm512_sub_epi32(_mm512_setzero_si512(), a);
460}
461template <>
462EIGEN_STRONG_INLINE Packet8l pnegate(const Packet8l& a) {
463 return _mm512_sub_epi64(_mm512_setzero_si512(), a);
464}
465
466template <>
467EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) {
468 return a;
469}
470template <>
471EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) {
472 return a;
473}
474template <>
475EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) {
476 return a;
477}
478template <>
479EIGEN_STRONG_INLINE Packet8l pconj(const Packet8l& a) {
480 return a;
481}
482
483template <>
484EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a, const Packet16f& b) {
485 return _mm512_mul_ps(a, b);
486}
487template <>
488EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a, const Packet8d& b) {
489 return _mm512_mul_pd(a, b);
490}
491template <>
492EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(const Packet16i& a, const Packet16i& b) {
493 return _mm512_mullo_epi32(a, b);
494}
495template <>
496EIGEN_STRONG_INLINE Packet8l pmul<Packet8l>(const Packet8l& a, const Packet8l& b) {
497#ifdef EIGEN_VECTORIZE_AVX512DQ
498 return _mm512_mullo_epi64(a, b);
499#else
500 return _mm512_mullox_epi64(a, b);
501#endif
502}
503
504template <>
505EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a, const Packet16f& b) {
506 return _mm512_div_ps(a, b);
507}
508
509template <>
510EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a, const Packet8d& b) {
511 return _mm512_div_pd(a, b);
512}
513
514template <>
515EIGEN_STRONG_INLINE Packet16i pdiv<Packet16i>(const Packet16i& a, const Packet16i& b) {
516 Packet8i q_lo = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 0), _mm512_extracti64x4_epi64(b, 0));
517 Packet8i q_hi = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 1), _mm512_extracti64x4_epi64(b, 1));
518 return _mm512_inserti64x4(_mm512_castsi256_si512(q_lo), q_hi, 1);
519}
520
521#ifdef EIGEN_VECTORIZE_FMA
522template <>
523EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
524 return _mm512_fmadd_ps(a, b, c);
525}
526template <>
527EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
528 return _mm512_fmadd_pd(a, b, c);
529}
530
531template <>
532EIGEN_STRONG_INLINE Packet16f pmsub(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
533 return _mm512_fmsub_ps(a, b, c);
534}
535template <>
536EIGEN_STRONG_INLINE Packet8d pmsub(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
537 return _mm512_fmsub_pd(a, b, c);
538}
539
540template <>
541EIGEN_STRONG_INLINE Packet16f pnmadd(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
542 return _mm512_fnmadd_ps(a, b, c);
543}
544template <>
545EIGEN_STRONG_INLINE Packet8d pnmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
546 return _mm512_fnmadd_pd(a, b, c);
547}
548
549template <>
550EIGEN_STRONG_INLINE Packet16f pnmsub(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
551 return _mm512_fnmsub_ps(a, b, c);
552}
553template <>
554EIGEN_STRONG_INLINE Packet8d pnmsub(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
555 return _mm512_fnmsub_pd(a, b, c);
556}
557#endif
558
559template <>
560EIGEN_DEVICE_FUNC inline Packet16f pselect(const Packet16f& mask, const Packet16f& a, const Packet16f& b) {
561 __mmask16 mask16 = _mm512_cmpeq_epi32_mask(_mm512_castps_si512(mask), _mm512_setzero_epi32());
562 return _mm512_mask_blend_ps(mask16, a, b);
563}
564
565template <>
566EIGEN_DEVICE_FUNC inline Packet16i pselect(const Packet16i& mask, const Packet16i& a, const Packet16i& b) {
567 __mmask16 mask16 = _mm512_cmpeq_epi32_mask(mask, _mm512_setzero_epi32());
568 return _mm512_mask_blend_epi32(mask16, a, b);
569}
570
571template <>
572EIGEN_DEVICE_FUNC inline Packet8l pselect(const Packet8l& mask, const Packet8l& a, const Packet8l& b) {
573 __mmask8 mask8 = _mm512_cmpeq_epi64_mask(mask, _mm512_setzero_si512());
574 return _mm512_mask_blend_epi64(mask8, a, b);
575}
576
577template <>
578EIGEN_DEVICE_FUNC inline Packet8d pselect(const Packet8d& mask, const Packet8d& a, const Packet8d& b) {
579 __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
580 return _mm512_mask_blend_pd(mask8, a, b);
581}
582
583template <>
584EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a, const Packet16f& b) {
585 // Arguments are reversed to match NaN propagation behavior of std::min.
586 return _mm512_min_ps(b, a);
587}
588template <>
589EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a, const Packet8d& b) {
590 // Arguments are reversed to match NaN propagation behavior of std::min.
591 return _mm512_min_pd(b, a);
592}
593template <>
594EIGEN_STRONG_INLINE Packet16i pmin<Packet16i>(const Packet16i& a, const Packet16i& b) {
595 return _mm512_min_epi32(b, a);
596}
597template <>
598EIGEN_STRONG_INLINE Packet8l pmin<Packet8l>(const Packet8l& a, const Packet8l& b) {
599 return _mm512_min_epi64(b, a);
600}
601
602template <>
603EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a, const Packet16f& b) {
604 // Arguments are reversed to match NaN propagation behavior of std::max.
605 return _mm512_max_ps(b, a);
606}
607template <>
608EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a, const Packet8d& b) {
609 // Arguments are reversed to match NaN propagation behavior of std::max.
610 return _mm512_max_pd(b, a);
611}
612template <>
613EIGEN_STRONG_INLINE Packet16i pmax<Packet16i>(const Packet16i& a, const Packet16i& b) {
614 return _mm512_max_epi32(b, a);
615}
616template <>
617EIGEN_STRONG_INLINE Packet8l pmax<Packet8l>(const Packet8l& a, const Packet8l& b) {
618 return _mm512_max_epi64(b, a);
619}
620
621// Add specializations for min/max with prescribed NaN propagation.
622template <>
623EIGEN_STRONG_INLINE Packet16f pmin<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
624 return pminmax_propagate_numbers(a, b, pmin<Packet16f>);
625}
626template <>
627EIGEN_STRONG_INLINE Packet8d pmin<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
628 return pminmax_propagate_numbers(a, b, pmin<Packet8d>);
629}
630template <>
631EIGEN_STRONG_INLINE Packet16f pmax<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
632 return pminmax_propagate_numbers(a, b, pmax<Packet16f>);
633}
634template <>
635EIGEN_STRONG_INLINE Packet8d pmax<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
636 return pminmax_propagate_numbers(a, b, pmax<Packet8d>);
637}
638template <>
639EIGEN_STRONG_INLINE Packet16f pmin<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
640 return pminmax_propagate_nan(a, b, pmin<Packet16f>);
641}
642template <>
643EIGEN_STRONG_INLINE Packet8d pmin<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
644 return pminmax_propagate_nan(a, b, pmin<Packet8d>);
645}
646template <>
647EIGEN_STRONG_INLINE Packet16f pmax<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
648 return pminmax_propagate_nan(a, b, pmax<Packet16f>);
649}
650template <>
651EIGEN_STRONG_INLINE Packet8d pmax<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
652 return pminmax_propagate_nan(a, b, pmax<Packet8d>);
653}
654
655#ifdef EIGEN_VECTORIZE_AVX512DQ
656template <int I_>
657EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) {
658 return _mm512_extractf32x8_ps(x, I_);
659}
660template <int I_>
661EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) {
662 return _mm512_extractf64x2_pd(x, I_);
663}
664EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) {
665 return _mm512_insertf32x8(_mm512_castps256_ps512(a), b, 1);
666}
667EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) {
668 return _mm512_inserti32x8(_mm512_castsi256_si512(a), b, 1);
669}
670#else
671// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
672template <int I_>
673EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) {
674 return _mm256_castsi256_ps(_mm512_extracti64x4_epi64(_mm512_castps_si512(x), I_));
675}
676
677// AVX512F does not define _mm512_extractf64x2_pd to extract _m128 from _m512
678template <int I_>
679EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) {
680 return _mm_castsi128_pd(_mm512_extracti32x4_epi32(_mm512_castpd_si512(x), I_));
681}
682
683EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) {
684 return _mm512_castsi512_ps(
685 _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), _mm256_castps_si256(b), 1));
686}
687EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) {
688 return _mm512_inserti64x4(_mm512_castsi256_si512(a), b, 1);
689}
690#endif
691
692// Helper function for bit packing snippet of low precision comparison.
693// It packs the flags from 32x16 to 16x16.
694EIGEN_STRONG_INLINE __m256i Pack32To16(Packet16f rf) {
695 // Split data into small pieces and handle with AVX instructions
696 // to guarantee internal order of vector.
697 // Operation:
698 // dst[15:0] := Saturate16(rf[31:0])
699 // dst[31:16] := Saturate16(rf[63:32])
700 // ...
701 // dst[255:240] := Saturate16(rf[255:224])
702 __m256i lo = _mm256_castps_si256(extract256<0>(rf));
703 __m256i hi = _mm256_castps_si256(extract256<1>(rf));
704 __m128i result_lo = _mm_packs_epi32(_mm256_extractf128_si256(lo, 0), _mm256_extractf128_si256(lo, 1));
705 __m128i result_hi = _mm_packs_epi32(_mm256_extractf128_si256(hi, 0), _mm256_extractf128_si256(hi, 1));
706 return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1);
707}
708
709template <>
710EIGEN_STRONG_INLINE Packet16f pisnan(const Packet16f& a) {
711 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_UNORD_Q);
712 return _mm512_castsi512_ps(_mm512_maskz_set1_epi32(mask, int32_t(-1)));
713}
714
715template <>
716EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) {
717 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ);
718 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
719}
720template <>
721EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) {
722 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ);
723 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
724}
725
726template <>
727EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) {
728 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ);
729 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
730}
731
732template <>
733EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) {
734 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ);
735 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
736}
737
738template <>
739EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) {
740 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_EQ);
741 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
742}
743template <>
744EIGEN_STRONG_INLINE Packet16i pcmp_le(const Packet16i& a, const Packet16i& b) {
745 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE);
746 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
747}
748template <>
749EIGEN_STRONG_INLINE Packet16i pcmp_lt(const Packet16i& a, const Packet16i& b) {
750 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT);
751 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
752}
753
754template <>
755EIGEN_STRONG_INLINE Packet8l pcmp_eq(const Packet8l& a, const Packet8l& b) {
756 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_EQ);
757 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
758}
759template <>
760EIGEN_STRONG_INLINE Packet8l pcmp_le(const Packet8l& a, const Packet8l& b) {
761 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LE);
762 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
763}
764template <>
765EIGEN_STRONG_INLINE Packet8l pcmp_lt(const Packet8l& a, const Packet8l& b) {
766 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LT);
767 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
768}
769
770template <>
771EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) {
772 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ);
773 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
774}
775template <>
776EIGEN_STRONG_INLINE Packet8d pcmp_le(const Packet8d& a, const Packet8d& b) {
777 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ);
778 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
779}
780template <>
781EIGEN_STRONG_INLINE Packet8d pcmp_lt(const Packet8d& a, const Packet8d& b) {
782 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ);
783 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
784}
785template <>
786EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(const Packet8d& a, const Packet8d& b) {
787 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ);
788 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
789}
790
791template <>
792EIGEN_STRONG_INLINE Packet16f print<Packet16f>(const Packet16f& a) {
793 return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION);
794}
795template <>
796EIGEN_STRONG_INLINE Packet8d print<Packet8d>(const Packet8d& a) {
797 return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION);
798}
799
800template <>
801EIGEN_STRONG_INLINE Packet16f pceil<Packet16f>(const Packet16f& a) {
802 return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF);
803}
804template <>
805EIGEN_STRONG_INLINE Packet8d pceil<Packet8d>(const Packet8d& a) {
806 return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF);
807}
808
809template <>
810EIGEN_STRONG_INLINE Packet16f pfloor<Packet16f>(const Packet16f& a) {
811 return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF);
812}
813template <>
814EIGEN_STRONG_INLINE Packet8d pfloor<Packet8d>(const Packet8d& a) {
815 return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF);
816}
817
818template <>
819EIGEN_STRONG_INLINE Packet16f ptrunc<Packet16f>(const Packet16f& a) {
820 return _mm512_roundscale_ps(a, _MM_FROUND_TO_ZERO);
821}
822template <>
823EIGEN_STRONG_INLINE Packet8d ptrunc<Packet8d>(const Packet8d& a) {
824 return _mm512_roundscale_pd(a, _MM_FROUND_TO_ZERO);
825}
826
827template <>
828EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(const Packet16i& /*a*/) {
829 return _mm512_set1_epi32(int32_t(-1));
830}
831
832template <>
833EIGEN_STRONG_INLINE Packet8l ptrue<Packet8l>(const Packet8l& /*a*/) {
834 return _mm512_set1_epi64(int64_t(-1));
835}
836
837template <>
838EIGEN_STRONG_INLINE Packet16f ptrue<Packet16f>(const Packet16f& a) {
839 return _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a)));
840}
841
842template <>
843EIGEN_STRONG_INLINE Packet8d ptrue<Packet8d>(const Packet8d& a) {
844 return _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a)));
845}
846
847template <>
848EIGEN_STRONG_INLINE Packet16i pand<Packet16i>(const Packet16i& a, const Packet16i& b) {
849 return _mm512_and_si512(a, b);
850}
851
852template <>
853EIGEN_STRONG_INLINE Packet8l pand<Packet8l>(const Packet8l& a, const Packet8l& b) {
854 return _mm512_and_si512(a, b);
855}
856
857template <>
858EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a, const Packet16f& b) {
859#ifdef EIGEN_VECTORIZE_AVX512DQ
860 return _mm512_and_ps(a, b);
861#else
862 return _mm512_castsi512_ps(pand(_mm512_castps_si512(a), _mm512_castps_si512(b)));
863#endif
864}
865template <>
866EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a, const Packet8d& b) {
867#ifdef EIGEN_VECTORIZE_AVX512DQ
868 return _mm512_and_pd(a, b);
869#else
870 Packet8d res = _mm512_undefined_pd();
871 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
872 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
873 res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
874
875 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
876 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
877 return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
878#endif
879}
880
881template <>
882EIGEN_STRONG_INLINE Packet16i por<Packet16i>(const Packet16i& a, const Packet16i& b) {
883 return _mm512_or_si512(a, b);
884}
885
886template <>
887EIGEN_STRONG_INLINE Packet8l por<Packet8l>(const Packet8l& a, const Packet8l& b) {
888 return _mm512_or_si512(a, b);
889}
890
891template <>
892EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a, const Packet16f& b) {
893#ifdef EIGEN_VECTORIZE_AVX512DQ
894 return _mm512_or_ps(a, b);
895#else
896 return _mm512_castsi512_ps(por(_mm512_castps_si512(a), _mm512_castps_si512(b)));
897#endif
898}
899
900template <>
901EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a, const Packet8d& b) {
902#ifdef EIGEN_VECTORIZE_AVX512DQ
903 return _mm512_or_pd(a, b);
904#else
905 return _mm512_castsi512_pd(por(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
906#endif
907}
908
909template <>
910EIGEN_STRONG_INLINE Packet16i pxor<Packet16i>(const Packet16i& a, const Packet16i& b) {
911 return _mm512_xor_si512(a, b);
912}
913
914template <>
915EIGEN_STRONG_INLINE Packet8l pxor<Packet8l>(const Packet8l& a, const Packet8l& b) {
916 return _mm512_xor_si512(a, b);
917}
918
919template <>
920EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a, const Packet16f& b) {
921#ifdef EIGEN_VECTORIZE_AVX512DQ
922 return _mm512_xor_ps(a, b);
923#else
924 return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a), _mm512_castps_si512(b)));
925#endif
926}
927
928template <>
929EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a, const Packet8d& b) {
930#ifdef EIGEN_VECTORIZE_AVX512DQ
931 return _mm512_xor_pd(a, b);
932#else
933 return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
934#endif
935}
936
937template <>
938EIGEN_STRONG_INLINE Packet16i pandnot<Packet16i>(const Packet16i& a, const Packet16i& b) {
939 return _mm512_andnot_si512(b, a);
940}
941
942template <>
943EIGEN_STRONG_INLINE Packet8l pandnot<Packet8l>(const Packet8l& a, const Packet8l& b) {
944 return _mm512_andnot_si512(b, a);
945}
946
947template <>
948EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a, const Packet16f& b) {
949#ifdef EIGEN_VECTORIZE_AVX512DQ
950 return _mm512_andnot_ps(b, a);
951#else
952 return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a), _mm512_castps_si512(b)));
953#endif
954}
955template <>
956EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a, const Packet8d& b) {
957#ifdef EIGEN_VECTORIZE_AVX512DQ
958 return _mm512_andnot_pd(b, a);
959#else
960 return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
961#endif
962}
963
964template <>
965EIGEN_STRONG_INLINE Packet16f pround<Packet16f>(const Packet16f& a) {
966 // Work-around for default std::round rounding mode.
967 const Packet16f mask = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x80000000u));
968 const Packet16f prev0dot5 = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
969 return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
970}
971template <>
972EIGEN_STRONG_INLINE Packet8d pround<Packet8d>(const Packet8d& a) {
973 // Work-around for default std::round rounding mode.
974 const Packet8d mask = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
975 const Packet8d prev0dot5 = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
976 return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
977}
978
979template <int N>
980EIGEN_STRONG_INLINE Packet16i parithmetic_shift_right(Packet16i a) {
981 return _mm512_srai_epi32(a, N);
982}
983
984template <int N>
985EIGEN_STRONG_INLINE Packet16i plogical_shift_right(Packet16i a) {
986 return _mm512_srli_epi32(a, N);
987}
988
989template <int N>
990EIGEN_STRONG_INLINE Packet16i plogical_shift_left(Packet16i a) {
991 return _mm512_slli_epi32(a, N);
992}
993
994template <int N>
995EIGEN_STRONG_INLINE Packet8l parithmetic_shift_right(Packet8l a) {
996 return _mm512_srai_epi64(a, N);
997}
998
999template <int N>
1000EIGEN_STRONG_INLINE Packet8l plogical_shift_right(Packet8l a) {
1001 return _mm512_srli_epi64(a, N);
1002}
1003
1004template <int N>
1005EIGEN_STRONG_INLINE Packet8l plogical_shift_left(Packet8l a) {
1006 return _mm512_slli_epi64(a, N);
1007}
1008
1009template <>
1010EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
1011 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
1012}
1013template <>
1014EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) {
1015 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
1016}
1017template <>
1018EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) {
1019 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
1020}
1021template <>
1022EIGEN_STRONG_INLINE Packet8l pload<Packet8l>(const int64_t* from) {
1023 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
1024}
1025
1026template <>
1027EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
1028 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
1029}
1030template <>
1031EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
1032 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
1033}
1034template <>
1035EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
1036 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi32(from);
1037}
1038template <>
1039EIGEN_STRONG_INLINE Packet8l ploadu<Packet8l>(const int64_t* from) {
1040 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_epi64(from);
1041}
1042
1043template <>
1044EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from, uint16_t umask) {
1045 __mmask16 mask = static_cast<__mmask16>(umask);
1046 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from);
1047}
1048template <>
1049EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from, uint8_t umask) {
1050 __mmask8 mask = static_cast<__mmask8>(umask);
1051 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_pd(mask, from);
1052}
1053
1054// Loads 8 floats from memory a returns the packet
1055// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
1056template <>
1057EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) {
1058 // an unaligned load is required here as there is no requirement
1059 // on the alignment of input pointer 'from'
1060 __m256i low_half = _mm256_castps_si256(_mm256_loadu_ps(from));
1061 __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1062 __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1063 return pairs;
1064}
1065
1066// Loads 4 doubles from memory a returns the packet {a0, a0, a1, a1, a2, a2, a3,
1067// a3}
1068template <>
1069EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
1070 Packet8d tmp = _mm512_castpd256_pd512(ploadu<Packet4d>(from));
1071 const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1072 return _mm512_permutexvar_pd(scatter_mask, tmp);
1073}
1074
1075// Loads 4 int64_t from memory a returns the packet {a0, a0, a1, a1, a2, a2, a3,
1076// a3}
1077template <>
1078EIGEN_STRONG_INLINE Packet8l ploaddup<Packet8l>(const int64_t* from) {
1079 Packet8l tmp = _mm512_castsi256_si512(ploadu<Packet4l>(from));
1080 const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1081 return _mm512_permutexvar_epi64(scatter_mask, tmp);
1082}
1083
1084// Loads 8 integers from memory and returns the packet
1085// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
1086template <>
1087EIGEN_STRONG_INLINE Packet16i ploaddup<Packet16i>(const int* from) {
1088 __m256i low_half = _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1089 __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1090 __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1091 return _mm512_castps_si512(pairs);
1092}
1093
1094// Loads 4 floats from memory a returns the packet
1095// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
1096template <>
1097EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) {
1098 Packet16f tmp = _mm512_castps128_ps512(ploadu<Packet4f>(from));
1099 const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1100 return _mm512_permutexvar_ps(scatter_mask, tmp);
1101}
1102
1103// Loads 2 doubles from memory a returns the packet
1104// {a0, a0 a0, a0, a1, a1, a1, a1}
1105template <>
1106EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) {
1107 __m256d lane0 = _mm256_set1_pd(*from);
1108 __m256d lane1 = _mm256_set1_pd(*(from + 1));
1109 __m512d tmp = _mm512_undefined_pd();
1110 tmp = _mm512_insertf64x4(tmp, lane0, 0);
1111 return _mm512_insertf64x4(tmp, lane1, 1);
1112}
1113
1114// Loads 2 int64_t from memory a returns the packet
1115// {a0, a0 a0, a0, a1, a1, a1, a1}
1116template <>
1117EIGEN_STRONG_INLINE Packet8l ploadquad<Packet8l>(const int64_t* from) {
1118 __m256i lane0 = _mm256_set1_epi64x(*from);
1119 __m256i lane1 = _mm256_set1_epi64x(*(from + 1));
1120 __m512i tmp = _mm512_undefined_epi32();
1121 tmp = _mm512_inserti64x4(tmp, lane0, 0);
1122 return _mm512_inserti64x4(tmp, lane1, 1);
1123}
1124
1125// Loads 4 integers from memory and returns the packet
1126// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
1127template <>
1128EIGEN_STRONG_INLINE Packet16i ploadquad<Packet16i>(const int* from) {
1129 Packet16i tmp = _mm512_castsi128_si512(ploadu<Packet4i>(from));
1130 const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1131 return _mm512_permutexvar_epi32(scatter_mask, tmp);
1132}
1133
1134template <>
1135EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
1136 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
1137}
1138template <>
1139EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) {
1140 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
1141}
1142template <>
1143EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) {
1144 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi32(to, from);
1145}
1146template <>
1147EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet8l& from) {
1148 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi64(to, from);
1149}
1150
1151template <>
1152EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
1153 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
1154}
1155template <>
1156EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
1157 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
1158}
1159template <>
1160EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
1161 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi32(to, from);
1162}
1163template <>
1164EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet8l& from) {
1165 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_epi64(to, from);
1166}
1167template <>
1168EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from, uint16_t umask) {
1169 __mmask16 mask = static_cast<__mmask16>(umask);
1170 EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from);
1171}
1172template <>
1173EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from, uint8_t umask) {
1174 __mmask8 mask = static_cast<__mmask8>(umask);
1175 EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from);
1176}
1177
1178template <typename Scalar, typename Packet>
1179EIGEN_DEVICE_FUNC inline Packet pgather(const Packet& src, const Scalar* from, Index stride,
1180 typename unpacket_traits<Packet>::mask_t umask);
1181template <>
1182EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const Packet16f& src, const float* from, Index stride,
1183 uint16_t umask) {
1184 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1185 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1186 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1187 __mmask16 mask = static_cast<__mmask16>(umask);
1188
1189 return _mm512_mask_i32gather_ps(src, mask, indices, from, 4);
1190}
1191template <>
1192EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const Packet8d& src, const double* from, Index stride,
1193 uint8_t umask) {
1194 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1195 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1196 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1197 __mmask8 mask = static_cast<__mmask8>(umask);
1198
1199 return _mm512_mask_i32gather_pd(src, mask, indices, from, 8);
1200}
1201
1202template <>
1203EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from, Index stride) {
1204 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1205 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1206 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1207
1208 return _mm512_i32gather_ps(indices, from, 4);
1209}
1210template <>
1211EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from, Index stride) {
1212 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1213 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1214 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1215
1216 return _mm512_i32gather_pd(indices, from, 8);
1217}
1218template <>
1219EIGEN_DEVICE_FUNC inline Packet8l pgather<int64_t, Packet8l>(const int64_t* from, Index stride) {
1220 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1221 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1222 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1223
1224 return _mm512_i32gather_epi64(indices, from, 8);
1225}
1226template <>
1227EIGEN_DEVICE_FUNC inline Packet16i pgather<int, Packet16i>(const int* from, Index stride) {
1228 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1229 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1230 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1231 return _mm512_i32gather_epi32(indices, from, 4);
1232}
1233
1234template <typename Scalar, typename Packet>
1235EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index stride,
1236 typename unpacket_traits<Packet>::mask_t umask);
1237template <>
1238EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride,
1239 uint16_t umask) {
1240 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1241 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1242 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1243 __mmask16 mask = static_cast<__mmask16>(umask);
1244 _mm512_mask_i32scatter_ps(to, mask, indices, from, 4);
1245}
1246template <>
1247EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride,
1248 uint8_t umask) {
1249 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1250 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1251 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1252 __mmask8 mask = static_cast<__mmask8>(umask);
1253 _mm512_mask_i32scatter_pd(to, mask, indices, from, 8);
1254}
1255template <>
1256EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride) {
1257 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1258 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1259 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1260 _mm512_i32scatter_ps(to, indices, from, 4);
1261}
1262template <>
1263EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride) {
1264 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1265 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1266 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1267 _mm512_i32scatter_pd(to, indices, from, 8);
1268}
1269template <>
1270EIGEN_DEVICE_FUNC inline void pscatter<int64_t, Packet8l>(int64_t* to, const Packet8l& from, Index stride) {
1271 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1272 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1273 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1274 _mm512_i32scatter_epi64(to, indices, from, 8);
1275}
1276template <>
1277EIGEN_DEVICE_FUNC inline void pscatter<int, Packet16i>(int* to, const Packet16i& from, Index stride) {
1278 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1279 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1280 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1281 _mm512_i32scatter_epi32(to, indices, from, 4);
1282}
1283
1284template <>
1285EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) {
1286 Packet16f pa = pset1<Packet16f>(a);
1287 pstore(to, pa);
1288}
1289template <>
1290EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) {
1291 Packet8d pa = pset1<Packet8d>(a);
1292 pstore(to, pa);
1293}
1294template <>
1295EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) {
1296 Packet16i pa = pset1<Packet16i>(a);
1297 pstore(to, pa);
1298}
1299template <>
1300EIGEN_STRONG_INLINE void pstore1<Packet8l>(int64_t* to, const int64_t& a) {
1301 Packet8l pa = pset1<Packet8l>(a);
1302 pstore(to, pa);
1303}
1304
1305template <>
1306EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1307 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1308}
1309template <>
1310EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1311 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1312}
1313template <>
1314EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
1315 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1316}
1317
1318template <>
1319EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) {
1320 return _mm512_cvtss_f32(a);
1321}
1322template <>
1323EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) {
1324 return _mm512_cvtsd_f64(a);
1325}
1326template <>
1327EIGEN_STRONG_INLINE int64_t pfirst<Packet8l>(const Packet8l& a) {
1328 int64_t x = _mm_extract_epi64_0(_mm512_extracti32x4_epi32(a, 0));
1329 return x;
1330}
1331template <>
1332EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) {
1333#if EIGEN_GNUC_STRICT_LESS_THAN(11, 0, 0)
1334 return _mm_cvtsi128_si32(_mm512_castsi512_si128(a));
1335#else
1336 return _mm512_cvtsi512_si32(a);
1337#endif
1338}
1339
1340template <>
1341EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) {
1342 return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1343}
1344
1345template <>
1346EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a) {
1347 return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
1348}
1349
1350template <>
1351EIGEN_STRONG_INLINE Packet16i preverse(const Packet16i& a) {
1352 return _mm512_permutexvar_epi32(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1353}
1354
1355template <>
1356EIGEN_STRONG_INLINE Packet8l preverse(const Packet8l& a) {
1357 return _mm512_permutexvar_epi64(_mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7), a);
1358}
1359
1360template <>
1361EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) {
1362 // _mm512_abs_ps intrinsic not found, so hack around it
1363 return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff)));
1364}
1365template <>
1366EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) {
1367 // _mm512_abs_ps intrinsic not found, so hack around it
1368 return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), _mm512_set1_epi64(0x7fffffffffffffff)));
1369}
1370template <>
1371EIGEN_STRONG_INLINE Packet16i pabs(const Packet16i& a) {
1372 return _mm512_abs_epi32(a);
1373}
1374template <>
1375EIGEN_STRONG_INLINE Packet8l pabs(const Packet8l& a) {
1376 return _mm512_abs_epi64(a);
1377}
1378
1379#ifndef EIGEN_VECTORIZE_AVX512FP16
1380template <>
1381EIGEN_STRONG_INLINE Packet16h psignbit(const Packet16h& a) {
1382 return _mm256_srai_epi16(a, 15);
1383}
1384#endif // EIGEN_VECTORIZE_AVX512FP16
1385
1386template <>
1387EIGEN_STRONG_INLINE Packet16bf psignbit(const Packet16bf& a) {
1388 return _mm256_srai_epi16(a, 15);
1389}
1390template <>
1391EIGEN_STRONG_INLINE Packet16f psignbit(const Packet16f& a) {
1392 return _mm512_castsi512_ps(_mm512_srai_epi32(_mm512_castps_si512(a), 31));
1393}
1394template <>
1395EIGEN_STRONG_INLINE Packet8d psignbit(const Packet8d& a) {
1396 return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), 63));
1397}
1398
1399template <>
1400EIGEN_STRONG_INLINE Packet16f pfrexp<Packet16f>(const Packet16f& a, Packet16f& exponent) {
1401 return pfrexp_generic(a, exponent);
1402}
1403
1404// Extract exponent without existence of Packet8l.
1405template <>
1406EIGEN_STRONG_INLINE Packet8d pfrexp_generic_get_biased_exponent(const Packet8d& a) {
1407 const Packet8d cst_exp_mask = pset1frombits<Packet8d>(static_cast<uint64_t>(0x7ff0000000000000ull));
1408#ifdef EIGEN_VECTORIZE_AVX512DQ
1409 return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52));
1410#else
1411 return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)));
1412#endif
1413}
1414
1415template <>
1416EIGEN_STRONG_INLINE Packet8d pfrexp<Packet8d>(const Packet8d& a, Packet8d& exponent) {
1417 return pfrexp_generic(a, exponent);
1418}
1419
1420template <>
1421EIGEN_STRONG_INLINE Packet16f pldexp<Packet16f>(const Packet16f& a, const Packet16f& exponent) {
1422 return pldexp_generic(a, exponent);
1423}
1424
1425template <>
1426EIGEN_STRONG_INLINE Packet8d pldexp<Packet8d>(const Packet8d& a, const Packet8d& exponent) {
1427 // Clamp exponent to [-2099, 2099]
1428 const Packet8d max_exponent = pset1<Packet8d>(2099.0);
1429 const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
1430
1431 // Split 2^e into four factors and multiply.
1432 const Packet8i bias = pset1<Packet8i>(1023);
1433 Packet8i b = parithmetic_shift_right<2>(e); // floor(e/4)
1434
1435 // 2^b
1436 const Packet8i permute_idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
1437 Packet8i hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
1438 Packet8i lo = _mm256_slli_epi64(hi, 52);
1439 hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
1440 Packet8d c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
1441 Packet8d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b)
1442
1443 // 2^(e - 3b)
1444 b = psub(psub(psub(e, b), b), b); // e - 3b
1445 hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
1446 lo = _mm256_slli_epi64(hi, 52);
1447 hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
1448 c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
1449 out = pmul(out, c); // a * 2^e
1450 return out;
1451}
1452
1453#ifdef EIGEN_VECTORIZE_AVX512DQ
1454// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
1455#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1456 __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \
1457 __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1)
1458
1459// AVX512F does not define _mm512_extracti32x8_epi32 to extract _m256i from _m512i
1460#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1461 __m256i OUTPUT##_0 = _mm512_extracti32x8_epi32(INPUT, 0); \
1462 __m256i OUTPUT##_1 = _mm512_extracti32x8_epi32(INPUT, 1)
1463#else
1464#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1465 __m256 OUTPUT##_0 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
1466 _mm512_extractf32x4_ps(INPUT, 1), 1); \
1467 __m256 OUTPUT##_1 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
1468 _mm512_extractf32x4_ps(INPUT, 3), 1)
1469
1470#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1471 __m256i OUTPUT##_0 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 0)), \
1472 _mm512_extracti32x4_epi32(INPUT, 1), 1); \
1473 __m256i OUTPUT##_1 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 2)), \
1474 _mm512_extracti32x4_epi32(INPUT, 3), 1)
1475#endif
1476
1477#ifdef EIGEN_VECTORIZE_AVX512DQ
1478#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1479 OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1);
1480
1481#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1482 OUTPUT = _mm512_inserti32x8(_mm512_castsi256_si512(INPUTA), INPUTB, 1);
1483#else
1484#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1485 OUTPUT = _mm512_undefined_ps(); \
1486 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
1487 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
1488 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
1489 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
1490
1491#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1492 OUTPUT = _mm512_undefined_epi32(); \
1493 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 0), 0); \
1494 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 1), 1); \
1495 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 0), 2); \
1496 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 1), 3);
1497#endif
1498
1499template <>
1500EIGEN_STRONG_INLINE Packet8f predux_half_dowto4<Packet16f>(const Packet16f& a) {
1501#ifdef EIGEN_VECTORIZE_AVX512DQ
1502 __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
1503 __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
1504 return _mm256_add_ps(lane0, lane1);
1505#else
1506 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1507 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1508 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1509 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1510 __m128 sum0 = _mm_add_ps(lane0, lane2);
1511 __m128 sum1 = _mm_add_ps(lane1, lane3);
1512 return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
1513#endif
1514}
1515template <>
1516EIGEN_STRONG_INLINE Packet4d predux_half_dowto4<Packet8d>(const Packet8d& a) {
1517 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1518 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1519 return _mm256_add_pd(lane0, lane1);
1520}
1521template <>
1522EIGEN_STRONG_INLINE Packet8i predux_half_dowto4<Packet16i>(const Packet16i& a) {
1523#ifdef EIGEN_VECTORIZE_AVX512DQ
1524 __m256i lane0 = _mm512_extracti32x8_epi32(a, 0);
1525 __m256i lane1 = _mm512_extracti32x8_epi32(a, 1);
1526 return _mm256_add_epi32(lane0, lane1);
1527#else
1528 __m128i lane0 = _mm512_extracti32x4_epi32(a, 0);
1529 __m128i lane1 = _mm512_extracti32x4_epi32(a, 1);
1530 __m128i lane2 = _mm512_extracti32x4_epi32(a, 2);
1531 __m128i lane3 = _mm512_extracti32x4_epi32(a, 3);
1532 __m128i sum0 = _mm_add_epi32(lane0, lane2);
1533 __m128i sum1 = _mm_add_epi32(lane1, lane3);
1534 return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1);
1535#endif
1536}
1537
1538template <>
1539EIGEN_STRONG_INLINE Packet4l predux_half_dowto4<Packet8l>(const Packet8l& a) {
1540 __m256i lane0 = _mm512_extracti64x4_epi64(a, 0);
1541 __m256i lane1 = _mm512_extracti64x4_epi64(a, 1);
1542 return _mm256_add_epi64(lane0, lane1);
1543}
1544
1545#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1546 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1547
1548EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1549 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1550 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1551 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1552 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1553 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1554 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1555 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1556 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1557 __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1558 __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1559 __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1560 __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1561 __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1562 __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1563 __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1564 __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1565 __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1566 __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1567 __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1568 __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1569 __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1570 __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1571 __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1572 __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1573 __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1574 __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1575 __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1576 __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1577 __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1578 __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1579 __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1580 __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1581
1582 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1583 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1584 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1585 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1586 EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1587 EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1588 EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1589 EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1590 EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1591 EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1592 EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1593 EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1594 EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1595 EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1596 EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1597 EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1598
1599 PacketBlock<Packet8f, 32> tmp;
1600
1601 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1602 tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1603 tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1604 tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1605 tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1606 tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1607 tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1608 tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1609
1610 tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1611 tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1612 tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1613 tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1614 tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1615 tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1616 tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1617 tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1618
1619 // Second set of _m256 outputs
1620 tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1621 tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1622 tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1623 tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1624 tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1625 tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1626 tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1627 tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1628
1629 tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1630 tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1631 tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1632 tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1633 tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1634 tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1635 tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1636 tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1637
1638 // Pack them into the output
1639 PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1640 PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1641 PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1642 PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1643
1644 PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1645 PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1646 PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1647 PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1648
1649 PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1650 PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1651 PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1652 PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1653
1654 PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1655 PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1656 PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1657 PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1658}
1659#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1660 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1661
1662EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 8>& kernel) {
1663 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1664 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1665 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1666 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1667 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1668 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1669 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1670 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1671
1672 kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1673 kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1674 kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1675 kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1676 kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1677 kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1678 kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1679 kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1680
1681 T0 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0x44);
1682 T1 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0xee);
1683 T2 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0x44);
1684 T3 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0xee);
1685 T4 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0x44);
1686 T5 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0xee);
1687 T6 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0x44);
1688 T7 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0xee);
1689
1690 kernel.packet[0] = _mm512_shuffle_f32x4(T0, T2, 0x88);
1691 kernel.packet[2] = _mm512_shuffle_f32x4(T0, T2, 0xdd);
1692 kernel.packet[1] = _mm512_shuffle_f32x4(T4, T6, 0x88);
1693 kernel.packet[3] = _mm512_shuffle_f32x4(T4, T6, 0xdd);
1694 kernel.packet[4] = _mm512_shuffle_f32x4(T1, T3, 0x88);
1695 kernel.packet[6] = _mm512_shuffle_f32x4(T1, T3, 0xdd);
1696 kernel.packet[5] = _mm512_shuffle_f32x4(T5, T7, 0x88);
1697 kernel.packet[7] = _mm512_shuffle_f32x4(T5, T7, 0xdd);
1698}
1699
1700EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1701 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1702 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1703 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1704 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1705
1706 __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1707 __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1708 __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1709 __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1710
1711 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1712 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1713 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1714 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1715
1716 PacketBlock<Packet8f, 8> tmp;
1717
1718 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1719 tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1720 tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1721 tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1722
1723 tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1724 tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1725 tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1726 tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1727
1728 PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1729 PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1730 PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1731 PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1732}
1733
1734#define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1735 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1736 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1737
1738#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1739 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1740 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1741
1742#define PACK_OUTPUT_L(OUTPUT, INPUT, INDEX, STRIDE) \
1743 OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1744 OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1745
1746EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1747 __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1748 __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1749 __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1750 __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1751
1752 PacketBlock<Packet4d, 8> tmp;
1753
1754 tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x20);
1755 tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x20);
1756 tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x31);
1757 tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x31);
1758
1759 tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x20);
1760 tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x20);
1761 tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x31);
1762 tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x31);
1763
1764 PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1765 PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1766 PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1767 PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1768}
1769
1770EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1771 __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1772 __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1773 __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
1774 __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
1775 __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
1776 __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
1777 __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
1778 __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
1779
1780 kernel.packet[0] = _mm512_permutex_pd(T2, 0x4E);
1781 kernel.packet[0] = _mm512_mask_blend_pd(0xCC, T0, kernel.packet[0]);
1782 kernel.packet[2] = _mm512_permutex_pd(T0, 0x4E);
1783 kernel.packet[2] = _mm512_mask_blend_pd(0xCC, kernel.packet[2], T2);
1784 kernel.packet[1] = _mm512_permutex_pd(T3, 0x4E);
1785 kernel.packet[1] = _mm512_mask_blend_pd(0xCC, T1, kernel.packet[1]);
1786 kernel.packet[3] = _mm512_permutex_pd(T1, 0x4E);
1787 kernel.packet[3] = _mm512_mask_blend_pd(0xCC, kernel.packet[3], T3);
1788 kernel.packet[4] = _mm512_permutex_pd(T6, 0x4E);
1789 kernel.packet[4] = _mm512_mask_blend_pd(0xCC, T4, kernel.packet[4]);
1790 kernel.packet[6] = _mm512_permutex_pd(T4, 0x4E);
1791 kernel.packet[6] = _mm512_mask_blend_pd(0xCC, kernel.packet[6], T6);
1792 kernel.packet[5] = _mm512_permutex_pd(T7, 0x4E);
1793 kernel.packet[5] = _mm512_mask_blend_pd(0xCC, T5, kernel.packet[5]);
1794 kernel.packet[7] = _mm512_permutex_pd(T5, 0x4E);
1795 kernel.packet[7] = _mm512_mask_blend_pd(0xCC, kernel.packet[7], T7);
1796
1797 T0 = _mm512_shuffle_f64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1798 T0 = _mm512_mask_blend_pd(0xF0, kernel.packet[0], T0);
1799 T4 = _mm512_shuffle_f64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1800 T4 = _mm512_mask_blend_pd(0xF0, T4, kernel.packet[4]);
1801 T1 = _mm512_shuffle_f64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1802 T1 = _mm512_mask_blend_pd(0xF0, kernel.packet[1], T1);
1803 T5 = _mm512_shuffle_f64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1804 T5 = _mm512_mask_blend_pd(0xF0, T5, kernel.packet[5]);
1805 T2 = _mm512_shuffle_f64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1806 T2 = _mm512_mask_blend_pd(0xF0, kernel.packet[2], T2);
1807 T6 = _mm512_shuffle_f64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1808 T6 = _mm512_mask_blend_pd(0xF0, T6, kernel.packet[6]);
1809 T3 = _mm512_shuffle_f64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1810 T3 = _mm512_mask_blend_pd(0xF0, kernel.packet[3], T3);
1811 T7 = _mm512_shuffle_f64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1812 T7 = _mm512_mask_blend_pd(0xF0, T7, kernel.packet[7]);
1813
1814 kernel.packet[0] = T0;
1815 kernel.packet[1] = T1;
1816 kernel.packet[2] = T2;
1817 kernel.packet[3] = T3;
1818 kernel.packet[4] = T4;
1819 kernel.packet[5] = T5;
1820 kernel.packet[6] = T6;
1821 kernel.packet[7] = T7;
1822}
1823
1824EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8l, 4>& kernel) {
1825 __m512i T0 = _mm512_castpd_si512(
1826 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0));
1827 __m512i T1 = _mm512_castpd_si512(
1828 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0xff));
1829 __m512i T2 = _mm512_castpd_si512(
1830 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0));
1831 __m512i T3 = _mm512_castpd_si512(
1832 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0xff));
1833
1834 PacketBlock<Packet4l, 8> tmp;
1835
1836 tmp.packet[0] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x20);
1837 tmp.packet[1] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x20);
1838 tmp.packet[2] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x31);
1839 tmp.packet[3] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x31);
1840
1841 tmp.packet[4] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x20);
1842 tmp.packet[5] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x20);
1843 tmp.packet[6] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x31);
1844 tmp.packet[7] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x31);
1845
1846 PACK_OUTPUT_L(kernel.packet, tmp.packet, 0, 1);
1847 PACK_OUTPUT_L(kernel.packet, tmp.packet, 1, 1);
1848 PACK_OUTPUT_L(kernel.packet, tmp.packet, 2, 1);
1849 PACK_OUTPUT_L(kernel.packet, tmp.packet, 3, 1);
1850}
1851
1852EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8l, 8>& kernel) {
1853 __m512i T0 = _mm512_unpacklo_epi64(kernel.packet[0], kernel.packet[1]);
1854 __m512i T1 = _mm512_unpackhi_epi64(kernel.packet[0], kernel.packet[1]);
1855 __m512i T2 = _mm512_unpacklo_epi64(kernel.packet[2], kernel.packet[3]);
1856 __m512i T3 = _mm512_unpackhi_epi64(kernel.packet[2], kernel.packet[3]);
1857 __m512i T4 = _mm512_unpacklo_epi64(kernel.packet[4], kernel.packet[5]);
1858 __m512i T5 = _mm512_unpackhi_epi64(kernel.packet[4], kernel.packet[5]);
1859 __m512i T6 = _mm512_unpacklo_epi64(kernel.packet[6], kernel.packet[7]);
1860 __m512i T7 = _mm512_unpackhi_epi64(kernel.packet[6], kernel.packet[7]);
1861
1862 kernel.packet[0] = _mm512_permutex_epi64(T2, 0x4E);
1863 kernel.packet[0] = _mm512_mask_blend_epi64(0xCC, T0, kernel.packet[0]);
1864 kernel.packet[2] = _mm512_permutex_epi64(T0, 0x4E);
1865 kernel.packet[2] = _mm512_mask_blend_epi64(0xCC, kernel.packet[2], T2);
1866 kernel.packet[1] = _mm512_permutex_epi64(T3, 0x4E);
1867 kernel.packet[1] = _mm512_mask_blend_epi64(0xCC, T1, kernel.packet[1]);
1868 kernel.packet[3] = _mm512_permutex_epi64(T1, 0x4E);
1869 kernel.packet[3] = _mm512_mask_blend_epi64(0xCC, kernel.packet[3], T3);
1870 kernel.packet[4] = _mm512_permutex_epi64(T6, 0x4E);
1871 kernel.packet[4] = _mm512_mask_blend_epi64(0xCC, T4, kernel.packet[4]);
1872 kernel.packet[6] = _mm512_permutex_epi64(T4, 0x4E);
1873 kernel.packet[6] = _mm512_mask_blend_epi64(0xCC, kernel.packet[6], T6);
1874 kernel.packet[5] = _mm512_permutex_epi64(T7, 0x4E);
1875 kernel.packet[5] = _mm512_mask_blend_epi64(0xCC, T5, kernel.packet[5]);
1876 kernel.packet[7] = _mm512_permutex_epi64(T5, 0x4E);
1877 kernel.packet[7] = _mm512_mask_blend_epi64(0xCC, kernel.packet[7], T7);
1878
1879 T0 = _mm512_shuffle_i64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1880 T0 = _mm512_mask_blend_epi64(0xF0, kernel.packet[0], T0);
1881 T4 = _mm512_shuffle_i64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1882 T4 = _mm512_mask_blend_epi64(0xF0, T4, kernel.packet[4]);
1883 T1 = _mm512_shuffle_i64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1884 T1 = _mm512_mask_blend_epi64(0xF0, kernel.packet[1], T1);
1885 T5 = _mm512_shuffle_i64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1886 T5 = _mm512_mask_blend_epi64(0xF0, T5, kernel.packet[5]);
1887 T2 = _mm512_shuffle_i64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1888 T2 = _mm512_mask_blend_epi64(0xF0, kernel.packet[2], T2);
1889 T6 = _mm512_shuffle_i64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1890 T6 = _mm512_mask_blend_epi64(0xF0, T6, kernel.packet[6]);
1891 T3 = _mm512_shuffle_i64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1892 T3 = _mm512_mask_blend_epi64(0xF0, kernel.packet[3], T3);
1893 T7 = _mm512_shuffle_i64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1894 T7 = _mm512_mask_blend_epi64(0xF0, T7, kernel.packet[7]);
1895
1896 kernel.packet[0] = T0;
1897 kernel.packet[1] = T1;
1898 kernel.packet[2] = T2;
1899 kernel.packet[3] = T3;
1900 kernel.packet[4] = T4;
1901 kernel.packet[5] = T5;
1902 kernel.packet[6] = T6;
1903 kernel.packet[7] = T7;
1904}
1905
1906#define PACK_OUTPUT_I32(OUTPUT, INPUT, INDEX, STRIDE) \
1907 EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1908
1909#define PACK_OUTPUT_I32_2(OUTPUT, INPUT, INDEX, STRIDE) \
1910 EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1911
1912#define SHUFFLE_EPI32(A, B, M) _mm512_castps_si512(_mm512_shuffle_ps(_mm512_castsi512_ps(A), _mm512_castsi512_ps(B), M))
1913
1914EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 16>& kernel) {
1915 __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
1916 __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
1917 __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
1918 __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
1919 __m512i T4 = _mm512_unpacklo_epi32(kernel.packet[4], kernel.packet[5]);
1920 __m512i T5 = _mm512_unpackhi_epi32(kernel.packet[4], kernel.packet[5]);
1921 __m512i T6 = _mm512_unpacklo_epi32(kernel.packet[6], kernel.packet[7]);
1922 __m512i T7 = _mm512_unpackhi_epi32(kernel.packet[6], kernel.packet[7]);
1923 __m512i T8 = _mm512_unpacklo_epi32(kernel.packet[8], kernel.packet[9]);
1924 __m512i T9 = _mm512_unpackhi_epi32(kernel.packet[8], kernel.packet[9]);
1925 __m512i T10 = _mm512_unpacklo_epi32(kernel.packet[10], kernel.packet[11]);
1926 __m512i T11 = _mm512_unpackhi_epi32(kernel.packet[10], kernel.packet[11]);
1927 __m512i T12 = _mm512_unpacklo_epi32(kernel.packet[12], kernel.packet[13]);
1928 __m512i T13 = _mm512_unpackhi_epi32(kernel.packet[12], kernel.packet[13]);
1929 __m512i T14 = _mm512_unpacklo_epi32(kernel.packet[14], kernel.packet[15]);
1930 __m512i T15 = _mm512_unpackhi_epi32(kernel.packet[14], kernel.packet[15]);
1931 __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1932 __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1933 __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1934 __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1935 __m512i S4 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1936 __m512i S5 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1937 __m512i S6 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1938 __m512i S7 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1939 __m512i S8 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1940 __m512i S9 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1941 __m512i S10 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1942 __m512i S11 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1943 __m512i S12 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1944 __m512i S13 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1945 __m512i S14 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1946 __m512i S15 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1947
1948 EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
1949 EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
1950 EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
1951 EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
1952 EIGEN_EXTRACT_8i_FROM_16i(S4, S4);
1953 EIGEN_EXTRACT_8i_FROM_16i(S5, S5);
1954 EIGEN_EXTRACT_8i_FROM_16i(S6, S6);
1955 EIGEN_EXTRACT_8i_FROM_16i(S7, S7);
1956 EIGEN_EXTRACT_8i_FROM_16i(S8, S8);
1957 EIGEN_EXTRACT_8i_FROM_16i(S9, S9);
1958 EIGEN_EXTRACT_8i_FROM_16i(S10, S10);
1959 EIGEN_EXTRACT_8i_FROM_16i(S11, S11);
1960 EIGEN_EXTRACT_8i_FROM_16i(S12, S12);
1961 EIGEN_EXTRACT_8i_FROM_16i(S13, S13);
1962 EIGEN_EXTRACT_8i_FROM_16i(S14, S14);
1963 EIGEN_EXTRACT_8i_FROM_16i(S15, S15);
1964
1965 PacketBlock<Packet8i, 32> tmp;
1966
1967 tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S4_0, 0x20);
1968 tmp.packet[1] = _mm256_permute2f128_si256(S1_0, S5_0, 0x20);
1969 tmp.packet[2] = _mm256_permute2f128_si256(S2_0, S6_0, 0x20);
1970 tmp.packet[3] = _mm256_permute2f128_si256(S3_0, S7_0, 0x20);
1971 tmp.packet[4] = _mm256_permute2f128_si256(S0_0, S4_0, 0x31);
1972 tmp.packet[5] = _mm256_permute2f128_si256(S1_0, S5_0, 0x31);
1973 tmp.packet[6] = _mm256_permute2f128_si256(S2_0, S6_0, 0x31);
1974 tmp.packet[7] = _mm256_permute2f128_si256(S3_0, S7_0, 0x31);
1975
1976 tmp.packet[8] = _mm256_permute2f128_si256(S0_1, S4_1, 0x20);
1977 tmp.packet[9] = _mm256_permute2f128_si256(S1_1, S5_1, 0x20);
1978 tmp.packet[10] = _mm256_permute2f128_si256(S2_1, S6_1, 0x20);
1979 tmp.packet[11] = _mm256_permute2f128_si256(S3_1, S7_1, 0x20);
1980 tmp.packet[12] = _mm256_permute2f128_si256(S0_1, S4_1, 0x31);
1981 tmp.packet[13] = _mm256_permute2f128_si256(S1_1, S5_1, 0x31);
1982 tmp.packet[14] = _mm256_permute2f128_si256(S2_1, S6_1, 0x31);
1983 tmp.packet[15] = _mm256_permute2f128_si256(S3_1, S7_1, 0x31);
1984
1985 // Second set of _m256 outputs
1986 tmp.packet[16] = _mm256_permute2f128_si256(S8_0, S12_0, 0x20);
1987 tmp.packet[17] = _mm256_permute2f128_si256(S9_0, S13_0, 0x20);
1988 tmp.packet[18] = _mm256_permute2f128_si256(S10_0, S14_0, 0x20);
1989 tmp.packet[19] = _mm256_permute2f128_si256(S11_0, S15_0, 0x20);
1990 tmp.packet[20] = _mm256_permute2f128_si256(S8_0, S12_0, 0x31);
1991 tmp.packet[21] = _mm256_permute2f128_si256(S9_0, S13_0, 0x31);
1992 tmp.packet[22] = _mm256_permute2f128_si256(S10_0, S14_0, 0x31);
1993 tmp.packet[23] = _mm256_permute2f128_si256(S11_0, S15_0, 0x31);
1994
1995 tmp.packet[24] = _mm256_permute2f128_si256(S8_1, S12_1, 0x20);
1996 tmp.packet[25] = _mm256_permute2f128_si256(S9_1, S13_1, 0x20);
1997 tmp.packet[26] = _mm256_permute2f128_si256(S10_1, S14_1, 0x20);
1998 tmp.packet[27] = _mm256_permute2f128_si256(S11_1, S15_1, 0x20);
1999 tmp.packet[28] = _mm256_permute2f128_si256(S8_1, S12_1, 0x31);
2000 tmp.packet[29] = _mm256_permute2f128_si256(S9_1, S13_1, 0x31);
2001 tmp.packet[30] = _mm256_permute2f128_si256(S10_1, S14_1, 0x31);
2002 tmp.packet[31] = _mm256_permute2f128_si256(S11_1, S15_1, 0x31);
2003
2004 // Pack them into the output
2005 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 0, 16);
2006 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 1, 16);
2007 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 2, 16);
2008 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 3, 16);
2009
2010 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 4, 16);
2011 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 5, 16);
2012 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 6, 16);
2013 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 7, 16);
2014
2015 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 8, 16);
2016 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 9, 16);
2017 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 10, 16);
2018 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 11, 16);
2019
2020 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 12, 16);
2021 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 13, 16);
2022 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 14, 16);
2023 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 15, 16);
2024}
2025
2026EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 4>& kernel) {
2027 __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
2028 __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
2029 __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
2030 __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
2031
2032 __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
2033 __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
2034 __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
2035 __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
2036
2037 EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
2038 EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
2039 EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
2040 EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
2041
2042 PacketBlock<Packet8i, 8> tmp;
2043
2044 tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S1_0, 0x20);
2045 tmp.packet[1] = _mm256_permute2f128_si256(S2_0, S3_0, 0x20);
2046 tmp.packet[2] = _mm256_permute2f128_si256(S0_0, S1_0, 0x31);
2047 tmp.packet[3] = _mm256_permute2f128_si256(S2_0, S3_0, 0x31);
2048
2049 tmp.packet[4] = _mm256_permute2f128_si256(S0_1, S1_1, 0x20);
2050 tmp.packet[5] = _mm256_permute2f128_si256(S2_1, S3_1, 0x20);
2051 tmp.packet[6] = _mm256_permute2f128_si256(S0_1, S1_1, 0x31);
2052 tmp.packet[7] = _mm256_permute2f128_si256(S2_1, S3_1, 0x31);
2053
2054 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 0, 1);
2055 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 1, 1);
2056 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 2, 1);
2057 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 3, 1);
2058}
2059
2060template <size_t N>
2061EIGEN_STRONG_INLINE int avx512_blend_mask(const Selector<N>& ifPacket) {
2062 alignas(__m128i) uint8_t aux[sizeof(__m128i)];
2063 for (size_t i = 0; i < N; i++) aux[i] = static_cast<uint8_t>(ifPacket.select[i]);
2064 __m128i paux = _mm_sub_epi8(_mm_setzero_si128(), _mm_load_si128(reinterpret_cast<const __m128i*>(aux)));
2065 return _mm_movemask_epi8(paux);
2066}
2067
2068template <>
2069EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& ifPacket, const Packet16f& thenPacket,
2070 const Packet16f& elsePacket) {
2071 __mmask16 m = avx512_blend_mask(ifPacket);
2072 return _mm512_mask_blend_ps(m, elsePacket, thenPacket);
2073}
2074template <>
2075EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket, const Packet8d& thenPacket,
2076 const Packet8d& elsePacket) {
2077 __mmask8 m = avx512_blend_mask(ifPacket);
2078 return _mm512_mask_blend_pd(m, elsePacket, thenPacket);
2079}
2080
2081// Packet math for Eigen::half
2082#ifndef EIGEN_VECTORIZE_AVX512FP16
2083template <>
2084EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) {
2085 return _mm256_set1_epi16(from.x);
2086}
2087
2088template <>
2089EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) {
2090 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from, 0)));
2091}
2092
2093template <>
2094EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) {
2095 return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
2096}
2097
2098template <>
2099EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) {
2100 return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
2101}
2102
2103template <>
2104EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) {
2105 // (void*) -> workaround clang warning:
2106 // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
2107 EIGEN_DEBUG_ALIGNED_STORE
2108 _mm256_store_si256((__m256i*)(void*)to, from);
2109}
2110
2111template <>
2112EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) {
2113 // (void*) -> workaround clang warning:
2114 // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
2115 EIGEN_DEBUG_UNALIGNED_STORE
2116 _mm256_storeu_si256((__m256i*)(void*)to, from);
2117}
2118
2119template <>
2120EIGEN_STRONG_INLINE Packet16h ploaddup<Packet16h>(const Eigen::half* from) {
2121 unsigned short a = from[0].x;
2122 unsigned short b = from[1].x;
2123 unsigned short c = from[2].x;
2124 unsigned short d = from[3].x;
2125 unsigned short e = from[4].x;
2126 unsigned short f = from[5].x;
2127 unsigned short g = from[6].x;
2128 unsigned short h = from[7].x;
2129 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2130}
2131
2132template <>
2133EIGEN_STRONG_INLINE Packet16h ploadquad(const Eigen::half* from) {
2134 unsigned short a = from[0].x;
2135 unsigned short b = from[1].x;
2136 unsigned short c = from[2].x;
2137 unsigned short d = from[3].x;
2138 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2139}
2140
2141EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { return _mm512_cvtph_ps(a); }
2142
2143EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) {
2144 return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
2145}
2146
2147template <>
2148EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) {
2149 return Packet16h(ptrue(Packet8i(a)));
2150}
2151
2152template <>
2153EIGEN_STRONG_INLINE Packet16h pabs(const Packet16h& a) {
2154 const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
2155 return _mm256_andnot_si256(sign_mask, a);
2156}
2157
2158template <>
2159EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(const Packet16h& a, const Packet16h& b) {
2160 return float2half(pmin<Packet16f>(half2float(a), half2float(b)));
2161}
2162
2163template <>
2164EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(const Packet16h& a, const Packet16h& b) {
2165 return float2half(pmax<Packet16f>(half2float(a), half2float(b)));
2166}
2167
2168template <>
2169EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(const half& a) {
2170 return float2half(plset<Packet16f>(static_cast<float>(a)));
2171}
2172
2173template <>
2174EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a, const Packet16h& b) {
2175 // in some cases Packet8i is a wrapper around __m256i, so we need to
2176 // cast to Packet8i to call the correct overload.
2177 return Packet16h(por(Packet8i(a), Packet8i(b)));
2178}
2179template <>
2180EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a, const Packet16h& b) {
2181 return Packet16h(pxor(Packet8i(a), Packet8i(b)));
2182}
2183template <>
2184EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a, const Packet16h& b) {
2185 return Packet16h(pand(Packet8i(a), Packet8i(b)));
2186}
2187template <>
2188EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a, const Packet16h& b) {
2189 return Packet16h(pandnot(Packet8i(a), Packet8i(b)));
2190}
2191
2192template <>
2193EIGEN_STRONG_INLINE Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) {
2194 return _mm256_blendv_epi8(b, a, mask);
2195}
2196
2197template <>
2198EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(const Packet16h& a) {
2199 return float2half(pround<Packet16f>(half2float(a)));
2200}
2201
2202template <>
2203EIGEN_STRONG_INLINE Packet16h print<Packet16h>(const Packet16h& a) {
2204 return float2half(print<Packet16f>(half2float(a)));
2205}
2206
2207template <>
2208EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(const Packet16h& a) {
2209 return float2half(pceil<Packet16f>(half2float(a)));
2210}
2211
2212template <>
2213EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) {
2214 return float2half(pfloor<Packet16f>(half2float(a)));
2215}
2216
2217template <>
2218EIGEN_STRONG_INLINE Packet16h ptrunc<Packet16h>(const Packet16h& a) {
2219 return float2half(ptrunc<Packet16f>(half2float(a)));
2220}
2221
2222template <>
2223EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a, const Packet16h& b) {
2224 Packet16f af = half2float(a);
2225 Packet16f bf = half2float(b);
2226 return Pack32To16(pcmp_eq(af, bf));
2227}
2228
2229template <>
2230EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a, const Packet16h& b) {
2231 return Pack32To16(pcmp_le(half2float(a), half2float(b)));
2232}
2233
2234template <>
2235EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a, const Packet16h& b) {
2236 return Pack32To16(pcmp_lt(half2float(a), half2float(b)));
2237}
2238
2239template <>
2240EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a, const Packet16h& b) {
2241 return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b)));
2242}
2243
2244template <>
2245EIGEN_STRONG_INLINE Packet16h pconj(const Packet16h& a) {
2246 return a;
2247}
2248
2249template <>
2250EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) {
2251 Packet16h sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
2252 return _mm256_xor_si256(a, sign_mask);
2253}
2254
2255template <>
2256EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) {
2257 Packet16f af = half2float(a);
2258 Packet16f bf = half2float(b);
2259 Packet16f rf = padd(af, bf);
2260 return float2half(rf);
2261}
2262
2263template <>
2264EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) {
2265 Packet16f af = half2float(a);
2266 Packet16f bf = half2float(b);
2267 Packet16f rf = psub(af, bf);
2268 return float2half(rf);
2269}
2270
2271template <>
2272EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) {
2273 Packet16f af = half2float(a);
2274 Packet16f bf = half2float(b);
2275 Packet16f rf = pmul(af, bf);
2276 return float2half(rf);
2277}
2278
2279template <>
2280EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) {
2281 Packet16f af = half2float(a);
2282 Packet16f bf = half2float(b);
2283 Packet16f rf = pdiv(af, bf);
2284 return float2half(rf);
2285}
2286
2287template <>
2288EIGEN_STRONG_INLINE Packet16h pmadd<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2289 return float2half(pmadd(half2float(a), half2float(b), half2float(c)));
2290}
2291
2292template <>
2293EIGEN_STRONG_INLINE Packet16h pmsub<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2294 return float2half(pmsub(half2float(a), half2float(b), half2float(c)));
2295}
2296
2297template <>
2298EIGEN_STRONG_INLINE Packet16h pnmadd<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2299 return float2half(pnmadd(half2float(a), half2float(b), half2float(c)));
2300}
2301
2302template <>
2303EIGEN_STRONG_INLINE Packet16h pnmsub<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2304 return float2half(pnmsub(half2float(a), half2float(b), half2float(c)));
2305}
2306
2307template <>
2308EIGEN_STRONG_INLINE Packet8h predux_half_dowto4<Packet16h>(const Packet16h& a) {
2309 Packet8h lane0 = _mm256_extractf128_si256(a, 0);
2310 Packet8h lane1 = _mm256_extractf128_si256(a, 1);
2311 return padd<Packet8h>(lane0, lane1);
2312}
2313
2314template <>
2315EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) {
2316 __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
2317 return _mm256_insertf128_si256(_mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a, 1), m)),
2318 _mm_shuffle_epi8(_mm256_extractf128_si256(a, 0), m), 1);
2319}
2320
2321template <>
2322EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride) {
2323 return _mm256_set_epi16(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x,
2324 from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x,
2325 from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x,
2326 from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x);
2327}
2328
2329template <>
2330EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride) {
2331 EIGEN_ALIGN64 half aux[16];
2332 pstore(aux, from);
2333 to[stride * 0] = aux[0];
2334 to[stride * 1] = aux[1];
2335 to[stride * 2] = aux[2];
2336 to[stride * 3] = aux[3];
2337 to[stride * 4] = aux[4];
2338 to[stride * 5] = aux[5];
2339 to[stride * 6] = aux[6];
2340 to[stride * 7] = aux[7];
2341 to[stride * 8] = aux[8];
2342 to[stride * 9] = aux[9];
2343 to[stride * 10] = aux[10];
2344 to[stride * 11] = aux[11];
2345 to[stride * 12] = aux[12];
2346 to[stride * 13] = aux[13];
2347 to[stride * 14] = aux[14];
2348 to[stride * 15] = aux[15];
2349}
2350
2351EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 16>& kernel) {
2352 __m256i a = kernel.packet[0];
2353 __m256i b = kernel.packet[1];
2354 __m256i c = kernel.packet[2];
2355 __m256i d = kernel.packet[3];
2356 __m256i e = kernel.packet[4];
2357 __m256i f = kernel.packet[5];
2358 __m256i g = kernel.packet[6];
2359 __m256i h = kernel.packet[7];
2360 __m256i i = kernel.packet[8];
2361 __m256i j = kernel.packet[9];
2362 __m256i k = kernel.packet[10];
2363 __m256i l = kernel.packet[11];
2364 __m256i m = kernel.packet[12];
2365 __m256i n = kernel.packet[13];
2366 __m256i o = kernel.packet[14];
2367 __m256i p = kernel.packet[15];
2368
2369 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2370 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2371 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2372 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2373 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2374 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2375 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2376 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2377
2378 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2379 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2380 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2381 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2382 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2383 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2384 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2385 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2386
2387 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2388 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2389 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2390 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2391 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2392 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2393 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2394 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2395
2396 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2397 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2398 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2399 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2400 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2401 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2402 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2403 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2404
2405 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2406 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2407 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2408 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2409 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2410 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2411 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2412 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2413 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2414 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2415 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2416 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2417 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2418 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2419 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2420 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2421
2422 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2423 __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2424 __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2425 __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2426 __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2427 __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2428 __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2429 __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2430 __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2431 __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2432 __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2433 __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2434 __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2435 __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2436 __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2437 __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2438 __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2439
2440 kernel.packet[0] = a_p_0;
2441 kernel.packet[1] = a_p_1;
2442 kernel.packet[2] = a_p_2;
2443 kernel.packet[3] = a_p_3;
2444 kernel.packet[4] = a_p_4;
2445 kernel.packet[5] = a_p_5;
2446 kernel.packet[6] = a_p_6;
2447 kernel.packet[7] = a_p_7;
2448 kernel.packet[8] = a_p_8;
2449 kernel.packet[9] = a_p_9;
2450 kernel.packet[10] = a_p_a;
2451 kernel.packet[11] = a_p_b;
2452 kernel.packet[12] = a_p_c;
2453 kernel.packet[13] = a_p_d;
2454 kernel.packet[14] = a_p_e;
2455 kernel.packet[15] = a_p_f;
2456}
2457
2458EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 8>& kernel) {
2459 EIGEN_ALIGN64 half in[8][16];
2460 pstore<half>(in[0], kernel.packet[0]);
2461 pstore<half>(in[1], kernel.packet[1]);
2462 pstore<half>(in[2], kernel.packet[2]);
2463 pstore<half>(in[3], kernel.packet[3]);
2464 pstore<half>(in[4], kernel.packet[4]);
2465 pstore<half>(in[5], kernel.packet[5]);
2466 pstore<half>(in[6], kernel.packet[6]);
2467 pstore<half>(in[7], kernel.packet[7]);
2468
2469 EIGEN_ALIGN64 half out[8][16];
2470
2471 for (int i = 0; i < 8; ++i) {
2472 for (int j = 0; j < 8; ++j) {
2473 out[i][j] = in[j][2 * i];
2474 }
2475 for (int j = 0; j < 8; ++j) {
2476 out[i][j + 8] = in[j][2 * i + 1];
2477 }
2478 }
2479
2480 kernel.packet[0] = pload<Packet16h>(out[0]);
2481 kernel.packet[1] = pload<Packet16h>(out[1]);
2482 kernel.packet[2] = pload<Packet16h>(out[2]);
2483 kernel.packet[3] = pload<Packet16h>(out[3]);
2484 kernel.packet[4] = pload<Packet16h>(out[4]);
2485 kernel.packet[5] = pload<Packet16h>(out[5]);
2486 kernel.packet[6] = pload<Packet16h>(out[6]);
2487 kernel.packet[7] = pload<Packet16h>(out[7]);
2488}
2489
2490EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 4>& kernel) {
2491 EIGEN_ALIGN64 half in[4][16];
2492 pstore<half>(in[0], kernel.packet[0]);
2493 pstore<half>(in[1], kernel.packet[1]);
2494 pstore<half>(in[2], kernel.packet[2]);
2495 pstore<half>(in[3], kernel.packet[3]);
2496
2497 EIGEN_ALIGN64 half out[4][16];
2498
2499 for (int i = 0; i < 4; ++i) {
2500 for (int j = 0; j < 4; ++j) {
2501 out[i][j] = in[j][4 * i];
2502 }
2503 for (int j = 0; j < 4; ++j) {
2504 out[i][j + 4] = in[j][4 * i + 1];
2505 }
2506 for (int j = 0; j < 4; ++j) {
2507 out[i][j + 8] = in[j][4 * i + 2];
2508 }
2509 for (int j = 0; j < 4; ++j) {
2510 out[i][j + 12] = in[j][4 * i + 3];
2511 }
2512 }
2513
2514 kernel.packet[0] = pload<Packet16h>(out[0]);
2515 kernel.packet[1] = pload<Packet16h>(out[1]);
2516 kernel.packet[2] = pload<Packet16h>(out[2]);
2517 kernel.packet[3] = pload<Packet16h>(out[3]);
2518}
2519
2520#endif // EIGEN_VECTORIZE_AVX512FP16
2521
2522template <>
2523struct is_arithmetic<Packet16bf> {
2524 enum { value = true };
2525};
2526
2527template <>
2528struct packet_traits<bfloat16> : default_packet_traits {
2529 typedef Packet16bf type;
2530 typedef Packet8bf half;
2531 enum {
2532 Vectorizable = 1,
2533 AlignedOnScalar = 1,
2534 size = 16,
2535 HasBlend = 0,
2536 HasInsert = 1,
2537 HasSin = EIGEN_FAST_MATH,
2538 HasCos = EIGEN_FAST_MATH,
2539 HasSqrt = 1,
2540 HasRsqrt = 1,
2541#ifdef EIGEN_VECTORIZE_AVX512DQ
2542 HasLog = 1, // Currently fails test with bad accuracy.
2543 HasLog1p = 1,
2544 HasExpm1 = 1,
2545 HasNdtri = 1,
2546 HasBessel = 1,
2547#endif
2548 HasExp = 1,
2549 HasTanh = EIGEN_FAST_MATH,
2550 HasErf = EIGEN_FAST_MATH,
2551 HasCmp = 1,
2552 HasDiv = 1
2553 };
2554};
2555
2556template <>
2557struct unpacket_traits<Packet16bf> {
2558 typedef bfloat16 type;
2559 enum {
2560 size = 16,
2561 alignment = Aligned32,
2562 vectorizable = true,
2563 masked_load_available = false,
2564 masked_store_available = false
2565 };
2566 typedef Packet8bf half;
2567};
2568
2569template <>
2570EIGEN_STRONG_INLINE Packet16bf pset1<Packet16bf>(const bfloat16& from) {
2571 return _mm256_set1_epi16(from.value);
2572}
2573
2574template <>
2575EIGEN_STRONG_INLINE bfloat16 pfirst<Packet16bf>(const Packet16bf& from) {
2576 bfloat16 t;
2577 t.value = static_cast<unsigned short>(_mm256_extract_epi16(from, 0));
2578 return t;
2579}
2580
2581template <>
2582EIGEN_STRONG_INLINE Packet16bf pload<Packet16bf>(const bfloat16* from) {
2583 return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
2584}
2585
2586template <>
2587EIGEN_STRONG_INLINE Packet16bf ploadu<Packet16bf>(const bfloat16* from) {
2588 return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
2589}
2590
2591template <>
2592EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet16bf& from) {
2593 EIGEN_DEBUG_ALIGNED_STORE
2594 _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
2595}
2596
2597template <>
2598EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet16bf& from) {
2599 EIGEN_DEBUG_UNALIGNED_STORE
2600 _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
2601}
2602
2603template <>
2604EIGEN_STRONG_INLINE Packet16bf ploaddup<Packet16bf>(const bfloat16* from) {
2605 unsigned short a = from[0].value;
2606 unsigned short b = from[1].value;
2607 unsigned short c = from[2].value;
2608 unsigned short d = from[3].value;
2609 unsigned short e = from[4].value;
2610 unsigned short f = from[5].value;
2611 unsigned short g = from[6].value;
2612 unsigned short h = from[7].value;
2613 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2614}
2615
2616template <>
2617EIGEN_STRONG_INLINE Packet16bf ploadquad(const bfloat16* from) {
2618 unsigned short a = from[0].value;
2619 unsigned short b = from[1].value;
2620 unsigned short c = from[2].value;
2621 unsigned short d = from[3].value;
2622 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2623}
2624
2625EIGEN_STRONG_INLINE Packet16f Bf16ToF32(const Packet16bf& a) {
2626 return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
2627}
2628
2629// Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm.
2630EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) {
2631 Packet16bf r;
2632
2633#if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_STRICT_AT_LEAST(10, 1, 0)
2634 // Since GCC 10.1 supports avx512bf16 and C style explicit cast
2635 // (C++ static_cast is not supported yet), do conversion via intrinsic
2636 // and register path for performance.
2637 r = (__m256i)(_mm512_cvtneps_pbh(a));
2638
2639#else
2640 __m512i t;
2641 __m512i input = _mm512_castps_si512(a);
2642 __m512i nan = _mm512_set1_epi32(0x7fc0);
2643
2644 // uint32_t lsb = (input >> 16) & 1;
2645 t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1));
2646 // uint32_t rounding_bias = 0x7fff + lsb;
2647 t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff));
2648 // input += rounding_bias;
2649 t = _mm512_add_epi32(t, input);
2650 // input = input >> 16;
2651 t = _mm512_srli_epi32(t, 16);
2652
2653 // Check NaN before converting back to bf16
2654 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q);
2655
2656 t = _mm512_mask_blend_epi32(mask, nan, t);
2657 // output.value = static_cast<uint16_t>(input);
2658 r = _mm512_cvtepi32_epi16(t);
2659#endif // EIGEN_VECTORIZE_AVX512BF16
2660
2661 return r;
2662}
2663
2664template <>
2665EIGEN_STRONG_INLINE Packet16bf ptrue(const Packet16bf& a) {
2666 return Packet16bf(ptrue<Packet8i>(Packet8i(a)));
2667}
2668
2669template <>
2670EIGEN_STRONG_INLINE Packet16bf por(const Packet16bf& a, const Packet16bf& b) {
2671 return Packet16bf(por<Packet8i>(Packet8i(a), Packet8i(b)));
2672}
2673
2674template <>
2675EIGEN_STRONG_INLINE Packet16bf pxor(const Packet16bf& a, const Packet16bf& b) {
2676 return Packet16bf(pxor<Packet8i>(Packet8i(a), Packet8i(b)));
2677}
2678
2679template <>
2680EIGEN_STRONG_INLINE Packet16bf pand(const Packet16bf& a, const Packet16bf& b) {
2681 return Packet16bf(pand<Packet8i>(Packet8i(a), Packet8i(b)));
2682}
2683
2684template <>
2685EIGEN_STRONG_INLINE Packet16bf pandnot(const Packet16bf& a, const Packet16bf& b) {
2686 return Packet16bf(pandnot<Packet8i>(Packet8i(a), Packet8i(b)));
2687}
2688
2689template <>
2690EIGEN_STRONG_INLINE Packet16bf pselect(const Packet16bf& mask, const Packet16bf& a, const Packet16bf& b) {
2691 // Input mask is expected to be all 0/1, handle it with 8-bit
2692 // intrinsic for performance.
2693 return _mm256_blendv_epi8(b, a, mask);
2694}
2695
2696template <>
2697EIGEN_STRONG_INLINE Packet16bf pround<Packet16bf>(const Packet16bf& a) {
2698 return F32ToBf16(pround<Packet16f>(Bf16ToF32(a)));
2699}
2700
2701template <>
2702EIGEN_STRONG_INLINE Packet16bf print<Packet16bf>(const Packet16bf& a) {
2703 return F32ToBf16(print<Packet16f>(Bf16ToF32(a)));
2704}
2705
2706template <>
2707EIGEN_STRONG_INLINE Packet16bf pceil<Packet16bf>(const Packet16bf& a) {
2708 return F32ToBf16(pceil<Packet16f>(Bf16ToF32(a)));
2709}
2710
2711template <>
2712EIGEN_STRONG_INLINE Packet16bf pfloor<Packet16bf>(const Packet16bf& a) {
2713 return F32ToBf16(pfloor<Packet16f>(Bf16ToF32(a)));
2714}
2715
2716template <>
2717EIGEN_STRONG_INLINE Packet16bf ptrunc<Packet16bf>(const Packet16bf& a) {
2718 return F32ToBf16(ptrunc<Packet16f>(Bf16ToF32(a)));
2719}
2720
2721template <>
2722EIGEN_STRONG_INLINE Packet16bf pcmp_eq(const Packet16bf& a, const Packet16bf& b) {
2723 return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
2724}
2725
2726template <>
2727EIGEN_STRONG_INLINE Packet16bf pcmp_le(const Packet16bf& a, const Packet16bf& b) {
2728 return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
2729}
2730
2731template <>
2732EIGEN_STRONG_INLINE Packet16bf pcmp_lt(const Packet16bf& a, const Packet16bf& b) {
2733 return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
2734}
2735
2736template <>
2737EIGEN_STRONG_INLINE Packet16bf pcmp_lt_or_nan(const Packet16bf& a, const Packet16bf& b) {
2738 return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
2739}
2740
2741template <>
2742EIGEN_STRONG_INLINE Packet16bf pnegate(const Packet16bf& a) {
2743 Packet16bf sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
2744 return _mm256_xor_si256(a, sign_mask);
2745}
2746
2747template <>
2748EIGEN_STRONG_INLINE Packet16bf pconj(const Packet16bf& a) {
2749 return a;
2750}
2751
2752template <>
2753EIGEN_STRONG_INLINE Packet16bf pabs(const Packet16bf& a) {
2754 const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
2755 return _mm256_andnot_si256(sign_mask, a);
2756}
2757
2758template <>
2759EIGEN_STRONG_INLINE Packet16bf padd<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2760 return F32ToBf16(padd<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2761}
2762
2763template <>
2764EIGEN_STRONG_INLINE Packet16bf psub<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2765 return F32ToBf16(psub<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2766}
2767
2768template <>
2769EIGEN_STRONG_INLINE Packet16bf pmul<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2770 return F32ToBf16(pmul(Bf16ToF32(a), Bf16ToF32(b)));
2771}
2772
2773template <>
2774EIGEN_STRONG_INLINE Packet16bf pmadd<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2775 return F32ToBf16(pmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2776}
2777
2778template <>
2779EIGEN_STRONG_INLINE Packet16bf pmsub<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2780 return F32ToBf16(pmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2781}
2782
2783template <>
2784EIGEN_STRONG_INLINE Packet16bf pnmadd<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2785 return F32ToBf16(pnmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2786}
2787
2788template <>
2789EIGEN_STRONG_INLINE Packet16bf pnmsub<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2790 return F32ToBf16(pnmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2791}
2792
2793template <>
2794EIGEN_STRONG_INLINE Packet16bf pdiv<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2795 return F32ToBf16(pdiv<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2796}
2797
2798template <>
2799EIGEN_STRONG_INLINE Packet16bf pmin<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2800 return F32ToBf16(pmin<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2801}
2802
2803template <>
2804EIGEN_STRONG_INLINE Packet16bf pmax<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2805 return F32ToBf16(pmax<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2806}
2807
2808template <>
2809EIGEN_STRONG_INLINE Packet16bf plset<Packet16bf>(const bfloat16& a) {
2810 return F32ToBf16(plset<Packet16f>(static_cast<float>(a)));
2811}
2812
2813template <>
2814EIGEN_STRONG_INLINE Packet8bf predux_half_dowto4<Packet16bf>(const Packet16bf& a) {
2815 Packet8bf lane0 = _mm256_extractf128_si256(a, 0);
2816 Packet8bf lane1 = _mm256_extractf128_si256(a, 1);
2817 return padd<Packet8bf>(lane0, lane1);
2818}
2819
2820template <>
2821EIGEN_STRONG_INLINE Packet16bf preverse(const Packet16bf& a) {
2822 __m256i m = _mm256_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7,
2823 4, 5, 2, 3, 0, 1);
2824
2825 Packet16bf res;
2826 // Swap hi and lo first because shuffle is in 128-bit lanes.
2827 res = _mm256_permute2x128_si256(a, a, 1);
2828 // Shuffle 8-bit values in src within 2*128-bit lanes.
2829 return _mm256_shuffle_epi8(res, m);
2830}
2831
2832template <>
2833EIGEN_STRONG_INLINE Packet16bf pgather<bfloat16, Packet16bf>(const bfloat16* from, Index stride) {
2834 return _mm256_set_epi16(
2835 from[15 * stride].value, from[14 * stride].value, from[13 * stride].value, from[12 * stride].value,
2836 from[11 * stride].value, from[10 * stride].value, from[9 * stride].value, from[8 * stride].value,
2837 from[7 * stride].value, from[6 * stride].value, from[5 * stride].value, from[4 * stride].value,
2838 from[3 * stride].value, from[2 * stride].value, from[1 * stride].value, from[0 * stride].value);
2839}
2840
2841template <>
2842EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet16bf>(bfloat16* to, const Packet16bf& from, Index stride) {
2843 EIGEN_ALIGN64 bfloat16 aux[16];
2844 pstore(aux, from);
2845 to[stride * 0] = aux[0];
2846 to[stride * 1] = aux[1];
2847 to[stride * 2] = aux[2];
2848 to[stride * 3] = aux[3];
2849 to[stride * 4] = aux[4];
2850 to[stride * 5] = aux[5];
2851 to[stride * 6] = aux[6];
2852 to[stride * 7] = aux[7];
2853 to[stride * 8] = aux[8];
2854 to[stride * 9] = aux[9];
2855 to[stride * 10] = aux[10];
2856 to[stride * 11] = aux[11];
2857 to[stride * 12] = aux[12];
2858 to[stride * 13] = aux[13];
2859 to[stride * 14] = aux[14];
2860 to[stride * 15] = aux[15];
2861}
2862
2863EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf, 16>& kernel) {
2864 __m256i a = kernel.packet[0];
2865 __m256i b = kernel.packet[1];
2866 __m256i c = kernel.packet[2];
2867 __m256i d = kernel.packet[3];
2868 __m256i e = kernel.packet[4];
2869 __m256i f = kernel.packet[5];
2870 __m256i g = kernel.packet[6];
2871 __m256i h = kernel.packet[7];
2872 __m256i i = kernel.packet[8];
2873 __m256i j = kernel.packet[9];
2874 __m256i k = kernel.packet[10];
2875 __m256i l = kernel.packet[11];
2876 __m256i m = kernel.packet[12];
2877 __m256i n = kernel.packet[13];
2878 __m256i o = kernel.packet[14];
2879 __m256i p = kernel.packet[15];
2880
2881 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2882 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2883 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2884 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2885 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2886 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2887 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2888 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2889
2890 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2891 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2892 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2893 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2894 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2895 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2896 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2897 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2898
2899 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2900 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2901 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2902 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2903 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2904 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2905 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2906 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2907
2908 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2909 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2910 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2911 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2912 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2913 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2914 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2915 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2916
2917 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2918 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2919 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2920 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2921 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2922 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2923 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2924 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2925 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2926 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2927 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2928 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2929 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2930 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2931 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2932 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2933
2934 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2935 kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2936 kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2937 kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2938 kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2939 kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2940 kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2941 kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2942 kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2943 kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2944 kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2945 kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2946 kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2947 kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2948 kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2949 kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2950 kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2951}
2952
2953EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf, 4>& kernel) {
2954 __m256i a = kernel.packet[0];
2955 __m256i b = kernel.packet[1];
2956 __m256i c = kernel.packet[2];
2957 __m256i d = kernel.packet[3];
2958
2959 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2960 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2961 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2962 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2963
2964 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2965 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2966 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2967 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2968
2969 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2970 kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20);
2971 kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20);
2972 kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31);
2973 kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31);
2974}
2975
2976// Minimal implementation of 16-bit int packets for use in pfrexp, pldexp.
2977
2978template <>
2979EIGEN_STRONG_INLINE Packet32s pset1<Packet32s>(const numext::int16_t& x) {
2980 return _mm512_set1_epi16(x);
2981}
2982
2983template <>
2984EIGEN_STRONG_INLINE Packet16s pset1<Packet16s>(const numext::int16_t& x) {
2985 return _mm256_set1_epi16(x);
2986}
2987
2988template <>
2989EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const numext::int16_t& x) {
2990 return _mm_set1_epi16(x);
2991}
2992
2993template <>
2994EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet32s>(numext::int16_t* out, const Packet32s& x) {
2995 EIGEN_DEBUG_ALIGNED_STORE
2996 _mm512_store_epi32(out, x);
2997}
2998
2999template <>
3000EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet16s>(numext::int16_t* out, const Packet16s& x) {
3001 EIGEN_DEBUG_ALIGNED_STORE
3002#if defined(EIGEN_VECTORIZE_AVX512F) && defined(EIGEN_VECTORIZE_AVX512VL)
3003 _mm256_store_epi32(out, x);
3004#else
3005 _mm256_store_si256(reinterpret_cast<__m256i*>(out), x);
3006#endif
3007}
3008
3009template <>
3010EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet8s>(numext::int16_t* out, const Packet8s& x) {
3011 EIGEN_DEBUG_ALIGNED_STORE
3012#if defined(EIGEN_VECTORIZE_AVX512F) && defined(EIGEN_VECTORIZE_AVX512VL)
3013 _mm256_store_epi32(out, x);
3014#else
3015 _mm_store_si128(reinterpret_cast<__m128i*>(out), x);
3016#endif
3017}
3018
3019template <>
3020EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet32s>(numext::int16_t* out, const Packet32s& x) {
3021 EIGEN_DEBUG_UNALIGNED_STORE
3022 _mm512_storeu_epi32(out, x);
3023}
3024
3025template <>
3026EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet16s>(numext::int16_t* out, const Packet16s& x) {
3027 EIGEN_DEBUG_UNALIGNED_STORE
3028 _mm256_storeu_epi32(out, x);
3029}
3030
3031template <>
3032EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet8s>(numext::int16_t* out, const Packet8s& x) {
3033 EIGEN_DEBUG_UNALIGNED_STORE
3034 _mm_storeu_epi32(out, x);
3035}
3036
3037template <>
3038EIGEN_STRONG_INLINE Packet32s padd(const Packet32s& a, const Packet32s& b) {
3039 return _mm512_add_epi16(a, b);
3040}
3041
3042template <>
3043EIGEN_STRONG_INLINE Packet16s padd(const Packet16s& a, const Packet16s& b) {
3044 return _mm256_add_epi16(a, b);
3045}
3046
3047template <>
3048EIGEN_STRONG_INLINE Packet8s padd(const Packet8s& a, const Packet8s& b) {
3049 return _mm_add_epi16(a, b);
3050}
3051
3052template <>
3053EIGEN_STRONG_INLINE Packet32s psub(const Packet32s& a, const Packet32s& b) {
3054 return _mm512_sub_epi16(a, b);
3055}
3056
3057template <>
3058EIGEN_STRONG_INLINE Packet16s psub(const Packet16s& a, const Packet16s& b) {
3059 return _mm256_sub_epi16(a, b);
3060}
3061
3062template <>
3063EIGEN_STRONG_INLINE Packet8s psub(const Packet8s& a, const Packet8s& b) {
3064 return _mm_sub_epi16(a, b);
3065}
3066
3067template <>
3068EIGEN_STRONG_INLINE Packet32s pmul(const Packet32s& a, const Packet32s& b) {
3069 return _mm512_mullo_epi16(a, b);
3070}
3071
3072template <>
3073EIGEN_STRONG_INLINE Packet16s pmul(const Packet16s& a, const Packet16s& b) {
3074 return _mm256_mullo_epi16(a, b);
3075}
3076
3077template <>
3078EIGEN_STRONG_INLINE Packet8s pmul(const Packet8s& a, const Packet8s& b) {
3079 return _mm_mullo_epi16(a, b);
3080}
3081
3082template <>
3083EIGEN_STRONG_INLINE Packet32s pnegate(const Packet32s& a) {
3084 return _mm512_sub_epi16(_mm512_setzero_si512(), a);
3085}
3086
3087template <>
3088EIGEN_STRONG_INLINE Packet16s pnegate(const Packet16s& a) {
3089 return _mm256_sub_epi16(_mm256_setzero_si256(), a);
3090}
3091
3092template <>
3093EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
3094 return _mm_sub_epi16(_mm_setzero_si128(), a);
3095}
3096
3097template <int N>
3098EIGEN_STRONG_INLINE Packet32s parithmetic_shift_right(Packet32s a) {
3099 return _mm512_srai_epi16(a, N);
3100}
3101
3102template <int N>
3103EIGEN_STRONG_INLINE Packet16s parithmetic_shift_right(Packet16s a) {
3104 return _mm256_srai_epi16(a, N);
3105}
3106
3107template <int N>
3108EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(Packet8s a) {
3109 return _mm_srai_epi16(a, N);
3110}
3111
3112template <int N>
3113EIGEN_STRONG_INLINE Packet32s plogical_shift_left(Packet32s a) {
3114 return _mm512_slli_epi16(a, N);
3115}
3116
3117template <int N>
3118EIGEN_STRONG_INLINE Packet16s plogical_shift_left(Packet16s a) {
3119 return _mm256_slli_epi16(a, N);
3120}
3121
3122template <int N>
3123EIGEN_STRONG_INLINE Packet8s plogical_shift_left(Packet8s a) {
3124 return _mm_slli_epi16(a, N);
3125}
3126
3127template <int N>
3128EIGEN_STRONG_INLINE Packet32s plogical_shift_right(Packet32s a) {
3129 return _mm512_srli_epi16(a, N);
3130}
3131
3132template <int N>
3133EIGEN_STRONG_INLINE Packet16s plogical_shift_right(Packet16s a) {
3134 return _mm256_srli_epi16(a, N);
3135}
3136
3137template <int N>
3138EIGEN_STRONG_INLINE Packet8s plogical_shift_right(Packet8s a) {
3139 return _mm_srli_epi16(a, N);
3140}
3141
3142} // end namespace internal
3143
3144} // end namespace Eigen
3145
3146#endif // EIGEN_PACKET_MATH_AVX512_H
Eigen::Aligned64
@ Aligned64
Definition Constants.h:239
Eigen::Aligned32
@ Aligned32
Definition Constants.h:238
Eigen::Aligned16
@ Aligned16
Definition Constants.h:237
Eigen
Namespace containing all symbols from the Eigen library.
Definition B01_Experimental.dox:1
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition Meta.h:82