Eigen  5.0.1
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_PACKET_MATH_AVX512_H
12#define EIGEN_PACKET_MATH_AVX512_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
23#endif
24
25#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
26#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
27#endif
28
29#ifdef EIGEN_VECTORIZE_FMA
30#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
31#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
32#endif
33#endif
34
35typedef __m512 Packet16f;
36typedef __m512i Packet16i;
37typedef __m512d Packet8d;
38typedef eigen_packet_wrapper<__m512i, 1> Packet8l;
39#ifndef EIGEN_VECTORIZE_AVX512FP16
40typedef eigen_packet_wrapper<__m256i, 1> Packet16h;
41#endif
42typedef eigen_packet_wrapper<__m256i, 2> Packet16bf;
43
44typedef eigen_packet_wrapper<__m512i, 6> Packet32s;
45typedef eigen_packet_wrapper<__m256i, 6> Packet16s;
46typedef eigen_packet_wrapper<__m128i, 6> Packet8s;
47
48#if EIGEN_COMP_NVHPC
49#define EIGEN_AVX512_SHUFFLE_PS(A, B, M) ((__m512)_mm512_shuffle_ps(A, B, M))
50#else
51#define EIGEN_AVX512_SHUFFLE_PS(A, B, M) _mm512_shuffle_ps(A, B, M)
52#endif
53
54EIGEN_STRONG_INLINE Packet16i _eigen_mm512_loadu_epi32(const int* from) {
55#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
56 return _mm512_loadu_si512(reinterpret_cast<const void*>(from));
57#else
58 return _mm512_loadu_epi32(from);
59#endif
60}
61
62EIGEN_STRONG_INLINE Packet16i _eigen_mm512_loadu_epi64(const int64_t* from) {
63#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
64 return _mm512_loadu_si512(reinterpret_cast<const void*>(from));
65#else
66 return _mm512_loadu_epi64(from);
67#endif
68}
69
70EIGEN_STRONG_INLINE void _eigen_mm512_storeu_epi32(void* to, const Packet16i& from) {
71#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
72 _mm512_storeu_si512(to, from);
73#else
74 _mm512_storeu_epi32(to, from);
75#endif
76}
77
78EIGEN_STRONG_INLINE void _eigen_mm512_storeu_epi64(void* to, const Packet16i& from) {
79#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
80 _mm512_storeu_si512(to, from);
81#else
82 _mm512_storeu_epi64(to, from);
83#endif
84}
85
86EIGEN_STRONG_INLINE void _eigen_mm256_storeu_epi32(void* to, const __m256i& from) {
87#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
88 _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
89#else
90 _mm256_storeu_epi32(to, from);
91#endif
92}
93
94EIGEN_STRONG_INLINE void _eigen_mm_storeu_epi32(void* to, const __m128i& from) {
95#if EIGEN_COMP_GNUC && (EIGEN_COMP_CLANG < 1000 || EIGEN_COMP_GNUC < 1010)
96 _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
97#else
98 _mm_storeu_epi32(to, from);
99#endif
100}
101
102template <>
103struct is_arithmetic<__m512> : std::true_type {};
104template <>
105struct is_arithmetic<__m512i> : std::true_type {};
106template <>
107struct is_arithmetic<__m512d> : std::true_type {};
108template <>
109struct is_arithmetic<Packet8l> : std::true_type {};
110
111#ifndef EIGEN_VECTORIZE_AVX512FP16
112template <>
113struct is_arithmetic<Packet16h> : std::true_type {};
114
115template <>
116struct packet_traits<half> : default_packet_traits {
117 typedef Packet16h type;
118 // There is no half-size packet for Packet16h.
119 typedef Packet16h half;
120 enum {
121 Vectorizable = 1,
122 AlignedOnScalar = 1,
123 size = 16,
124
125 HasCmp = 1,
126 HasAdd = 1,
127 HasSub = 1,
128 HasMul = 1,
129 HasDiv = 1,
130 HasNegate = 1,
131 HasAbs = 1,
132 HasMin = 1,
133 HasMax = 1,
134 HasConj = 1,
135 HasSetLinear = 0,
136 HasSqrt = 1,
137 HasRsqrt = 1,
138 HasLog = 1,
139 HasLog1p = 1,
140 HasExp = 1,
141 HasExpm1 = 1,
142 HasBessel = 1,
143 HasNdtri = 1,
144 HasSin = EIGEN_FAST_MATH,
145 HasCos = EIGEN_FAST_MATH,
146 HasTanh = EIGEN_FAST_MATH,
147 HasErf = EIGEN_FAST_MATH,
148 };
149};
150#endif
151
152template <>
153struct packet_traits<float> : default_packet_traits {
154 typedef Packet16f type;
155 typedef Packet8f half;
156 enum {
157 Vectorizable = 1,
158 AlignedOnScalar = 1,
159 size = 16,
160
161 HasAbs = 1,
162 HasMin = 1,
163 HasMax = 1,
164 HasConj = 1,
165 HasSin = EIGEN_FAST_MATH,
166 HasCos = EIGEN_FAST_MATH,
167 HasTan = EIGEN_FAST_MATH,
168 HasACos = 1,
169 HasASin = 1,
170 HasATan = 1,
171 HasATanh = 1,
172 HasSinh = 1,
173 HasCosh = 1,
174 HasASinh = 1,
175 HasACosh = 1,
176 HasSqrt = 1,
177 HasRsqrt = 1,
178 HasCbrt = 1,
179 HasLog = 1,
180 HasLog10 = 1,
181 HasLog1p = 1,
182 HasExpm1 = 1,
183 HasNdtri = 1,
184 HasBessel = 1,
185 HasExp = 1,
186 HasPow = 1,
187 HasReciprocal = EIGEN_FAST_MATH,
188 HasTanh = EIGEN_FAST_MATH,
189 HasErf = EIGEN_FAST_MATH,
190 HasErfc = EIGEN_FAST_MATH,
191 HasCmp = 1,
192 HasDiv = 1
193 };
194};
195template <>
196struct packet_traits<double> : default_packet_traits {
197 typedef Packet8d type;
198 typedef Packet4d half;
199 enum {
200 Vectorizable = 1,
201 AlignedOnScalar = 1,
202 size = 8,
203 HasSqrt = 1,
204 HasRsqrt = 1,
205 HasCbrt = 1,
206 HasSin = EIGEN_FAST_MATH,
207 HasCos = EIGEN_FAST_MATH,
208 HasTan = EIGEN_FAST_MATH,
209 HasSinh = 1,
210 HasCosh = 1,
211 HasASinh = 1,
212 HasACosh = 1,
213 HasLog = 1,
214 HasLog10 = 1,
215 HasExp = 1,
216 HasLog1p = 1,
217 HasExpm1 = 1,
218 HasPow = 1,
219 HasATan = 1,
220 HasTanh = EIGEN_FAST_MATH,
221 HasErf = EIGEN_FAST_MATH,
222 HasErfc = EIGEN_FAST_MATH,
223 HasATanh = 1,
224 HasCmp = 1,
225 HasDiv = 1
226 };
227};
228
229template <>
230struct packet_traits<int> : default_packet_traits {
231 typedef Packet16i type;
232 typedef Packet8i half;
233 enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, HasDiv = 1, size = 16 };
234};
235
236template <>
237struct packet_traits<int64_t> : default_packet_traits {
238 typedef Packet8l type;
239 typedef Packet4l half;
240 enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, size = 8 };
241};
242
243template <>
244struct unpacket_traits<Packet16f> {
245 typedef float type;
246 typedef Packet8f half;
247 typedef Packet16i integer_packet;
248 typedef uint16_t mask_t;
249 enum {
250 size = 16,
251 alignment = Aligned64,
252 vectorizable = true,
253 masked_load_available = true,
254 masked_store_available = true,
255 masked_fpops_available = true
256 };
257};
258template <>
259struct unpacket_traits<Packet8d> {
260 typedef double type;
261 typedef Packet4d half;
262 typedef Packet8l integer_packet;
263 typedef uint8_t mask_t;
264 enum {
265 size = 8,
266 alignment = Aligned64,
267 vectorizable = true,
268 masked_load_available = true,
269 masked_store_available = true,
270 masked_fpops_available = true
271 };
272};
273template <>
274struct unpacket_traits<Packet16i> {
275 typedef int type;
276 typedef Packet8i half;
277 enum {
278 size = 16,
279 alignment = Aligned64,
280 vectorizable = true,
281 masked_load_available = false,
282 masked_store_available = false
283 };
284};
285
286template <>
287struct unpacket_traits<Packet8l> {
288 typedef int64_t type;
289 typedef Packet4l half;
290 enum {
291 size = 8,
292 alignment = Aligned64,
293 vectorizable = true,
294 masked_load_available = false,
295 masked_store_available = false
296 };
297};
298
299#ifndef EIGEN_VECTORIZE_AVX512FP16
300template <>
301struct unpacket_traits<Packet16h> {
302 typedef Eigen::half type;
303 typedef Packet8h half;
304 enum {
305 size = 16,
306 alignment = Aligned32,
307 vectorizable = true,
308 masked_load_available = false,
309 masked_store_available = false
310 };
311};
312#endif
313
314template <>
315struct unpacket_traits<Packet32s> {
316 typedef numext::int16_t type;
317 typedef Packet16s half;
318 enum {
319 size = 32,
320 alignment = Aligned64,
321 vectorizable = false,
322 };
323};
324
325template <>
326struct unpacket_traits<Packet16s> {
327 typedef numext::int16_t type;
328 typedef Packet8s half;
329 enum {
330 size = 16,
331 alignment = Aligned32,
332 vectorizable = false,
333 };
334};
335
336template <>
337struct unpacket_traits<Packet8s> {
338 typedef numext::int16_t type;
339 typedef Packet8s half;
340 enum {
341 size = 8,
342 alignment = Aligned16,
343 vectorizable = false,
344 };
345};
346
347template <>
348EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
349 return _mm512_set1_ps(from);
350}
351template <>
352EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) {
353 return _mm512_set1_pd(from);
354}
355template <>
356EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) {
357 return _mm512_set1_epi32(from);
358}
359template <>
360EIGEN_STRONG_INLINE Packet8l pset1<Packet8l>(const int64_t& from) {
361 return _mm512_set1_epi64(from);
362}
363
364template <>
365EIGEN_STRONG_INLINE Packet16f pset1frombits<Packet16f>(unsigned int from) {
366 return _mm512_castsi512_ps(_mm512_set1_epi32(from));
367}
368
369template <>
370EIGEN_STRONG_INLINE Packet8d pset1frombits<Packet8d>(const numext::uint64_t from) {
371 return _mm512_castsi512_pd(_mm512_set1_epi64(from));
372}
373
374template <>
375EIGEN_STRONG_INLINE Packet16f pzero(const Packet16f& /*a*/) {
376 return _mm512_setzero_ps();
377}
378template <>
379EIGEN_STRONG_INLINE Packet8d pzero(const Packet8d& /*a*/) {
380 return _mm512_setzero_pd();
381}
382template <>
383EIGEN_STRONG_INLINE Packet16i pzero(const Packet16i& /*a*/) {
384 return _mm512_setzero_si512();
385}
386
387template <>
388EIGEN_STRONG_INLINE Packet8l pzero(const Packet8l& /*a*/) {
389 return _mm512_setzero_si512();
390}
391
392template <>
393EIGEN_STRONG_INLINE Packet16f peven_mask(const Packet16f& /*a*/) {
394 Packet16f r = _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1));
395 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
396 return r;
397}
398template <>
399EIGEN_STRONG_INLINE Packet16i peven_mask(const Packet16i& /*a*/) {
400 return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1);
401}
402template <>
403EIGEN_STRONG_INLINE Packet8d peven_mask(const Packet8d& /*a*/) {
404 Packet8d r = _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1));
405 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
406 return r;
407}
408template <>
409EIGEN_STRONG_INLINE Packet8l peven_mask(const Packet8l& /*a*/) {
410 return _mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1);
411}
412
413template <>
414EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) {
415#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
416 // Inline asm here helps reduce some register spilling in TRSM kernels.
417 // See note in unrolls::gemm::microKernel in TrsmKernel.h
418 Packet16f ret;
419 __asm__("vbroadcastss %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
420 return ret;
421#else
422 return _mm512_broadcastss_ps(_mm_load_ps1(from));
423#endif
424}
425template <>
426EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) {
427#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
428 Packet8d ret;
429 __asm__("vbroadcastsd %[mem], %[dst]" : [dst] "=v"(ret) : [mem] "m"(*from));
430 return ret;
431#else
432 return _mm512_set1_pd(*from);
433#endif
434}
435
436template <>
437EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) {
438 return _mm512_add_ps(_mm512_set1_ps(a), _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f,
439 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
440}
441template <>
442EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) {
443 return _mm512_add_pd(_mm512_set1_pd(a), _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
444}
445template <>
446EIGEN_STRONG_INLINE Packet16i plset<Packet16i>(const int& a) {
447 return _mm512_add_epi32(_mm512_set1_epi32(a), _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
448}
449template <>
450EIGEN_STRONG_INLINE Packet8l plset<Packet8l>(const int64_t& a) {
451 return _mm512_add_epi64(_mm512_set1_epi64(a), _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0));
452}
453
454template <>
455EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, const Packet16f& b) {
456 return _mm512_add_ps(a, b);
457}
458template <>
459EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, const Packet8d& b) {
460 return _mm512_add_pd(a, b);
461}
462template <>
463EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(const Packet16i& a, const Packet16i& b) {
464 return _mm512_add_epi32(a, b);
465}
466template <>
467EIGEN_STRONG_INLINE Packet8l padd<Packet8l>(const Packet8l& a, const Packet8l& b) {
468 return _mm512_add_epi64(a, b);
469}
470
471template <>
472EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, const Packet16f& b, uint16_t umask) {
473 __mmask16 mask = static_cast<__mmask16>(umask);
474 return _mm512_maskz_add_ps(mask, a, b);
475}
476template <>
477EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a, const Packet8d& b, uint8_t umask) {
478 __mmask8 mask = static_cast<__mmask8>(umask);
479 return _mm512_maskz_add_pd(mask, a, b);
480}
481
482template <>
483EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a, const Packet16f& b) {
484 return _mm512_sub_ps(a, b);
485}
486template <>
487EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a, const Packet8d& b) {
488 return _mm512_sub_pd(a, b);
489}
490template <>
491EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(const Packet16i& a, const Packet16i& b) {
492 return _mm512_sub_epi32(a, b);
493}
494template <>
495EIGEN_STRONG_INLINE Packet8l psub<Packet8l>(const Packet8l& a, const Packet8l& b) {
496 return _mm512_sub_epi64(a, b);
497}
498
499template <>
500EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
501 // NOTE: MSVC seems to struggle with _mm512_set1_epi32, leading to random results.
502 // The intel docs give it a relatively high latency as well, so we're probably
503 // better off with using _mm512_set_epi32 directly anyways.
504 const __m512i mask =
505 _mm512_set_epi32(SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32,
506 SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32,
507 SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32);
508 return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a), mask));
509}
510template <>
511EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) {
512 const __m512i mask = _mm512_set_epi64(SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64,
513 SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64);
514 return _mm512_castsi512_pd(_mm512_xor_epi64(_mm512_castpd_si512(a), mask));
515}
516template <>
517EIGEN_STRONG_INLINE Packet16i pnegate(const Packet16i& a) {
518 return _mm512_sub_epi32(_mm512_setzero_si512(), a);
519}
520template <>
521EIGEN_STRONG_INLINE Packet8l pnegate(const Packet8l& a) {
522 return _mm512_sub_epi64(_mm512_setzero_si512(), a);
523}
524
525template <>
526EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a, const Packet16f& b) {
527 return _mm512_mul_ps(a, b);
528}
529template <>
530EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a, const Packet8d& b) {
531 return _mm512_mul_pd(a, b);
532}
533template <>
534EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(const Packet16i& a, const Packet16i& b) {
535 return _mm512_mullo_epi32(a, b);
536}
537template <>
538EIGEN_STRONG_INLINE Packet8l pmul<Packet8l>(const Packet8l& a, const Packet8l& b) {
539#ifdef EIGEN_VECTORIZE_AVX512DQ
540 return _mm512_mullo_epi64(a, b);
541#else
542 return _mm512_mullox_epi64(a, b);
543#endif
544}
545
546template <>
547EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a, const Packet16f& b) {
548 return _mm512_div_ps(a, b);
549}
550
551template <>
552EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a, const Packet8d& b) {
553 return _mm512_div_pd(a, b);
554}
555
556template <>
557EIGEN_STRONG_INLINE Packet16i pdiv<Packet16i>(const Packet16i& a, const Packet16i& b) {
558 Packet8i q_lo = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 0), _mm512_extracti64x4_epi64(b, 0));
559 Packet8i q_hi = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 1), _mm512_extracti64x4_epi64(b, 1));
560 return _mm512_inserti64x4(_mm512_castsi256_si512(q_lo), q_hi, 1);
561}
562
563#ifdef EIGEN_VECTORIZE_FMA
564template <>
565EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
566 return _mm512_fmadd_ps(a, b, c);
567}
568template <>
569EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
570 return _mm512_fmadd_pd(a, b, c);
571}
572
573template <>
574EIGEN_STRONG_INLINE Packet16f pmsub(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
575 return _mm512_fmsub_ps(a, b, c);
576}
577template <>
578EIGEN_STRONG_INLINE Packet8d pmsub(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
579 return _mm512_fmsub_pd(a, b, c);
580}
581
582template <>
583EIGEN_STRONG_INLINE Packet16f pnmadd(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
584 return _mm512_fnmadd_ps(a, b, c);
585}
586template <>
587EIGEN_STRONG_INLINE Packet8d pnmadd(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
588 return _mm512_fnmadd_pd(a, b, c);
589}
590
591template <>
592EIGEN_STRONG_INLINE Packet16f pnmsub(const Packet16f& a, const Packet16f& b, const Packet16f& c) {
593 return _mm512_fnmsub_ps(a, b, c);
594}
595template <>
596EIGEN_STRONG_INLINE Packet8d pnmsub(const Packet8d& a, const Packet8d& b, const Packet8d& c) {
597 return _mm512_fnmsub_pd(a, b, c);
598}
599#endif
600
601template <>
602EIGEN_DEVICE_FUNC inline Packet16f pselect(const Packet16f& mask, const Packet16f& a, const Packet16f& b) {
603 __mmask16 mask16 = _mm512_cmpeq_epi32_mask(_mm512_castps_si512(mask), _mm512_setzero_epi32());
604 return _mm512_mask_blend_ps(mask16, a, b);
605}
606
607template <>
608EIGEN_DEVICE_FUNC inline Packet16i pselect(const Packet16i& mask, const Packet16i& a, const Packet16i& b) {
609 __mmask16 mask16 = _mm512_cmpeq_epi32_mask(mask, _mm512_setzero_epi32());
610 return _mm512_mask_blend_epi32(mask16, a, b);
611}
612
613template <>
614EIGEN_DEVICE_FUNC inline Packet8l pselect(const Packet8l& mask, const Packet8l& a, const Packet8l& b) {
615 __mmask8 mask8 = _mm512_cmpeq_epi64_mask(mask, _mm512_setzero_si512());
616 return _mm512_mask_blend_epi64(mask8, a, b);
617}
618
619template <>
620EIGEN_DEVICE_FUNC inline Packet8d pselect(const Packet8d& mask, const Packet8d& a, const Packet8d& b) {
621 __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
622 return _mm512_mask_blend_pd(mask8, a, b);
623}
624
625template <>
626EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a, const Packet16f& b) {
627 // Arguments are reversed to match NaN propagation behavior of std::min.
628 return _mm512_min_ps(b, a);
629}
630template <>
631EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a, const Packet8d& b) {
632 // Arguments are reversed to match NaN propagation behavior of std::min.
633 return _mm512_min_pd(b, a);
634}
635template <>
636EIGEN_STRONG_INLINE Packet16i pmin<Packet16i>(const Packet16i& a, const Packet16i& b) {
637 return _mm512_min_epi32(b, a);
638}
639template <>
640EIGEN_STRONG_INLINE Packet8l pmin<Packet8l>(const Packet8l& a, const Packet8l& b) {
641 return _mm512_min_epi64(b, a);
642}
643
644template <>
645EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a, const Packet16f& b) {
646 // Arguments are reversed to match NaN propagation behavior of std::max.
647 return _mm512_max_ps(b, a);
648}
649template <>
650EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a, const Packet8d& b) {
651 // Arguments are reversed to match NaN propagation behavior of std::max.
652 return _mm512_max_pd(b, a);
653}
654template <>
655EIGEN_STRONG_INLINE Packet16i pmax<Packet16i>(const Packet16i& a, const Packet16i& b) {
656 return _mm512_max_epi32(b, a);
657}
658template <>
659EIGEN_STRONG_INLINE Packet8l pmax<Packet8l>(const Packet8l& a, const Packet8l& b) {
660 return _mm512_max_epi64(b, a);
661}
662
663// Add specializations for min/max with prescribed NaN propagation.
664template <>
665EIGEN_STRONG_INLINE Packet16f pmin<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
666 return pminmax_propagate_numbers(a, b, pmin<Packet16f>);
667}
668template <>
669EIGEN_STRONG_INLINE Packet8d pmin<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
670 return pminmax_propagate_numbers(a, b, pmin<Packet8d>);
671}
672template <>
673EIGEN_STRONG_INLINE Packet16f pmax<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
674 return pminmax_propagate_numbers(a, b, pmax<Packet16f>);
675}
676template <>
677EIGEN_STRONG_INLINE Packet8d pmax<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
678 return pminmax_propagate_numbers(a, b, pmax<Packet8d>);
679}
680template <>
681EIGEN_STRONG_INLINE Packet16f pmin<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
682 return pminmax_propagate_nan(a, b, pmin<Packet16f>);
683}
684template <>
685EIGEN_STRONG_INLINE Packet8d pmin<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
686 return pminmax_propagate_nan(a, b, pmin<Packet8d>);
687}
688template <>
689EIGEN_STRONG_INLINE Packet16f pmax<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
690 return pminmax_propagate_nan(a, b, pmax<Packet16f>);
691}
692template <>
693EIGEN_STRONG_INLINE Packet8d pmax<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
694 return pminmax_propagate_nan(a, b, pmax<Packet8d>);
695}
696
697#ifdef EIGEN_VECTORIZE_AVX512DQ
698template <int I_>
699EIGEN_STRONG_INLINE Packet8f extract256(const Packet16f& x) {
700 return _mm512_extractf32x8_ps(x, I_);
701}
702template <int I_>
703EIGEN_STRONG_INLINE Packet2d extract128(const Packet8d& x) {
704 return _mm512_extractf64x2_pd(x, I_);
705}
706EIGEN_STRONG_INLINE Packet16f cat256(const Packet8f& a, const Packet8f& b) {
707 return _mm512_insertf32x8(_mm512_castps256_ps512(a), b, 1);
708}
709EIGEN_STRONG_INLINE Packet16i cat256i(const Packet8i& a, const Packet8i& b) {
710 return _mm512_inserti32x8(_mm512_castsi256_si512(a), b, 1);
711}
712#else
713// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
714template <int I_>
715EIGEN_STRONG_INLINE Packet8f extract256(const Packet16f& x) {
716 return _mm256_castsi256_ps(_mm512_extracti64x4_epi64(_mm512_castps_si512(x), I_));
717}
718
719// AVX512F does not define _mm512_extractf64x2_pd to extract _m128 from _m512
720template <int I_>
721EIGEN_STRONG_INLINE Packet2d extract128(const Packet8d& x) {
722 return _mm_castsi128_pd(_mm512_extracti32x4_epi32(_mm512_castpd_si512(x), I_));
723}
724
725EIGEN_STRONG_INLINE Packet16f cat256(const Packet8f& a, const Packet8f& b) {
726 return _mm512_castsi512_ps(
727 _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), _mm256_castps_si256(b), 1));
728}
729EIGEN_STRONG_INLINE Packet16i cat256i(const Packet8i& a, const Packet8i& b) {
730 return _mm512_inserti64x4(_mm512_castsi256_si512(a), b, 1);
731}
732#endif
733
734// Helper function for bit packing snippet of low precision comparison.
735// It packs the flags from 32x16 to 16x16.
736EIGEN_STRONG_INLINE __m256i Pack32To16(const Packet16f& rf) {
737 // Split data into small pieces and handle with AVX instructions
738 // to guarantee internal order of vector.
739 // Operation:
740 // dst[15:0] := Saturate16(rf[31:0])
741 // dst[31:16] := Saturate16(rf[63:32])
742 // ...
743 // dst[255:240] := Saturate16(rf[255:224])
744 __m256i lo = _mm256_castps_si256(extract256<0>(rf));
745 __m256i hi = _mm256_castps_si256(extract256<1>(rf));
746 __m128i result_lo = _mm_packs_epi32(_mm256_extractf128_si256(lo, 0), _mm256_extractf128_si256(lo, 1));
747 __m128i result_hi = _mm_packs_epi32(_mm256_extractf128_si256(hi, 0), _mm256_extractf128_si256(hi, 1));
748 return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1);
749}
750
751template <>
752EIGEN_STRONG_INLINE Packet16f pisnan(const Packet16f& a) {
753 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_UNORD_Q);
754 return _mm512_castsi512_ps(_mm512_maskz_set1_epi32(mask, int32_t(-1)));
755}
756
757template <>
758EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) {
759 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ);
760 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
761}
762template <>
763EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) {
764 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ);
765 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
766}
767
768template <>
769EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) {
770 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ);
771 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
772}
773
774template <>
775EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) {
776 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ);
777 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
778}
779
780template <>
781EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) {
782 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_EQ);
783 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
784}
785template <>
786EIGEN_STRONG_INLINE Packet16i pcmp_le(const Packet16i& a, const Packet16i& b) {
787 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE);
788 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
789}
790template <>
791EIGEN_STRONG_INLINE Packet16i pcmp_lt(const Packet16i& a, const Packet16i& b) {
792 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT);
793 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
794}
795
796template <>
797EIGEN_STRONG_INLINE Packet8l pcmp_eq(const Packet8l& a, const Packet8l& b) {
798 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_EQ);
799 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
800}
801template <>
802EIGEN_STRONG_INLINE Packet8l pcmp_le(const Packet8l& a, const Packet8l& b) {
803 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LE);
804 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
805}
806template <>
807EIGEN_STRONG_INLINE Packet8l pcmp_lt(const Packet8l& a, const Packet8l& b) {
808 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LT);
809 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
810}
811
812template <>
813EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) {
814 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ);
815 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
816}
817template <>
818EIGEN_STRONG_INLINE Packet8d pcmp_le(const Packet8d& a, const Packet8d& b) {
819 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ);
820 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
821}
822template <>
823EIGEN_STRONG_INLINE Packet8d pcmp_lt(const Packet8d& a, const Packet8d& b) {
824 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ);
825 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
826}
827template <>
828EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(const Packet8d& a, const Packet8d& b) {
829 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ);
830 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
831}
832
833template <>
834EIGEN_STRONG_INLINE Packet16f print<Packet16f>(const Packet16f& a) {
835 return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION);
836}
837template <>
838EIGEN_STRONG_INLINE Packet8d print<Packet8d>(const Packet8d& a) {
839 return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION);
840}
841
842template <>
843EIGEN_STRONG_INLINE Packet16f pceil<Packet16f>(const Packet16f& a) {
844 return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF);
845}
846template <>
847EIGEN_STRONG_INLINE Packet8d pceil<Packet8d>(const Packet8d& a) {
848 return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF);
849}
850
851template <>
852EIGEN_STRONG_INLINE Packet16f pfloor<Packet16f>(const Packet16f& a) {
853 return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF);
854}
855template <>
856EIGEN_STRONG_INLINE Packet8d pfloor<Packet8d>(const Packet8d& a) {
857 return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF);
858}
859
860template <>
861EIGEN_STRONG_INLINE Packet16f ptrunc<Packet16f>(const Packet16f& a) {
862 return _mm512_roundscale_ps(a, _MM_FROUND_TO_ZERO);
863}
864template <>
865EIGEN_STRONG_INLINE Packet8d ptrunc<Packet8d>(const Packet8d& a) {
866 return _mm512_roundscale_pd(a, _MM_FROUND_TO_ZERO);
867}
868
869template <>
870EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(const Packet16i& /*a*/) {
871 return _mm512_set1_epi32(int32_t(-1));
872}
873
874template <>
875EIGEN_STRONG_INLINE Packet8l ptrue<Packet8l>(const Packet8l& /*a*/) {
876 return _mm512_set1_epi64(int64_t(-1));
877}
878
879template <>
880EIGEN_STRONG_INLINE Packet16f ptrue<Packet16f>(const Packet16f& a) {
881 Packet16f r = _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a)));
882 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
883 return r;
884}
885
886template <>
887EIGEN_STRONG_INLINE Packet8d ptrue<Packet8d>(const Packet8d& a) {
888 Packet8d r = _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a)));
889 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
890 return r;
891}
892
893template <>
894EIGEN_STRONG_INLINE Packet16i pand<Packet16i>(const Packet16i& a, const Packet16i& b) {
895 return _mm512_and_si512(a, b);
896}
897
898template <>
899EIGEN_STRONG_INLINE Packet8l pand<Packet8l>(const Packet8l& a, const Packet8l& b) {
900 return _mm512_and_si512(a, b);
901}
902
903template <>
904EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a, const Packet16f& b) {
905#ifdef EIGEN_VECTORIZE_AVX512DQ
906 return _mm512_and_ps(a, b);
907#else
908 return _mm512_castsi512_ps(pand(_mm512_castps_si512(a), _mm512_castps_si512(b)));
909#endif
910}
911template <>
912EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a, const Packet8d& b) {
913#ifdef EIGEN_VECTORIZE_AVX512DQ
914 return _mm512_and_pd(a, b);
915#else
916 Packet8d res = _mm512_undefined_pd();
917 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
918 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
919 res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
920
921 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
922 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
923 return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
924#endif
925}
926
927template <>
928EIGEN_STRONG_INLINE Packet16i por<Packet16i>(const Packet16i& a, const Packet16i& b) {
929 return _mm512_or_si512(a, b);
930}
931
932template <>
933EIGEN_STRONG_INLINE Packet8l por<Packet8l>(const Packet8l& a, const Packet8l& b) {
934 return _mm512_or_si512(a, b);
935}
936
937template <>
938EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a, const Packet16f& b) {
939#ifdef EIGEN_VECTORIZE_AVX512DQ
940 return _mm512_or_ps(a, b);
941#else
942 return _mm512_castsi512_ps(por(_mm512_castps_si512(a), _mm512_castps_si512(b)));
943#endif
944}
945
946template <>
947EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a, const Packet8d& b) {
948#ifdef EIGEN_VECTORIZE_AVX512DQ
949 return _mm512_or_pd(a, b);
950#else
951 return _mm512_castsi512_pd(por(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
952#endif
953}
954
955template <>
956EIGEN_STRONG_INLINE Packet16i pxor<Packet16i>(const Packet16i& a, const Packet16i& b) {
957 return _mm512_xor_si512(a, b);
958}
959
960template <>
961EIGEN_STRONG_INLINE Packet8l pxor<Packet8l>(const Packet8l& a, const Packet8l& b) {
962 return _mm512_xor_si512(a, b);
963}
964
965template <>
966EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a, const Packet16f& b) {
967#ifdef EIGEN_VECTORIZE_AVX512DQ
968 return _mm512_xor_ps(a, b);
969#else
970 return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a), _mm512_castps_si512(b)));
971#endif
972}
973
974template <>
975EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a, const Packet8d& b) {
976#ifdef EIGEN_VECTORIZE_AVX512DQ
977 return _mm512_xor_pd(a, b);
978#else
979 return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
980#endif
981}
982
983template <>
984EIGEN_STRONG_INLINE Packet16i pandnot<Packet16i>(const Packet16i& a, const Packet16i& b) {
985 return _mm512_andnot_si512(b, a);
986}
987
988template <>
989EIGEN_STRONG_INLINE Packet8l pandnot<Packet8l>(const Packet8l& a, const Packet8l& b) {
990 return _mm512_andnot_si512(b, a);
991}
992
993template <>
994EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a, const Packet16f& b) {
995#ifdef EIGEN_VECTORIZE_AVX512DQ
996 return _mm512_andnot_ps(b, a);
997#else
998 return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a), _mm512_castps_si512(b)));
999#endif
1000}
1001template <>
1002EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a, const Packet8d& b) {
1003#ifdef EIGEN_VECTORIZE_AVX512DQ
1004 return _mm512_andnot_pd(b, a);
1005#else
1006 return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
1007#endif
1008}
1009
1010template <>
1011EIGEN_STRONG_INLINE Packet16f pround<Packet16f>(const Packet16f& a) {
1012 // Work-around for default std::round rounding mode.
1013 const Packet16f mask = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x80000000u));
1014 const Packet16f prev0dot5 = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
1015 return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
1016}
1017template <>
1018EIGEN_STRONG_INLINE Packet8d pround<Packet8d>(const Packet8d& a) {
1019 // Work-around for default std::round rounding mode.
1020 const Packet8d mask = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
1021 const Packet8d prev0dot5 = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
1022 return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
1023}
1024
1025template <int N>
1026EIGEN_STRONG_INLINE Packet16i parithmetic_shift_right(const Packet16i& a) {
1027 return _mm512_srai_epi32(a, N);
1028}
1029
1030template <int N>
1031EIGEN_STRONG_INLINE Packet16i plogical_shift_right(const Packet16i& a) {
1032 return _mm512_srli_epi32(a, N);
1033}
1034
1035template <int N>
1036EIGEN_STRONG_INLINE Packet16i plogical_shift_left(const Packet16i& a) {
1037 return _mm512_slli_epi32(a, N);
1038}
1039
1040template <int N>
1041EIGEN_STRONG_INLINE Packet8l parithmetic_shift_right(const Packet8l& a) {
1042 return _mm512_srai_epi64(a, N);
1043}
1044
1045template <int N>
1046EIGEN_STRONG_INLINE Packet8l plogical_shift_right(const Packet8l& a) {
1047 return _mm512_srli_epi64(a, N);
1048}
1049
1050template <int N>
1051EIGEN_STRONG_INLINE Packet8l plogical_shift_left(const Packet8l& a) {
1052 return _mm512_slli_epi64(a, N);
1053}
1054
1055template <>
1056EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
1057 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
1058}
1059template <>
1060EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) {
1061 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
1062}
1063template <>
1064EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) {
1065 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
1066}
1067template <>
1068EIGEN_STRONG_INLINE Packet8l pload<Packet8l>(const int64_t* from) {
1069 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_epi64(from);
1070}
1071
1072template <>
1073EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
1074 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
1075}
1076template <>
1077EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
1078 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
1079}
1080template <>
1081EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
1082 EIGEN_DEBUG_UNALIGNED_LOAD return _eigen_mm512_loadu_epi32(from);
1083}
1084template <>
1085EIGEN_STRONG_INLINE Packet8l ploadu<Packet8l>(const int64_t* from) {
1086 EIGEN_DEBUG_UNALIGNED_LOAD return _eigen_mm512_loadu_epi64(from);
1087}
1088
1089template <>
1090EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from, uint16_t umask) {
1091 __mmask16 mask = static_cast<__mmask16>(umask);
1092 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from);
1093}
1094template <>
1095EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from, uint8_t umask) {
1096 __mmask8 mask = static_cast<__mmask8>(umask);
1097 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_pd(mask, from);
1098}
1099
1100// Loads 8 floats from memory and returns the packet
1101// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
1102template <>
1103EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) {
1104 // an unaligned load is required here as there is no requirement
1105 // on the alignment of input pointer 'from'
1106 __m256i low_half = _mm256_castps_si256(_mm256_loadu_ps(from));
1107 __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1108 __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1109 return pairs;
1110}
1111
1112// Loads 4 doubles from memory and returns the packet {a0, a0, a1, a1, a2, a2, a3,
1113// a3}
1114template <>
1115EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
1116 Packet8d tmp = _mm512_castpd256_pd512(ploadu<Packet4d>(from));
1117 const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1118 return _mm512_permutexvar_pd(scatter_mask, tmp);
1119}
1120
1121// Loads 4 int64_t from memory and returns the packet {a0, a0, a1, a1, a2, a2, a3,
1122// a3}
1123template <>
1124EIGEN_STRONG_INLINE Packet8l ploaddup<Packet8l>(const int64_t* from) {
1125 Packet8l tmp = _mm512_castsi256_si512(ploadu<Packet4l>(from));
1126 const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1127 return _mm512_permutexvar_epi64(scatter_mask, tmp);
1128}
1129
1130// Loads 8 integers from memory and returns the packet
1131// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
1132template <>
1133EIGEN_STRONG_INLINE Packet16i ploaddup<Packet16i>(const int* from) {
1134 __m256i low_half = _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1135 __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1136 __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1137 return _mm512_castps_si512(pairs);
1138}
1139
1140// Loads 4 floats from memory and returns the packet
1141// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
1142template <>
1143EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) {
1144 Packet16f tmp = _mm512_castps128_ps512(ploadu<Packet4f>(from));
1145 const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1146 return _mm512_permutexvar_ps(scatter_mask, tmp);
1147}
1148
1149// Loads 2 doubles from memory and returns the packet
1150// {a0, a0 a0, a0, a1, a1, a1, a1}
1151template <>
1152EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) {
1153 __m256d lane0 = _mm256_set1_pd(*from);
1154 __m256d lane1 = _mm256_set1_pd(*(from + 1));
1155 __m512d tmp = _mm512_undefined_pd();
1156 tmp = _mm512_insertf64x4(tmp, lane0, 0);
1157 return _mm512_insertf64x4(tmp, lane1, 1);
1158}
1159
1160// Loads 2 int64_t from memory and returns the packet
1161// {a0, a0 a0, a0, a1, a1, a1, a1}
1162template <>
1163EIGEN_STRONG_INLINE Packet8l ploadquad<Packet8l>(const int64_t* from) {
1164 __m256i lane0 = _mm256_set1_epi64x(*from);
1165 __m256i lane1 = _mm256_set1_epi64x(*(from + 1));
1166 __m512i tmp = _mm512_undefined_epi32();
1167 tmp = _mm512_inserti64x4(tmp, lane0, 0);
1168 return _mm512_inserti64x4(tmp, lane1, 1);
1169}
1170
1171// Loads 4 integers from memory and returns the packet
1172// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
1173template <>
1174EIGEN_STRONG_INLINE Packet16i ploadquad<Packet16i>(const int* from) {
1175 Packet16i tmp = _mm512_castsi128_si512(ploadu<Packet4i>(from));
1176 const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1177 return _mm512_permutexvar_epi32(scatter_mask, tmp);
1178}
1179
1180template <>
1181EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
1182 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
1183}
1184template <>
1185EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) {
1186 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
1187}
1188template <>
1189EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) {
1190 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi32(to, from);
1191}
1192template <>
1193EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet8l& from) {
1194 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi64(to, from);
1195}
1196
1197template <>
1198EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
1199 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
1200}
1201template <>
1202EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
1203 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
1204}
1205template <>
1206EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
1207 EIGEN_DEBUG_UNALIGNED_STORE _eigen_mm512_storeu_epi32(to, from);
1208}
1209template <>
1210EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet8l& from) {
1211 EIGEN_DEBUG_UNALIGNED_STORE _eigen_mm512_storeu_epi64(to, from);
1212}
1213template <>
1214EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from, uint16_t umask) {
1215 __mmask16 mask = static_cast<__mmask16>(umask);
1216 EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from);
1217}
1218template <>
1219EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from, uint8_t umask) {
1220 __mmask8 mask = static_cast<__mmask8>(umask);
1221 EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from);
1222}
1223
1224template <typename Scalar, typename Packet>
1225EIGEN_DEVICE_FUNC inline Packet pgather(const Packet& src, const Scalar* from, Index stride,
1226 typename unpacket_traits<Packet>::mask_t umask);
1227template <>
1228EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const Packet16f& src, const float* from, Index stride,
1229 uint16_t umask) {
1230 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1231 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1232 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1233 __mmask16 mask = static_cast<__mmask16>(umask);
1234
1235 return _mm512_mask_i32gather_ps(src, mask, indices, from, 4);
1236}
1237template <>
1238EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const Packet8d& src, const double* from, Index stride,
1239 uint8_t umask) {
1240 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1241 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1242 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1243 __mmask8 mask = static_cast<__mmask8>(umask);
1244
1245 return _mm512_mask_i32gather_pd(src, mask, indices, from, 8);
1246}
1247
1248template <>
1249EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from, Index stride) {
1250 if (stride == 2) {
1251 // The overlapping loads end at from[30], not the unused coefficient from[31].
1252 const Packet16f low = _mm512_loadu_ps(from);
1253 const Packet16f high = _mm512_loadu_ps(from + 15);
1254 const Packet16i indices = _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 17, 19, 21, 23, 25, 27, 29, 31);
1255 return _mm512_permutex2var_ps(low, indices, high);
1256 }
1257 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1258 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1259 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1260
1261 return _mm512_i32gather_ps(indices, from, 4);
1262}
1263template <>
1264EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from, Index stride) {
1265 if (stride == 2) {
1266 const Packet8d low = _mm512_loadu_pd(from);
1267 const Packet8d high = _mm512_loadu_pd(from + 7);
1268 const __m512i indices = _mm512_setr_epi64(0, 2, 4, 6, 9, 11, 13, 15);
1269 return _mm512_permutex2var_pd(low, indices, high);
1270 }
1271 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1272 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1273 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1274
1275 return _mm512_i32gather_pd(indices, from, 8);
1276}
1277template <>
1278EIGEN_DEVICE_FUNC inline Packet8l pgather<int64_t, Packet8l>(const int64_t* from, Index stride) {
1279 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1280 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1281 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1282
1283 return _mm512_i32gather_epi64(indices, from, 8);
1284}
1285template <>
1286EIGEN_DEVICE_FUNC inline Packet16i pgather<int, Packet16i>(const int* from, Index stride) {
1287 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1288 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1289 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1290 return _mm512_i32gather_epi32(indices, from, 4);
1291}
1292
1293template <typename Scalar, typename Packet>
1294EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index stride,
1295 typename unpacket_traits<Packet>::mask_t umask);
1296template <>
1297EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride,
1298 uint16_t umask) {
1299 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1300 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1301 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1302 __mmask16 mask = static_cast<__mmask16>(umask);
1303 _mm512_mask_i32scatter_ps(to, mask, indices, from, 4);
1304}
1305template <>
1306EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride,
1307 uint8_t umask) {
1308 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1309 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1310 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1311 __mmask8 mask = static_cast<__mmask8>(umask);
1312 _mm512_mask_i32scatter_pd(to, mask, indices, from, 8);
1313}
1314template <>
1315EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride) {
1316 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1317 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1318 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1319 _mm512_i32scatter_ps(to, indices, from, 4);
1320}
1321template <>
1322EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to, const Packet8d& from, Index stride) {
1323 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1324 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1325 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1326 _mm512_i32scatter_pd(to, indices, from, 8);
1327}
1328template <>
1329EIGEN_DEVICE_FUNC inline void pscatter<int64_t, Packet8l>(int64_t* to, const Packet8l& from, Index stride) {
1330 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1331 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1332 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1333 _mm512_i32scatter_epi64(to, indices, from, 8);
1334}
1335template <>
1336EIGEN_DEVICE_FUNC inline void pscatter<int, Packet16i>(int* to, const Packet16i& from, Index stride) {
1337 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1338 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1339 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1340 _mm512_i32scatter_epi32(to, indices, from, 4);
1341}
1342
1343template <>
1344EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) {
1345 Packet16f pa = pset1<Packet16f>(a);
1346 pstore(to, pa);
1347}
1348template <>
1349EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) {
1350 Packet8d pa = pset1<Packet8d>(a);
1351 pstore(to, pa);
1352}
1353template <>
1354EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) {
1355 Packet16i pa = pset1<Packet16i>(a);
1356 pstore(to, pa);
1357}
1358template <>
1359EIGEN_STRONG_INLINE void pstore1<Packet8l>(int64_t* to, const int64_t& a) {
1360 Packet8l pa = pset1<Packet8l>(a);
1361 pstore(to, pa);
1362}
1363
1364template <>
1365EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1366 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1367}
1368template <>
1369EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1370 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1371}
1372template <>
1373EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
1374 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1375}
1376
1377template <>
1378EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) {
1379 return _mm512_cvtss_f32(a);
1380}
1381template <>
1382EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) {
1383 return _mm512_cvtsd_f64(a);
1384}
1385template <>
1386EIGEN_STRONG_INLINE int64_t pfirst<Packet8l>(const Packet8l& a) {
1387 int64_t x = _mm_extract_epi64_0(_mm512_extracti32x4_epi32(a, 0));
1388 return x;
1389}
1390template <>
1391EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) {
1392#if EIGEN_GNUC_STRICT_LESS_THAN(11, 0, 0)
1393 return _mm_cvtsi128_si32(_mm512_castsi512_si128(a));
1394#else
1395 return _mm512_cvtsi512_si32(a);
1396#endif
1397}
1398
1399template <>
1400EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) {
1401 return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1402}
1403
1404template <>
1405EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a) {
1406 return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
1407}
1408
1409template <>
1410EIGEN_STRONG_INLINE Packet16i preverse(const Packet16i& a) {
1411 return _mm512_permutexvar_epi32(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1412}
1413
1414template <>
1415EIGEN_STRONG_INLINE Packet8l preverse(const Packet8l& a) {
1416 return _mm512_permutexvar_epi64(_mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7), a);
1417}
1418
1419template <>
1420EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) {
1421 // _mm512_abs_ps intrinsic not found, so implement via bitwise AND with sign-bit mask.
1422 return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff)));
1423}
1424template <>
1425EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) {
1426 // _mm512_abs_pd intrinsic not found, so implement via bitwise AND with sign-bit mask.
1427 return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), _mm512_set1_epi64(0x7fffffffffffffff)));
1428}
1429template <>
1430EIGEN_STRONG_INLINE Packet16i pabs(const Packet16i& a) {
1431 return _mm512_abs_epi32(a);
1432}
1433template <>
1434EIGEN_STRONG_INLINE Packet8l pabs(const Packet8l& a) {
1435 return _mm512_abs_epi64(a);
1436}
1437
1438#ifndef EIGEN_VECTORIZE_AVX512FP16
1439template <>
1440EIGEN_STRONG_INLINE Packet16h psignbit(const Packet16h& a) {
1441 return _mm256_srai_epi16(a, 15);
1442}
1443#endif // EIGEN_VECTORIZE_AVX512FP16
1444
1445template <>
1446EIGEN_STRONG_INLINE Packet16bf psignbit(const Packet16bf& a) {
1447 return _mm256_srai_epi16(a, 15);
1448}
1449template <>
1450EIGEN_STRONG_INLINE Packet16f psignbit(const Packet16f& a) {
1451 return _mm512_castsi512_ps(_mm512_srai_epi32(_mm512_castps_si512(a), 31));
1452}
1453template <>
1454EIGEN_STRONG_INLINE Packet8d psignbit(const Packet8d& a) {
1455 return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), 63));
1456}
1457
1458template <>
1459EIGEN_STRONG_INLINE Packet16f pfrexp<Packet16f>(const Packet16f& a, Packet16f& exponent) {
1460 return pfrexp_generic(a, exponent);
1461}
1462
1463// Extract exponent without existence of Packet8l.
1464template <>
1465EIGEN_STRONG_INLINE Packet8d pfrexp_generic_get_biased_exponent(const Packet8d& a) {
1466 const Packet8d cst_exp_mask = pset1frombits<Packet8d>(static_cast<uint64_t>(0x7ff0000000000000ull));
1467#ifdef EIGEN_VECTORIZE_AVX512DQ
1468 return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52));
1469#else
1470 return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)));
1471#endif
1472}
1473
1474template <>
1475EIGEN_STRONG_INLINE Packet8d pfrexp<Packet8d>(const Packet8d& a, Packet8d& exponent) {
1476 return pfrexp_generic(a, exponent);
1477}
1478
1479template <>
1480EIGEN_STRONG_INLINE Packet16f pldexp<Packet16f>(const Packet16f& a, const Packet16f& exponent) {
1481 // vscalef is a * 2^floor(exponent), rounded once, and pldexp takes (int)exponent: truncate first. Clamp as
1482 // pldexp_generic does: vscalef gives NaN for 0 * 2^inf and inf * 2^-inf, and a number for NaN * 2^(+-inf), and an
1483 // int exponent beyond the range of half is infinite by the time it reaches here through half2float.
1484 const Packet16f max_exponent = pset1<Packet16f>(278.0f);
1485 const Packet16f e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent);
1486 return _mm512_scalef_ps(a, _mm512_roundscale_ps(e, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC));
1487}
1488
1489template <>
1490EIGEN_STRONG_INLINE Packet8d pldexp<Packet8d>(const Packet8d& a, const Packet8d& exponent) {
1491 const Packet8d max_exponent = pset1<Packet8d>(2099.0); // see pldexp<Packet16f>
1492 const Packet8d e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent);
1493 return _mm512_scalef_pd(a, _mm512_roundscale_pd(e, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC));
1494}
1495
1496// pldexp_fast's callers pass integral exponents, and vscalef floors, so neither the clamp nor the truncation.
1497template <>
1498EIGEN_STRONG_INLINE Packet16f pldexp_fast<Packet16f>(const Packet16f& a, const Packet16f& exponent) {
1499 return _mm512_scalef_ps(a, exponent);
1500}
1501
1502template <>
1503EIGEN_STRONG_INLINE Packet8d pldexp_fast<Packet8d>(const Packet8d& a, const Packet8d& exponent) {
1504 return _mm512_scalef_pd(a, exponent);
1505}
1506
1507#ifdef EIGEN_VECTORIZE_AVX512DQ
1508// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
1509#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1510 __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \
1511 __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1)
1512
1513// AVX512F does not define _mm512_extracti32x8_epi32 to extract _m256i from _m512i
1514#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1515 __m256i OUTPUT##_0 = _mm512_extracti32x8_epi32(INPUT, 0); \
1516 __m256i OUTPUT##_1 = _mm512_extracti32x8_epi32(INPUT, 1)
1517#else
1518#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1519 __m256 OUTPUT##_0 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
1520 _mm512_extractf32x4_ps(INPUT, 1), 1); \
1521 __m256 OUTPUT##_1 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
1522 _mm512_extractf32x4_ps(INPUT, 3), 1)
1523
1524#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1525 __m256i OUTPUT##_0 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 0)), \
1526 _mm512_extracti32x4_epi32(INPUT, 1), 1); \
1527 __m256i OUTPUT##_1 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 2)), \
1528 _mm512_extracti32x4_epi32(INPUT, 3), 1)
1529#endif
1530
1531#ifdef EIGEN_VECTORIZE_AVX512DQ
1532#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1533 OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1);
1534
1535#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1536 OUTPUT = _mm512_inserti32x8(_mm512_castsi256_si512(INPUTA), INPUTB, 1);
1537#else
1538#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1539 OUTPUT = _mm512_undefined_ps(); \
1540 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
1541 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
1542 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
1543 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
1544
1545#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1546 OUTPUT = _mm512_undefined_epi32(); \
1547 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 0), 0); \
1548 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 1), 1); \
1549 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 0), 2); \
1550 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 1), 3);
1551#endif
1552
1553template <>
1554EIGEN_STRONG_INLINE Packet8f predux_half<Packet16f>(const Packet16f& a) {
1555#ifdef EIGEN_VECTORIZE_AVX512DQ
1556 __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
1557 __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
1558 return _mm256_add_ps(lane0, lane1);
1559#else
1560 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1561 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1562 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1563 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1564 __m128 sum0 = _mm_add_ps(lane0, lane2);
1565 __m128 sum1 = _mm_add_ps(lane1, lane3);
1566 return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
1567#endif
1568}
1569template <>
1570EIGEN_STRONG_INLINE Packet4d predux_half<Packet8d>(const Packet8d& a) {
1571 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1572 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1573 return _mm256_add_pd(lane0, lane1);
1574}
1575template <>
1576EIGEN_STRONG_INLINE Packet8i predux_half<Packet16i>(const Packet16i& a) {
1577#ifdef EIGEN_VECTORIZE_AVX512DQ
1578 __m256i lane0 = _mm512_extracti32x8_epi32(a, 0);
1579 __m256i lane1 = _mm512_extracti32x8_epi32(a, 1);
1580 return _mm256_add_epi32(lane0, lane1);
1581#else
1582 __m128i lane0 = _mm512_extracti32x4_epi32(a, 0);
1583 __m128i lane1 = _mm512_extracti32x4_epi32(a, 1);
1584 __m128i lane2 = _mm512_extracti32x4_epi32(a, 2);
1585 __m128i lane3 = _mm512_extracti32x4_epi32(a, 3);
1586 __m128i sum0 = _mm_add_epi32(lane0, lane2);
1587 __m128i sum1 = _mm_add_epi32(lane1, lane3);
1588 return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1);
1589#endif
1590}
1591
1592template <>
1593EIGEN_STRONG_INLINE Packet4l predux_half<Packet8l>(const Packet8l& a) {
1594 __m256i lane0 = _mm512_extracti64x4_epi64(a, 0);
1595 __m256i lane1 = _mm512_extracti64x4_epi64(a, 1);
1596 return _mm256_add_epi64(lane0, lane1);
1597}
1598
1599#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1600 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1601
1602EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1603 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1604 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1605 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1606 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1607 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1608 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1609 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1610 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1611 __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1612 __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1613 __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1614 __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1615 __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1616 __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1617 __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1618 __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1619 __m512 S0 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1620 __m512 S1 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1621 __m512 S2 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1622 __m512 S3 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1623 __m512 S4 = EIGEN_AVX512_SHUFFLE_PS(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1624 __m512 S5 = EIGEN_AVX512_SHUFFLE_PS(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1625 __m512 S6 = EIGEN_AVX512_SHUFFLE_PS(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1626 __m512 S7 = EIGEN_AVX512_SHUFFLE_PS(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1627 __m512 S8 = EIGEN_AVX512_SHUFFLE_PS(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1628 __m512 S9 = EIGEN_AVX512_SHUFFLE_PS(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1629 __m512 S10 = EIGEN_AVX512_SHUFFLE_PS(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1630 __m512 S11 = EIGEN_AVX512_SHUFFLE_PS(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1631 __m512 S12 = EIGEN_AVX512_SHUFFLE_PS(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1632 __m512 S13 = EIGEN_AVX512_SHUFFLE_PS(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1633 __m512 S14 = EIGEN_AVX512_SHUFFLE_PS(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1634 __m512 S15 = EIGEN_AVX512_SHUFFLE_PS(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1635
1636 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1637 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1638 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1639 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1640 EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1641 EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1642 EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1643 EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1644 EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1645 EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1646 EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1647 EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1648 EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1649 EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1650 EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1651 EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1652
1653 PacketBlock<Packet8f, 32> tmp;
1654
1655 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1656 tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1657 tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1658 tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1659 tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1660 tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1661 tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1662 tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1663
1664 tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1665 tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1666 tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1667 tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1668 tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1669 tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1670 tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1671 tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1672
1673 // Second set of _m256 outputs
1674 tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1675 tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1676 tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1677 tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1678 tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1679 tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1680 tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1681 tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1682
1683 tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1684 tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1685 tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1686 tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1687 tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1688 tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1689 tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1690 tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1691
1692 // Pack them into the output
1693 PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1694 PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1695 PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1696 PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1697
1698 PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1699 PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1700 PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1701 PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1702
1703 PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1704 PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1705 PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1706 PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1707
1708 PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1709 PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1710 PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1711 PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1712}
1713#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1714 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1715
1716EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 8>& kernel) {
1717 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1718 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1719 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1720 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1721 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1722 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1723 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1724 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1725
1726 kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1727 kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1728 kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1729 kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1730 kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1731 kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1732 kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1733 kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1734
1735 T0 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0x44);
1736 T1 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0xee);
1737 T2 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0x44);
1738 T3 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0xee);
1739 T4 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0x44);
1740 T5 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0xee);
1741 T6 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0x44);
1742 T7 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0xee);
1743
1744 kernel.packet[0] = _mm512_shuffle_f32x4(T0, T2, 0x88);
1745 kernel.packet[2] = _mm512_shuffle_f32x4(T0, T2, 0xdd);
1746 kernel.packet[1] = _mm512_shuffle_f32x4(T4, T6, 0x88);
1747 kernel.packet[3] = _mm512_shuffle_f32x4(T4, T6, 0xdd);
1748 kernel.packet[4] = _mm512_shuffle_f32x4(T1, T3, 0x88);
1749 kernel.packet[6] = _mm512_shuffle_f32x4(T1, T3, 0xdd);
1750 kernel.packet[5] = _mm512_shuffle_f32x4(T5, T7, 0x88);
1751 kernel.packet[7] = _mm512_shuffle_f32x4(T5, T7, 0xdd);
1752}
1753
1754EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1755 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1756 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1757 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1758 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1759
1760 __m512 S0 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1761 __m512 S1 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1762 __m512 S2 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1763 __m512 S3 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1764
1765 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1766 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1767 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1768 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1769
1770 PacketBlock<Packet8f, 8> tmp;
1771
1772 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1773 tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1774 tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1775 tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1776
1777 tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1778 tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1779 tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1780 tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1781
1782 PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1783 PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1784 PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1785 PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1786}
1787
1788#define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1789 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1790 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1791
1792#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1793 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1794 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1795
1796#define PACK_OUTPUT_L(OUTPUT, INPUT, INDEX, STRIDE) \
1797 OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1798 OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1799
1800EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1801 __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1802 __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1803 __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1804 __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1805
1806 PacketBlock<Packet4d, 8> tmp;
1807
1808 tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x20);
1809 tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x20);
1810 tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x31);
1811 tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x31);
1812
1813 tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x20);
1814 tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x20);
1815 tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x31);
1816 tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x31);
1817
1818 PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1819 PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1820 PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1821 PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1822}
1823
1824EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1825 __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1826 __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1827 __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
1828 __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
1829 __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
1830 __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
1831 __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
1832 __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
1833
1834 kernel.packet[0] = _mm512_permutex_pd(T2, 0x4E);
1835 kernel.packet[0] = _mm512_mask_blend_pd(0xCC, T0, kernel.packet[0]);
1836 kernel.packet[2] = _mm512_permutex_pd(T0, 0x4E);
1837 kernel.packet[2] = _mm512_mask_blend_pd(0xCC, kernel.packet[2], T2);
1838 kernel.packet[1] = _mm512_permutex_pd(T3, 0x4E);
1839 kernel.packet[1] = _mm512_mask_blend_pd(0xCC, T1, kernel.packet[1]);
1840 kernel.packet[3] = _mm512_permutex_pd(T1, 0x4E);
1841 kernel.packet[3] = _mm512_mask_blend_pd(0xCC, kernel.packet[3], T3);
1842 kernel.packet[4] = _mm512_permutex_pd(T6, 0x4E);
1843 kernel.packet[4] = _mm512_mask_blend_pd(0xCC, T4, kernel.packet[4]);
1844 kernel.packet[6] = _mm512_permutex_pd(T4, 0x4E);
1845 kernel.packet[6] = _mm512_mask_blend_pd(0xCC, kernel.packet[6], T6);
1846 kernel.packet[5] = _mm512_permutex_pd(T7, 0x4E);
1847 kernel.packet[5] = _mm512_mask_blend_pd(0xCC, T5, kernel.packet[5]);
1848 kernel.packet[7] = _mm512_permutex_pd(T5, 0x4E);
1849 kernel.packet[7] = _mm512_mask_blend_pd(0xCC, kernel.packet[7], T7);
1850
1851 T0 = _mm512_shuffle_f64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1852 T0 = _mm512_mask_blend_pd(0xF0, kernel.packet[0], T0);
1853 T4 = _mm512_shuffle_f64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1854 T4 = _mm512_mask_blend_pd(0xF0, T4, kernel.packet[4]);
1855 T1 = _mm512_shuffle_f64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1856 T1 = _mm512_mask_blend_pd(0xF0, kernel.packet[1], T1);
1857 T5 = _mm512_shuffle_f64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1858 T5 = _mm512_mask_blend_pd(0xF0, T5, kernel.packet[5]);
1859 T2 = _mm512_shuffle_f64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1860 T2 = _mm512_mask_blend_pd(0xF0, kernel.packet[2], T2);
1861 T6 = _mm512_shuffle_f64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1862 T6 = _mm512_mask_blend_pd(0xF0, T6, kernel.packet[6]);
1863 T3 = _mm512_shuffle_f64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1864 T3 = _mm512_mask_blend_pd(0xF0, kernel.packet[3], T3);
1865 T7 = _mm512_shuffle_f64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1866 T7 = _mm512_mask_blend_pd(0xF0, T7, kernel.packet[7]);
1867
1868 kernel.packet[0] = T0;
1869 kernel.packet[1] = T1;
1870 kernel.packet[2] = T2;
1871 kernel.packet[3] = T3;
1872 kernel.packet[4] = T4;
1873 kernel.packet[5] = T5;
1874 kernel.packet[6] = T6;
1875 kernel.packet[7] = T7;
1876}
1877
1878EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8l, 4>& kernel) {
1879 __m512i T0 = _mm512_castpd_si512(
1880 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0));
1881 __m512i T1 = _mm512_castpd_si512(
1882 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0xff));
1883 __m512i T2 = _mm512_castpd_si512(
1884 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0));
1885 __m512i T3 = _mm512_castpd_si512(
1886 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0xff));
1887
1888 PacketBlock<Packet4l, 8> tmp;
1889
1890 tmp.packet[0] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x20);
1891 tmp.packet[1] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x20);
1892 tmp.packet[2] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x31);
1893 tmp.packet[3] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x31);
1894
1895 tmp.packet[4] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x20);
1896 tmp.packet[5] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x20);
1897 tmp.packet[6] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x31);
1898 tmp.packet[7] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x31);
1899
1900 PACK_OUTPUT_L(kernel.packet, tmp.packet, 0, 1);
1901 PACK_OUTPUT_L(kernel.packet, tmp.packet, 1, 1);
1902 PACK_OUTPUT_L(kernel.packet, tmp.packet, 2, 1);
1903 PACK_OUTPUT_L(kernel.packet, tmp.packet, 3, 1);
1904}
1905
1906EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8l, 8>& kernel) {
1907 __m512i T0 = _mm512_unpacklo_epi64(kernel.packet[0], kernel.packet[1]);
1908 __m512i T1 = _mm512_unpackhi_epi64(kernel.packet[0], kernel.packet[1]);
1909 __m512i T2 = _mm512_unpacklo_epi64(kernel.packet[2], kernel.packet[3]);
1910 __m512i T3 = _mm512_unpackhi_epi64(kernel.packet[2], kernel.packet[3]);
1911 __m512i T4 = _mm512_unpacklo_epi64(kernel.packet[4], kernel.packet[5]);
1912 __m512i T5 = _mm512_unpackhi_epi64(kernel.packet[4], kernel.packet[5]);
1913 __m512i T6 = _mm512_unpacklo_epi64(kernel.packet[6], kernel.packet[7]);
1914 __m512i T7 = _mm512_unpackhi_epi64(kernel.packet[6], kernel.packet[7]);
1915
1916 kernel.packet[0] = _mm512_permutex_epi64(T2, 0x4E);
1917 kernel.packet[0] = _mm512_mask_blend_epi64(0xCC, T0, kernel.packet[0]);
1918 kernel.packet[2] = _mm512_permutex_epi64(T0, 0x4E);
1919 kernel.packet[2] = _mm512_mask_blend_epi64(0xCC, kernel.packet[2], T2);
1920 kernel.packet[1] = _mm512_permutex_epi64(T3, 0x4E);
1921 kernel.packet[1] = _mm512_mask_blend_epi64(0xCC, T1, kernel.packet[1]);
1922 kernel.packet[3] = _mm512_permutex_epi64(T1, 0x4E);
1923 kernel.packet[3] = _mm512_mask_blend_epi64(0xCC, kernel.packet[3], T3);
1924 kernel.packet[4] = _mm512_permutex_epi64(T6, 0x4E);
1925 kernel.packet[4] = _mm512_mask_blend_epi64(0xCC, T4, kernel.packet[4]);
1926 kernel.packet[6] = _mm512_permutex_epi64(T4, 0x4E);
1927 kernel.packet[6] = _mm512_mask_blend_epi64(0xCC, kernel.packet[6], T6);
1928 kernel.packet[5] = _mm512_permutex_epi64(T7, 0x4E);
1929 kernel.packet[5] = _mm512_mask_blend_epi64(0xCC, T5, kernel.packet[5]);
1930 kernel.packet[7] = _mm512_permutex_epi64(T5, 0x4E);
1931 kernel.packet[7] = _mm512_mask_blend_epi64(0xCC, kernel.packet[7], T7);
1932
1933 T0 = _mm512_shuffle_i64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1934 T0 = _mm512_mask_blend_epi64(0xF0, kernel.packet[0], T0);
1935 T4 = _mm512_shuffle_i64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1936 T4 = _mm512_mask_blend_epi64(0xF0, T4, kernel.packet[4]);
1937 T1 = _mm512_shuffle_i64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1938 T1 = _mm512_mask_blend_epi64(0xF0, kernel.packet[1], T1);
1939 T5 = _mm512_shuffle_i64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1940 T5 = _mm512_mask_blend_epi64(0xF0, T5, kernel.packet[5]);
1941 T2 = _mm512_shuffle_i64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1942 T2 = _mm512_mask_blend_epi64(0xF0, kernel.packet[2], T2);
1943 T6 = _mm512_shuffle_i64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1944 T6 = _mm512_mask_blend_epi64(0xF0, T6, kernel.packet[6]);
1945 T3 = _mm512_shuffle_i64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1946 T3 = _mm512_mask_blend_epi64(0xF0, kernel.packet[3], T3);
1947 T7 = _mm512_shuffle_i64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1948 T7 = _mm512_mask_blend_epi64(0xF0, T7, kernel.packet[7]);
1949
1950 kernel.packet[0] = T0;
1951 kernel.packet[1] = T1;
1952 kernel.packet[2] = T2;
1953 kernel.packet[3] = T3;
1954 kernel.packet[4] = T4;
1955 kernel.packet[5] = T5;
1956 kernel.packet[6] = T6;
1957 kernel.packet[7] = T7;
1958}
1959
1960#define PACK_OUTPUT_I32(OUTPUT, INPUT, INDEX, STRIDE) \
1961 EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1962
1963#define PACK_OUTPUT_I32_2(OUTPUT, INPUT, INDEX, STRIDE) \
1964 EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1965
1966#define SHUFFLE_EPI32(A, B, M) \
1967 _mm512_castps_si512(EIGEN_AVX512_SHUFFLE_PS(_mm512_castsi512_ps(A), _mm512_castsi512_ps(B), M))
1968
1969EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 16>& kernel) {
1970 __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
1971 __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
1972 __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
1973 __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
1974 __m512i T4 = _mm512_unpacklo_epi32(kernel.packet[4], kernel.packet[5]);
1975 __m512i T5 = _mm512_unpackhi_epi32(kernel.packet[4], kernel.packet[5]);
1976 __m512i T6 = _mm512_unpacklo_epi32(kernel.packet[6], kernel.packet[7]);
1977 __m512i T7 = _mm512_unpackhi_epi32(kernel.packet[6], kernel.packet[7]);
1978 __m512i T8 = _mm512_unpacklo_epi32(kernel.packet[8], kernel.packet[9]);
1979 __m512i T9 = _mm512_unpackhi_epi32(kernel.packet[8], kernel.packet[9]);
1980 __m512i T10 = _mm512_unpacklo_epi32(kernel.packet[10], kernel.packet[11]);
1981 __m512i T11 = _mm512_unpackhi_epi32(kernel.packet[10], kernel.packet[11]);
1982 __m512i T12 = _mm512_unpacklo_epi32(kernel.packet[12], kernel.packet[13]);
1983 __m512i T13 = _mm512_unpackhi_epi32(kernel.packet[12], kernel.packet[13]);
1984 __m512i T14 = _mm512_unpacklo_epi32(kernel.packet[14], kernel.packet[15]);
1985 __m512i T15 = _mm512_unpackhi_epi32(kernel.packet[14], kernel.packet[15]);
1986 __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1987 __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1988 __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1989 __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1990 __m512i S4 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1991 __m512i S5 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1992 __m512i S6 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1993 __m512i S7 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1994 __m512i S8 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1995 __m512i S9 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1996 __m512i S10 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1997 __m512i S11 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1998 __m512i S12 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1999 __m512i S13 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
2000 __m512i S14 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
2001 __m512i S15 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
2002
2003 EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
2004 EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
2005 EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
2006 EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
2007 EIGEN_EXTRACT_8i_FROM_16i(S4, S4);
2008 EIGEN_EXTRACT_8i_FROM_16i(S5, S5);
2009 EIGEN_EXTRACT_8i_FROM_16i(S6, S6);
2010 EIGEN_EXTRACT_8i_FROM_16i(S7, S7);
2011 EIGEN_EXTRACT_8i_FROM_16i(S8, S8);
2012 EIGEN_EXTRACT_8i_FROM_16i(S9, S9);
2013 EIGEN_EXTRACT_8i_FROM_16i(S10, S10);
2014 EIGEN_EXTRACT_8i_FROM_16i(S11, S11);
2015 EIGEN_EXTRACT_8i_FROM_16i(S12, S12);
2016 EIGEN_EXTRACT_8i_FROM_16i(S13, S13);
2017 EIGEN_EXTRACT_8i_FROM_16i(S14, S14);
2018 EIGEN_EXTRACT_8i_FROM_16i(S15, S15);
2019
2020 PacketBlock<Packet8i, 32> tmp;
2021
2022 tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S4_0, 0x20);
2023 tmp.packet[1] = _mm256_permute2f128_si256(S1_0, S5_0, 0x20);
2024 tmp.packet[2] = _mm256_permute2f128_si256(S2_0, S6_0, 0x20);
2025 tmp.packet[3] = _mm256_permute2f128_si256(S3_0, S7_0, 0x20);
2026 tmp.packet[4] = _mm256_permute2f128_si256(S0_0, S4_0, 0x31);
2027 tmp.packet[5] = _mm256_permute2f128_si256(S1_0, S5_0, 0x31);
2028 tmp.packet[6] = _mm256_permute2f128_si256(S2_0, S6_0, 0x31);
2029 tmp.packet[7] = _mm256_permute2f128_si256(S3_0, S7_0, 0x31);
2030
2031 tmp.packet[8] = _mm256_permute2f128_si256(S0_1, S4_1, 0x20);
2032 tmp.packet[9] = _mm256_permute2f128_si256(S1_1, S5_1, 0x20);
2033 tmp.packet[10] = _mm256_permute2f128_si256(S2_1, S6_1, 0x20);
2034 tmp.packet[11] = _mm256_permute2f128_si256(S3_1, S7_1, 0x20);
2035 tmp.packet[12] = _mm256_permute2f128_si256(S0_1, S4_1, 0x31);
2036 tmp.packet[13] = _mm256_permute2f128_si256(S1_1, S5_1, 0x31);
2037 tmp.packet[14] = _mm256_permute2f128_si256(S2_1, S6_1, 0x31);
2038 tmp.packet[15] = _mm256_permute2f128_si256(S3_1, S7_1, 0x31);
2039
2040 // Second set of _m256 outputs
2041 tmp.packet[16] = _mm256_permute2f128_si256(S8_0, S12_0, 0x20);
2042 tmp.packet[17] = _mm256_permute2f128_si256(S9_0, S13_0, 0x20);
2043 tmp.packet[18] = _mm256_permute2f128_si256(S10_0, S14_0, 0x20);
2044 tmp.packet[19] = _mm256_permute2f128_si256(S11_0, S15_0, 0x20);
2045 tmp.packet[20] = _mm256_permute2f128_si256(S8_0, S12_0, 0x31);
2046 tmp.packet[21] = _mm256_permute2f128_si256(S9_0, S13_0, 0x31);
2047 tmp.packet[22] = _mm256_permute2f128_si256(S10_0, S14_0, 0x31);
2048 tmp.packet[23] = _mm256_permute2f128_si256(S11_0, S15_0, 0x31);
2049
2050 tmp.packet[24] = _mm256_permute2f128_si256(S8_1, S12_1, 0x20);
2051 tmp.packet[25] = _mm256_permute2f128_si256(S9_1, S13_1, 0x20);
2052 tmp.packet[26] = _mm256_permute2f128_si256(S10_1, S14_1, 0x20);
2053 tmp.packet[27] = _mm256_permute2f128_si256(S11_1, S15_1, 0x20);
2054 tmp.packet[28] = _mm256_permute2f128_si256(S8_1, S12_1, 0x31);
2055 tmp.packet[29] = _mm256_permute2f128_si256(S9_1, S13_1, 0x31);
2056 tmp.packet[30] = _mm256_permute2f128_si256(S10_1, S14_1, 0x31);
2057 tmp.packet[31] = _mm256_permute2f128_si256(S11_1, S15_1, 0x31);
2058
2059 // Pack them into the output
2060 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 0, 16);
2061 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 1, 16);
2062 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 2, 16);
2063 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 3, 16);
2064
2065 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 4, 16);
2066 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 5, 16);
2067 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 6, 16);
2068 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 7, 16);
2069
2070 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 8, 16);
2071 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 9, 16);
2072 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 10, 16);
2073 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 11, 16);
2074
2075 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 12, 16);
2076 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 13, 16);
2077 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 14, 16);
2078 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 15, 16);
2079}
2080
2081EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 4>& kernel) {
2082 __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
2083 __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
2084 __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
2085 __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
2086
2087 __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
2088 __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
2089 __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
2090 __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
2091
2092 EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
2093 EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
2094 EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
2095 EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
2096
2097 PacketBlock<Packet8i, 8> tmp;
2098
2099 tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S1_0, 0x20);
2100 tmp.packet[1] = _mm256_permute2f128_si256(S2_0, S3_0, 0x20);
2101 tmp.packet[2] = _mm256_permute2f128_si256(S0_0, S1_0, 0x31);
2102 tmp.packet[3] = _mm256_permute2f128_si256(S2_0, S3_0, 0x31);
2103
2104 tmp.packet[4] = _mm256_permute2f128_si256(S0_1, S1_1, 0x20);
2105 tmp.packet[5] = _mm256_permute2f128_si256(S2_1, S3_1, 0x20);
2106 tmp.packet[6] = _mm256_permute2f128_si256(S0_1, S1_1, 0x31);
2107 tmp.packet[7] = _mm256_permute2f128_si256(S2_1, S3_1, 0x31);
2108
2109 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 0, 1);
2110 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 1, 1);
2111 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 2, 1);
2112 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 3, 1);
2113}
2114
2115// Packet math for Eigen::half
2116#ifndef EIGEN_VECTORIZE_AVX512FP16
2117template <>
2118EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) {
2119 return _mm256_set1_epi16(from.x);
2120}
2121
2122template <>
2123EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) {
2124 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from, 0)));
2125}
2126
2127template <>
2128EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) {
2129 return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
2130}
2131
2132template <>
2133EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) {
2134 return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
2135}
2136
2137template <>
2138EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) {
2139 // (void*) -> workaround clang warning:
2140 // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
2141 EIGEN_DEBUG_ALIGNED_STORE
2142 _mm256_store_si256((__m256i*)(void*)to, from);
2143}
2144
2145template <>
2146EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) {
2147 // (void*) -> workaround clang warning:
2148 // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
2149 EIGEN_DEBUG_UNALIGNED_STORE
2150 _mm256_storeu_si256((__m256i*)(void*)to, from);
2151}
2152
2153template <>
2154EIGEN_STRONG_INLINE Packet16h ploaddup<Packet16h>(const Eigen::half* from) {
2155 unsigned short a = from[0].x;
2156 unsigned short b = from[1].x;
2157 unsigned short c = from[2].x;
2158 unsigned short d = from[3].x;
2159 unsigned short e = from[4].x;
2160 unsigned short f = from[5].x;
2161 unsigned short g = from[6].x;
2162 unsigned short h = from[7].x;
2163 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2164}
2165
2166template <>
2167EIGEN_STRONG_INLINE Packet16h ploadquad(const Eigen::half* from) {
2168 unsigned short a = from[0].x;
2169 unsigned short b = from[1].x;
2170 unsigned short c = from[2].x;
2171 unsigned short d = from[3].x;
2172 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2173}
2174
2175EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { return _mm512_cvtph_ps(a); }
2176
2177EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) {
2178 return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
2179}
2180
2181template <>
2182EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) {
2183 return Packet16h(ptrue(Packet8i(a)));
2184}
2185
2186template <>
2187EIGEN_STRONG_INLINE Packet16h pabs(const Packet16h& a) {
2188 const __m256i sign_mask = _mm256_set1_epi16(static_cast<short>(0x8000u));
2189 return _mm256_andnot_si256(sign_mask, a);
2190}
2191
2192template <>
2193EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(const Packet16h& a, const Packet16h& b) {
2194 return float2half(pmin<Packet16f>(half2float(a), half2float(b)));
2195}
2196
2197template <>
2198EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(const Packet16h& a, const Packet16h& b) {
2199 return float2half(pmax<Packet16f>(half2float(a), half2float(b)));
2200}
2201
2202template <>
2203EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(const half& a) {
2204 return float2half(plset<Packet16f>(static_cast<float>(a)));
2205}
2206
2207template <>
2208EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a, const Packet16h& b) {
2209 // in some cases Packet8i is a wrapper around __m256i, so we need to
2210 // cast to Packet8i to call the correct overload.
2211 return Packet16h(por(Packet8i(a), Packet8i(b)));
2212}
2213template <>
2214EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a, const Packet16h& b) {
2215 return Packet16h(pxor(Packet8i(a), Packet8i(b)));
2216}
2217template <>
2218EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a, const Packet16h& b) {
2219 return Packet16h(pand(Packet8i(a), Packet8i(b)));
2220}
2221template <>
2222EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a, const Packet16h& b) {
2223 return Packet16h(pandnot(Packet8i(a), Packet8i(b)));
2224}
2225
2226template <>
2227EIGEN_STRONG_INLINE Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) {
2228 return _mm256_blendv_epi8(b, a, mask);
2229}
2230
2231template <>
2232EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(const Packet16h& a) {
2233 return float2half(pround<Packet16f>(half2float(a)));
2234}
2235
2236template <>
2237EIGEN_STRONG_INLINE Packet16h print<Packet16h>(const Packet16h& a) {
2238 return float2half(print<Packet16f>(half2float(a)));
2239}
2240
2241template <>
2242EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(const Packet16h& a) {
2243 return float2half(pceil<Packet16f>(half2float(a)));
2244}
2245
2246template <>
2247EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) {
2248 return float2half(pfloor<Packet16f>(half2float(a)));
2249}
2250
2251template <>
2252EIGEN_STRONG_INLINE Packet16h ptrunc<Packet16h>(const Packet16h& a) {
2253 return float2half(ptrunc<Packet16f>(half2float(a)));
2254}
2255
2256template <>
2257EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a, const Packet16h& b) {
2258 Packet16f af = half2float(a);
2259 Packet16f bf = half2float(b);
2260 return Pack32To16(pcmp_eq(af, bf));
2261}
2262
2263template <>
2264EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a, const Packet16h& b) {
2265 return Pack32To16(pcmp_le(half2float(a), half2float(b)));
2266}
2267
2268template <>
2269EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a, const Packet16h& b) {
2270 return Pack32To16(pcmp_lt(half2float(a), half2float(b)));
2271}
2272
2273template <>
2274EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a, const Packet16h& b) {
2275 return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b)));
2276}
2277
2278template <>
2279EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) {
2280 Packet16h sign_mask = _mm256_set1_epi16(static_cast<short>(0x8000u));
2281 return _mm256_xor_si256(a, sign_mask);
2282}
2283
2284template <>
2285EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) {
2286 Packet16f af = half2float(a);
2287 Packet16f bf = half2float(b);
2288 Packet16f rf = padd(af, bf);
2289 return float2half(rf);
2290}
2291
2292template <>
2293EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) {
2294 Packet16f af = half2float(a);
2295 Packet16f bf = half2float(b);
2296 Packet16f rf = psub(af, bf);
2297 return float2half(rf);
2298}
2299
2300template <>
2301EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) {
2302 Packet16f af = half2float(a);
2303 Packet16f bf = half2float(b);
2304 Packet16f rf = pmul(af, bf);
2305 return float2half(rf);
2306}
2307
2308template <>
2309EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) {
2310 Packet16f af = half2float(a);
2311 Packet16f bf = half2float(b);
2312 Packet16f rf = pdiv(af, bf);
2313 return float2half(rf);
2314}
2315
2316template <>
2317EIGEN_STRONG_INLINE Packet16h pmadd<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2318 return float2half(pmadd(half2float(a), half2float(b), half2float(c)));
2319}
2320
2321template <>
2322EIGEN_STRONG_INLINE Packet16h pmsub<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2323 return float2half(pmsub(half2float(a), half2float(b), half2float(c)));
2324}
2325
2326template <>
2327EIGEN_STRONG_INLINE Packet16h pnmadd<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2328 return float2half(pnmadd(half2float(a), half2float(b), half2float(c)));
2329}
2330
2331template <>
2332EIGEN_STRONG_INLINE Packet16h pnmsub<Packet16h>(const Packet16h& a, const Packet16h& b, const Packet16h& c) {
2333 return float2half(pnmsub(half2float(a), half2float(b), half2float(c)));
2334}
2335
2336template <>
2337EIGEN_STRONG_INLINE Packet8h predux_half<Packet16h>(const Packet16h& a) {
2338 Packet8h lane0 = _mm256_extractf128_si256(a, 0);
2339 Packet8h lane1 = _mm256_extractf128_si256(a, 1);
2340 return padd<Packet8h>(lane0, lane1);
2341}
2342
2343template <>
2344EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) {
2345 __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
2346 return _mm256_insertf128_si256(_mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a, 1), m)),
2347 _mm_shuffle_epi8(_mm256_extractf128_si256(a, 0), m), 1);
2348}
2349
2350template <>
2351EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride) {
2352 return _mm256_set_epi16(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x,
2353 from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x,
2354 from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x,
2355 from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x);
2356}
2357
2358template <>
2359EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride) {
2360 EIGEN_ALIGN64 half aux[16];
2361 pstore(aux, from);
2362 to[stride * 0] = aux[0];
2363 to[stride * 1] = aux[1];
2364 to[stride * 2] = aux[2];
2365 to[stride * 3] = aux[3];
2366 to[stride * 4] = aux[4];
2367 to[stride * 5] = aux[5];
2368 to[stride * 6] = aux[6];
2369 to[stride * 7] = aux[7];
2370 to[stride * 8] = aux[8];
2371 to[stride * 9] = aux[9];
2372 to[stride * 10] = aux[10];
2373 to[stride * 11] = aux[11];
2374 to[stride * 12] = aux[12];
2375 to[stride * 13] = aux[13];
2376 to[stride * 14] = aux[14];
2377 to[stride * 15] = aux[15];
2378}
2379
2380EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 16>& kernel) {
2381 __m256i a = kernel.packet[0];
2382 __m256i b = kernel.packet[1];
2383 __m256i c = kernel.packet[2];
2384 __m256i d = kernel.packet[3];
2385 __m256i e = kernel.packet[4];
2386 __m256i f = kernel.packet[5];
2387 __m256i g = kernel.packet[6];
2388 __m256i h = kernel.packet[7];
2389 __m256i i = kernel.packet[8];
2390 __m256i j = kernel.packet[9];
2391 __m256i k = kernel.packet[10];
2392 __m256i l = kernel.packet[11];
2393 __m256i m = kernel.packet[12];
2394 __m256i n = kernel.packet[13];
2395 __m256i o = kernel.packet[14];
2396 __m256i p = kernel.packet[15];
2397
2398 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2399 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2400 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2401 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2402 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2403 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2404 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2405 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2406
2407 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2408 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2409 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2410 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2411 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2412 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2413 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2414 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2415
2416 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2417 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2418 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2419 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2420 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2421 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2422 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2423 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2424
2425 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2426 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2427 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2428 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2429 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2430 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2431 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2432 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2433
2434 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2435 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2436 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2437 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2438 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2439 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2440 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2441 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2442 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2443 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2444 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2445 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2446 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2447 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2448 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2449 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2450
2451 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2452 __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2453 __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2454 __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2455 __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2456 __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2457 __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2458 __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2459 __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2460 __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2461 __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2462 __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2463 __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2464 __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2465 __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2466 __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2467 __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2468
2469 kernel.packet[0] = a_p_0;
2470 kernel.packet[1] = a_p_1;
2471 kernel.packet[2] = a_p_2;
2472 kernel.packet[3] = a_p_3;
2473 kernel.packet[4] = a_p_4;
2474 kernel.packet[5] = a_p_5;
2475 kernel.packet[6] = a_p_6;
2476 kernel.packet[7] = a_p_7;
2477 kernel.packet[8] = a_p_8;
2478 kernel.packet[9] = a_p_9;
2479 kernel.packet[10] = a_p_a;
2480 kernel.packet[11] = a_p_b;
2481 kernel.packet[12] = a_p_c;
2482 kernel.packet[13] = a_p_d;
2483 kernel.packet[14] = a_p_e;
2484 kernel.packet[15] = a_p_f;
2485}
2486
2487EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 8>& kernel) {
2488 EIGEN_ALIGN64 half in[8][16];
2489 pstore<half>(in[0], kernel.packet[0]);
2490 pstore<half>(in[1], kernel.packet[1]);
2491 pstore<half>(in[2], kernel.packet[2]);
2492 pstore<half>(in[3], kernel.packet[3]);
2493 pstore<half>(in[4], kernel.packet[4]);
2494 pstore<half>(in[5], kernel.packet[5]);
2495 pstore<half>(in[6], kernel.packet[6]);
2496 pstore<half>(in[7], kernel.packet[7]);
2497
2498 EIGEN_ALIGN64 half out[8][16];
2499
2500 for (int i = 0; i < 8; ++i) {
2501 for (int j = 0; j < 8; ++j) {
2502 out[i][j] = in[j][2 * i];
2503 }
2504 for (int j = 0; j < 8; ++j) {
2505 out[i][j + 8] = in[j][2 * i + 1];
2506 }
2507 }
2508
2509 kernel.packet[0] = pload<Packet16h>(out[0]);
2510 kernel.packet[1] = pload<Packet16h>(out[1]);
2511 kernel.packet[2] = pload<Packet16h>(out[2]);
2512 kernel.packet[3] = pload<Packet16h>(out[3]);
2513 kernel.packet[4] = pload<Packet16h>(out[4]);
2514 kernel.packet[5] = pload<Packet16h>(out[5]);
2515 kernel.packet[6] = pload<Packet16h>(out[6]);
2516 kernel.packet[7] = pload<Packet16h>(out[7]);
2517}
2518
2519EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16h, 4>& kernel) {
2520 EIGEN_ALIGN64 half in[4][16];
2521 pstore<half>(in[0], kernel.packet[0]);
2522 pstore<half>(in[1], kernel.packet[1]);
2523 pstore<half>(in[2], kernel.packet[2]);
2524 pstore<half>(in[3], kernel.packet[3]);
2525
2526 EIGEN_ALIGN64 half out[4][16];
2527
2528 for (int i = 0; i < 4; ++i) {
2529 for (int j = 0; j < 4; ++j) {
2530 out[i][j] = in[j][4 * i];
2531 }
2532 for (int j = 0; j < 4; ++j) {
2533 out[i][j + 4] = in[j][4 * i + 1];
2534 }
2535 for (int j = 0; j < 4; ++j) {
2536 out[i][j + 8] = in[j][4 * i + 2];
2537 }
2538 for (int j = 0; j < 4; ++j) {
2539 out[i][j + 12] = in[j][4 * i + 3];
2540 }
2541 }
2542
2543 kernel.packet[0] = pload<Packet16h>(out[0]);
2544 kernel.packet[1] = pload<Packet16h>(out[1]);
2545 kernel.packet[2] = pload<Packet16h>(out[2]);
2546 kernel.packet[3] = pload<Packet16h>(out[3]);
2547}
2548
2549#endif // EIGEN_VECTORIZE_AVX512FP16
2550
2551template <>
2552struct is_arithmetic<Packet16bf> : std::true_type {};
2553
2554template <>
2555struct packet_traits<bfloat16> : default_packet_traits {
2556 typedef Packet16bf type;
2557 typedef Packet8bf half;
2558 enum {
2559 Vectorizable = 1,
2560 AlignedOnScalar = 1,
2561 size = 16,
2562 HasInsert = 1,
2563 HasSin = EIGEN_FAST_MATH,
2564 HasCos = EIGEN_FAST_MATH,
2565 HasSqrt = 1,
2566 HasRsqrt = 1,
2567#ifdef EIGEN_VECTORIZE_AVX512DQ
2568 HasLog = 1, // Currently fails test with bad accuracy.
2569 HasLog1p = 1,
2570 HasExpm1 = 1,
2571 HasNdtri = 1,
2572 HasBessel = 1,
2573#endif
2574 HasExp = 1,
2575 HasTanh = EIGEN_FAST_MATH,
2576 HasErf = EIGEN_FAST_MATH,
2577 HasCmp = 1,
2578 HasDiv = 1
2579 };
2580};
2581
2582template <>
2583struct unpacket_traits<Packet16bf> {
2584 typedef bfloat16 type;
2585 enum {
2586 size = 16,
2587 alignment = Aligned32,
2588 vectorizable = true,
2589 masked_load_available = false,
2590 masked_store_available = false
2591 };
2592 typedef Packet8bf half;
2593};
2594
2595template <>
2596EIGEN_STRONG_INLINE Packet16bf pset1<Packet16bf>(const bfloat16& from) {
2597 return _mm256_set1_epi16(from.value);
2598}
2599
2600template <>
2601EIGEN_STRONG_INLINE bfloat16 pfirst<Packet16bf>(const Packet16bf& from) {
2602 bfloat16 t;
2603 t.value = static_cast<unsigned short>(_mm256_extract_epi16(from, 0));
2604 return t;
2605}
2606
2607template <>
2608EIGEN_STRONG_INLINE Packet16bf pload<Packet16bf>(const bfloat16* from) {
2609 return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
2610}
2611
2612template <>
2613EIGEN_STRONG_INLINE Packet16bf ploadu<Packet16bf>(const bfloat16* from) {
2614 return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
2615}
2616
2617template <>
2618EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet16bf& from) {
2619 EIGEN_DEBUG_ALIGNED_STORE
2620 _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
2621}
2622
2623template <>
2624EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet16bf& from) {
2625 EIGEN_DEBUG_UNALIGNED_STORE
2626 _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
2627}
2628
2629template <>
2630EIGEN_STRONG_INLINE Packet16bf ploaddup<Packet16bf>(const bfloat16* from) {
2631 unsigned short a = from[0].value;
2632 unsigned short b = from[1].value;
2633 unsigned short c = from[2].value;
2634 unsigned short d = from[3].value;
2635 unsigned short e = from[4].value;
2636 unsigned short f = from[5].value;
2637 unsigned short g = from[6].value;
2638 unsigned short h = from[7].value;
2639 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2640}
2641
2642template <>
2643EIGEN_STRONG_INLINE Packet16bf ploadquad(const bfloat16* from) {
2644 unsigned short a = from[0].value;
2645 unsigned short b = from[1].value;
2646 unsigned short c = from[2].value;
2647 unsigned short d = from[3].value;
2648 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2649}
2650
2651EIGEN_STRONG_INLINE Packet16f Bf16ToF32(const Packet16bf& a) {
2652 return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
2653}
2654
2655// Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm.
2656EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) {
2657 Packet16bf r;
2658
2659#if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_STRICT_AT_LEAST(10, 1, 0)
2660 // Since GCC 10.1 supports avx512bf16 and C style explicit cast
2661 // (C++ static_cast is not supported yet), do conversion via intrinsic
2662 // and register path for performance.
2663 r = (__m256i)(_mm512_cvtneps_pbh(a));
2664
2665#else
2666 __m512i t;
2667 __m512i input = _mm512_castps_si512(a);
2668 __m512i nan = _mm512_set1_epi32(0x7fc0);
2669
2670 // uint32_t lsb = (input >> 16) & 1;
2671 t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1));
2672 // uint32_t rounding_bias = 0x7fff + lsb;
2673 t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff));
2674 // input += rounding_bias;
2675 t = _mm512_add_epi32(t, input);
2676 // input = input >> 16;
2677 t = _mm512_srli_epi32(t, 16);
2678
2679 // Check NaN before converting back to bf16
2680 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q);
2681
2682 t = _mm512_mask_blend_epi32(mask, nan, t);
2683 // output.value = static_cast<uint16_t>(input);
2684 r = _mm512_cvtepi32_epi16(t);
2685#endif // EIGEN_VECTORIZE_AVX512BF16
2686
2687 return r;
2688}
2689
2690// Discard the low 16 bits of each float. Only valid when every lane already holds an exact
2691// bfloat16 value, in which case this agrees with F32ToBf16 but skips its rounding and NaN
2692// canonicalization. pmin/pmax qualify: they return one of their operands bit for bit.
2693EIGEN_STRONG_INLINE Packet16bf F32ToBf16Truncate(const Packet16f& a) {
2694 return _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(a), 16));
2695}
2696
2697template <>
2698EIGEN_STRONG_INLINE Packet16bf ptrue(const Packet16bf& a) {
2699 return Packet16bf(ptrue<Packet8i>(Packet8i(a)));
2700}
2701
2702template <>
2703EIGEN_STRONG_INLINE Packet16bf por(const Packet16bf& a, const Packet16bf& b) {
2704 return Packet16bf(por<Packet8i>(Packet8i(a), Packet8i(b)));
2705}
2706
2707template <>
2708EIGEN_STRONG_INLINE Packet16bf pxor(const Packet16bf& a, const Packet16bf& b) {
2709 return Packet16bf(pxor<Packet8i>(Packet8i(a), Packet8i(b)));
2710}
2711
2712template <>
2713EIGEN_STRONG_INLINE Packet16bf pand(const Packet16bf& a, const Packet16bf& b) {
2714 return Packet16bf(pand<Packet8i>(Packet8i(a), Packet8i(b)));
2715}
2716
2717template <>
2718EIGEN_STRONG_INLINE Packet16bf pandnot(const Packet16bf& a, const Packet16bf& b) {
2719 return Packet16bf(pandnot<Packet8i>(Packet8i(a), Packet8i(b)));
2720}
2721
2722template <>
2723EIGEN_STRONG_INLINE Packet16bf pselect(const Packet16bf& mask, const Packet16bf& a, const Packet16bf& b) {
2724 // Input mask is expected to be all 0/1, handle it with 8-bit
2725 // intrinsic for performance.
2726 return _mm256_blendv_epi8(b, a, mask);
2727}
2728
2729template <>
2730EIGEN_STRONG_INLINE Packet16bf pround<Packet16bf>(const Packet16bf& a) {
2731 return F32ToBf16Truncate(pround<Packet16f>(Bf16ToF32(a)));
2732}
2733
2734template <>
2735EIGEN_STRONG_INLINE Packet16bf print<Packet16bf>(const Packet16bf& a) {
2736 return F32ToBf16Truncate(print<Packet16f>(Bf16ToF32(a)));
2737}
2738
2739template <>
2740EIGEN_STRONG_INLINE Packet16bf pceil<Packet16bf>(const Packet16bf& a) {
2741 return F32ToBf16Truncate(pceil<Packet16f>(Bf16ToF32(a)));
2742}
2743
2744template <>
2745EIGEN_STRONG_INLINE Packet16bf pfloor<Packet16bf>(const Packet16bf& a) {
2746 return F32ToBf16Truncate(pfloor<Packet16f>(Bf16ToF32(a)));
2747}
2748
2749template <>
2750EIGEN_STRONG_INLINE Packet16bf ptrunc<Packet16bf>(const Packet16bf& a) {
2751 return F32ToBf16Truncate(ptrunc<Packet16f>(Bf16ToF32(a)));
2752}
2753
2754template <>
2755EIGEN_STRONG_INLINE Packet16bf pcmp_eq(const Packet16bf& a, const Packet16bf& b) {
2756 return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
2757}
2758
2759template <>
2760EIGEN_STRONG_INLINE Packet16bf pcmp_le(const Packet16bf& a, const Packet16bf& b) {
2761 return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
2762}
2763
2764template <>
2765EIGEN_STRONG_INLINE Packet16bf pcmp_lt(const Packet16bf& a, const Packet16bf& b) {
2766 return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
2767}
2768
2769template <>
2770EIGEN_STRONG_INLINE Packet16bf pcmp_lt_or_nan(const Packet16bf& a, const Packet16bf& b) {
2771 return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
2772}
2773
2774// Classify on the raw bits, as the scalar isinf/isnan/isfinite do: |a| ==, >, < 0x7f80.
2775template <>
2776EIGEN_STRONG_INLINE Packet16bf pisinf<Packet16bf>(const Packet16bf& a) {
2777 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2778 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2779 return _mm256_cmpeq_epi16(_mm256_and_si256(a, _mm256_set1_epi16(kAbsMask)), _mm256_set1_epi16(kInf));
2780}
2781
2782template <>
2783EIGEN_STRONG_INLINE Packet16bf pisnan<Packet16bf>(const Packet16bf& a) {
2784 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2785 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2786 return _mm256_cmpgt_epi16(_mm256_and_si256(a, _mm256_set1_epi16(kAbsMask)), _mm256_set1_epi16(kInf));
2787}
2788
2789// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ.
2790template <>
2791EIGEN_STRONG_INLINE Packet16bf psign<Packet16bf>(const Packet16bf& a) {
2792 const __m256i magnitude = _mm256_and_si256(a, _mm256_set1_epi16(0x7fff));
2793 const __m256i is_nan = _mm256_cmpgt_epi16(magnitude, _mm256_set1_epi16(0x7f80));
2794 const __m256i keep = _mm256_or_si256(is_nan, _mm256_set1_epi16(static_cast<short>(0xbf80u)));
2795 return _mm256_sign_epi16(_mm256_and_si256(_mm256_or_si256(a, _mm256_set1_epi16(0x3f80)), keep), magnitude);
2796}
2797
2798template <>
2799EIGEN_STRONG_INLINE Packet16bf pisfinite<Packet16bf>(const Packet16bf& a) {
2800 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2801 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2802 return _mm256_cmpgt_epi16(_mm256_set1_epi16(kInf), _mm256_and_si256(a, _mm256_set1_epi16(kAbsMask)));
2803}
2804
2805template <>
2806EIGEN_STRONG_INLINE Packet16bf pnegate(const Packet16bf& a) {
2807 Packet16bf sign_mask = _mm256_set1_epi16(static_cast<short>(0x8000u));
2808 return _mm256_xor_si256(a, sign_mask);
2809}
2810
2811template <>
2812EIGEN_STRONG_INLINE Packet16bf pabs(const Packet16bf& a) {
2813 const __m256i sign_mask = _mm256_set1_epi16(static_cast<short>(0x8000u));
2814 return _mm256_andnot_si256(sign_mask, a);
2815}
2816
2817template <>
2818EIGEN_STRONG_INLINE Packet16bf padd<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2819 return F32ToBf16(padd<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2820}
2821
2822template <>
2823EIGEN_STRONG_INLINE Packet16bf psub<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2824 return F32ToBf16(psub<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2825}
2826
2827template <>
2828EIGEN_STRONG_INLINE Packet16bf pmul<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2829 return F32ToBf16(pmul(Bf16ToF32(a), Bf16ToF32(b)));
2830}
2831
2832template <>
2833EIGEN_STRONG_INLINE Packet16bf pmadd<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2834 return F32ToBf16(pmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2835}
2836
2837template <>
2838EIGEN_STRONG_INLINE Packet16bf pmsub<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2839 return F32ToBf16(pmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2840}
2841
2842template <>
2843EIGEN_STRONG_INLINE Packet16bf pnmadd<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2844 return F32ToBf16(pnmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2845}
2846
2847template <>
2848EIGEN_STRONG_INLINE Packet16bf pnmsub<Packet16bf>(const Packet16bf& a, const Packet16bf& b, const Packet16bf& c) {
2849 return F32ToBf16(pnmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2850}
2851
2852template <>
2853EIGEN_STRONG_INLINE Packet16bf pdiv<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2854 return F32ToBf16(pdiv<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2855}
2856
2857template <>
2858EIGEN_STRONG_INLINE Packet16bf pmin<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2859 return F32ToBf16Truncate(pmin<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2860}
2861
2862template <>
2863EIGEN_STRONG_INLINE Packet16bf pmax<Packet16bf>(const Packet16bf& a, const Packet16bf& b) {
2864 return F32ToBf16Truncate(pmax<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2865}
2866
2867template <>
2868EIGEN_STRONG_INLINE Packet16bf plset<Packet16bf>(const bfloat16& a) {
2869 return F32ToBf16(plset<Packet16f>(static_cast<float>(a)));
2870}
2871
2872template <>
2873EIGEN_STRONG_INLINE Packet8bf predux_half<Packet16bf>(const Packet16bf& a) {
2874 Packet8bf lane0 = _mm256_extractf128_si256(a, 0);
2875 Packet8bf lane1 = _mm256_extractf128_si256(a, 1);
2876 return padd<Packet8bf>(lane0, lane1);
2877}
2878
2879template <>
2880EIGEN_STRONG_INLINE Packet16bf preverse(const Packet16bf& a) {
2881 __m256i m = _mm256_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7,
2882 4, 5, 2, 3, 0, 1);
2883
2884 Packet16bf res;
2885 // Swap hi and lo first because shuffle is in 128-bit lanes.
2886 res = _mm256_permute2x128_si256(a, a, 1);
2887 // Shuffle 8-bit values in src within 2*128-bit lanes.
2888 return _mm256_shuffle_epi8(res, m);
2889}
2890
2891template <>
2892EIGEN_STRONG_INLINE Packet16bf pgather<bfloat16, Packet16bf>(const bfloat16* from, Index stride) {
2893 return _mm256_set_epi16(
2894 from[15 * stride].value, from[14 * stride].value, from[13 * stride].value, from[12 * stride].value,
2895 from[11 * stride].value, from[10 * stride].value, from[9 * stride].value, from[8 * stride].value,
2896 from[7 * stride].value, from[6 * stride].value, from[5 * stride].value, from[4 * stride].value,
2897 from[3 * stride].value, from[2 * stride].value, from[1 * stride].value, from[0 * stride].value);
2898}
2899
2900template <>
2901EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet16bf>(bfloat16* to, const Packet16bf& from, Index stride) {
2902 EIGEN_ALIGN64 bfloat16 aux[16];
2903 pstore(aux, from);
2904 to[stride * 0] = aux[0];
2905 to[stride * 1] = aux[1];
2906 to[stride * 2] = aux[2];
2907 to[stride * 3] = aux[3];
2908 to[stride * 4] = aux[4];
2909 to[stride * 5] = aux[5];
2910 to[stride * 6] = aux[6];
2911 to[stride * 7] = aux[7];
2912 to[stride * 8] = aux[8];
2913 to[stride * 9] = aux[9];
2914 to[stride * 10] = aux[10];
2915 to[stride * 11] = aux[11];
2916 to[stride * 12] = aux[12];
2917 to[stride * 13] = aux[13];
2918 to[stride * 14] = aux[14];
2919 to[stride * 15] = aux[15];
2920}
2921
2922EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf, 16>& kernel) {
2923 __m256i a = kernel.packet[0];
2924 __m256i b = kernel.packet[1];
2925 __m256i c = kernel.packet[2];
2926 __m256i d = kernel.packet[3];
2927 __m256i e = kernel.packet[4];
2928 __m256i f = kernel.packet[5];
2929 __m256i g = kernel.packet[6];
2930 __m256i h = kernel.packet[7];
2931 __m256i i = kernel.packet[8];
2932 __m256i j = kernel.packet[9];
2933 __m256i k = kernel.packet[10];
2934 __m256i l = kernel.packet[11];
2935 __m256i m = kernel.packet[12];
2936 __m256i n = kernel.packet[13];
2937 __m256i o = kernel.packet[14];
2938 __m256i p = kernel.packet[15];
2939
2940 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2941 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2942 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2943 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2944 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2945 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2946 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2947 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2948
2949 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2950 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2951 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2952 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2953 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2954 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2955 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2956 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2957
2958 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2959 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2960 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2961 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2962 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2963 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2964 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2965 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2966
2967 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2968 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2969 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2970 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2971 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2972 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2973 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2974 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2975
2976 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2977 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2978 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2979 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2980 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2981 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2982 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2983 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2984 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2985 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2986 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2987 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2988 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2989 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2990 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2991 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2992
2993 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2994 kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2995 kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2996 kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2997 kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2998 kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2999 kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
3000 kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
3001 kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
3002 kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
3003 kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
3004 kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
3005 kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
3006 kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
3007 kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
3008 kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
3009 kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
3010}
3011
3012EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf, 4>& kernel) {
3013 __m256i a = kernel.packet[0];
3014 __m256i b = kernel.packet[1];
3015 __m256i c = kernel.packet[2];
3016 __m256i d = kernel.packet[3];
3017
3018 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
3019 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
3020 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
3021 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
3022
3023 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
3024 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
3025 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
3026 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
3027
3028 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
3029 kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20);
3030 kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20);
3031 kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31);
3032 kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31);
3033}
3034
3035// Minimal implementation of 16-bit int packets for use in pfrexp, pldexp.
3036
3037template <>
3038EIGEN_STRONG_INLINE Packet32s pset1<Packet32s>(const numext::int16_t& x) {
3039 return _mm512_set1_epi16(x);
3040}
3041
3042template <>
3043EIGEN_STRONG_INLINE Packet16s pset1<Packet16s>(const numext::int16_t& x) {
3044 return _mm256_set1_epi16(x);
3045}
3046
3047template <>
3048EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const numext::int16_t& x) {
3049 return _mm_set1_epi16(x);
3050}
3051
3052template <>
3053EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet32s>(numext::int16_t* out, const Packet32s& x) {
3054 EIGEN_DEBUG_ALIGNED_STORE
3055 _mm512_store_epi32(out, x);
3056}
3057
3058template <>
3059EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet16s>(numext::int16_t* out, const Packet16s& x) {
3060 EIGEN_DEBUG_ALIGNED_STORE
3061#if defined(EIGEN_VECTORIZE_AVX512F) && defined(EIGEN_VECTORIZE_AVX512VL)
3062 _mm256_store_epi32(out, x);
3063#else
3064 _mm256_store_si256(reinterpret_cast<__m256i*>(out), x);
3065#endif
3066}
3067
3068template <>
3069EIGEN_STRONG_INLINE void pstore<numext::int16_t, Packet8s>(numext::int16_t* out, const Packet8s& x) {
3070 EIGEN_DEBUG_ALIGNED_STORE
3071#if defined(EIGEN_VECTORIZE_AVX512F) && defined(EIGEN_VECTORIZE_AVX512VL)
3072 _mm256_store_epi32(out, x);
3073#else
3074 _mm_store_si128(reinterpret_cast<__m128i*>(out), x);
3075#endif
3076}
3077
3078template <>
3079EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet32s>(numext::int16_t* out, const Packet32s& x) {
3080 EIGEN_DEBUG_UNALIGNED_STORE
3081 _eigen_mm512_storeu_epi32(out, x);
3082}
3083
3084template <>
3085EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet16s>(numext::int16_t* out, const Packet16s& x) {
3086 EIGEN_DEBUG_UNALIGNED_STORE
3087 _eigen_mm256_storeu_epi32(out, x);
3088}
3089
3090template <>
3091EIGEN_STRONG_INLINE void pstoreu<numext::int16_t, Packet8s>(numext::int16_t* out, const Packet8s& x) {
3092 EIGEN_DEBUG_UNALIGNED_STORE
3093 _eigen_mm_storeu_epi32(out, x);
3094}
3095
3096template <>
3097EIGEN_STRONG_INLINE Packet32s padd(const Packet32s& a, const Packet32s& b) {
3098 return _mm512_add_epi16(a, b);
3099}
3100
3101template <>
3102EIGEN_STRONG_INLINE Packet16s padd(const Packet16s& a, const Packet16s& b) {
3103 return _mm256_add_epi16(a, b);
3104}
3105
3106template <>
3107EIGEN_STRONG_INLINE Packet8s padd(const Packet8s& a, const Packet8s& b) {
3108 return _mm_add_epi16(a, b);
3109}
3110
3111template <>
3112EIGEN_STRONG_INLINE Packet32s psub(const Packet32s& a, const Packet32s& b) {
3113 return _mm512_sub_epi16(a, b);
3114}
3115
3116template <>
3117EIGEN_STRONG_INLINE Packet16s psub(const Packet16s& a, const Packet16s& b) {
3118 return _mm256_sub_epi16(a, b);
3119}
3120
3121template <>
3122EIGEN_STRONG_INLINE Packet8s psub(const Packet8s& a, const Packet8s& b) {
3123 return _mm_sub_epi16(a, b);
3124}
3125
3126template <>
3127EIGEN_STRONG_INLINE Packet32s pmul(const Packet32s& a, const Packet32s& b) {
3128 return _mm512_mullo_epi16(a, b);
3129}
3130
3131template <>
3132EIGEN_STRONG_INLINE Packet16s pmul(const Packet16s& a, const Packet16s& b) {
3133 return _mm256_mullo_epi16(a, b);
3134}
3135
3136template <>
3137EIGEN_STRONG_INLINE Packet8s pmul(const Packet8s& a, const Packet8s& b) {
3138 return _mm_mullo_epi16(a, b);
3139}
3140
3141template <>
3142EIGEN_STRONG_INLINE Packet32s pnegate(const Packet32s& a) {
3143 return _mm512_sub_epi16(_mm512_setzero_si512(), a);
3144}
3145
3146template <>
3147EIGEN_STRONG_INLINE Packet16s pnegate(const Packet16s& a) {
3148 return _mm256_sub_epi16(_mm256_setzero_si256(), a);
3149}
3150
3151template <>
3152EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
3153 return _mm_sub_epi16(_mm_setzero_si128(), a);
3154}
3155
3156template <int N>
3157EIGEN_STRONG_INLINE Packet32s parithmetic_shift_right(const Packet32s& a) {
3158 return _mm512_srai_epi16(a, N);
3159}
3160
3161template <int N>
3162EIGEN_STRONG_INLINE Packet16s parithmetic_shift_right(const Packet16s& a) {
3163 return _mm256_srai_epi16(a, N);
3164}
3165
3166template <int N>
3167EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(const Packet8s& a) {
3168 return _mm_srai_epi16(a, N);
3169}
3170
3171template <int N>
3172EIGEN_STRONG_INLINE Packet32s plogical_shift_left(const Packet32s& a) {
3173 return _mm512_slli_epi16(a, N);
3174}
3175
3176template <int N>
3177EIGEN_STRONG_INLINE Packet16s plogical_shift_left(const Packet16s& a) {
3178 return _mm256_slli_epi16(a, N);
3179}
3180
3181template <int N>
3182EIGEN_STRONG_INLINE Packet8s plogical_shift_left(const Packet8s& a) {
3183 return _mm_slli_epi16(a, N);
3184}
3185
3186template <int N>
3187EIGEN_STRONG_INLINE Packet32s plogical_shift_right(const Packet32s& a) {
3188 return _mm512_srli_epi16(a, N);
3189}
3190
3191template <int N>
3192EIGEN_STRONG_INLINE Packet16s plogical_shift_right(const Packet16s& a) {
3193 return _mm256_srli_epi16(a, N);
3194}
3195
3196template <int N>
3197EIGEN_STRONG_INLINE Packet8s plogical_shift_right(const Packet8s& a) {
3198 return _mm_srli_epi16(a, N);
3199}
3200
3201/*---------------- load/store segment support ----------------*/
3202
3203// Returns a k-mask with bits [begin, begin + count) set. 64-bit arithmetic lets count reach the lane
3204// count of the widest AVX-512 packet (32) without shifting a 1 past the top bit.
3205EIGEN_STRONG_INLINE uint64_t segment_kmask(Index begin, Index count) {
3206 eigen_assert(begin >= 0 && count >= 0 && begin + count <= 32);
3207 return ((uint64_t(1) << count) - 1) << begin;
3208}
3209
3210/*---------------- float ----------------*/
3211
3212template <>
3213struct has_packet_segment<Packet16f> : std::true_type {};
3214
3215template <>
3216inline Packet16f ploaduSegment<Packet16f>(const float* from, Index begin, Index count) {
3217 return _mm512_maskz_loadu_ps(static_cast<__mmask16>(segment_kmask(begin, count)), from);
3218}
3219
3220template <>
3221inline void pstoreuSegment<float, Packet16f>(float* to, const Packet16f& from, Index begin, Index count) {
3222 _mm512_mask_storeu_ps(to, static_cast<__mmask16>(segment_kmask(begin, count)), from);
3223}
3224
3225/*---------------- double ----------------*/
3226
3227template <>
3228struct has_packet_segment<Packet8d> : std::true_type {};
3229
3230template <>
3231inline Packet8d ploaduSegment<Packet8d>(const double* from, Index begin, Index count) {
3232 return _mm512_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3233}
3234
3235template <>
3236inline void pstoreuSegment<double, Packet8d>(double* to, const Packet8d& from, Index begin, Index count) {
3237 _mm512_mask_storeu_pd(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3238}
3239
3240/*---------------- int32 ----------------*/
3241
3242template <>
3243struct has_packet_segment<Packet16i> : std::true_type {};
3244
3245template <>
3246inline Packet16i ploaduSegment<Packet16i>(const int* from, Index begin, Index count) {
3247 return _mm512_maskz_loadu_epi32(static_cast<__mmask16>(segment_kmask(begin, count)), from);
3248}
3249
3250template <>
3251inline void pstoreuSegment<int, Packet16i>(int* to, const Packet16i& from, Index begin, Index count) {
3252 _mm512_mask_storeu_epi32(to, static_cast<__mmask16>(segment_kmask(begin, count)), from);
3253}
3254
3255/*---------------- int64_t ----------------*/
3256
3257template <>
3258struct has_packet_segment<Packet8l> : std::true_type {};
3259
3260template <>
3261inline Packet8l ploaduSegment<Packet8l>(const int64_t* from, Index begin, Index count) {
3262 return _mm512_maskz_loadu_epi64(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3263}
3264
3265template <>
3266inline void pstoreuSegment<int64_t, Packet8l>(int64_t* to, const Packet8l& from, Index begin, Index count) {
3267 _mm512_mask_storeu_epi64(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3268}
3269
3270#ifdef EIGEN_VECTORIZE_AVX512VL
3271
3272// The 256- and 128-bit packets declare has_packet_segment in AVX/PacketMath.h; with AVX-512VL their
3273// segments are k-masked as well, which avoids vmaskmov and the vector mask it needs.
3274
3275/*---------------- float ----------------*/
3276
3277template <>
3278inline Packet8f ploaduSegment<Packet8f>(const float* from, Index begin, Index count) {
3279 return _mm256_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3280}
3281
3282template <>
3283inline void pstoreuSegment<float, Packet8f>(float* to, const Packet8f& from, Index begin, Index count) {
3284 _mm256_mask_storeu_ps(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3285}
3286
3287template <>
3288inline Packet4f ploaduSegment<Packet4f>(const float* from, Index begin, Index count) {
3289 return _mm_maskz_loadu_ps(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3290}
3291
3292template <>
3293inline void pstoreuSegment<float, Packet4f>(float* to, const Packet4f& from, Index begin, Index count) {
3294 _mm_mask_storeu_ps(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3295}
3296
3297/*---------------- double ----------------*/
3298
3299template <>
3300inline Packet4d ploaduSegment<Packet4d>(const double* from, Index begin, Index count) {
3301 return _mm256_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3302}
3303
3304template <>
3305inline void pstoreuSegment<double, Packet4d>(double* to, const Packet4d& from, Index begin, Index count) {
3306 _mm256_mask_storeu_pd(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3307}
3308
3309template <>
3310inline Packet2d ploaduSegment<Packet2d>(const double* from, Index begin, Index count) {
3311 return _mm_maskz_loadu_pd(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3312}
3313
3314template <>
3315inline void pstoreuSegment<double, Packet2d>(double* to, const Packet2d& from, Index begin, Index count) {
3316 _mm_mask_storeu_pd(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3317}
3318
3319/*---------------- int32 ----------------*/
3320
3321template <>
3322inline Packet8i ploaduSegment<Packet8i>(const int* from, Index begin, Index count) {
3323 return _mm256_maskz_loadu_epi32(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3324}
3325
3326template <>
3327inline void pstoreuSegment<int, Packet8i>(int* to, const Packet8i& from, Index begin, Index count) {
3328 _mm256_mask_storeu_epi32(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3329}
3330
3331template <>
3332inline Packet4i ploaduSegment<Packet4i>(const int* from, Index begin, Index count) {
3333 return _mm_maskz_loadu_epi32(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3334}
3335
3336template <>
3337inline void pstoreuSegment<int, Packet4i>(int* to, const Packet4i& from, Index begin, Index count) {
3338 _mm_mask_storeu_epi32(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3339}
3340
3341/*---------------- uint32 ----------------*/
3342
3343template <>
3344inline Packet8ui ploaduSegment<Packet8ui>(const uint32_t* from, Index begin, Index count) {
3345 return Packet8ui(ploaduSegment<Packet8i>(reinterpret_cast<const int*>(from), begin, count));
3346}
3347
3348template <>
3349inline void pstoreuSegment<uint32_t, Packet8ui>(uint32_t* to, const Packet8ui& from, Index begin, Index count) {
3350 pstoreuSegment<int, Packet8i>(reinterpret_cast<int*>(to), Packet8i(from), begin, count);
3351}
3352
3353template <>
3354inline Packet4ui ploaduSegment<Packet4ui>(const uint32_t* from, Index begin, Index count) {
3355 return Packet4ui(ploaduSegment<Packet4i>(reinterpret_cast<const int*>(from), begin, count));
3356}
3357
3358template <>
3359inline void pstoreuSegment<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from, Index begin, Index count) {
3360 pstoreuSegment<int, Packet4i>(reinterpret_cast<int*>(to), Packet4i(from), begin, count);
3361}
3362
3363/*---------------- int64_t ----------------*/
3364
3365template <>
3366inline Packet4l ploaduSegment<Packet4l>(const int64_t* from, Index begin, Index count) {
3367 return _mm256_maskz_loadu_epi64(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3368}
3369
3370template <>
3371inline void pstoreuSegment<int64_t, Packet4l>(int64_t* to, const Packet4l& from, Index begin, Index count) {
3372 _mm256_mask_storeu_epi64(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3373}
3374
3375template <>
3376inline Packet2l ploaduSegment<Packet2l>(const int64_t* from, Index begin, Index count) {
3377 return _mm_maskz_loadu_epi64(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3378}
3379
3380template <>
3381inline void pstoreuSegment<int64_t, Packet2l>(int64_t* to, const Packet2l& from, Index begin, Index count) {
3382 _mm_mask_storeu_epi64(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3383}
3384
3385/*---------------- uint64_t ----------------*/
3386
3387template <>
3388inline Packet4ul ploaduSegment<Packet4ul>(const uint64_t* from, Index begin, Index count) {
3389 return Packet4ul(ploaduSegment<Packet4l>(reinterpret_cast<const int64_t*>(from), begin, count));
3390}
3391
3392template <>
3393inline void pstoreuSegment<uint64_t, Packet4ul>(uint64_t* to, const Packet4ul& from, Index begin, Index count) {
3394 pstoreuSegment<int64_t, Packet4l>(reinterpret_cast<int64_t*>(to), Packet4l(from), begin, count);
3395}
3396
3397#if defined(__AVX512BW__)
3398
3399// 16-bit lanes need the AVX-512BW word-granular masked moves.
3400
3401/*---------------- bfloat16 ----------------*/
3402
3403template <>
3404struct has_packet_segment<Packet16bf> : std::true_type {};
3405
3406template <>
3407struct has_packet_segment<Packet8bf> : std::true_type {};
3408
3409template <>
3410inline Packet16bf ploaduSegment<Packet16bf>(const bfloat16* from, Index begin, Index count) {
3411 return _mm256_maskz_loadu_epi16(static_cast<__mmask16>(segment_kmask(begin, count)), from);
3412}
3413
3414template <>
3415inline void pstoreuSegment<bfloat16, Packet16bf>(bfloat16* to, const Packet16bf& from, Index begin, Index count) {
3416 _mm256_mask_storeu_epi16(to, static_cast<__mmask16>(segment_kmask(begin, count)), from);
3417}
3418
3419template <>
3420inline Packet8bf ploaduSegment<Packet8bf>(const bfloat16* from, Index begin, Index count) {
3421 return _mm_maskz_loadu_epi16(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3422}
3423
3424template <>
3425inline void pstoreuSegment<bfloat16, Packet8bf>(bfloat16* to, const Packet8bf& from, Index begin, Index count) {
3426 _mm_mask_storeu_epi16(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3427}
3428
3429#ifndef EIGEN_VECTORIZE_AVX512FP16
3430
3431/*---------------- half ----------------*/
3432
3433template <>
3434struct has_packet_segment<Packet16h> : std::true_type {};
3435
3436template <>
3437struct has_packet_segment<Packet8h> : std::true_type {};
3438
3439template <>
3440inline Packet16h ploaduSegment<Packet16h>(const Eigen::half* from, Index begin, Index count) {
3441 return _mm256_maskz_loadu_epi16(static_cast<__mmask16>(segment_kmask(begin, count)), from);
3442}
3443
3444template <>
3445inline void pstoreuSegment<Eigen::half, Packet16h>(Eigen::half* to, const Packet16h& from, Index begin, Index count) {
3446 _mm256_mask_storeu_epi16(to, static_cast<__mmask16>(segment_kmask(begin, count)), from);
3447}
3448
3449template <>
3450inline Packet8h ploaduSegment<Packet8h>(const Eigen::half* from, Index begin, Index count) {
3451 return _mm_maskz_loadu_epi16(static_cast<__mmask8>(segment_kmask(begin, count)), from);
3452}
3453
3454template <>
3455inline void pstoreuSegment<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index begin, Index count) {
3456 _mm_mask_storeu_epi16(to, static_cast<__mmask8>(segment_kmask(begin, count)), from);
3457}
3458
3459#endif // EIGEN_VECTORIZE_AVX512FP16
3460#endif // __AVX512BW__
3461#endif // EIGEN_VECTORIZE_AVX512VL
3462
3463/*---------------- end load/store segment support ----------------*/
3464
3465} // end namespace internal
3466
3467} // end namespace Eigen
3468
3469#endif // EIGEN_PACKET_MATH_AVX512_H
Eigen::Aligned64
@ Aligned64
Definition Constants.h:240
Eigen::Aligned32
@ Aligned32
Definition Constants.h:239
Eigen::Aligned16
@ Aligned16
Definition Constants.h:238