11#ifndef EIGEN_PACKET_MATH_AVX512_H
12#define EIGEN_PACKET_MATH_AVX512_H
15#include "../../InternalHeaderCheck.h"
21#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
25#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
26#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
29#ifdef EIGEN_VECTORIZE_FMA
30#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
31#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
35using Packet16f = __m512;
36using Packet16i = __m512i;
37using Packet8d = __m512d;
38using Packet8l = eigen_packet_wrapper<__m512i, 1>;
39#ifndef EIGEN_VECTORIZE_AVX512FP16
40using Packet16h = eigen_packet_wrapper<__m256i, 1>;
42using Packet16bf = eigen_packet_wrapper<__m256i, 2>;
44using Packet32s = eigen_packet_wrapper<__m512i, 6>;
45using Packet16s = eigen_packet_wrapper<__m256i, 6>;
46using Packet8s = eigen_packet_wrapper<__m128i, 6>;
49#define EIGEN_AVX512_SHUFFLE_PS(A, B, M) ((__m512)_mm512_shuffle_ps(A, B, M))
51#define EIGEN_AVX512_SHUFFLE_PS(A, B, M) _mm512_shuffle_ps(A, B, M)
54EIGEN_STRONG_INLINE Packet16i _eigen_mm512_loadu_epi32(
const int* from) {
55 return _mm512_loadu_si512(
reinterpret_cast<const void*
>(from));
58EIGEN_STRONG_INLINE Packet16i _eigen_mm512_loadu_epi64(
const int64_t* from) {
59 return _mm512_loadu_si512(
reinterpret_cast<const void*
>(from));
62EIGEN_STRONG_INLINE
void _eigen_mm512_storeu_epi32(
void* to,
const Packet16i& from) { _mm512_storeu_si512(to, from); }
64EIGEN_STRONG_INLINE
void _eigen_mm512_storeu_epi64(
void* to,
const Packet16i& from) { _mm512_storeu_si512(to, from); }
66EIGEN_STRONG_INLINE
void _eigen_mm256_storeu_epi32(
void* to,
const __m256i& from) {
67 _mm256_storeu_si256(
reinterpret_cast<__m256i*
>(to), from);
70EIGEN_STRONG_INLINE
void _eigen_mm_storeu_epi32(
void* to,
const __m128i& from) {
71 _mm_storeu_si128(
reinterpret_cast<__m128i*
>(to), from);
75struct is_arithmetic<__m512> : std::true_type {};
77struct is_arithmetic<__m512i> : std::true_type {};
79struct is_arithmetic<__m512d> : std::true_type {};
81struct is_arithmetic<Packet8l> : std::true_type {};
83#ifndef EIGEN_VECTORIZE_AVX512FP16
85struct is_arithmetic<Packet16h> : std::true_type {};
88struct packet_traits<half> : default_packet_traits {
89 using type = Packet16h;
91 using half = Packet16h;
116 HasSin = EIGEN_FAST_MATH,
117 HasCos = EIGEN_FAST_MATH,
118 HasTanh = EIGEN_FAST_MATH,
119 HasErf = EIGEN_FAST_MATH,
125struct packet_traits<float> : default_packet_traits {
126 using type = Packet16f;
127 using half = Packet8f;
137 HasSin = EIGEN_FAST_MATH,
138 HasCos = EIGEN_FAST_MATH,
139 HasTan = EIGEN_FAST_MATH,
159 HasReciprocal = EIGEN_FAST_MATH,
160 HasTanh = EIGEN_FAST_MATH,
161 HasErf = EIGEN_FAST_MATH,
162 HasErfc = EIGEN_FAST_MATH,
168struct packet_traits<double> : default_packet_traits {
169 using type = Packet8d;
170 using half = Packet4d;
178 HasSin = EIGEN_FAST_MATH,
179 HasCos = EIGEN_FAST_MATH,
180 HasTan = EIGEN_FAST_MATH,
192 HasTanh = EIGEN_FAST_MATH,
193 HasErf = EIGEN_FAST_MATH,
194 HasErfc = EIGEN_FAST_MATH,
202struct packet_traits<int> : default_packet_traits {
203 using type = Packet16i;
204 using half = Packet8i;
205 enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, HasDiv = 1, HasFastIntDiv = 1, size = 16 };
209struct packet_traits<int64_t> : default_packet_traits {
210 using type = Packet8l;
211 using half = Packet4l;
217 HasFastIntDiv = int_div::use_multiply_add<uint64_t>::value,
223struct unpacket_traits<Packet16f> {
225 using half = Packet8f;
226 using integer_packet = Packet16i;
227 using mask_t = uint16_t;
232 masked_load_available =
true,
233 masked_store_available =
true,
234 masked_fpops_available =
true
238struct unpacket_traits<Packet8d> {
240 using half = Packet4d;
241 using integer_packet = Packet8l;
242 using mask_t = uint8_t;
247 masked_load_available =
true,
248 masked_store_available =
true,
249 masked_fpops_available =
true
253struct unpacket_traits<Packet16i> {
255 using half = Packet8i;
260 masked_load_available =
false,
261 masked_store_available =
false
266struct unpacket_traits<Packet8l> {
267 using type = int64_t;
268 using half = Packet4l;
273 masked_load_available =
false,
274 masked_store_available =
false
278#ifndef EIGEN_VECTORIZE_AVX512FP16
280struct unpacket_traits<Packet16h> {
281 using type = Eigen::half;
282 using half = Packet8h;
287 masked_load_available =
false,
288 masked_store_available =
false
294struct unpacket_traits<Packet32s> {
295 using type = numext::int16_t;
296 using half = Packet16s;
300 vectorizable =
false,
305struct unpacket_traits<Packet16s> {
306 using type = numext::int16_t;
307 using half = Packet8s;
311 vectorizable =
false,
316struct unpacket_traits<Packet8s> {
317 using type = numext::int16_t;
318 using half = Packet8s;
322 vectorizable =
false,
327EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(
const float& from) {
328 return _mm512_set1_ps(from);
331EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(
const double& from) {
332 return _mm512_set1_pd(from);
335EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(
const int& from) {
336 return _mm512_set1_epi32(from);
339EIGEN_STRONG_INLINE Packet8l pset1<Packet8l>(
const int64_t& from) {
340 return _mm512_set1_epi64(from);
344EIGEN_STRONG_INLINE Packet16f pset1frombits<Packet16f>(
unsigned int from) {
345 return _mm512_castsi512_ps(_mm512_set1_epi32(from));
349EIGEN_STRONG_INLINE Packet8d pset1frombits<Packet8d>(
const numext::uint64_t from) {
350 return _mm512_castsi512_pd(_mm512_set1_epi64(from));
354EIGEN_STRONG_INLINE Packet16f pzero(
const Packet16f& ) {
355 return _mm512_setzero_ps();
358EIGEN_STRONG_INLINE Packet8d pzero(
const Packet8d& ) {
359 return _mm512_setzero_pd();
362EIGEN_STRONG_INLINE Packet16i pzero(
const Packet16i& ) {
363 return _mm512_setzero_si512();
367EIGEN_STRONG_INLINE Packet8l pzero(
const Packet8l& ) {
368 return _mm512_setzero_si512();
372EIGEN_STRONG_INLINE Packet16f peven_mask(
const Packet16f& ) {
373 Packet16f r = _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1));
374 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
378EIGEN_STRONG_INLINE Packet16i peven_mask(
const Packet16i& ) {
379 return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1);
382EIGEN_STRONG_INLINE Packet8d peven_mask(
const Packet8d& ) {
383 Packet8d r = _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1));
384 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
388EIGEN_STRONG_INLINE Packet8l peven_mask(
const Packet8l& ) {
389 return _mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1);
393EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(
const float* from) {
394#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
398 __asm__(
"vbroadcastss %[mem], %[dst]" : [dst]
"=v"(ret) : [mem]
"m"(*from));
401 return _mm512_broadcastss_ps(_mm_load_ps1(from));
405EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(
const double* from) {
406#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
408 __asm__(
"vbroadcastsd %[mem], %[dst]" : [dst]
"=v"(ret) : [mem]
"m"(*from));
411 return _mm512_set1_pd(*from);
416EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(
const float& a) {
417 return _mm512_add_ps(_mm512_set1_ps(a), _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f,
418 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
421EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(
const double& a) {
422 return _mm512_add_pd(_mm512_set1_pd(a), _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
425EIGEN_STRONG_INLINE Packet16i plset<Packet16i>(
const int& a) {
426 return _mm512_add_epi32(_mm512_set1_epi32(a), _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
429EIGEN_STRONG_INLINE Packet8l plset<Packet8l>(
const int64_t& a) {
430 return _mm512_add_epi64(_mm512_set1_epi64(a), _mm512_set_epi64(7, 6, 5, 4, 3, 2, 1, 0));
434EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
435 return _mm512_add_ps(a, b);
438EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
439 return _mm512_add_pd(a, b);
442EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
443 return _mm512_add_epi32(a, b);
446EIGEN_STRONG_INLINE Packet8l padd<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
447 return _mm512_add_epi64(a, b);
451EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(
const Packet16f& a,
const Packet16f& b, uint16_t umask) {
452 __mmask16 mask =
static_cast<__mmask16
>(umask);
453 return _mm512_maskz_add_ps(mask, a, b);
456EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(
const Packet8d& a,
const Packet8d& b, uint8_t umask) {
457 __mmask8 mask =
static_cast<__mmask8
>(umask);
458 return _mm512_maskz_add_pd(mask, a, b);
462EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
463 return _mm512_sub_ps(a, b);
466EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
467 return _mm512_sub_pd(a, b);
470EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
471 return _mm512_sub_epi32(a, b);
474EIGEN_STRONG_INLINE Packet8l psub<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
475 return _mm512_sub_epi64(a, b);
479EIGEN_STRONG_INLINE Packet16f pnegate(
const Packet16f& a) {
484 _mm512_set_epi32(SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32,
485 SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32,
486 SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32);
487 return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a), mask));
490EIGEN_STRONG_INLINE Packet8d pnegate(
const Packet8d& a) {
491 const __m512i mask = _mm512_set_epi64(SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64,
492 SIGN_MASK_I64, SIGN_MASK_I64, SIGN_MASK_I64);
493 return _mm512_castsi512_pd(_mm512_xor_epi64(_mm512_castpd_si512(a), mask));
496EIGEN_STRONG_INLINE Packet16i pnegate(
const Packet16i& a) {
497 return _mm512_sub_epi32(_mm512_setzero_si512(), a);
500EIGEN_STRONG_INLINE Packet8l pnegate(
const Packet8l& a) {
501 return _mm512_sub_epi64(_mm512_setzero_si512(), a);
505EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
506 return _mm512_mul_ps(a, b);
509EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
510 return _mm512_mul_pd(a, b);
513EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
514 return _mm512_mullo_epi32(a, b);
517EIGEN_STRONG_INLINE Packet8l pmul<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
518#ifdef EIGEN_VECTORIZE_AVX512DQ
519 return _mm512_mullo_epi64(a, b);
521 return _mm512_mullox_epi64(a, b);
526EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
527 return _mm512_div_ps(a, b);
531EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
532 return _mm512_div_pd(a, b);
536EIGEN_STRONG_INLINE Packet16i pdiv<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
537 Packet8i q_lo = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 0), _mm512_extracti64x4_epi64(b, 0));
538 Packet8i q_hi = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 1), _mm512_extracti64x4_epi64(b, 1));
539 return _mm512_inserti64x4(_mm512_castsi256_si512(q_lo), q_hi, 1);
542#ifdef EIGEN_VECTORIZE_FMA
544struct is_fused_madd<Packet16f> : true_type {};
546EIGEN_STRONG_INLINE Packet16f pmadd(
const Packet16f& a,
const Packet16f& b,
const Packet16f& c) {
547 return _mm512_fmadd_ps(a, b, c);
550struct is_fused_madd<Packet8d> : true_type {};
552EIGEN_STRONG_INLINE Packet8d pmadd(
const Packet8d& a,
const Packet8d& b,
const Packet8d& c) {
553 return _mm512_fmadd_pd(a, b, c);
557EIGEN_STRONG_INLINE Packet16f pmsub(
const Packet16f& a,
const Packet16f& b,
const Packet16f& c) {
558 return _mm512_fmsub_ps(a, b, c);
561EIGEN_STRONG_INLINE Packet8d pmsub(
const Packet8d& a,
const Packet8d& b,
const Packet8d& c) {
562 return _mm512_fmsub_pd(a, b, c);
566EIGEN_STRONG_INLINE Packet16f pnmadd(
const Packet16f& a,
const Packet16f& b,
const Packet16f& c) {
567 return _mm512_fnmadd_ps(a, b, c);
570EIGEN_STRONG_INLINE Packet8d pnmadd(
const Packet8d& a,
const Packet8d& b,
const Packet8d& c) {
571 return _mm512_fnmadd_pd(a, b, c);
575EIGEN_STRONG_INLINE Packet16f pnmsub(
const Packet16f& a,
const Packet16f& b,
const Packet16f& c) {
576 return _mm512_fnmsub_ps(a, b, c);
579EIGEN_STRONG_INLINE Packet8d pnmsub(
const Packet8d& a,
const Packet8d& b,
const Packet8d& c) {
580 return _mm512_fnmsub_pd(a, b, c);
585EIGEN_DEVICE_FUNC
inline Packet16f pselect(
const Packet16f& mask,
const Packet16f& a,
const Packet16f& b) {
586 __mmask16 mask16 = _mm512_cmpeq_epi32_mask(_mm512_castps_si512(mask), _mm512_setzero_epi32());
587 return _mm512_mask_blend_ps(mask16, a, b);
591EIGEN_DEVICE_FUNC
inline Packet16i pselect(
const Packet16i& mask,
const Packet16i& a,
const Packet16i& b) {
592 __mmask16 mask16 = _mm512_cmpeq_epi32_mask(mask, _mm512_setzero_epi32());
593 return _mm512_mask_blend_epi32(mask16, a, b);
597EIGEN_DEVICE_FUNC
inline Packet8l pselect(
const Packet8l& mask,
const Packet8l& a,
const Packet8l& b) {
598 __mmask8 mask8 = _mm512_cmpeq_epi64_mask(mask, _mm512_setzero_si512());
599 return _mm512_mask_blend_epi64(mask8, a, b);
603EIGEN_DEVICE_FUNC
inline Packet8d pselect(
const Packet8d& mask,
const Packet8d& a,
const Packet8d& b) {
604 __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ);
605 return _mm512_mask_blend_pd(mask8, a, b);
609EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
611 return _mm512_min_ps(b, a);
614EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
616 return _mm512_min_pd(b, a);
619EIGEN_STRONG_INLINE Packet16i pmin<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
620 return _mm512_min_epi32(b, a);
623EIGEN_STRONG_INLINE Packet8l pmin<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
624 return _mm512_min_epi64(b, a);
628EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
630 return _mm512_max_ps(b, a);
633EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
635 return _mm512_max_pd(b, a);
638EIGEN_STRONG_INLINE Packet16i pmax<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
639 return _mm512_max_epi32(b, a);
642EIGEN_STRONG_INLINE Packet8l pmax<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
643 return _mm512_max_epi64(b, a);
648EIGEN_STRONG_INLINE Packet16f pmin<PropagateNumbers, Packet16f>(
const Packet16f& a,
const Packet16f& b) {
649 return pminmax_propagate_numbers(a, b, pmin<Packet16f>);
652EIGEN_STRONG_INLINE Packet8d pmin<PropagateNumbers, Packet8d>(
const Packet8d& a,
const Packet8d& b) {
653 return pminmax_propagate_numbers(a, b, pmin<Packet8d>);
656EIGEN_STRONG_INLINE Packet16f pmax<PropagateNumbers, Packet16f>(
const Packet16f& a,
const Packet16f& b) {
657 return pminmax_propagate_numbers(a, b, pmax<Packet16f>);
660EIGEN_STRONG_INLINE Packet8d pmax<PropagateNumbers, Packet8d>(
const Packet8d& a,
const Packet8d& b) {
661 return pminmax_propagate_numbers(a, b, pmax<Packet8d>);
664EIGEN_STRONG_INLINE Packet16f pmin<PropagateNaN, Packet16f>(
const Packet16f& a,
const Packet16f& b) {
665 return pminmax_propagate_nan(a, b, pmin<Packet16f>);
668EIGEN_STRONG_INLINE Packet8d pmin<PropagateNaN, Packet8d>(
const Packet8d& a,
const Packet8d& b) {
669 return pminmax_propagate_nan(a, b, pmin<Packet8d>);
672EIGEN_STRONG_INLINE Packet16f pmax<PropagateNaN, Packet16f>(
const Packet16f& a,
const Packet16f& b) {
673 return pminmax_propagate_nan(a, b, pmax<Packet16f>);
676EIGEN_STRONG_INLINE Packet8d pmax<PropagateNaN, Packet8d>(
const Packet8d& a,
const Packet8d& b) {
677 return pminmax_propagate_nan(a, b, pmax<Packet8d>);
680#ifdef EIGEN_VECTORIZE_AVX512DQ
682EIGEN_STRONG_INLINE Packet8f extract256(
const Packet16f& x) {
683 return _mm512_extractf32x8_ps(x, I_);
686EIGEN_STRONG_INLINE Packet2d extract128(
const Packet8d& x) {
687 return _mm512_extractf64x2_pd(x, I_);
689EIGEN_STRONG_INLINE Packet16f cat256(
const Packet8f& a,
const Packet8f& b) {
690 return _mm512_insertf32x8(_mm512_castps256_ps512(a), b, 1);
692EIGEN_STRONG_INLINE Packet16i cat256i(
const Packet8i& a,
const Packet8i& b) {
693 return _mm512_inserti32x8(_mm512_castsi256_si512(a), b, 1);
698EIGEN_STRONG_INLINE Packet8f extract256(
const Packet16f& x) {
699 return _mm256_castsi256_ps(_mm512_extracti64x4_epi64(_mm512_castps_si512(x), I_));
704EIGEN_STRONG_INLINE Packet2d extract128(
const Packet8d& x) {
705 return _mm_castsi128_pd(_mm512_extracti32x4_epi32(_mm512_castpd_si512(x), I_));
708EIGEN_STRONG_INLINE Packet16f cat256(
const Packet8f& a,
const Packet8f& b) {
709 return _mm512_castsi512_ps(
710 _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)), _mm256_castps_si256(b), 1));
712EIGEN_STRONG_INLINE Packet16i cat256i(
const Packet8i& a,
const Packet8i& b) {
713 return _mm512_inserti64x4(_mm512_castsi256_si512(a), b, 1);
719EIGEN_STRONG_INLINE __m256i Pack32To16(
const Packet16f& rf) {
return _mm512_cvtepi32_epi16(_mm512_castps_si512(rf)); }
722EIGEN_STRONG_INLINE Packet16f pisnan(
const Packet16f& a) {
723 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_UNORD_Q);
724 return _mm512_castsi512_ps(_mm512_maskz_set1_epi32(mask, int32_t(-1)));
728EIGEN_STRONG_INLINE Packet16f pcmp_eq(
const Packet16f& a,
const Packet16f& b) {
729 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ);
730 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
733EIGEN_STRONG_INLINE Packet16f pcmp_le(
const Packet16f& a,
const Packet16f& b) {
734 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ);
735 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
739EIGEN_STRONG_INLINE Packet16f pcmp_lt(
const Packet16f& a,
const Packet16f& b) {
740 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ);
741 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
745EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(
const Packet16f& a,
const Packet16f& b) {
746 __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ);
747 return _mm512_castsi512_ps(_mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1)));
751EIGEN_STRONG_INLINE Packet16i pcmp_eq(
const Packet16i& a,
const Packet16i& b) {
752 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_EQ);
753 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
756EIGEN_STRONG_INLINE Packet16i pcmp_le(
const Packet16i& a,
const Packet16i& b) {
757 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE);
758 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
761EIGEN_STRONG_INLINE Packet16i pcmp_lt(
const Packet16i& a,
const Packet16i& b) {
762 __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT);
763 return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, int32_t(-1));
767EIGEN_STRONG_INLINE Packet8l pcmp_eq(
const Packet8l& a,
const Packet8l& b) {
768 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_EQ);
769 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
772EIGEN_STRONG_INLINE Packet8l pcmp_le(
const Packet8l& a,
const Packet8l& b) {
773 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LE);
774 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
777EIGEN_STRONG_INLINE Packet8l pcmp_lt(
const Packet8l& a,
const Packet8l& b) {
778 __mmask8 mask = _mm512_cmp_epi64_mask(a, b, _MM_CMPINT_LT);
779 return _mm512_mask_set1_epi64(_mm512_setzero_si512(), mask, int64_t(-1));
783EIGEN_STRONG_INLINE Packet8d pcmp_eq(
const Packet8d& a,
const Packet8d& b) {
784 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ);
785 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
788EIGEN_STRONG_INLINE Packet8d pcmp_le(
const Packet8d& a,
const Packet8d& b) {
789 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ);
790 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
793EIGEN_STRONG_INLINE Packet8d pcmp_lt(
const Packet8d& a,
const Packet8d& b) {
794 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ);
795 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
798EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(
const Packet8d& a,
const Packet8d& b) {
799 __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ);
800 return _mm512_castsi512_pd(_mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, int64_t(-1)));
804EIGEN_STRONG_INLINE Packet16f print<Packet16f>(
const Packet16f& a) {
805 return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION);
808EIGEN_STRONG_INLINE Packet8d print<Packet8d>(
const Packet8d& a) {
809 return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION);
813EIGEN_STRONG_INLINE Packet16f pceil<Packet16f>(
const Packet16f& a) {
814 return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF);
817EIGEN_STRONG_INLINE Packet8d pceil<Packet8d>(
const Packet8d& a) {
818 return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF);
822EIGEN_STRONG_INLINE Packet16f pfloor<Packet16f>(
const Packet16f& a) {
823 return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF);
826EIGEN_STRONG_INLINE Packet8d pfloor<Packet8d>(
const Packet8d& a) {
827 return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF);
831EIGEN_STRONG_INLINE Packet16f ptrunc<Packet16f>(
const Packet16f& a) {
832 return _mm512_roundscale_ps(a, _MM_FROUND_TO_ZERO);
835EIGEN_STRONG_INLINE Packet8d ptrunc<Packet8d>(
const Packet8d& a) {
836 return _mm512_roundscale_pd(a, _MM_FROUND_TO_ZERO);
840EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(
const Packet16i& ) {
841 return _mm512_set1_epi32(int32_t(-1));
845EIGEN_STRONG_INLINE Packet8l ptrue<Packet8l>(
const Packet8l& ) {
846 return _mm512_set1_epi64(int64_t(-1));
850EIGEN_STRONG_INLINE Packet16f ptrue<Packet16f>(
const Packet16f& a) {
851 Packet16f r = _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a)));
852 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
857EIGEN_STRONG_INLINE Packet8d ptrue<Packet8d>(
const Packet8d& a) {
858 Packet8d r = _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a)));
859 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
864EIGEN_STRONG_INLINE Packet16i pand<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
865 return _mm512_and_si512(a, b);
869EIGEN_STRONG_INLINE Packet8l pand<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
870 return _mm512_and_si512(a, b);
874EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
875#ifdef EIGEN_VECTORIZE_AVX512DQ
876 return _mm512_and_ps(a, b);
878 return _mm512_castsi512_ps(pand(_mm512_castps_si512(a), _mm512_castps_si512(b)));
882EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
883#ifdef EIGEN_VECTORIZE_AVX512DQ
884 return _mm512_and_pd(a, b);
886 Packet8d res = _mm512_undefined_pd();
887 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
888 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
889 res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
891 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
892 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
893 return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
898EIGEN_STRONG_INLINE Packet16i por<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
899 return _mm512_or_si512(a, b);
903EIGEN_STRONG_INLINE Packet8l por<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
904 return _mm512_or_si512(a, b);
908EIGEN_STRONG_INLINE Packet16f por<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
909#ifdef EIGEN_VECTORIZE_AVX512DQ
910 return _mm512_or_ps(a, b);
912 return _mm512_castsi512_ps(por(_mm512_castps_si512(a), _mm512_castps_si512(b)));
917EIGEN_STRONG_INLINE Packet8d por<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
918#ifdef EIGEN_VECTORIZE_AVX512DQ
919 return _mm512_or_pd(a, b);
921 return _mm512_castsi512_pd(por(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
926EIGEN_STRONG_INLINE Packet16i pxor<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
927 return _mm512_xor_si512(a, b);
931EIGEN_STRONG_INLINE Packet8l pxor<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
932 return _mm512_xor_si512(a, b);
936EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
937#ifdef EIGEN_VECTORIZE_AVX512DQ
938 return _mm512_xor_ps(a, b);
940 return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a), _mm512_castps_si512(b)));
945EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
946#ifdef EIGEN_VECTORIZE_AVX512DQ
947 return _mm512_xor_pd(a, b);
949 return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
954EIGEN_STRONG_INLINE Packet16i pandnot<Packet16i>(
const Packet16i& a,
const Packet16i& b) {
955 return _mm512_andnot_si512(b, a);
959EIGEN_STRONG_INLINE Packet8l pandnot<Packet8l>(
const Packet8l& a,
const Packet8l& b) {
960 return _mm512_andnot_si512(b, a);
964EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(
const Packet16f& a,
const Packet16f& b) {
965#ifdef EIGEN_VECTORIZE_AVX512DQ
966 return _mm512_andnot_ps(b, a);
968 return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a), _mm512_castps_si512(b)));
972EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(
const Packet8d& a,
const Packet8d& b) {
973#ifdef EIGEN_VECTORIZE_AVX512DQ
974 return _mm512_andnot_pd(b, a);
976 return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a), _mm512_castpd_si512(b)));
981EIGEN_STRONG_INLINE Packet16f pround<Packet16f>(
const Packet16f& a) {
983 const Packet16f mask = pset1frombits<Packet16f>(
static_cast<numext::uint32_t
>(0x80000000u));
984 const Packet16f prev0dot5 = pset1frombits<Packet16f>(
static_cast<numext::uint32_t
>(0x3EFFFFFFu));
985 return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
988EIGEN_STRONG_INLINE Packet8d pround<Packet8d>(
const Packet8d& a) {
990 const Packet8d mask = pset1frombits<Packet8d>(
static_cast<numext::uint64_t
>(0x8000000000000000ull));
991 const Packet8d prev0dot5 = pset1frombits<Packet8d>(
static_cast<numext::uint64_t
>(0x3FDFFFFFFFFFFFFFull));
992 return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
996EIGEN_STRONG_INLINE Packet16i parithmetic_shift_right(
const Packet16i& a) {
997 return _mm512_srai_epi32(a, N);
1001EIGEN_STRONG_INLINE Packet16i plogical_shift_right(
const Packet16i& a) {
1002 return _mm512_srli_epi32(a, N);
1006EIGEN_STRONG_INLINE Packet16i plogical_shift_left(
const Packet16i& a) {
1007 return _mm512_slli_epi32(a, N);
1011EIGEN_STRONG_INLINE Packet8l parithmetic_shift_right(
const Packet8l& a) {
1012 return _mm512_srai_epi64(a, N);
1016EIGEN_STRONG_INLINE Packet8l plogical_shift_right(
const Packet8l& a) {
1017 return _mm512_srli_epi64(a, N);
1021EIGEN_STRONG_INLINE Packet8l plogical_shift_left(
const Packet8l& a) {
1022 return _mm512_slli_epi64(a, N);
1026EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(
const float* from) {
1027 EIGEN_DEBUG_ALIGNED_LOAD
return _mm512_load_ps(from);
1030EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(
const double* from) {
1031 EIGEN_DEBUG_ALIGNED_LOAD
return _mm512_load_pd(from);
1034EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(
const int* from) {
1035 EIGEN_DEBUG_ALIGNED_LOAD
return _mm512_load_epi64(from);
1038EIGEN_STRONG_INLINE Packet8l pload<Packet8l>(
const int64_t* from) {
1039 EIGEN_DEBUG_ALIGNED_LOAD
return _mm512_load_epi64(from);
1043EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(
const float* from) {
1044 EIGEN_DEBUG_UNALIGNED_LOAD
return _mm512_loadu_ps(from);
1047EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(
const double* from) {
1048 EIGEN_DEBUG_UNALIGNED_LOAD
return _mm512_loadu_pd(from);
1051EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(
const int* from) {
1052 EIGEN_DEBUG_UNALIGNED_LOAD
return _eigen_mm512_loadu_epi32(from);
1055EIGEN_STRONG_INLINE Packet8l ploadu<Packet8l>(
const int64_t* from) {
1056 EIGEN_DEBUG_UNALIGNED_LOAD
return _eigen_mm512_loadu_epi64(from);
1060EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(
const float* from, uint16_t umask) {
1061 __mmask16 mask =
static_cast<__mmask16
>(umask);
1062 EIGEN_DEBUG_UNALIGNED_LOAD
return _mm512_maskz_loadu_ps(mask, from);
1065EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(
const double* from, uint8_t umask) {
1066 __mmask8 mask =
static_cast<__mmask8
>(umask);
1067 EIGEN_DEBUG_UNALIGNED_LOAD
return _mm512_maskz_loadu_pd(mask, from);
1073EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(
const float* from) {
1076 __m256i low_half = _mm256_castps_si256(_mm256_loadu_ps(from));
1077 __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1078 __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1085EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(
const double* from) {
1086 Packet8d tmp = _mm512_castpd256_pd512(ploadu<Packet4d>(from));
1087 const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1088 return _mm512_permutexvar_pd(scatter_mask, tmp);
1094EIGEN_STRONG_INLINE Packet8l ploaddup<Packet8l>(
const int64_t* from) {
1095 Packet8l tmp = _mm512_castsi256_si512(ploadu<Packet4l>(from));
1096 const Packet8l scatter_mask = _mm512_set_epi64(3, 3, 2, 2, 1, 1, 0, 0);
1097 return _mm512_permutexvar_epi64(scatter_mask, tmp);
1103EIGEN_STRONG_INLINE Packet16i ploaddup<Packet16i>(
const int* from) {
1104 __m256i low_half = _mm256_loadu_si256(
reinterpret_cast<const __m256i*
>(from));
1105 __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
1106 __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
1107 return _mm512_castps_si512(pairs);
1113EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(
const float* from) {
1114 Packet16f tmp = _mm512_castps128_ps512(ploadu<Packet4f>(from));
1115 const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1116 return _mm512_permutexvar_ps(scatter_mask, tmp);
1122EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(
const double* from) {
1123 __m256d lane0 = _mm256_set1_pd(*from);
1124 __m256d lane1 = _mm256_set1_pd(*(from + 1));
1125 __m512d tmp = _mm512_undefined_pd();
1126 tmp = _mm512_insertf64x4(tmp, lane0, 0);
1127 return _mm512_insertf64x4(tmp, lane1, 1);
1133EIGEN_STRONG_INLINE Packet8l ploadquad<Packet8l>(
const int64_t* from) {
1134 __m256i lane0 = _mm256_set1_epi64x(*from);
1135 __m256i lane1 = _mm256_set1_epi64x(*(from + 1));
1136 __m512i tmp = _mm512_undefined_epi32();
1137 tmp = _mm512_inserti64x4(tmp, lane0, 0);
1138 return _mm512_inserti64x4(tmp, lane1, 1);
1144EIGEN_STRONG_INLINE Packet16i ploadquad<Packet16i>(
const int* from) {
1145 Packet16i tmp = _mm512_castsi128_si512(ploadu<Packet4i>(from));
1146 const Packet16i scatter_mask = _mm512_set_epi32(3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0);
1147 return _mm512_permutexvar_epi32(scatter_mask, tmp);
1151EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet16f& from) {
1152 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
1155EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet8d& from) {
1156 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
1159EIGEN_STRONG_INLINE
void pstore<int>(
int* to,
const Packet16i& from) {
1160 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi32(to, from);
1163EIGEN_STRONG_INLINE
void pstore<int64_t>(int64_t* to,
const Packet8l& from) {
1164 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_epi64(to, from);
1168EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet16f& from) {
1169 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
1172EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet8d& from) {
1173 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
1176EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet16i& from) {
1177 EIGEN_DEBUG_UNALIGNED_STORE _eigen_mm512_storeu_epi32(to, from);
1180EIGEN_STRONG_INLINE
void pstoreu<int64_t>(int64_t* to,
const Packet8l& from) {
1181 EIGEN_DEBUG_UNALIGNED_STORE _eigen_mm512_storeu_epi64(to, from);
1184EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet16f& from, uint16_t umask) {
1185 __mmask16 mask =
static_cast<__mmask16
>(umask);
1186 EIGEN_DEBUG_UNALIGNED_STORE
return _mm512_mask_storeu_ps(to, mask, from);
1189EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet8d& from, uint8_t umask) {
1190 __mmask8 mask =
static_cast<__mmask8
>(umask);
1191 EIGEN_DEBUG_UNALIGNED_STORE
return _mm512_mask_storeu_pd(to, mask, from);
1196template <
typename Packet>
1197EIGEN_STRONG_INLINE
bool avx512_lane_offsets_fit(Index stride) {
1198 const Index max_stride = Index(NumTraits<int>::highest()) / Index(unpacket_traits<Packet>::size - 1);
1199 return stride <= max_stride && stride >= -max_stride;
1202template <
typename Scalar,
typename Packet>
1203EIGEN_STRONG_INLINE Packet avx512_pgather_lanes(
const Packet& src,
const Scalar* from, Index stride,
1204 typename unpacket_traits<Packet>::mask_t umask) {
1205 EIGEN_ALIGN64 Scalar values[unpacket_traits<Packet>::size];
1206 pstore(values, src);
1207 for (
int i = 0; i < unpacket_traits<Packet>::size; ++i)
1208 if ((umask >> i) & 1) values[i] = from[i * stride];
1209 return pload<Packet>(values);
1212template <
typename Scalar,
typename Packet>
1213EIGEN_STRONG_INLINE
void avx512_pscatter_lanes(Scalar* to,
const Packet& from, Index stride,
1214 typename unpacket_traits<Packet>::mask_t umask) {
1215 EIGEN_ALIGN64 Scalar values[unpacket_traits<Packet>::size];
1216 pstore(values, from);
1217 for (
int i = 0; i < unpacket_traits<Packet>::size; ++i)
1218 if ((umask >> i) & 1) to[i * stride] = values[i];
1221template <
typename Scalar,
typename Packet>
1222EIGEN_DEVICE_FUNC
inline Packet pgather(
const Packet& src,
const Scalar* from, Index stride,
1223 typename unpacket_traits<Packet>::mask_t umask);
1225EIGEN_DEVICE_FUNC
inline Packet16f pgather<float, Packet16f>(
const Packet16f& src,
const float* from, Index stride,
1227 if (!avx512_lane_offsets_fit<Packet16f>(stride))
return avx512_pgather_lanes(src, from, stride, umask);
1228 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1229 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1230 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1231 __mmask16 mask =
static_cast<__mmask16
>(umask);
1233 return _mm512_mask_i32gather_ps(src, mask, indices, from, 4);
1236EIGEN_DEVICE_FUNC
inline Packet8d pgather<double, Packet8d>(
const Packet8d& src,
const double* from, Index stride,
1238 if (!avx512_lane_offsets_fit<Packet8d>(stride))
return avx512_pgather_lanes(src, from, stride, umask);
1239 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1240 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1241 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1242 __mmask8 mask =
static_cast<__mmask8
>(umask);
1244 return _mm512_mask_i32gather_pd(src, mask, indices, from, 8);
1248EIGEN_DEVICE_FUNC
inline Packet16f pgather<float, Packet16f>(
const float* from, Index stride) {
1251 const Packet16f low = _mm512_loadu_ps(from);
1252 const Packet16f high = _mm512_loadu_ps(from + 15);
1253 const Packet16i indices = _mm512_setr_epi32(0, 2, 4, 6, 8, 10, 12, 14, 17, 19, 21, 23, 25, 27, 29, 31);
1254 return _mm512_permutex2var_ps(low, indices, high);
1256 if (!avx512_lane_offsets_fit<Packet16f>(stride))
return pgather_partial<float, Packet16f>(from, stride, 16);
1257 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1258 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1259 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1261 return _mm512_i32gather_ps(indices, from, 4);
1264EIGEN_DEVICE_FUNC
inline Packet8d pgather<double, Packet8d>(
const double* from, Index stride) {
1266 const Packet8d low = _mm512_loadu_pd(from);
1267 const Packet8d high = _mm512_loadu_pd(from + 7);
1268 const __m512i indices = _mm512_setr_epi64(0, 2, 4, 6, 9, 11, 13, 15);
1269 return _mm512_permutex2var_pd(low, indices, high);
1271 if (!avx512_lane_offsets_fit<Packet8d>(stride))
return pgather_partial<double, Packet8d>(from, stride, 8);
1272 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1273 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1274 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1276 return _mm512_i32gather_pd(indices, from, 8);
1279EIGEN_DEVICE_FUNC
inline Packet8l pgather<int64_t, Packet8l>(
const int64_t* from, Index stride) {
1280 if (!avx512_lane_offsets_fit<Packet8l>(stride))
return pgather_partial<int64_t, Packet8l>(from, stride, 8);
1281 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1282 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1283 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1285 return _mm512_i32gather_epi64(indices, from, 8);
1288EIGEN_DEVICE_FUNC
inline Packet16i pgather<int, Packet16i>(
const int* from, Index stride) {
1289 if (!avx512_lane_offsets_fit<Packet16i>(stride))
return pgather_partial<int, Packet16i>(from, stride, 16);
1290 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1291 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1292 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1293 return _mm512_i32gather_epi32(indices, from, 4);
1296template <
typename Scalar,
typename Packet>
1297EIGEN_DEVICE_FUNC
inline void pscatter(Scalar* to,
const Packet& from, Index stride,
1298 typename unpacket_traits<Packet>::mask_t umask);
1300EIGEN_DEVICE_FUNC
inline void pscatter<float, Packet16f>(
float* to,
const Packet16f& from, Index stride,
1302 if (!avx512_lane_offsets_fit<Packet16f>(stride))
return avx512_pscatter_lanes(to, from, stride, umask);
1303 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1304 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1305 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1306 __mmask16 mask =
static_cast<__mmask16
>(umask);
1307 _mm512_mask_i32scatter_ps(to, mask, indices, from, 4);
1310EIGEN_DEVICE_FUNC
inline void pscatter<double, Packet8d>(
double* to,
const Packet8d& from, Index stride,
1312 if (!avx512_lane_offsets_fit<Packet8d>(stride))
return avx512_pscatter_lanes(to, from, stride, umask);
1313 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1314 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1315 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1316 __mmask8 mask =
static_cast<__mmask8
>(umask);
1317 _mm512_mask_i32scatter_pd(to, mask, indices, from, 8);
1320EIGEN_DEVICE_FUNC
inline void pscatter<float, Packet16f>(
float* to,
const Packet16f& from, Index stride) {
1321 if (!avx512_lane_offsets_fit<Packet16f>(stride))
return pscatter_partial<float, Packet16f>(to, from, stride, 16);
1322 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1323 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1324 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1325 _mm512_i32scatter_ps(to, indices, from, 4);
1328EIGEN_DEVICE_FUNC
inline void pscatter<double, Packet8d>(
double* to,
const Packet8d& from, Index stride) {
1329 if (!avx512_lane_offsets_fit<Packet8d>(stride))
return pscatter_partial<double, Packet8d>(to, from, stride, 8);
1330 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1331 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1332 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1333 _mm512_i32scatter_pd(to, indices, from, 8);
1336EIGEN_DEVICE_FUNC
inline void pscatter<int64_t, Packet8l>(int64_t* to,
const Packet8l& from, Index stride) {
1337 if (!avx512_lane_offsets_fit<Packet8l>(stride))
return pscatter_partial<int64_t, Packet8l>(to, from, stride, 8);
1338 Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1339 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1340 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1341 _mm512_i32scatter_epi64(to, indices, from, 8);
1344EIGEN_DEVICE_FUNC
inline void pscatter<int, Packet16i>(
int* to,
const Packet16i& from, Index stride) {
1345 if (!avx512_lane_offsets_fit<Packet16i>(stride))
return pscatter_partial<int, Packet16i>(to, from, stride, 16);
1346 Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1347 Packet16i stride_multiplier = _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1348 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1349 _mm512_i32scatter_epi32(to, indices, from, 4);
1353EIGEN_STRONG_INLINE
void pstore1<Packet16f>(
float* to,
const float& a) {
1354 Packet16f pa = pset1<Packet16f>(a);
1358EIGEN_STRONG_INLINE
void pstore1<Packet8d>(
double* to,
const double& a) {
1359 Packet8d pa = pset1<Packet8d>(a);
1363EIGEN_STRONG_INLINE
void pstore1<Packet16i>(
int* to,
const int& a) {
1364 Packet16i pa = pset1<Packet16i>(a);
1368EIGEN_STRONG_INLINE
void pstore1<Packet8l>(int64_t* to,
const int64_t& a) {
1369 Packet8l pa = pset1<Packet8l>(a);
1374EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) {
1375 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1378EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) {
1379 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1382EIGEN_STRONG_INLINE
void prefetch<int>(
const int* addr) {
1383 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1387EIGEN_STRONG_INLINE
float pfirst<Packet16f>(
const Packet16f& a) {
1388 return _mm512_cvtss_f32(a);
1391EIGEN_STRONG_INLINE
double pfirst<Packet8d>(
const Packet8d& a) {
1392 return _mm512_cvtsd_f64(a);
1395EIGEN_STRONG_INLINE int64_t pfirst<Packet8l>(
const Packet8l& a) {
1396 int64_t x = _mm_extract_epi64_0(_mm512_extracti32x4_epi32(a, 0));
1400EIGEN_STRONG_INLINE
int pfirst<Packet16i>(
const Packet16i& a) {
1401#if EIGEN_GNUC_STRICT_LESS_THAN(11, 0, 0)
1402 return _mm_cvtsi128_si32(_mm512_castsi512_si128(a));
1404 return _mm512_cvtsi512_si32(a);
1409EIGEN_STRONG_INLINE Packet16f preverse(
const Packet16f& a) {
1410 return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1414EIGEN_STRONG_INLINE Packet8d preverse(
const Packet8d& a) {
1415 return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
1419EIGEN_STRONG_INLINE Packet16i preverse(
const Packet16i& a) {
1420 return _mm512_permutexvar_epi32(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1424EIGEN_STRONG_INLINE Packet8l preverse(
const Packet8l& a) {
1425 return _mm512_permutexvar_epi64(_mm512_set_epi64(0, 1, 2, 3, 4, 5, 6, 7), a);
1429EIGEN_STRONG_INLINE Packet16f pabs(
const Packet16f& a) {
1431 return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff)));
1434EIGEN_STRONG_INLINE Packet8d pabs(
const Packet8d& a) {
1436 return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a), _mm512_set1_epi64(0x7fffffffffffffff)));
1439EIGEN_STRONG_INLINE Packet16i pabs(
const Packet16i& a) {
1440 return _mm512_abs_epi32(a);
1443EIGEN_STRONG_INLINE Packet8l pabs(
const Packet8l& a) {
1444 return _mm512_abs_epi64(a);
1447#ifndef EIGEN_VECTORIZE_AVX512FP16
1449EIGEN_STRONG_INLINE Packet16h psignbit(
const Packet16h& a) {
1450 return _mm256_srai_epi16(a, 15);
1455EIGEN_STRONG_INLINE Packet16bf psignbit(
const Packet16bf& a) {
1456 return _mm256_srai_epi16(a, 15);
1459EIGEN_STRONG_INLINE Packet16f psignbit(
const Packet16f& a) {
1460 return _mm512_castsi512_ps(_mm512_srai_epi32(_mm512_castps_si512(a), 31));
1463EIGEN_STRONG_INLINE Packet8d psignbit(
const Packet8d& a) {
1464 return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), 63));
1468EIGEN_STRONG_INLINE Packet16f pfrexp<Packet16f>(
const Packet16f& a, Packet16f& exponent) {
1469 return pfrexp_generic(a, exponent);
1474EIGEN_STRONG_INLINE Packet8d pfrexp_generic_get_biased_exponent(
const Packet8d& a) {
1475 const Packet8d cst_exp_mask = pset1frombits<Packet8d>(
static_cast<uint64_t
>(0x7ff0000000000000ull));
1476#ifdef EIGEN_VECTORIZE_AVX512DQ
1477 return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52));
1479 return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)));
1484EIGEN_STRONG_INLINE Packet8d pfrexp<Packet8d>(
const Packet8d& a, Packet8d& exponent) {
1485 return pfrexp_generic(a, exponent);
1489EIGEN_STRONG_INLINE Packet16f pldexp<Packet16f>(
const Packet16f& a,
const Packet16f& exponent) {
1493 const Packet16f max_exponent = pset1<Packet16f>(278.0f);
1494 const Packet16f e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent);
1495 return _mm512_scalef_ps(a, _mm512_roundscale_ps(e, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC));
1499EIGEN_STRONG_INLINE Packet8d pldexp<Packet8d>(
const Packet8d& a,
const Packet8d& exponent) {
1500 const Packet8d max_exponent = pset1<Packet8d>(2099.0);
1501 const Packet8d e = pmin(pmax(exponent, pnegate(max_exponent)), max_exponent);
1502 return _mm512_scalef_pd(a, _mm512_roundscale_pd(e, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC));
1507EIGEN_STRONG_INLINE Packet16f pldexp_fast<Packet16f>(
const Packet16f& a,
const Packet16f& exponent) {
1508 return _mm512_scalef_ps(a, exponent);
1512EIGEN_STRONG_INLINE Packet8d pldexp_fast<Packet8d>(
const Packet8d& a,
const Packet8d& exponent) {
1513 return _mm512_scalef_pd(a, exponent);
1516#ifdef EIGEN_VECTORIZE_AVX512DQ
1518#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1519 __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \
1520 __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1)
1523#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1524 __m256i OUTPUT##_0 = _mm512_extracti32x8_epi32(INPUT, 0); \
1525 __m256i OUTPUT##_1 = _mm512_extracti32x8_epi32(INPUT, 1)
1527#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1528 __m256 OUTPUT##_0 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
1529 _mm512_extractf32x4_ps(INPUT, 1), 1); \
1530 __m256 OUTPUT##_1 = _mm256_insertf128_ps(_mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
1531 _mm512_extractf32x4_ps(INPUT, 3), 1)
1533#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1534 __m256i OUTPUT##_0 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 0)), \
1535 _mm512_extracti32x4_epi32(INPUT, 1), 1); \
1536 __m256i OUTPUT##_1 = _mm256_insertf128_si256(_mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 2)), \
1537 _mm512_extracti32x4_epi32(INPUT, 3), 1)
1540#ifdef EIGEN_VECTORIZE_AVX512DQ
1541#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1542 OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1);
1544#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1545 OUTPUT = _mm512_inserti32x8(_mm512_castsi256_si512(INPUTA), INPUTB, 1);
1547#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1548 OUTPUT = _mm512_undefined_ps(); \
1549 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
1550 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
1551 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
1552 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
1554#define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1555 OUTPUT = _mm512_undefined_epi32(); \
1556 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 0), 0); \
1557 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 1), 1); \
1558 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 0), 2); \
1559 OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 1), 3);
1563EIGEN_STRONG_INLINE Packet8f predux_half<Packet16f>(
const Packet16f& a) {
1564#ifdef EIGEN_VECTORIZE_AVX512DQ
1565 __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
1566 __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
1567 return _mm256_add_ps(lane0, lane1);
1569 __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1570 __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1571 __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1572 __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1573 __m128 sum0 = _mm_add_ps(lane0, lane2);
1574 __m128 sum1 = _mm_add_ps(lane1, lane3);
1575 return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
1579EIGEN_STRONG_INLINE Packet4d predux_half<Packet8d>(
const Packet8d& a) {
1580 __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1581 __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1582 return _mm256_add_pd(lane0, lane1);
1585EIGEN_STRONG_INLINE Packet8i predux_half<Packet16i>(
const Packet16i& a) {
1586#ifdef EIGEN_VECTORIZE_AVX512DQ
1587 __m256i lane0 = _mm512_extracti32x8_epi32(a, 0);
1588 __m256i lane1 = _mm512_extracti32x8_epi32(a, 1);
1589 return _mm256_add_epi32(lane0, lane1);
1591 __m128i lane0 = _mm512_extracti32x4_epi32(a, 0);
1592 __m128i lane1 = _mm512_extracti32x4_epi32(a, 1);
1593 __m128i lane2 = _mm512_extracti32x4_epi32(a, 2);
1594 __m128i lane3 = _mm512_extracti32x4_epi32(a, 3);
1595 __m128i sum0 = _mm_add_epi32(lane0, lane2);
1596 __m128i sum1 = _mm_add_epi32(lane1, lane3);
1597 return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1);
1602EIGEN_STRONG_INLINE Packet4l predux_half<Packet8l>(
const Packet8l& a) {
1603 __m256i lane0 = _mm512_extracti64x4_epi64(a, 0);
1604 __m256i lane1 = _mm512_extracti64x4_epi64(a, 1);
1605 return _mm256_add_epi64(lane0, lane1);
1608#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1609 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1611EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1612 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1613 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1614 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1615 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1616 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1617 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1618 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1619 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1620 __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1621 __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1622 __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1623 __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1624 __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1625 __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1626 __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1627 __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1628 __m512 S0 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1629 __m512 S1 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1630 __m512 S2 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1631 __m512 S3 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1632 __m512 S4 = EIGEN_AVX512_SHUFFLE_PS(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1633 __m512 S5 = EIGEN_AVX512_SHUFFLE_PS(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1634 __m512 S6 = EIGEN_AVX512_SHUFFLE_PS(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1635 __m512 S7 = EIGEN_AVX512_SHUFFLE_PS(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1636 __m512 S8 = EIGEN_AVX512_SHUFFLE_PS(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1637 __m512 S9 = EIGEN_AVX512_SHUFFLE_PS(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1638 __m512 S10 = EIGEN_AVX512_SHUFFLE_PS(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1639 __m512 S11 = EIGEN_AVX512_SHUFFLE_PS(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1640 __m512 S12 = EIGEN_AVX512_SHUFFLE_PS(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1641 __m512 S13 = EIGEN_AVX512_SHUFFLE_PS(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1642 __m512 S14 = EIGEN_AVX512_SHUFFLE_PS(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1643 __m512 S15 = EIGEN_AVX512_SHUFFLE_PS(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1645 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1646 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1647 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1648 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1649 EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1650 EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1651 EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1652 EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1653 EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1654 EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1655 EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1656 EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1657 EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1658 EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1659 EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1660 EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1662 PacketBlock<Packet8f, 32> tmp;
1664 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1665 tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1666 tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1667 tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1668 tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1669 tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1670 tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1671 tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1673 tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1674 tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1675 tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1676 tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1677 tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1678 tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1679 tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1680 tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1683 tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1684 tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1685 tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1686 tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1687 tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1688 tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1689 tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1690 tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1692 tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1693 tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1694 tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1695 tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1696 tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1697 tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1698 tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1699 tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1702 PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1703 PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1704 PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1705 PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1707 PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1708 PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1709 PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1710 PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1712 PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1713 PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1714 PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1715 PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1717 PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1718 PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1719 PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1720 PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1722#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1723 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1725EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16f, 8>& kernel) {
1726 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1727 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1728 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1729 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1730 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1731 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1732 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1733 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1735 kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1736 kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2)));
1737 kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1738 kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3)));
1739 kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1740 kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6)));
1741 kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1742 kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7)));
1744 T0 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0x44);
1745 T1 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0xee);
1746 T2 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0x44);
1747 T3 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0xee);
1748 T4 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0x44);
1749 T5 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0xee);
1750 T6 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0x44);
1751 T7 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0xee);
1753 kernel.packet[0] = _mm512_shuffle_f32x4(T0, T2, 0x88);
1754 kernel.packet[2] = _mm512_shuffle_f32x4(T0, T2, 0xdd);
1755 kernel.packet[1] = _mm512_shuffle_f32x4(T4, T6, 0x88);
1756 kernel.packet[3] = _mm512_shuffle_f32x4(T4, T6, 0xdd);
1757 kernel.packet[4] = _mm512_shuffle_f32x4(T1, T3, 0x88);
1758 kernel.packet[6] = _mm512_shuffle_f32x4(T1, T3, 0xdd);
1759 kernel.packet[5] = _mm512_shuffle_f32x4(T5, T7, 0x88);
1760 kernel.packet[7] = _mm512_shuffle_f32x4(T5, T7, 0xdd);
1763EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1764 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1765 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1766 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1767 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1769 __m512 S0 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1770 __m512 S1 = EIGEN_AVX512_SHUFFLE_PS(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1771 __m512 S2 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1772 __m512 S3 = EIGEN_AVX512_SHUFFLE_PS(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1774 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1775 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1776 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1777 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1779 PacketBlock<Packet8f, 8> tmp;
1781 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1782 tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1783 tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1784 tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1786 tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1787 tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1788 tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1789 tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1791 PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1792 PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1793 PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1794 PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1797#define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1798 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1799 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1801#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1802 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1803 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1805#define PACK_OUTPUT_L(OUTPUT, INPUT, INDEX, STRIDE) \
1806 OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1807 OUTPUT[INDEX] = _mm512_inserti64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1809EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1810 __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1811 __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1812 __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1813 __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1815 PacketBlock<Packet4d, 8> tmp;
1817 tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x20);
1818 tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x20);
1819 tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0), _mm512_extractf64x4_pd(T2, 0), 0x31);
1820 tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0), _mm512_extractf64x4_pd(T3, 0), 0x31);
1822 tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x20);
1823 tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x20);
1824 tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1), _mm512_extractf64x4_pd(T2, 1), 0x31);
1825 tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1), _mm512_extractf64x4_pd(T3, 1), 0x31);
1827 PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1828 PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1829 PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1830 PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1833EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1834 __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1835 __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1836 __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
1837 __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
1838 __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
1839 __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
1840 __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
1841 __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
1843 kernel.packet[0] = _mm512_permutex_pd(T2, 0x4E);
1844 kernel.packet[0] = _mm512_mask_blend_pd(0xCC, T0, kernel.packet[0]);
1845 kernel.packet[2] = _mm512_permutex_pd(T0, 0x4E);
1846 kernel.packet[2] = _mm512_mask_blend_pd(0xCC, kernel.packet[2], T2);
1847 kernel.packet[1] = _mm512_permutex_pd(T3, 0x4E);
1848 kernel.packet[1] = _mm512_mask_blend_pd(0xCC, T1, kernel.packet[1]);
1849 kernel.packet[3] = _mm512_permutex_pd(T1, 0x4E);
1850 kernel.packet[3] = _mm512_mask_blend_pd(0xCC, kernel.packet[3], T3);
1851 kernel.packet[4] = _mm512_permutex_pd(T6, 0x4E);
1852 kernel.packet[4] = _mm512_mask_blend_pd(0xCC, T4, kernel.packet[4]);
1853 kernel.packet[6] = _mm512_permutex_pd(T4, 0x4E);
1854 kernel.packet[6] = _mm512_mask_blend_pd(0xCC, kernel.packet[6], T6);
1855 kernel.packet[5] = _mm512_permutex_pd(T7, 0x4E);
1856 kernel.packet[5] = _mm512_mask_blend_pd(0xCC, T5, kernel.packet[5]);
1857 kernel.packet[7] = _mm512_permutex_pd(T5, 0x4E);
1858 kernel.packet[7] = _mm512_mask_blend_pd(0xCC, kernel.packet[7], T7);
1860 T0 = _mm512_shuffle_f64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1861 T0 = _mm512_mask_blend_pd(0xF0, kernel.packet[0], T0);
1862 T4 = _mm512_shuffle_f64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1863 T4 = _mm512_mask_blend_pd(0xF0, T4, kernel.packet[4]);
1864 T1 = _mm512_shuffle_f64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1865 T1 = _mm512_mask_blend_pd(0xF0, kernel.packet[1], T1);
1866 T5 = _mm512_shuffle_f64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1867 T5 = _mm512_mask_blend_pd(0xF0, T5, kernel.packet[5]);
1868 T2 = _mm512_shuffle_f64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1869 T2 = _mm512_mask_blend_pd(0xF0, kernel.packet[2], T2);
1870 T6 = _mm512_shuffle_f64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1871 T6 = _mm512_mask_blend_pd(0xF0, T6, kernel.packet[6]);
1872 T3 = _mm512_shuffle_f64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1873 T3 = _mm512_mask_blend_pd(0xF0, kernel.packet[3], T3);
1874 T7 = _mm512_shuffle_f64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1875 T7 = _mm512_mask_blend_pd(0xF0, T7, kernel.packet[7]);
1877 kernel.packet[0] = T0;
1878 kernel.packet[1] = T1;
1879 kernel.packet[2] = T2;
1880 kernel.packet[3] = T3;
1881 kernel.packet[4] = T4;
1882 kernel.packet[5] = T5;
1883 kernel.packet[6] = T6;
1884 kernel.packet[7] = T7;
1887EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8l, 4>& kernel) {
1888 __m512i T0 = _mm512_castpd_si512(
1889 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0));
1890 __m512i T1 = _mm512_castpd_si512(
1891 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[0]), _mm512_castsi512_pd(kernel.packet[1]), 0xff));
1892 __m512i T2 = _mm512_castpd_si512(
1893 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0));
1894 __m512i T3 = _mm512_castpd_si512(
1895 _mm512_shuffle_pd(_mm512_castsi512_pd(kernel.packet[2]), _mm512_castsi512_pd(kernel.packet[3]), 0xff));
1897 PacketBlock<Packet4l, 8> tmp;
1899 tmp.packet[0] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x20);
1900 tmp.packet[1] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x20);
1901 tmp.packet[2] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 0), _mm512_extracti64x4_epi64(T2, 0), 0x31);
1902 tmp.packet[3] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 0), _mm512_extracti64x4_epi64(T3, 0), 0x31);
1904 tmp.packet[4] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x20);
1905 tmp.packet[5] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x20);
1906 tmp.packet[6] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T0, 1), _mm512_extracti64x4_epi64(T2, 1), 0x31);
1907 tmp.packet[7] = _mm256_permute2x128_si256(_mm512_extracti64x4_epi64(T1, 1), _mm512_extracti64x4_epi64(T3, 1), 0x31);
1909 PACK_OUTPUT_L(kernel.packet, tmp.packet, 0, 1);
1910 PACK_OUTPUT_L(kernel.packet, tmp.packet, 1, 1);
1911 PACK_OUTPUT_L(kernel.packet, tmp.packet, 2, 1);
1912 PACK_OUTPUT_L(kernel.packet, tmp.packet, 3, 1);
1915EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet8l, 8>& kernel) {
1916 __m512i T0 = _mm512_unpacklo_epi64(kernel.packet[0], kernel.packet[1]);
1917 __m512i T1 = _mm512_unpackhi_epi64(kernel.packet[0], kernel.packet[1]);
1918 __m512i T2 = _mm512_unpacklo_epi64(kernel.packet[2], kernel.packet[3]);
1919 __m512i T3 = _mm512_unpackhi_epi64(kernel.packet[2], kernel.packet[3]);
1920 __m512i T4 = _mm512_unpacklo_epi64(kernel.packet[4], kernel.packet[5]);
1921 __m512i T5 = _mm512_unpackhi_epi64(kernel.packet[4], kernel.packet[5]);
1922 __m512i T6 = _mm512_unpacklo_epi64(kernel.packet[6], kernel.packet[7]);
1923 __m512i T7 = _mm512_unpackhi_epi64(kernel.packet[6], kernel.packet[7]);
1925 kernel.packet[0] = _mm512_permutex_epi64(T2, 0x4E);
1926 kernel.packet[0] = _mm512_mask_blend_epi64(0xCC, T0, kernel.packet[0]);
1927 kernel.packet[2] = _mm512_permutex_epi64(T0, 0x4E);
1928 kernel.packet[2] = _mm512_mask_blend_epi64(0xCC, kernel.packet[2], T2);
1929 kernel.packet[1] = _mm512_permutex_epi64(T3, 0x4E);
1930 kernel.packet[1] = _mm512_mask_blend_epi64(0xCC, T1, kernel.packet[1]);
1931 kernel.packet[3] = _mm512_permutex_epi64(T1, 0x4E);
1932 kernel.packet[3] = _mm512_mask_blend_epi64(0xCC, kernel.packet[3], T3);
1933 kernel.packet[4] = _mm512_permutex_epi64(T6, 0x4E);
1934 kernel.packet[4] = _mm512_mask_blend_epi64(0xCC, T4, kernel.packet[4]);
1935 kernel.packet[6] = _mm512_permutex_epi64(T4, 0x4E);
1936 kernel.packet[6] = _mm512_mask_blend_epi64(0xCC, kernel.packet[6], T6);
1937 kernel.packet[5] = _mm512_permutex_epi64(T7, 0x4E);
1938 kernel.packet[5] = _mm512_mask_blend_epi64(0xCC, T5, kernel.packet[5]);
1939 kernel.packet[7] = _mm512_permutex_epi64(T5, 0x4E);
1940 kernel.packet[7] = _mm512_mask_blend_epi64(0xCC, kernel.packet[7], T7);
1942 T0 = _mm512_shuffle_i64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1943 T0 = _mm512_mask_blend_epi64(0xF0, kernel.packet[0], T0);
1944 T4 = _mm512_shuffle_i64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1945 T4 = _mm512_mask_blend_epi64(0xF0, T4, kernel.packet[4]);
1946 T1 = _mm512_shuffle_i64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1947 T1 = _mm512_mask_blend_epi64(0xF0, kernel.packet[1], T1);
1948 T5 = _mm512_shuffle_i64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1949 T5 = _mm512_mask_blend_epi64(0xF0, T5, kernel.packet[5]);
1950 T2 = _mm512_shuffle_i64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1951 T2 = _mm512_mask_blend_epi64(0xF0, kernel.packet[2], T2);
1952 T6 = _mm512_shuffle_i64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1953 T6 = _mm512_mask_blend_epi64(0xF0, T6, kernel.packet[6]);
1954 T3 = _mm512_shuffle_i64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1955 T3 = _mm512_mask_blend_epi64(0xF0, kernel.packet[3], T3);
1956 T7 = _mm512_shuffle_i64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1957 T7 = _mm512_mask_blend_epi64(0xF0, T7, kernel.packet[7]);
1959 kernel.packet[0] = T0;
1960 kernel.packet[1] = T1;
1961 kernel.packet[2] = T2;
1962 kernel.packet[3] = T3;
1963 kernel.packet[4] = T4;
1964 kernel.packet[5] = T5;
1965 kernel.packet[6] = T6;
1966 kernel.packet[7] = T7;
1969#define PACK_OUTPUT_I32(OUTPUT, INPUT, INDEX, STRIDE) \
1970 EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1972#define PACK_OUTPUT_I32_2(OUTPUT, INPUT, INDEX, STRIDE) \
1973 EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[2 * INDEX], INPUT[2 * INDEX + STRIDE]);
1975#define SHUFFLE_EPI32(A, B, M) \
1976 _mm512_castps_si512(EIGEN_AVX512_SHUFFLE_PS(_mm512_castsi512_ps(A), _mm512_castsi512_ps(B), M))
1978EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16i, 16>& kernel) {
1979 __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
1980 __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
1981 __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
1982 __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
1983 __m512i T4 = _mm512_unpacklo_epi32(kernel.packet[4], kernel.packet[5]);
1984 __m512i T5 = _mm512_unpackhi_epi32(kernel.packet[4], kernel.packet[5]);
1985 __m512i T6 = _mm512_unpacklo_epi32(kernel.packet[6], kernel.packet[7]);
1986 __m512i T7 = _mm512_unpackhi_epi32(kernel.packet[6], kernel.packet[7]);
1987 __m512i T8 = _mm512_unpacklo_epi32(kernel.packet[8], kernel.packet[9]);
1988 __m512i T9 = _mm512_unpackhi_epi32(kernel.packet[8], kernel.packet[9]);
1989 __m512i T10 = _mm512_unpacklo_epi32(kernel.packet[10], kernel.packet[11]);
1990 __m512i T11 = _mm512_unpackhi_epi32(kernel.packet[10], kernel.packet[11]);
1991 __m512i T12 = _mm512_unpacklo_epi32(kernel.packet[12], kernel.packet[13]);
1992 __m512i T13 = _mm512_unpackhi_epi32(kernel.packet[12], kernel.packet[13]);
1993 __m512i T14 = _mm512_unpacklo_epi32(kernel.packet[14], kernel.packet[15]);
1994 __m512i T15 = _mm512_unpackhi_epi32(kernel.packet[14], kernel.packet[15]);
1995 __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1996 __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1997 __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1998 __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1999 __m512i S4 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
2000 __m512i S5 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
2001 __m512i S6 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
2002 __m512i S7 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
2003 __m512i S8 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
2004 __m512i S9 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
2005 __m512i S10 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
2006 __m512i S11 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
2007 __m512i S12 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
2008 __m512i S13 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
2009 __m512i S14 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
2010 __m512i S15 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
2012 EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
2013 EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
2014 EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
2015 EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
2016 EIGEN_EXTRACT_8i_FROM_16i(S4, S4);
2017 EIGEN_EXTRACT_8i_FROM_16i(S5, S5);
2018 EIGEN_EXTRACT_8i_FROM_16i(S6, S6);
2019 EIGEN_EXTRACT_8i_FROM_16i(S7, S7);
2020 EIGEN_EXTRACT_8i_FROM_16i(S8, S8);
2021 EIGEN_EXTRACT_8i_FROM_16i(S9, S9);
2022 EIGEN_EXTRACT_8i_FROM_16i(S10, S10);
2023 EIGEN_EXTRACT_8i_FROM_16i(S11, S11);
2024 EIGEN_EXTRACT_8i_FROM_16i(S12, S12);
2025 EIGEN_EXTRACT_8i_FROM_16i(S13, S13);
2026 EIGEN_EXTRACT_8i_FROM_16i(S14, S14);
2027 EIGEN_EXTRACT_8i_FROM_16i(S15, S15);
2029 PacketBlock<Packet8i, 32> tmp;
2031 tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S4_0, 0x20);
2032 tmp.packet[1] = _mm256_permute2f128_si256(S1_0, S5_0, 0x20);
2033 tmp.packet[2] = _mm256_permute2f128_si256(S2_0, S6_0, 0x20);
2034 tmp.packet[3] = _mm256_permute2f128_si256(S3_0, S7_0, 0x20);
2035 tmp.packet[4] = _mm256_permute2f128_si256(S0_0, S4_0, 0x31);
2036 tmp.packet[5] = _mm256_permute2f128_si256(S1_0, S5_0, 0x31);
2037 tmp.packet[6] = _mm256_permute2f128_si256(S2_0, S6_0, 0x31);
2038 tmp.packet[7] = _mm256_permute2f128_si256(S3_0, S7_0, 0x31);
2040 tmp.packet[8] = _mm256_permute2f128_si256(S0_1, S4_1, 0x20);
2041 tmp.packet[9] = _mm256_permute2f128_si256(S1_1, S5_1, 0x20);
2042 tmp.packet[10] = _mm256_permute2f128_si256(S2_1, S6_1, 0x20);
2043 tmp.packet[11] = _mm256_permute2f128_si256(S3_1, S7_1, 0x20);
2044 tmp.packet[12] = _mm256_permute2f128_si256(S0_1, S4_1, 0x31);
2045 tmp.packet[13] = _mm256_permute2f128_si256(S1_1, S5_1, 0x31);
2046 tmp.packet[14] = _mm256_permute2f128_si256(S2_1, S6_1, 0x31);
2047 tmp.packet[15] = _mm256_permute2f128_si256(S3_1, S7_1, 0x31);
2050 tmp.packet[16] = _mm256_permute2f128_si256(S8_0, S12_0, 0x20);
2051 tmp.packet[17] = _mm256_permute2f128_si256(S9_0, S13_0, 0x20);
2052 tmp.packet[18] = _mm256_permute2f128_si256(S10_0, S14_0, 0x20);
2053 tmp.packet[19] = _mm256_permute2f128_si256(S11_0, S15_0, 0x20);
2054 tmp.packet[20] = _mm256_permute2f128_si256(S8_0, S12_0, 0x31);
2055 tmp.packet[21] = _mm256_permute2f128_si256(S9_0, S13_0, 0x31);
2056 tmp.packet[22] = _mm256_permute2f128_si256(S10_0, S14_0, 0x31);
2057 tmp.packet[23] = _mm256_permute2f128_si256(S11_0, S15_0, 0x31);
2059 tmp.packet[24] = _mm256_permute2f128_si256(S8_1, S12_1, 0x20);
2060 tmp.packet[25] = _mm256_permute2f128_si256(S9_1, S13_1, 0x20);
2061 tmp.packet[26] = _mm256_permute2f128_si256(S10_1, S14_1, 0x20);
2062 tmp.packet[27] = _mm256_permute2f128_si256(S11_1, S15_1, 0x20);
2063 tmp.packet[28] = _mm256_permute2f128_si256(S8_1, S12_1, 0x31);
2064 tmp.packet[29] = _mm256_permute2f128_si256(S9_1, S13_1, 0x31);
2065 tmp.packet[30] = _mm256_permute2f128_si256(S10_1, S14_1, 0x31);
2066 tmp.packet[31] = _mm256_permute2f128_si256(S11_1, S15_1, 0x31);
2069 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 0, 16);
2070 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 1, 16);
2071 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 2, 16);
2072 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 3, 16);
2074 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 4, 16);
2075 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 5, 16);
2076 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 6, 16);
2077 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 7, 16);
2079 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 8, 16);
2080 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 9, 16);
2081 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 10, 16);
2082 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 11, 16);
2084 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 12, 16);
2085 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 13, 16);
2086 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 14, 16);
2087 PACK_OUTPUT_I32(kernel.packet, tmp.packet, 15, 16);
2090EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet16i, 4>& kernel) {
2091 __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
2092 __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
2093 __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
2094 __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
2096 __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
2097 __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
2098 __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
2099 __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
2101 EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
2102 EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
2103 EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
2104 EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
2106 PacketBlock<Packet8i, 8> tmp;
2108 tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S1_0, 0x20);
2109 tmp.packet[1] = _mm256_permute2f128_si256(S2_0, S3_0, 0x20);
2110 tmp.packet[2] = _mm256_permute2f128_si256(S0_0, S1_0, 0x31);
2111 tmp.packet[3] = _mm256_permute2f128_si256(S2_0, S3_0, 0x31);
2113 tmp.packet[4] = _mm256_permute2f128_si256(S0_1, S1_1, 0x20);
2114 tmp.packet[5] = _mm256_permute2f128_si256(S2_1, S3_1, 0x20);
2115 tmp.packet[6] = _mm256_permute2f128_si256(S0_1, S1_1, 0x31);
2116 tmp.packet[7] = _mm256_permute2f128_si256(S2_1, S3_1, 0x31);
2118 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 0, 1);
2119 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 1, 1);
2120 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 2, 1);
2121 PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 3, 1);
2125#ifndef EIGEN_VECTORIZE_AVX512FP16
2127EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(
const Eigen::half& from) {
2128 return _mm256_set1_epi16(from.x);
2132EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(
const Packet16h& from) {
2133 return half_impl::raw_uint16_to_half(
static_cast<unsigned short>(_mm256_extract_epi16(from, 0)));
2137EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(
const Eigen::half* from) {
2138 return _mm256_load_si256(
reinterpret_cast<const __m256i*
>(from));
2142EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(
const Eigen::half* from) {
2143 return _mm256_loadu_si256(
reinterpret_cast<const __m256i*
>(from));
2147EIGEN_STRONG_INLINE
void pstore<half>(Eigen::half* to,
const Packet16h& from) {
2150 EIGEN_DEBUG_ALIGNED_STORE
2151 _mm256_store_si256((__m256i*)(
void*)to, from);
2155EIGEN_STRONG_INLINE
void pstoreu<half>(Eigen::half* to,
const Packet16h& from) {
2158 EIGEN_DEBUG_UNALIGNED_STORE
2159 _mm256_storeu_si256((__m256i*)(
void*)to, from);
2163EIGEN_STRONG_INLINE Packet16h ploaddup<Packet16h>(
const Eigen::half* from) {
2164 unsigned short a = from[0].x;
2165 unsigned short b = from[1].x;
2166 unsigned short c = from[2].x;
2167 unsigned short d = from[3].x;
2168 unsigned short e = from[4].x;
2169 unsigned short f = from[5].x;
2170 unsigned short g = from[6].x;
2171 unsigned short h = from[7].x;
2172 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2176EIGEN_STRONG_INLINE Packet16h ploadquad(
const Eigen::half* from) {
2177 unsigned short a = from[0].x;
2178 unsigned short b = from[1].x;
2179 unsigned short c = from[2].x;
2180 unsigned short d = from[3].x;
2181 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2184EIGEN_STRONG_INLINE Packet16f half2float(
const Packet16h& a) {
return _mm512_cvtph_ps(a); }
2186EIGEN_STRONG_INLINE Packet16h float2half(
const Packet16f& a) {
2187 return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC);
2191EIGEN_STRONG_INLINE Packet16h ptrue(
const Packet16h& a) {
2192 return Packet16h(ptrue(Packet8i(a)));
2196EIGEN_STRONG_INLINE Packet16h pabs(
const Packet16h& a) {
2197 const __m256i sign_mask = _mm256_set1_epi16(
static_cast<short>(0x8000u));
2198 return _mm256_andnot_si256(sign_mask, a);
2202EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(
const Packet16h& a,
const Packet16h& b) {
2203 return float2half(pmin<Packet16f>(half2float(a), half2float(b)));
2207EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(
const Packet16h& a,
const Packet16h& b) {
2208 return float2half(pmax<Packet16f>(half2float(a), half2float(b)));
2212EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(
const half& a) {
2213 return float2half(plset<Packet16f>(
static_cast<float>(a)));
2217EIGEN_STRONG_INLINE Packet16h por(
const Packet16h& a,
const Packet16h& b) {
2220 return Packet16h(por(Packet8i(a), Packet8i(b)));
2223EIGEN_STRONG_INLINE Packet16h pxor(
const Packet16h& a,
const Packet16h& b) {
2224 return Packet16h(pxor(Packet8i(a), Packet8i(b)));
2227EIGEN_STRONG_INLINE Packet16h pand(
const Packet16h& a,
const Packet16h& b) {
2228 return Packet16h(pand(Packet8i(a), Packet8i(b)));
2231EIGEN_STRONG_INLINE Packet16h pandnot(
const Packet16h& a,
const Packet16h& b) {
2232 return Packet16h(pandnot(Packet8i(a), Packet8i(b)));
2236EIGEN_STRONG_INLINE Packet16h pselect(
const Packet16h& mask,
const Packet16h& a,
const Packet16h& b) {
2237 return _mm256_blendv_epi8(b, a, mask);
2241EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(
const Packet16h& a) {
2242 return float2half(pround<Packet16f>(half2float(a)));
2246EIGEN_STRONG_INLINE Packet16h print<Packet16h>(
const Packet16h& a) {
2247 return float2half(print<Packet16f>(half2float(a)));
2251EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(
const Packet16h& a) {
2252 return float2half(pceil<Packet16f>(half2float(a)));
2256EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(
const Packet16h& a) {
2257 return float2half(pfloor<Packet16f>(half2float(a)));
2261EIGEN_STRONG_INLINE Packet16h ptrunc<Packet16h>(
const Packet16h& a) {
2262 return float2half(ptrunc<Packet16f>(half2float(a)));
2266EIGEN_STRONG_INLINE Packet16h pcmp_eq(
const Packet16h& a,
const Packet16h& b) {
2267 Packet16f af = half2float(a);
2268 Packet16f bf = half2float(b);
2269 return Pack32To16(pcmp_eq(af, bf));
2273EIGEN_STRONG_INLINE Packet16h pcmp_le(
const Packet16h& a,
const Packet16h& b) {
2274 return Pack32To16(pcmp_le(half2float(a), half2float(b)));
2278EIGEN_STRONG_INLINE Packet16h pcmp_lt(
const Packet16h& a,
const Packet16h& b) {
2279 return Pack32To16(pcmp_lt(half2float(a), half2float(b)));
2283EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(
const Packet16h& a,
const Packet16h& b) {
2284 return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b)));
2288EIGEN_STRONG_INLINE Packet16h pnegate(
const Packet16h& a) {
2289 Packet16h sign_mask = _mm256_set1_epi16(
static_cast<short>(0x8000u));
2290 return _mm256_xor_si256(a, sign_mask);
2294EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(
const Packet16h& a,
const Packet16h& b) {
2295 Packet16f af = half2float(a);
2296 Packet16f bf = half2float(b);
2297 Packet16f rf = padd(af, bf);
2298 return float2half(rf);
2302EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(
const Packet16h& a,
const Packet16h& b) {
2303 Packet16f af = half2float(a);
2304 Packet16f bf = half2float(b);
2305 Packet16f rf = psub(af, bf);
2306 return float2half(rf);
2310EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(
const Packet16h& a,
const Packet16h& b) {
2311 Packet16f af = half2float(a);
2312 Packet16f bf = half2float(b);
2313 Packet16f rf = pmul(af, bf);
2314 return float2half(rf);
2318EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(
const Packet16h& a,
const Packet16h& b) {
2319 Packet16f af = half2float(a);
2320 Packet16f bf = half2float(b);
2321 Packet16f rf = pdiv(af, bf);
2322 return float2half(rf);
2326EIGEN_STRONG_INLINE Packet16h pmadd<Packet16h>(
const Packet16h& a,
const Packet16h& b,
const Packet16h& c) {
2327 return float2half(pmadd(half2float(a), half2float(b), half2float(c)));
2331EIGEN_STRONG_INLINE Packet16h pmsub<Packet16h>(
const Packet16h& a,
const Packet16h& b,
const Packet16h& c) {
2332 return float2half(pmsub(half2float(a), half2float(b), half2float(c)));
2336EIGEN_STRONG_INLINE Packet16h pnmadd<Packet16h>(
const Packet16h& a,
const Packet16h& b,
const Packet16h& c) {
2337 return float2half(pnmadd(half2float(a), half2float(b), half2float(c)));
2341EIGEN_STRONG_INLINE Packet16h pnmsub<Packet16h>(
const Packet16h& a,
const Packet16h& b,
const Packet16h& c) {
2342 return float2half(pnmsub(half2float(a), half2float(b), half2float(c)));
2346EIGEN_STRONG_INLINE Packet8h predux_half<Packet16h>(
const Packet16h& a) {
2347 Packet8h lane0 = _mm256_extractf128_si256(a, 0);
2348 Packet8h lane1 = _mm256_extractf128_si256(a, 1);
2349 return padd<Packet8h>(lane0, lane1);
2353EIGEN_STRONG_INLINE Packet16h preverse(
const Packet16h& a) {
2354 __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
2355 return _mm256_insertf128_si256(_mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a, 1), m)),
2356 _mm_shuffle_epi8(_mm256_extractf128_si256(a, 0), m), 1);
2360EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(
const Eigen::half* from, Index stride) {
2361 return _mm256_set_epi16(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x,
2362 from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x,
2363 from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x,
2364 from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x);
2368EIGEN_STRONG_INLINE
void pscatter<half, Packet16h>(half* to,
const Packet16h& from, Index stride) {
2369 EIGEN_ALIGN64 half aux[16];
2371 to[stride * 0] = aux[0];
2372 to[stride * 1] = aux[1];
2373 to[stride * 2] = aux[2];
2374 to[stride * 3] = aux[3];
2375 to[stride * 4] = aux[4];
2376 to[stride * 5] = aux[5];
2377 to[stride * 6] = aux[6];
2378 to[stride * 7] = aux[7];
2379 to[stride * 8] = aux[8];
2380 to[stride * 9] = aux[9];
2381 to[stride * 10] = aux[10];
2382 to[stride * 11] = aux[11];
2383 to[stride * 12] = aux[12];
2384 to[stride * 13] = aux[13];
2385 to[stride * 14] = aux[14];
2386 to[stride * 15] = aux[15];
2389EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16h, 16>& kernel) {
2390 __m256i a = kernel.packet[0];
2391 __m256i b = kernel.packet[1];
2392 __m256i c = kernel.packet[2];
2393 __m256i d = kernel.packet[3];
2394 __m256i e = kernel.packet[4];
2395 __m256i f = kernel.packet[5];
2396 __m256i g = kernel.packet[6];
2397 __m256i h = kernel.packet[7];
2398 __m256i i = kernel.packet[8];
2399 __m256i j = kernel.packet[9];
2400 __m256i k = kernel.packet[10];
2401 __m256i l = kernel.packet[11];
2402 __m256i m = kernel.packet[12];
2403 __m256i n = kernel.packet[13];
2404 __m256i o = kernel.packet[14];
2405 __m256i p = kernel.packet[15];
2407 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2408 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2409 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2410 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2411 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2412 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2413 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2414 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2416 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2417 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2418 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2419 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2420 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2421 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2422 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2423 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2425 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2426 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2427 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2428 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2429 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2430 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2431 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2432 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2434 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2435 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2436 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2437 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2438 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2439 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2440 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2441 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2443 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2444 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2445 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2446 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2447 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2448 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2449 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2450 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2451 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2452 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2453 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2454 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2455 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2456 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2457 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2458 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2461 __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2462 __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2463 __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2464 __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2465 __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2466 __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2467 __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2468 __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2469 __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2470 __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2471 __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2472 __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2473 __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2474 __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2475 __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2476 __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2478 kernel.packet[0] = a_p_0;
2479 kernel.packet[1] = a_p_1;
2480 kernel.packet[2] = a_p_2;
2481 kernel.packet[3] = a_p_3;
2482 kernel.packet[4] = a_p_4;
2483 kernel.packet[5] = a_p_5;
2484 kernel.packet[6] = a_p_6;
2485 kernel.packet[7] = a_p_7;
2486 kernel.packet[8] = a_p_8;
2487 kernel.packet[9] = a_p_9;
2488 kernel.packet[10] = a_p_a;
2489 kernel.packet[11] = a_p_b;
2490 kernel.packet[12] = a_p_c;
2491 kernel.packet[13] = a_p_d;
2492 kernel.packet[14] = a_p_e;
2493 kernel.packet[15] = a_p_f;
2496EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16h, 8>& kernel) {
2497 EIGEN_ALIGN64 half in[8][16];
2498 pstore<half>(in[0], kernel.packet[0]);
2499 pstore<half>(in[1], kernel.packet[1]);
2500 pstore<half>(in[2], kernel.packet[2]);
2501 pstore<half>(in[3], kernel.packet[3]);
2502 pstore<half>(in[4], kernel.packet[4]);
2503 pstore<half>(in[5], kernel.packet[5]);
2504 pstore<half>(in[6], kernel.packet[6]);
2505 pstore<half>(in[7], kernel.packet[7]);
2507 EIGEN_ALIGN64 half out[8][16];
2509 for (
int i = 0; i < 8; ++i) {
2510 for (
int j = 0; j < 8; ++j) {
2511 out[i][j] = in[j][2 * i];
2513 for (
int j = 0; j < 8; ++j) {
2514 out[i][j + 8] = in[j][2 * i + 1];
2518 kernel.packet[0] = pload<Packet16h>(out[0]);
2519 kernel.packet[1] = pload<Packet16h>(out[1]);
2520 kernel.packet[2] = pload<Packet16h>(out[2]);
2521 kernel.packet[3] = pload<Packet16h>(out[3]);
2522 kernel.packet[4] = pload<Packet16h>(out[4]);
2523 kernel.packet[5] = pload<Packet16h>(out[5]);
2524 kernel.packet[6] = pload<Packet16h>(out[6]);
2525 kernel.packet[7] = pload<Packet16h>(out[7]);
2528EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16h, 4>& kernel) {
2529 EIGEN_ALIGN64 half in[4][16];
2530 pstore<half>(in[0], kernel.packet[0]);
2531 pstore<half>(in[1], kernel.packet[1]);
2532 pstore<half>(in[2], kernel.packet[2]);
2533 pstore<half>(in[3], kernel.packet[3]);
2535 EIGEN_ALIGN64 half out[4][16];
2537 for (
int i = 0; i < 4; ++i) {
2538 for (
int j = 0; j < 4; ++j) {
2539 out[i][j] = in[j][4 * i];
2541 for (
int j = 0; j < 4; ++j) {
2542 out[i][j + 4] = in[j][4 * i + 1];
2544 for (
int j = 0; j < 4; ++j) {
2545 out[i][j + 8] = in[j][4 * i + 2];
2547 for (
int j = 0; j < 4; ++j) {
2548 out[i][j + 12] = in[j][4 * i + 3];
2552 kernel.packet[0] = pload<Packet16h>(out[0]);
2553 kernel.packet[1] = pload<Packet16h>(out[1]);
2554 kernel.packet[2] = pload<Packet16h>(out[2]);
2555 kernel.packet[3] = pload<Packet16h>(out[3]);
2561struct is_arithmetic<Packet16bf> : std::true_type {};
2564struct packet_traits<bfloat16> : default_packet_traits {
2565 using type = Packet16bf;
2566 using half = Packet8bf;
2569 AlignedOnScalar = 1,
2572 HasSin = EIGEN_FAST_MATH,
2573 HasCos = EIGEN_FAST_MATH,
2576#ifdef EIGEN_VECTORIZE_AVX512DQ
2584 HasTanh = EIGEN_FAST_MATH,
2585 HasErf = EIGEN_FAST_MATH,
2592struct unpacket_traits<Packet16bf> {
2593 using type = bfloat16;
2597 vectorizable =
true,
2598 masked_load_available =
false,
2599 masked_store_available =
false
2601 using half = Packet8bf;
2605EIGEN_STRONG_INLINE Packet16bf pset1<Packet16bf>(
const bfloat16& from) {
2606 return _mm256_set1_epi16(from.value);
2610EIGEN_STRONG_INLINE bfloat16 pfirst<Packet16bf>(
const Packet16bf& from) {
2612 t.value =
static_cast<unsigned short>(_mm256_extract_epi16(from, 0));
2617EIGEN_STRONG_INLINE Packet16bf pload<Packet16bf>(
const bfloat16* from) {
2618 return _mm256_load_si256(
reinterpret_cast<const __m256i*
>(from));
2622EIGEN_STRONG_INLINE Packet16bf ploadu<Packet16bf>(
const bfloat16* from) {
2623 return _mm256_loadu_si256(
reinterpret_cast<const __m256i*
>(from));
2627EIGEN_STRONG_INLINE
void pstore<bfloat16>(bfloat16* to,
const Packet16bf& from) {
2628 EIGEN_DEBUG_ALIGNED_STORE
2629 _mm256_store_si256(
reinterpret_cast<__m256i*
>(to), from);
2633EIGEN_STRONG_INLINE
void pstoreu<bfloat16>(bfloat16* to,
const Packet16bf& from) {
2634 EIGEN_DEBUG_UNALIGNED_STORE
2635 _mm256_storeu_si256(
reinterpret_cast<__m256i*
>(to), from);
2639EIGEN_STRONG_INLINE Packet16bf ploaddup<Packet16bf>(
const bfloat16* from) {
2640 unsigned short a = from[0].value;
2641 unsigned short b = from[1].value;
2642 unsigned short c = from[2].value;
2643 unsigned short d = from[3].value;
2644 unsigned short e = from[4].value;
2645 unsigned short f = from[5].value;
2646 unsigned short g = from[6].value;
2647 unsigned short h = from[7].value;
2648 return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2652EIGEN_STRONG_INLINE Packet16bf ploadquad(
const bfloat16* from) {
2653 unsigned short a = from[0].value;
2654 unsigned short b = from[1].value;
2655 unsigned short c = from[2].value;
2656 unsigned short d = from[3].value;
2657 return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2660EIGEN_STRONG_INLINE Packet16f Bf16ToF32(
const Packet16bf& a) {
2661 return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
2665EIGEN_STRONG_INLINE Packet16bf F32ToBf16(
const Packet16f& a) {
2671 __m512i input = _mm512_castps_si512(a);
2672 __m512i nan = _mm512_set1_epi32(0x7fc0);
2675 t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1));
2677 t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff));
2679 t = _mm512_add_epi32(t, input);
2681 t = _mm512_srli_epi32(t, 16);
2684 __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q);
2686 t = _mm512_mask_blend_epi32(mask, nan, t);
2688 r = _mm512_cvtepi32_epi16(t);
2696EIGEN_STRONG_INLINE Packet16bf F32ToBf16Truncate(
const Packet16f& a) {
2697 return _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(a), 16));
2701EIGEN_STRONG_INLINE Packet16bf ptrue(
const Packet16bf& a) {
2702 return Packet16bf(ptrue<Packet8i>(Packet8i(a)));
2706EIGEN_STRONG_INLINE Packet16bf por(
const Packet16bf& a,
const Packet16bf& b) {
2707 return Packet16bf(por<Packet8i>(Packet8i(a), Packet8i(b)));
2711EIGEN_STRONG_INLINE Packet16bf pxor(
const Packet16bf& a,
const Packet16bf& b) {
2712 return Packet16bf(pxor<Packet8i>(Packet8i(a), Packet8i(b)));
2716EIGEN_STRONG_INLINE Packet16bf pand(
const Packet16bf& a,
const Packet16bf& b) {
2717 return Packet16bf(pand<Packet8i>(Packet8i(a), Packet8i(b)));
2721EIGEN_STRONG_INLINE Packet16bf pandnot(
const Packet16bf& a,
const Packet16bf& b) {
2722 return Packet16bf(pandnot<Packet8i>(Packet8i(a), Packet8i(b)));
2726EIGEN_STRONG_INLINE Packet16bf pselect(
const Packet16bf& mask,
const Packet16bf& a,
const Packet16bf& b) {
2729 return _mm256_blendv_epi8(b, a, mask);
2733EIGEN_STRONG_INLINE Packet16bf pround<Packet16bf>(
const Packet16bf& a) {
2734 return F32ToBf16Truncate(pround<Packet16f>(Bf16ToF32(a)));
2738EIGEN_STRONG_INLINE Packet16bf print<Packet16bf>(
const Packet16bf& a) {
2739 return F32ToBf16Truncate(print<Packet16f>(Bf16ToF32(a)));
2743EIGEN_STRONG_INLINE Packet16bf pceil<Packet16bf>(
const Packet16bf& a) {
2744 return F32ToBf16Truncate(pceil<Packet16f>(Bf16ToF32(a)));
2748EIGEN_STRONG_INLINE Packet16bf pfloor<Packet16bf>(
const Packet16bf& a) {
2749 return F32ToBf16Truncate(pfloor<Packet16f>(Bf16ToF32(a)));
2753EIGEN_STRONG_INLINE Packet16bf ptrunc<Packet16bf>(
const Packet16bf& a) {
2754 return F32ToBf16Truncate(ptrunc<Packet16f>(Bf16ToF32(a)));
2758EIGEN_STRONG_INLINE Packet16bf pcmp_eq(
const Packet16bf& a,
const Packet16bf& b) {
2759 return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
2763EIGEN_STRONG_INLINE Packet16bf pcmp_le(
const Packet16bf& a,
const Packet16bf& b) {
2764 return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
2768EIGEN_STRONG_INLINE Packet16bf pcmp_lt(
const Packet16bf& a,
const Packet16bf& b) {
2769 return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
2773EIGEN_STRONG_INLINE Packet16bf pcmp_lt_or_nan(
const Packet16bf& a,
const Packet16bf& b) {
2774 return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
2779EIGEN_STRONG_INLINE Packet16bf pisinf<Packet16bf>(
const Packet16bf& a) {
2780 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2781 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2782 return _mm256_cmpeq_epi16(_mm256_and_si256(a, _mm256_set1_epi16(kAbsMask)), _mm256_set1_epi16(kInf));
2786EIGEN_STRONG_INLINE Packet16bf pisnan<Packet16bf>(
const Packet16bf& a) {
2787 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2788 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2789 return _mm256_cmpgt_epi16(_mm256_and_si256(a, _mm256_set1_epi16(kAbsMask)), _mm256_set1_epi16(kInf));
2794EIGEN_STRONG_INLINE Packet16bf psign<Packet16bf>(
const Packet16bf& a) {
2795 const __m256i magnitude = _mm256_and_si256(a, _mm256_set1_epi16(0x7fff));
2796 const __m256i is_nan = _mm256_cmpgt_epi16(magnitude, _mm256_set1_epi16(0x7f80));
2797 const __m256i keep = _mm256_or_si256(is_nan, _mm256_set1_epi16(
static_cast<short>(0xbf80u)));
2798 return _mm256_sign_epi16(_mm256_and_si256(_mm256_or_si256(a, _mm256_set1_epi16(0x3f80)), keep), magnitude);
2802EIGEN_STRONG_INLINE Packet16bf pisfinite<Packet16bf>(
const Packet16bf& a) {
2803 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2804 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2805 return _mm256_cmpgt_epi16(_mm256_set1_epi16(kInf), _mm256_and_si256(a, _mm256_set1_epi16(kAbsMask)));
2809EIGEN_STRONG_INLINE Packet16bf pnegate(
const Packet16bf& a) {
2810 Packet16bf sign_mask = _mm256_set1_epi16(
static_cast<short>(0x8000u));
2811 return _mm256_xor_si256(a, sign_mask);
2815EIGEN_STRONG_INLINE Packet16bf pabs(
const Packet16bf& a) {
2816 const __m256i sign_mask = _mm256_set1_epi16(
static_cast<short>(0x8000u));
2817 return _mm256_andnot_si256(sign_mask, a);
2821EIGEN_STRONG_INLINE Packet16bf padd<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b) {
2822 return F32ToBf16(padd<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2826EIGEN_STRONG_INLINE Packet16bf psub<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b) {
2827 return F32ToBf16(psub<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2831EIGEN_STRONG_INLINE Packet16bf pmul<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b) {
2832 return F32ToBf16(pmul(Bf16ToF32(a), Bf16ToF32(b)));
2836EIGEN_STRONG_INLINE Packet16bf pmadd<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b,
const Packet16bf& c) {
2837 return F32ToBf16(pmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2841EIGEN_STRONG_INLINE Packet16bf pmsub<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b,
const Packet16bf& c) {
2842 return F32ToBf16(pmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2846EIGEN_STRONG_INLINE Packet16bf pnmadd<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b,
const Packet16bf& c) {
2847 return F32ToBf16(pnmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2851EIGEN_STRONG_INLINE Packet16bf pnmsub<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b,
const Packet16bf& c) {
2852 return F32ToBf16(pnmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2856EIGEN_STRONG_INLINE Packet16bf pdiv<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b) {
2857 return F32ToBf16(pdiv<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2861EIGEN_STRONG_INLINE Packet16bf pmin<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b) {
2862 return F32ToBf16Truncate(pmin<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2866EIGEN_STRONG_INLINE Packet16bf pmax<Packet16bf>(
const Packet16bf& a,
const Packet16bf& b) {
2867 return F32ToBf16Truncate(pmax<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2871EIGEN_STRONG_INLINE Packet16bf plset<Packet16bf>(
const bfloat16& a) {
2872 return F32ToBf16(plset<Packet16f>(
static_cast<float>(a)));
2876EIGEN_STRONG_INLINE Packet8bf predux_half<Packet16bf>(
const Packet16bf& a) {
2877 Packet8bf lane0 = _mm256_extractf128_si256(a, 0);
2878 Packet8bf lane1 = _mm256_extractf128_si256(a, 1);
2879 return padd<Packet8bf>(lane0, lane1);
2883EIGEN_STRONG_INLINE Packet16bf preverse(
const Packet16bf& a) {
2884 __m256i m = _mm256_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9, 6, 7,
2889 res = _mm256_permute2x128_si256(a, a, 1);
2891 return _mm256_shuffle_epi8(res, m);
2895EIGEN_STRONG_INLINE Packet16bf pgather<bfloat16, Packet16bf>(
const bfloat16* from, Index stride) {
2896 return _mm256_set_epi16(
2897 from[15 * stride].value, from[14 * stride].value, from[13 * stride].value, from[12 * stride].value,
2898 from[11 * stride].value, from[10 * stride].value, from[9 * stride].value, from[8 * stride].value,
2899 from[7 * stride].value, from[6 * stride].value, from[5 * stride].value, from[4 * stride].value,
2900 from[3 * stride].value, from[2 * stride].value, from[1 * stride].value, from[0 * stride].value);
2904EIGEN_STRONG_INLINE
void pscatter<bfloat16, Packet16bf>(bfloat16* to,
const Packet16bf& from, Index stride) {
2905 EIGEN_ALIGN64 bfloat16 aux[16];
2907 to[stride * 0] = aux[0];
2908 to[stride * 1] = aux[1];
2909 to[stride * 2] = aux[2];
2910 to[stride * 3] = aux[3];
2911 to[stride * 4] = aux[4];
2912 to[stride * 5] = aux[5];
2913 to[stride * 6] = aux[6];
2914 to[stride * 7] = aux[7];
2915 to[stride * 8] = aux[8];
2916 to[stride * 9] = aux[9];
2917 to[stride * 10] = aux[10];
2918 to[stride * 11] = aux[11];
2919 to[stride * 12] = aux[12];
2920 to[stride * 13] = aux[13];
2921 to[stride * 14] = aux[14];
2922 to[stride * 15] = aux[15];
2925EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16bf, 16>& kernel) {
2926 __m256i a = kernel.packet[0];
2927 __m256i b = kernel.packet[1];
2928 __m256i c = kernel.packet[2];
2929 __m256i d = kernel.packet[3];
2930 __m256i e = kernel.packet[4];
2931 __m256i f = kernel.packet[5];
2932 __m256i g = kernel.packet[6];
2933 __m256i h = kernel.packet[7];
2934 __m256i i = kernel.packet[8];
2935 __m256i j = kernel.packet[9];
2936 __m256i k = kernel.packet[10];
2937 __m256i l = kernel.packet[11];
2938 __m256i m = kernel.packet[12];
2939 __m256i n = kernel.packet[13];
2940 __m256i o = kernel.packet[14];
2941 __m256i p = kernel.packet[15];
2943 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2944 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2945 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2946 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2947 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2948 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2949 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2950 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2952 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2953 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2954 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2955 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2956 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2957 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2958 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2959 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2961 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2962 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2963 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2964 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2965 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2966 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2967 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2968 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2970 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2971 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2972 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2973 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2974 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2975 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2976 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2977 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2979 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2980 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2981 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2982 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2983 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2984 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2985 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2986 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2987 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2988 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2989 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2990 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2991 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2992 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2993 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2994 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2997 kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2998 kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2999 kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
3000 kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
3001 kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
3002 kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
3003 kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
3004 kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
3005 kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
3006 kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
3007 kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
3008 kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
3009 kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
3010 kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
3011 kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
3012 kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
3015EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16bf, 4>& kernel) {
3016 __m256i a = kernel.packet[0];
3017 __m256i b = kernel.packet[1];
3018 __m256i c = kernel.packet[2];
3019 __m256i d = kernel.packet[3];
3021 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
3022 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
3023 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
3024 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
3026 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
3027 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
3028 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
3029 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
3032 kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20);
3033 kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20);
3034 kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31);
3035 kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31);
3041EIGEN_STRONG_INLINE Packet32s pset1<Packet32s>(
const numext::int16_t& x) {
3042 return _mm512_set1_epi16(x);
3046EIGEN_STRONG_INLINE Packet16s pset1<Packet16s>(
const numext::int16_t& x) {
3047 return _mm256_set1_epi16(x);
3051EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(
const numext::int16_t& x) {
3052 return _mm_set1_epi16(x);
3056EIGEN_STRONG_INLINE
void pstore<numext::int16_t, Packet32s>(numext::int16_t* out,
const Packet32s& x) {
3057 EIGEN_DEBUG_ALIGNED_STORE
3058 _mm512_store_epi32(out, x);
3062EIGEN_STRONG_INLINE
void pstore<numext::int16_t, Packet16s>(numext::int16_t* out,
const Packet16s& x) {
3063 EIGEN_DEBUG_ALIGNED_STORE
3064 _mm256_store_si256(
reinterpret_cast<__m256i*
>(out), x);
3068EIGEN_STRONG_INLINE
void pstore<numext::int16_t, Packet8s>(numext::int16_t* out,
const Packet8s& x) {
3069 EIGEN_DEBUG_ALIGNED_STORE
3070 _mm_store_si128(
reinterpret_cast<__m128i*
>(out), x);
3074EIGEN_STRONG_INLINE
void pstoreu<numext::int16_t, Packet32s>(numext::int16_t* out,
const Packet32s& x) {
3075 EIGEN_DEBUG_UNALIGNED_STORE
3076 _eigen_mm512_storeu_epi32(out, x);
3080EIGEN_STRONG_INLINE
void pstoreu<numext::int16_t, Packet16s>(numext::int16_t* out,
const Packet16s& x) {
3081 EIGEN_DEBUG_UNALIGNED_STORE
3082 _eigen_mm256_storeu_epi32(out, x);
3086EIGEN_STRONG_INLINE
void pstoreu<numext::int16_t, Packet8s>(numext::int16_t* out,
const Packet8s& x) {
3087 EIGEN_DEBUG_UNALIGNED_STORE
3088 _eigen_mm_storeu_epi32(out, x);
3092EIGEN_STRONG_INLINE Packet32s padd(
const Packet32s& a,
const Packet32s& b) {
3093 return _mm512_add_epi16(a, b);
3097EIGEN_STRONG_INLINE Packet16s padd(
const Packet16s& a,
const Packet16s& b) {
3098 return _mm256_add_epi16(a, b);
3102EIGEN_STRONG_INLINE Packet8s padd(
const Packet8s& a,
const Packet8s& b) {
3103 return _mm_add_epi16(a, b);
3107EIGEN_STRONG_INLINE Packet32s psub(
const Packet32s& a,
const Packet32s& b) {
3108 return _mm512_sub_epi16(a, b);
3112EIGEN_STRONG_INLINE Packet16s psub(
const Packet16s& a,
const Packet16s& b) {
3113 return _mm256_sub_epi16(a, b);
3117EIGEN_STRONG_INLINE Packet8s psub(
const Packet8s& a,
const Packet8s& b) {
3118 return _mm_sub_epi16(a, b);
3122EIGEN_STRONG_INLINE Packet32s pmul(
const Packet32s& a,
const Packet32s& b) {
3123 return _mm512_mullo_epi16(a, b);
3127EIGEN_STRONG_INLINE Packet16s pmul(
const Packet16s& a,
const Packet16s& b) {
3128 return _mm256_mullo_epi16(a, b);
3132EIGEN_STRONG_INLINE Packet8s pmul(
const Packet8s& a,
const Packet8s& b) {
3133 return _mm_mullo_epi16(a, b);
3137EIGEN_STRONG_INLINE Packet32s pnegate(
const Packet32s& a) {
3138 return _mm512_sub_epi16(_mm512_setzero_si512(), a);
3142EIGEN_STRONG_INLINE Packet16s pnegate(
const Packet16s& a) {
3143 return _mm256_sub_epi16(_mm256_setzero_si256(), a);
3147EIGEN_STRONG_INLINE Packet8s pnegate(
const Packet8s& a) {
3148 return _mm_sub_epi16(_mm_setzero_si128(), a);
3152EIGEN_STRONG_INLINE Packet32s parithmetic_shift_right(
const Packet32s& a) {
3153 return _mm512_srai_epi16(a, N);
3157EIGEN_STRONG_INLINE Packet16s parithmetic_shift_right(
const Packet16s& a) {
3158 return _mm256_srai_epi16(a, N);
3162EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(
const Packet8s& a) {
3163 return _mm_srai_epi16(a, N);
3167EIGEN_STRONG_INLINE Packet32s plogical_shift_left(
const Packet32s& a) {
3168 return _mm512_slli_epi16(a, N);
3172EIGEN_STRONG_INLINE Packet16s plogical_shift_left(
const Packet16s& a) {
3173 return _mm256_slli_epi16(a, N);
3177EIGEN_STRONG_INLINE Packet8s plogical_shift_left(
const Packet8s& a) {
3178 return _mm_slli_epi16(a, N);
3182EIGEN_STRONG_INLINE Packet32s plogical_shift_right(
const Packet32s& a) {
3183 return _mm512_srli_epi16(a, N);
3187EIGEN_STRONG_INLINE Packet16s plogical_shift_right(
const Packet16s& a) {
3188 return _mm256_srli_epi16(a, N);
3192EIGEN_STRONG_INLINE Packet8s plogical_shift_right(
const Packet8s& a) {
3193 return _mm_srli_epi16(a, N);
3200template <
typename Packet>
3201EIGEN_STRONG_INLINE uint64_t segment_kmask(Index begin, Index count) {
3202 eigen_assert(begin >= 0 && count >= 0 && begin + count <= unpacket_traits<Packet>::size);
3203 return ((uint64_t(1) << count) - 1) << begin;
3209struct has_packet_segment<Packet16f> : std::true_type {};
3212inline Packet16f ploaduSegment<Packet16f>(
const float* from, Index begin, Index count) {
3213 return _mm512_maskz_loadu_ps(
static_cast<__mmask16
>(segment_kmask<Packet16f>(begin, count)), from);
3217inline void pstoreuSegment<float, Packet16f>(
float* to,
const Packet16f& from, Index begin, Index count) {
3218 _mm512_mask_storeu_ps(to,
static_cast<__mmask16
>(segment_kmask<Packet16f>(begin, count)), from);
3224struct has_packet_segment<Packet8d> : std::true_type {};
3227inline Packet8d ploaduSegment<Packet8d>(
const double* from, Index begin, Index count) {
3228 return _mm512_maskz_loadu_pd(
static_cast<__mmask8
>(segment_kmask<Packet8d>(begin, count)), from);
3232inline void pstoreuSegment<double, Packet8d>(
double* to,
const Packet8d& from, Index begin, Index count) {
3233 _mm512_mask_storeu_pd(to,
static_cast<__mmask8
>(segment_kmask<Packet8d>(begin, count)), from);
3239struct has_packet_segment<Packet16i> : std::true_type {};
3242inline Packet16i ploaduSegment<Packet16i>(
const int* from, Index begin, Index count) {
3243 return _mm512_maskz_loadu_epi32(
static_cast<__mmask16
>(segment_kmask<Packet16i>(begin, count)), from);
3247inline void pstoreuSegment<int, Packet16i>(
int* to,
const Packet16i& from, Index begin, Index count) {
3248 _mm512_mask_storeu_epi32(to,
static_cast<__mmask16
>(segment_kmask<Packet16i>(begin, count)), from);
3254struct has_packet_segment<Packet8l> : std::true_type {};
3257inline Packet8l ploaduSegment<Packet8l>(
const int64_t* from, Index begin, Index count) {
3258 return _mm512_maskz_loadu_epi64(
static_cast<__mmask8
>(segment_kmask<Packet8l>(begin, count)), from);
3262inline void pstoreuSegment<int64_t, Packet8l>(int64_t* to,
const Packet8l& from, Index begin, Index count) {
3263 _mm512_mask_storeu_epi64(to,
static_cast<__mmask8
>(segment_kmask<Packet8l>(begin, count)), from);
3266#ifdef EIGEN_VECTORIZE_AVX512VL
3274inline Packet8f ploaduSegment<Packet8f>(
const float* from, Index begin, Index count) {
3275 return _mm256_maskz_loadu_ps(
static_cast<__mmask8
>(segment_kmask<Packet8f>(begin, count)), from);
3279inline void pstoreuSegment<float, Packet8f>(
float* to,
const Packet8f& from, Index begin, Index count) {
3280 _mm256_mask_storeu_ps(to,
static_cast<__mmask8
>(segment_kmask<Packet8f>(begin, count)), from);
3284inline Packet4f ploaduSegment<Packet4f>(
const float* from, Index begin, Index count) {
3285 return _mm_maskz_loadu_ps(
static_cast<__mmask8
>(segment_kmask<Packet4f>(begin, count)), from);
3289inline void pstoreuSegment<float, Packet4f>(
float* to,
const Packet4f& from, Index begin, Index count) {
3290 _mm_mask_storeu_ps(to,
static_cast<__mmask8
>(segment_kmask<Packet4f>(begin, count)), from);
3296inline Packet4d ploaduSegment<Packet4d>(
const double* from, Index begin, Index count) {
3297 return _mm256_maskz_loadu_pd(
static_cast<__mmask8
>(segment_kmask<Packet4d>(begin, count)), from);
3301inline void pstoreuSegment<double, Packet4d>(
double* to,
const Packet4d& from, Index begin, Index count) {
3302 _mm256_mask_storeu_pd(to,
static_cast<__mmask8
>(segment_kmask<Packet4d>(begin, count)), from);
3306inline Packet2d ploaduSegment<Packet2d>(
const double* from, Index begin, Index count) {
3307 return _mm_maskz_loadu_pd(
static_cast<__mmask8
>(segment_kmask<Packet2d>(begin, count)), from);
3311inline void pstoreuSegment<double, Packet2d>(
double* to,
const Packet2d& from, Index begin, Index count) {
3312 _mm_mask_storeu_pd(to,
static_cast<__mmask8
>(segment_kmask<Packet2d>(begin, count)), from);
3318inline Packet8i ploaduSegment<Packet8i>(
const int* from, Index begin, Index count) {
3319 return _mm256_maskz_loadu_epi32(
static_cast<__mmask8
>(segment_kmask<Packet8i>(begin, count)), from);
3323inline void pstoreuSegment<int, Packet8i>(
int* to,
const Packet8i& from, Index begin, Index count) {
3324 _mm256_mask_storeu_epi32(to,
static_cast<__mmask8
>(segment_kmask<Packet8i>(begin, count)), from);
3328inline Packet4i ploaduSegment<Packet4i>(
const int* from, Index begin, Index count) {
3329 return _mm_maskz_loadu_epi32(
static_cast<__mmask8
>(segment_kmask<Packet4i>(begin, count)), from);
3333inline void pstoreuSegment<int, Packet4i>(
int* to,
const Packet4i& from, Index begin, Index count) {
3334 _mm_mask_storeu_epi32(to,
static_cast<__mmask8
>(segment_kmask<Packet4i>(begin, count)), from);
3340inline Packet8ui ploaduSegment<Packet8ui>(
const uint32_t* from, Index begin, Index count) {
3341 return Packet8ui(ploaduSegment<Packet8i>(
reinterpret_cast<const int*
>(from), begin, count));
3345inline void pstoreuSegment<uint32_t, Packet8ui>(uint32_t* to,
const Packet8ui& from, Index begin, Index count) {
3346 pstoreuSegment<int, Packet8i>(
reinterpret_cast<int*
>(to), Packet8i(from), begin, count);
3350inline Packet4ui ploaduSegment<Packet4ui>(
const uint32_t* from, Index begin, Index count) {
3351 return Packet4ui(ploaduSegment<Packet4i>(
reinterpret_cast<const int*
>(from), begin, count));
3355inline void pstoreuSegment<uint32_t, Packet4ui>(uint32_t* to,
const Packet4ui& from, Index begin, Index count) {
3356 pstoreuSegment<int, Packet4i>(
reinterpret_cast<int*
>(to), Packet4i(from), begin, count);
3362inline Packet4l ploaduSegment<Packet4l>(
const int64_t* from, Index begin, Index count) {
3363 return _mm256_maskz_loadu_epi64(
static_cast<__mmask8
>(segment_kmask<Packet4l>(begin, count)), from);
3367inline void pstoreuSegment<int64_t, Packet4l>(int64_t* to,
const Packet4l& from, Index begin, Index count) {
3368 _mm256_mask_storeu_epi64(to,
static_cast<__mmask8
>(segment_kmask<Packet4l>(begin, count)), from);
3372inline Packet2l ploaduSegment<Packet2l>(
const int64_t* from, Index begin, Index count) {
3373 return _mm_maskz_loadu_epi64(
static_cast<__mmask8
>(segment_kmask<Packet2l>(begin, count)), from);
3377inline void pstoreuSegment<int64_t, Packet2l>(int64_t* to,
const Packet2l& from, Index begin, Index count) {
3378 _mm_mask_storeu_epi64(to,
static_cast<__mmask8
>(segment_kmask<Packet2l>(begin, count)), from);
3384inline Packet4ul ploaduSegment<Packet4ul>(
const uint64_t* from, Index begin, Index count) {
3385 return Packet4ul(ploaduSegment<Packet4l>(
reinterpret_cast<const int64_t*
>(from), begin, count));
3389inline void pstoreuSegment<uint64_t, Packet4ul>(uint64_t* to,
const Packet4ul& from, Index begin, Index count) {
3390 pstoreuSegment<int64_t, Packet4l>(
reinterpret_cast<int64_t*
>(to), Packet4l(from), begin, count);
3393#if defined(__AVX512BW__)
3400struct has_packet_segment<Packet16bf> : std::true_type {};
3403struct has_packet_segment<Packet8bf> : std::true_type {};
3406inline Packet16bf ploaduSegment<Packet16bf>(
const bfloat16* from, Index begin, Index count) {
3407 return _mm256_maskz_loadu_epi16(
static_cast<__mmask16
>(segment_kmask<Packet16bf>(begin, count)), from);
3411inline void pstoreuSegment<bfloat16, Packet16bf>(bfloat16* to,
const Packet16bf& from, Index begin, Index count) {
3412 _mm256_mask_storeu_epi16(to,
static_cast<__mmask16
>(segment_kmask<Packet16bf>(begin, count)), from);
3416inline Packet8bf ploaduSegment<Packet8bf>(
const bfloat16* from, Index begin, Index count) {
3417 return _mm_maskz_loadu_epi16(
static_cast<__mmask8
>(segment_kmask<Packet8bf>(begin, count)), from);
3421inline void pstoreuSegment<bfloat16, Packet8bf>(bfloat16* to,
const Packet8bf& from, Index begin, Index count) {
3422 _mm_mask_storeu_epi16(to,
static_cast<__mmask8
>(segment_kmask<Packet8bf>(begin, count)), from);
3425#ifndef EIGEN_VECTORIZE_AVX512FP16
3430struct has_packet_segment<Packet16h> : std::true_type {};
3433struct has_packet_segment<Packet8h> : std::true_type {};
3436inline Packet16h ploaduSegment<Packet16h>(
const Eigen::half* from, Index begin, Index count) {
3437 return _mm256_maskz_loadu_epi16(
static_cast<__mmask16
>(segment_kmask<Packet16h>(begin, count)), from);
3441inline void pstoreuSegment<Eigen::half, Packet16h>(Eigen::half* to,
const Packet16h& from, Index begin, Index count) {
3442 _mm256_mask_storeu_epi16(to,
static_cast<__mmask16
>(segment_kmask<Packet16h>(begin, count)), from);
3446inline Packet8h ploaduSegment<Packet8h>(
const Eigen::half* from, Index begin, Index count) {
3447 return _mm_maskz_loadu_epi16(
static_cast<__mmask8
>(segment_kmask<Packet8h>(begin, count)), from);
3451inline void pstoreuSegment<Eigen::half, Packet8h>(Eigen::half* to,
const Packet8h& from, Index begin, Index count) {
3452 _mm_mask_storeu_epi16(to,
static_cast<__mmask8
>(segment_kmask<Packet8h>(begin, count)), from);
3465EIGEN_STRONG_INLINE Packet16i psdiv_signed_magic<Packet16i>(
const Packet16i& a, uint32_t magic,
int shift,
3466 int ,
const Packet16i& divisor_sign) {
3467 __m512i m = _mm512_set1_epi32(
static_cast<int>(magic));
3468 __m512i even = _mm512_shuffle_epi32(_mm512_mul_epi32(a, m), _MM_PERM_DDBB);
3469 __m512i odd = _mm512_mul_epi32(_mm512_shuffle_epi32(a, _MM_PERM_DDBB), m);
3470 __m512i t = _mm512_add_epi32(a, _mm512_mask_blend_epi32(0xAAAA, even, odd));
3471 __m512i q = _mm512_sub_epi32(_mm512_srav_epi32(t, _mm512_set1_epi32(shift)), _mm512_srai_epi32(a, 31));
3472 return _mm512_sub_epi32(_mm512_xor_si512(q, divisor_sign), divisor_sign);
3476#if EIGEN_ARCH_i386_OR_x86_64
3478EIGEN_STRONG_INLINE Packet8l pudiv_magic<Packet8l>(
const Packet8l& a, uint64_t magic, uint64_t addend,
int shift) {
3479 __m512i m_lo = _mm512_set1_epi64(
static_cast<long long>(magic));
3480 __m512i m_hi = _mm512_set1_epi64(
static_cast<long long>(magic >> 32));
3481 __m512i c_lo = _mm512_set1_epi64(
static_cast<long long>(addend & 0xffffffffu));
3482 __m512i c_hi = _mm512_set1_epi64(
static_cast<long long>(addend >> 32));
3483 __m512i a_hi = _mm512_srli_epi64(a, 32);
3484 __m512i u0 = _mm512_add_epi64(_mm512_mul_epu32(a, m_lo), c_lo);
3485 __m512i u = _mm512_add_epi64(_mm512_mul_epu32(a_hi, m_lo), _mm512_srli_epi64(u0, 32));
3486 __m512i w = _mm512_add_epi64(_mm512_add_epi64(_mm512_mul_epu32(a, m_hi), _mm512_maskz_mov_epi32(0x5555, u)), c_hi);
3487 __m512i t = _mm512_add_epi64(_mm512_add_epi64(_mm512_mul_epu32(a_hi, m_hi), _mm512_srli_epi64(u, 32)),
3488 _mm512_srli_epi64(w, 32));
3489 return _mm512_srlv_epi64(t, _mm512_set1_epi64(shift));
@ Aligned64
Definition Constants.h:240
@ Aligned32
Definition Constants.h:239
@ Aligned16
Definition Constants.h:238