Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner (benoit.steiner.goog@gmail.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_PACKET_MATH_AVX_H
12#define EIGEN_PACKET_MATH_AVX_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
23#endif
24
25#if !defined(EIGEN_VECTORIZE_AVX512) && !defined(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS)
26#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16
27#endif
28
29#ifdef EIGEN_VECTORIZE_FMA
30#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
31#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
32#endif
33#endif
34
35using Packet8f = __m256;
36using Packet8i = eigen_packet_wrapper<__m256i, 0>;
37using Packet4d = __m256d;
38#ifndef EIGEN_VECTORIZE_AVX512FP16
39using Packet8h = eigen_packet_wrapper<__m128i, 2>;
40#endif
41using Packet8bf = eigen_packet_wrapper<__m128i, 3>;
42using Packet8ui = eigen_packet_wrapper<__m256i, 4>;
43
44#ifdef EIGEN_VECTORIZE_AVX2
45// Start from 3 to be compatible with AVX512
46using Packet4l = eigen_packet_wrapper<__m256i, 3>;
47using Packet4ul = eigen_packet_wrapper<__m256i, 5>;
48#endif
49
50#define SIGN_MASK_I64 static_cast<int64_t>(0x8000000000000000ULL)
51
52template <>
53struct is_arithmetic<__m256> : std::true_type {};
54template <>
55struct is_arithmetic<__m256i> : std::true_type {};
56template <>
57struct is_arithmetic<__m256d> : std::true_type {};
58template <>
59struct is_arithmetic<Packet8i> : std::true_type {};
60// Note that `Packet8ui` uses the underlying type `__m256i`, which is
61// interpreted as a vector of _signed_ `int32`s, which breaks some arithmetic
62// operations used in `GenericPacketMath.h`.
63template <>
64struct is_arithmetic<Packet8ui> : std::false_type {};
65#ifndef EIGEN_VECTORIZE_AVX512FP16
66template <>
67struct is_arithmetic<Packet8h> : std::true_type {};
68#endif
69template <>
70struct is_arithmetic<Packet8bf> : std::true_type {};
71#ifdef EIGEN_VECTORIZE_AVX2
72template <>
73struct is_arithmetic<Packet4l> : std::true_type {};
74// Note that `Packet4ul` uses the underlying type `__m256i`, which is
75// interpreted as a vector of _signed_ `int32`s, which breaks some arithmetic
76// operations used in `GenericPacketMath.h`.
77template <>
78struct is_arithmetic<Packet4ul> : std::false_type {};
79#endif
80
81// Use the packet_traits defined in AVX512/PacketMath.h instead if we're going
82// to leverage AVX512 instructions.
83#ifndef EIGEN_VECTORIZE_AVX512
84template <>
85struct packet_traits<float> : default_packet_traits {
86 using type = Packet8f;
87 using half = Packet4f;
88 enum {
89 Vectorizable = 1,
90 AlignedOnScalar = 1,
91 size = 8,
92
93 HasCmp = 1,
94 HasDiv = 1,
95 HasReciprocal = EIGEN_FAST_MATH,
96 HasSin = EIGEN_FAST_MATH,
97 HasCos = EIGEN_FAST_MATH,
98 HasTan = EIGEN_FAST_MATH,
99 HasACos = 1,
100 HasASin = 1,
101 HasATan = 1,
102 HasATanh = 1,
103 HasSinh = 1,
104 HasCosh = 1,
105 HasASinh = 1,
106 HasACosh = 1,
107 HasLog = 1,
108 HasLog10 = 1,
109 HasExp = 1,
110 HasLog1p = 1,
111 HasExpm1 = 1,
112 HasPow = 1,
113 HasNdtri = 1,
114 HasBessel = 1,
115 HasSqrt = 1,
116 HasRsqrt = 1,
117 HasCbrt = 1,
118 HasTanh = EIGEN_FAST_MATH,
119 HasErf = EIGEN_FAST_MATH,
120 HasErfc = EIGEN_FAST_MATH,
121 };
122};
123template <>
124struct packet_traits<double> : default_packet_traits {
125 using type = Packet4d;
126 using half = Packet2d;
127 enum {
128 Vectorizable = 1,
129 AlignedOnScalar = 1,
130 size = 4,
131
132 HasCmp = 1,
133 HasDiv = 1,
134#ifdef EIGEN_VECTORIZE_AVX2
135 HasSin = EIGEN_FAST_MATH,
136 HasCos = EIGEN_FAST_MATH,
137 HasTan = EIGEN_FAST_MATH,
138#endif
139 HasSinh = 1,
140 HasCosh = 1,
141 HasASinh = 1,
142 HasACosh = 1,
143 HasTanh = EIGEN_FAST_MATH,
144 HasErf = 1,
145 HasErfc = 1,
146 HasLog = 1,
147 HasLog10 = 1,
148 HasExp = 1,
149 HasLog1p = 1,
150 HasExpm1 = 1,
151 HasPow = 1,
152 HasSqrt = 1,
153 HasRsqrt = 1,
154 HasCbrt = 1,
155 HasATan = 1,
156 HasATanh = 1,
157 };
158};
159
160template <>
161struct packet_traits<Eigen::half> : default_packet_traits {
162 using type = Packet8h;
163 // There is no half-size packet for Packet8h.
164 using half = Packet8h;
165 enum {
166 Vectorizable = 1,
167 AlignedOnScalar = 1,
168 size = 8,
169
170 HasCmp = 1,
171 HasAdd = 1,
172 HasSub = 1,
173 HasMul = 1,
174 HasDiv = 1,
175 HasSin = EIGEN_FAST_MATH,
176 HasCos = EIGEN_FAST_MATH,
177 HasNegate = 1,
178 HasAbs = 1,
179 HasMin = 1,
180 HasMax = 1,
181 HasConj = 1,
182 HasSetLinear = 0,
183 HasLog = 1,
184 HasLog1p = 1,
185 HasExpm1 = 1,
186 HasExp = 1,
187 HasSqrt = 1,
188 HasRsqrt = 1,
189 HasTanh = EIGEN_FAST_MATH,
190 HasErf = EIGEN_FAST_MATH,
191 HasBessel = 1,
192 HasNdtri = 1
193 };
194};
195
196template <>
197struct packet_traits<bfloat16> : default_packet_traits {
198 using type = Packet8bf;
199 // There is no half-size packet for current Packet8bf.
200 // TODO: support as SSE path.
201 using half = Packet8bf;
202 enum {
203 Vectorizable = 1,
204 AlignedOnScalar = 1,
205 size = 8,
206
207 HasCmp = 1,
208 HasAdd = 1,
209 HasSub = 1,
210 HasMul = 1,
211 HasDiv = 1,
212 HasSin = EIGEN_FAST_MATH,
213 HasCos = EIGEN_FAST_MATH,
214 HasNegate = 1,
215 HasAbs = 1,
216 HasMin = 1,
217 HasMax = 1,
218 HasConj = 1,
219 HasSetLinear = 0,
220 HasLog = 1,
221 HasLog1p = 1,
222 HasExpm1 = 1,
223 HasExp = 1,
224 HasSqrt = 1,
225 HasRsqrt = 1,
226 HasTanh = EIGEN_FAST_MATH,
227 HasErf = EIGEN_FAST_MATH,
228 HasBessel = 1,
229 HasNdtri = 1
230 };
231};
232
233template <>
234struct packet_traits<int> : default_packet_traits {
235 using type = Packet8i;
236 using half = Packet4i;
237 enum { Vectorizable = 1, AlignedOnScalar = 1, HasCmp = 1, HasDiv = 1, HasFastIntDiv = 1, size = 8 };
238};
239template <>
240struct packet_traits<uint32_t> : default_packet_traits {
241 using type = Packet8ui;
242 using half = Packet4ui;
243 enum {
244 Vectorizable = 1,
245 AlignedOnScalar = 1,
246 size = 8,
247
248 HasDiv = 0,
249 HasNegate = 0,
250
251 HasCmp = 1,
252 HasMin = 1,
253 HasMax = 1,
254 HasFastIntDiv = 1,
255 HasShift = 1
256 };
257};
258
259#ifdef EIGEN_VECTORIZE_AVX2
260template <>
261struct packet_traits<int64_t> : default_packet_traits {
262 using type = Packet4l;
263 using half = Packet2l;
264 // The 64-bit kernels take multiply-add constants.
265 enum {
266 Vectorizable = 1,
267 AlignedOnScalar = 1,
268 HasCmp = 1,
269 HasFastIntDiv = int_div::use_multiply_add<uint64_t>::value,
270 size = 4
271 };
272};
273template <>
274struct packet_traits<uint64_t> : default_packet_traits {
275 using type = Packet4ul;
276 // There is no half-size packet for current Packet4ul.
277 // TODO: support as SSE path.
278 using half = Packet4ul;
279 enum {
280 Vectorizable = 1,
281 AlignedOnScalar = 1,
282 size = 4,
283
284 HasDiv = 0,
285 HasTranspose = 0,
286 HasNegate = 0,
287 HasCmp = 1,
288 HasFastIntDiv = int_div::use_multiply_add<uint64_t>::value,
289 HasShift = 1
290 };
291};
292#endif
293
294#endif
295
296template <>
297struct scalar_div_cost<float, true> : std::integral_constant<int, 14> {};
298template <>
299struct scalar_div_cost<double, true> : std::integral_constant<int, 16> {};
300
301template <>
302struct unpacket_traits<Packet8f> {
303 using type = float;
304 using half = Packet4f;
305 using integer_packet = Packet8i;
306 using mask_t = uint8_t;
307 enum {
308 size = 8,
309 alignment = Aligned32,
310 vectorizable = true,
311 masked_load_available = true,
312 masked_store_available = true
313#ifdef EIGEN_VECTORIZE_AVX512
314 ,
315 masked_fpops_available = true
316#endif
317 };
318};
319template <>
320struct unpacket_traits<Packet4d> {
321 using type = double;
322 using half = Packet2d;
323#ifdef EIGEN_VECTORIZE_AVX2
324 using integer_packet = Packet4l;
325#endif
326 enum {
327 size = 4,
328 alignment = Aligned32,
329 vectorizable = true,
330 masked_load_available = false,
331 masked_store_available = false
332 };
333};
334template <>
335struct unpacket_traits<Packet8i> {
336 using type = int;
337 using half = Packet4i;
338 enum {
339 size = 8,
340 alignment = Aligned32,
341 vectorizable = true,
342 masked_load_available = false,
343 masked_store_available = false
344 };
345};
346template <>
347struct unpacket_traits<Packet8ui> {
348 using type = uint32_t;
349 using half = Packet4ui;
350 enum {
351 size = 8,
352 alignment = Aligned32,
353 vectorizable = true,
354 masked_load_available = false,
355 masked_store_available = false
356 };
357};
358#ifdef EIGEN_VECTORIZE_AVX2
359template <>
360struct unpacket_traits<Packet4l> {
361 using type = int64_t;
362 using half = Packet2l;
363 enum {
364 size = 4,
365 alignment = Aligned32,
366 vectorizable = true,
367 masked_load_available = false,
368 masked_store_available = false
369 };
370};
371template <>
372struct unpacket_traits<Packet4ul> {
373 using type = uint64_t;
374 using half = Packet4ul;
375 enum {
376 size = 4,
377 alignment = Aligned32,
378 vectorizable = true,
379 masked_load_available = false,
380 masked_store_available = false
381 };
382};
383#endif
384template <>
385struct unpacket_traits<Packet8bf> {
386 using type = bfloat16;
387 using half = Packet8bf;
388 enum {
389 size = 8,
390 alignment = Aligned16,
391 vectorizable = true,
392 masked_load_available = false,
393 masked_store_available = false
394 };
395};
396
397// Helper function for bit packing snippet of low precision comparison.
398// It packs the flags from 16x16 to 8x16.
399EIGEN_STRONG_INLINE __m128i Pack16To8(const Packet8f& rf) {
400 return _mm_packs_epi32(_mm256_extractf128_si256(_mm256_castps_si256(rf), 0),
401 _mm256_extractf128_si256(_mm256_castps_si256(rf), 1));
402}
403
404#ifdef EIGEN_VECTORIZE_AVX2
405template <>
406EIGEN_STRONG_INLINE Packet4l pset1<Packet4l>(const int64_t& from) {
407 return _mm256_set1_epi64x(from);
408}
409template <>
410EIGEN_STRONG_INLINE Packet4ul pset1<Packet4ul>(const uint64_t& from) {
411 return _mm256_set1_epi64x(numext::bit_cast<uint64_t>(from));
412}
413template <>
414EIGEN_STRONG_INLINE Packet4l pzero(const Packet4l& /*a*/) {
415 return _mm256_setzero_si256();
416}
417template <>
418EIGEN_STRONG_INLINE Packet4ul pzero(const Packet4ul& /*a*/) {
419 return _mm256_setzero_si256();
420}
421template <>
422EIGEN_STRONG_INLINE Packet4l peven_mask(const Packet4l& /*a*/) {
423 return _mm256_set_epi64x(0ll, -1ll, 0ll, -1ll);
424}
425template <>
426EIGEN_STRONG_INLINE Packet4ul peven_mask(const Packet4ul& /*a*/) {
427 return _mm256_set_epi64x(0ll, -1ll, 0ll, -1ll);
428}
429template <>
430EIGEN_STRONG_INLINE Packet4l pload1<Packet4l>(const int64_t* from) {
431 return _mm256_set1_epi64x(*from);
432}
433template <>
434EIGEN_STRONG_INLINE Packet4ul pload1<Packet4ul>(const uint64_t* from) {
435 return _mm256_set1_epi64x(*from);
436}
437template <>
438EIGEN_STRONG_INLINE Packet4l padd<Packet4l>(const Packet4l& a, const Packet4l& b) {
439 return _mm256_add_epi64(a, b);
440}
441template <>
442EIGEN_STRONG_INLINE Packet4ul padd<Packet4ul>(const Packet4ul& a, const Packet4ul& b) {
443 return _mm256_add_epi64(a, b);
444}
445template <>
446EIGEN_STRONG_INLINE Packet4l plset<Packet4l>(const int64_t& a) {
447 return padd(pset1<Packet4l>(a), Packet4l(_mm256_set_epi64x(3ll, 2ll, 1ll, 0ll)));
448}
449template <>
450EIGEN_STRONG_INLINE Packet4ul plset<Packet4ul>(const uint64_t& a) {
451 return padd(pset1<Packet4ul>(a), Packet4ul(_mm256_set_epi64x(3ll, 2ll, 1ll, 0ll)));
452}
453template <>
454EIGEN_STRONG_INLINE Packet4l psub<Packet4l>(const Packet4l& a, const Packet4l& b) {
455 return _mm256_sub_epi64(a, b);
456}
457template <>
458EIGEN_STRONG_INLINE Packet4ul psub<Packet4ul>(const Packet4ul& a, const Packet4ul& b) {
459 return _mm256_sub_epi64(a, b);
460}
461template <>
462EIGEN_STRONG_INLINE Packet4l pnegate(const Packet4l& a) {
463 return psub(pzero(a), a);
464}
465template <>
466EIGEN_STRONG_INLINE Packet4l pcmp_le(const Packet4l& a, const Packet4l& b) {
467 return _mm256_xor_si256(_mm256_cmpgt_epi64(a, b), _mm256_set1_epi32(-1));
468}
469template <>
470EIGEN_STRONG_INLINE Packet4ul pcmp_le(const Packet4ul& a, const Packet4ul& b) {
471 return (Packet4ul)pcmp_le((Packet4l)psub(a, pset1<Packet4ul>(0x8000000000000000UL)),
472 (Packet4l)psub(b, pset1<Packet4ul>(0x8000000000000000UL)));
473}
474template <>
475EIGEN_STRONG_INLINE Packet4l pcmp_lt(const Packet4l& a, const Packet4l& b) {
476 return _mm256_cmpgt_epi64(b, a);
477}
478template <>
479EIGEN_STRONG_INLINE Packet4ul pcmp_lt(const Packet4ul& a, const Packet4ul& b) {
480 return (Packet4ul)pcmp_lt((Packet4l)psub(a, pset1<Packet4ul>(0x8000000000000000UL)),
481 (Packet4l)psub(b, pset1<Packet4ul>(0x8000000000000000UL)));
482}
483template <>
484EIGEN_STRONG_INLINE Packet4l pcmp_eq(const Packet4l& a, const Packet4l& b) {
485 return _mm256_cmpeq_epi64(a, b);
486}
487template <>
488EIGEN_STRONG_INLINE Packet4ul pcmp_eq(const Packet4ul& a, const Packet4ul& b) {
489 return _mm256_cmpeq_epi64(a, b);
490}
491template <>
492EIGEN_STRONG_INLINE Packet4l ptrue<Packet4l>(const Packet4l& a) {
493 return _mm256_cmpeq_epi64(a, a);
494}
495template <>
496EIGEN_STRONG_INLINE Packet4ul ptrue<Packet4ul>(const Packet4ul& a) {
497 return _mm256_cmpeq_epi64(a, a);
498}
499template <>
500EIGEN_STRONG_INLINE Packet4l pand<Packet4l>(const Packet4l& a, const Packet4l& b) {
501 return _mm256_and_si256(a, b);
502}
503template <>
504EIGEN_STRONG_INLINE Packet4l por<Packet4l>(const Packet4l& a, const Packet4l& b) {
505 return _mm256_or_si256(a, b);
506}
507template <>
508EIGEN_STRONG_INLINE Packet4l pxor<Packet4l>(const Packet4l& a, const Packet4l& b) {
509 return _mm256_xor_si256(a, b);
510}
511template <>
512EIGEN_STRONG_INLINE Packet4ul pxor<Packet4ul>(const Packet4ul& a, const Packet4ul& b) {
513 return _mm256_xor_si256(a, b);
514}
515template <>
516EIGEN_STRONG_INLINE Packet4l pandnot<Packet4l>(const Packet4l& a, const Packet4l& b) {
517 return _mm256_andnot_si256(b, a);
518}
519template <int N>
520EIGEN_STRONG_INLINE Packet4l plogical_shift_right(const Packet4l& a) {
521 return _mm256_srli_epi64(a, N);
522}
523template <int N>
524EIGEN_STRONG_INLINE Packet4l plogical_shift_left(const Packet4l& a) {
525 return _mm256_slli_epi64(a, N);
526}
527#ifdef EIGEN_VECTORIZE_AVX512FP16
528template <int N>
529EIGEN_STRONG_INLINE Packet4l parithmetic_shift_right(const Packet4l& a) {
530 return _mm256_srai_epi64(a, N);
531}
532#else
533template <int N>
534EIGEN_STRONG_INLINE std::enable_if_t<(N == 0), Packet4l> parithmetic_shift_right(const Packet4l& a) {
535 return a;
536}
537template <int N>
538EIGEN_STRONG_INLINE std::enable_if_t<(N > 0) && (N < 32), Packet4l> parithmetic_shift_right(const Packet4l& a) {
539 __m256i hi_word = _mm256_srai_epi32(a, N);
540 __m256i lo_word = _mm256_srli_epi64(a, N);
541 return _mm256_blend_epi32(hi_word, lo_word, 0b01010101);
542}
543template <int N>
544EIGEN_STRONG_INLINE std::enable_if_t<(N >= 32) && (N < 63), Packet4l> parithmetic_shift_right(const Packet4l& a) {
545 __m256i hi_word = _mm256_srai_epi32(a, 31);
546 __m256i lo_word = _mm256_shuffle_epi32(_mm256_srai_epi32(a, N - 32), (shuffle_mask<1, 1, 3, 3>::mask));
547 return _mm256_blend_epi32(hi_word, lo_word, 0b01010101);
548}
549template <int N>
550EIGEN_STRONG_INLINE std::enable_if_t<(N == 63), Packet4l> parithmetic_shift_right(const Packet4l& a) {
551 return _mm256_cmpgt_epi64(_mm256_setzero_si256(), a);
552}
553template <int N>
554EIGEN_STRONG_INLINE std::enable_if_t<(N < 0) || (N > 63), Packet4l> parithmetic_shift_right(const Packet4l& a) {
555 return parithmetic_shift_right<int(N & 63)>(a);
556}
557#endif
558template <int N>
559EIGEN_STRONG_INLINE Packet4ul parithmetic_shift_right(const Packet4ul& a) {
560 return _mm256_srli_epi64(a, N);
561}
562template <int N>
563EIGEN_STRONG_INLINE Packet4ul plogical_shift_right(const Packet4ul& a) {
564 return _mm256_srli_epi64(a, N);
565}
566template <int N>
567EIGEN_STRONG_INLINE Packet4ul plogical_shift_left(const Packet4ul& a) {
568 return _mm256_slli_epi64(a, N);
569}
570template <>
571EIGEN_STRONG_INLINE Packet4l pload<Packet4l>(const int64_t* from) {
572 EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
573}
574template <>
575EIGEN_STRONG_INLINE Packet4ul pload<Packet4ul>(const uint64_t* from) {
576 EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
577}
578template <>
579EIGEN_STRONG_INLINE Packet4l ploadu<Packet4l>(const int64_t* from) {
580 EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
581}
582template <>
583EIGEN_STRONG_INLINE Packet4ul ploadu<Packet4ul>(const uint64_t* from) {
584 EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
585}
586// Loads 2 int64_ts from memory a returns the packet {a0, a0, a1, a1}
587template <>
588EIGEN_STRONG_INLINE Packet4l ploaddup<Packet4l>(const int64_t* from) {
589 const Packet4l a = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast<const __m128i*>(from)));
590 return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 1, 0, 1, 2, 3, 2, 3));
591}
592// Loads 2 uint64_ts from memory a returns the packet {a0, a0, a1, a1}
593template <>
594EIGEN_STRONG_INLINE Packet4ul ploaddup<Packet4ul>(const uint64_t* from) {
595 const Packet4ul a = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast<const __m128i*>(from)));
596 return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 1, 0, 1, 2, 3, 2, 3));
597}
598template <>
599EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet4l& from) {
600 EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
601}
602template <>
603EIGEN_STRONG_INLINE void pstore<uint64_t>(uint64_t* to, const Packet4ul& from) {
604 EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
605}
606template <>
607EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet4l& from) {
608 EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
609}
610template <>
611EIGEN_STRONG_INLINE void pstoreu<uint64_t>(uint64_t* to, const Packet4ul& from) {
612 EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
613}
614template <>
615EIGEN_DEVICE_FUNC inline Packet4l pgather<int64_t, Packet4l>(const int64_t* from, Index stride) {
616 return _mm256_set_epi64x(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]);
617}
618template <>
619EIGEN_DEVICE_FUNC inline Packet4ul pgather<uint64_t, Packet4ul>(const uint64_t* from, Index stride) {
620 return _mm256_set_epi64x(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]);
621}
622template <>
623EIGEN_DEVICE_FUNC inline void pscatter<int64_t, Packet4l>(int64_t* to, const Packet4l& from, Index stride) {
624 __m128i low = _mm256_extractf128_si256(from, 0);
625 to[stride * 0] = _mm_extract_epi64_0(low);
626 to[stride * 1] = _mm_extract_epi64_1(low);
627
628 __m128i high = _mm256_extractf128_si256(from, 1);
629 to[stride * 2] = _mm_extract_epi64_0(high);
630 to[stride * 3] = _mm_extract_epi64_1(high);
631}
632template <>
633EIGEN_DEVICE_FUNC inline void pscatter<uint64_t, Packet4ul>(uint64_t* to, const Packet4ul& from, Index stride) {
634 __m128i low = _mm256_extractf128_si256(from, 0);
635 to[stride * 0] = _mm_extract_epi64_0(low);
636 to[stride * 1] = _mm_extract_epi64_1(low);
637
638 __m128i high = _mm256_extractf128_si256(from, 1);
639 to[stride * 2] = _mm_extract_epi64_0(high);
640 to[stride * 3] = _mm_extract_epi64_1(high);
641}
642template <>
643EIGEN_STRONG_INLINE void pstore1<Packet4l>(int64_t* to, const int64_t& a) {
644 Packet4l pa = pset1<Packet4l>(a);
645 pstore(to, pa);
646}
647template <>
648EIGEN_STRONG_INLINE void pstore1<Packet4ul>(uint64_t* to, const uint64_t& a) {
649 Packet4ul pa = pset1<Packet4ul>(a);
650 pstore(to, pa);
651}
652template <>
653EIGEN_STRONG_INLINE int64_t pfirst<Packet4l>(const Packet4l& a) {
654 return _mm_extract_epi64_0(_mm256_castsi256_si128(a));
655}
656template <>
657EIGEN_STRONG_INLINE uint64_t pfirst<Packet4ul>(const Packet4ul& a) {
658 return _mm_extract_epi64_0(_mm256_castsi256_si128(a));
659}
660
661#define MM256_SHUFFLE_EPI64(A, B, M) _mm256_shuffle_pd(_mm256_castsi256_pd(A), _mm256_castsi256_pd(B), M)
662EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4l, 4>& kernel) {
663 __m256d T0 = MM256_SHUFFLE_EPI64(kernel.packet[0], kernel.packet[1], 15);
664 __m256d T1 = MM256_SHUFFLE_EPI64(kernel.packet[0], kernel.packet[1], 0);
665 __m256d T2 = MM256_SHUFFLE_EPI64(kernel.packet[2], kernel.packet[3], 15);
666 __m256d T3 = MM256_SHUFFLE_EPI64(kernel.packet[2], kernel.packet[3], 0);
667
668 kernel.packet[1] = _mm256_castpd_si256(_mm256_permute2f128_pd(T0, T2, 32));
669 kernel.packet[3] = _mm256_castpd_si256(_mm256_permute2f128_pd(T0, T2, 49));
670 kernel.packet[0] = _mm256_castpd_si256(_mm256_permute2f128_pd(T1, T3, 32));
671 kernel.packet[2] = _mm256_castpd_si256(_mm256_permute2f128_pd(T1, T3, 49));
672}
673EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4ul, 4>& kernel) {
674 ptranspose((PacketBlock<Packet4l, 4>&)kernel);
675}
676template <>
677EIGEN_STRONG_INLINE Packet4l pmin<Packet4l>(const Packet4l& a, const Packet4l& b) {
678 __m256i cmp = _mm256_cmpgt_epi64(a, b);
679 __m256i a_min = _mm256_andnot_si256(cmp, a);
680 __m256i b_min = _mm256_and_si256(cmp, b);
681 return Packet4l(_mm256_or_si256(a_min, b_min));
682}
683template <>
684EIGEN_STRONG_INLINE Packet4ul pmin<Packet4ul>(const Packet4ul& a, const Packet4ul& b) {
685 return padd((Packet4ul)pmin((Packet4l)psub(a, pset1<Packet4ul>(0x8000000000000000UL)),
686 (Packet4l)psub(b, pset1<Packet4ul>(0x8000000000000000UL))),
687 pset1<Packet4ul>(0x8000000000000000UL));
688}
689template <>
690EIGEN_STRONG_INLINE Packet4l pmax<Packet4l>(const Packet4l& a, const Packet4l& b) {
691 __m256i cmp = _mm256_cmpgt_epi64(a, b);
692 __m256i a_min = _mm256_and_si256(cmp, a);
693 __m256i b_min = _mm256_andnot_si256(cmp, b);
694 return Packet4l(_mm256_or_si256(a_min, b_min));
695}
696template <>
697EIGEN_STRONG_INLINE Packet4ul pmax<Packet4ul>(const Packet4ul& a, const Packet4ul& b) {
698 return padd((Packet4ul)pmax((Packet4l)psub(a, pset1<Packet4ul>(0x8000000000000000UL)),
699 (Packet4l)psub(b, pset1<Packet4ul>(0x8000000000000000UL))),
700 pset1<Packet4ul>(0x8000000000000000UL));
701}
702template <>
703EIGEN_STRONG_INLINE Packet4l pabs<Packet4l>(const Packet4l& a) {
704 Packet4l pz = pzero<Packet4l>(a);
705 Packet4l cmp = _mm256_cmpgt_epi64(a, pz);
706 return psub(cmp, pxor(a, cmp));
707}
708template <>
709EIGEN_STRONG_INLINE Packet4l pmul<Packet4l>(const Packet4l& a, const Packet4l& b) {
710 // 64-bit mul requires avx512, so do this with 32-bit multiplication
711 __m256i upper32_a = _mm256_srli_epi64(a, 32);
712 __m256i upper32_b = _mm256_srli_epi64(b, 32);
713
714 // upper * lower
715 __m256i mul1 = _mm256_mul_epu32(upper32_a, b);
716 __m256i mul2 = _mm256_mul_epu32(upper32_b, a);
717 // Gives us both upper*upper and lower*lower
718 __m256i mul3 = _mm256_mul_epu32(a, b);
719
720 __m256i high = _mm256_slli_epi64(_mm256_add_epi64(mul1, mul2), 32);
721 return _mm256_add_epi64(high, mul3);
722}
723template <>
724EIGEN_STRONG_INLINE Packet4ul pmul<Packet4ul>(const Packet4ul& a, const Packet4ul& b) {
725 return (Packet4ul)pmul<Packet4l>((Packet4l)a, (Packet4l)b);
726}
727#endif
728
729template <>
730EIGEN_STRONG_INLINE Packet8f pset1<Packet8f>(const float& from) {
731 return _mm256_set1_ps(from);
732}
733template <>
734EIGEN_STRONG_INLINE Packet4d pset1<Packet4d>(const double& from) {
735 return _mm256_set1_pd(from);
736}
737template <>
738EIGEN_STRONG_INLINE Packet8i pset1<Packet8i>(const int& from) {
739 return _mm256_set1_epi32(from);
740}
741template <>
742EIGEN_STRONG_INLINE Packet8ui pset1<Packet8ui>(const uint32_t& from) {
743 return _mm256_set1_epi32(from);
744}
745
746template <>
747EIGEN_STRONG_INLINE Packet8f pset1frombits<Packet8f>(unsigned int from) {
748 return _mm256_castsi256_ps(pset1<Packet8i>(from));
749}
750template <>
751EIGEN_STRONG_INLINE Packet4d pset1frombits<Packet4d>(uint64_t from) {
752 return _mm256_castsi256_pd(_mm256_set1_epi64x(from));
753}
754
755template <>
756EIGEN_STRONG_INLINE Packet8f pzero(const Packet8f& /*a*/) {
757 return _mm256_setzero_ps();
758}
759template <>
760EIGEN_STRONG_INLINE Packet4d pzero(const Packet4d& /*a*/) {
761 return _mm256_setzero_pd();
762}
763template <>
764EIGEN_STRONG_INLINE Packet8i pzero(const Packet8i& /*a*/) {
765 return _mm256_setzero_si256();
766}
767template <>
768EIGEN_STRONG_INLINE Packet8ui pzero(const Packet8ui& /*a*/) {
769 return _mm256_setzero_si256();
770}
771
772template <>
773EIGEN_STRONG_INLINE Packet8f peven_mask(const Packet8f& /*a*/) {
774 Packet8f r = _mm256_castsi256_ps(_mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1));
775 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
776 return r;
777}
778template <>
779EIGEN_STRONG_INLINE Packet8i peven_mask(const Packet8i& /*a*/) {
780 return _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1);
781}
782template <>
783EIGEN_STRONG_INLINE Packet8ui peven_mask(const Packet8ui& /*a*/) {
784 return _mm256_set_epi32(0, -1, 0, -1, 0, -1, 0, -1);
785}
786template <>
787EIGEN_STRONG_INLINE Packet4d peven_mask(const Packet4d& /*a*/) {
788 Packet4d r = _mm256_castsi256_pd(_mm256_set_epi32(0, 0, -1, -1, 0, 0, -1, -1));
789 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
790 return r;
791}
792
793template <>
794EIGEN_STRONG_INLINE Packet8f pload1<Packet8f>(const float* from) {
795 return _mm256_broadcast_ss(from);
796}
797template <>
798EIGEN_STRONG_INLINE Packet4d pload1<Packet4d>(const double* from) {
799 return _mm256_broadcast_sd(from);
800}
801
802template <>
803EIGEN_STRONG_INLINE Packet8f padd<Packet8f>(const Packet8f& a, const Packet8f& b) {
804 return _mm256_add_ps(a, b);
805}
806#ifdef EIGEN_VECTORIZE_AVX512
807template <>
808EIGEN_STRONG_INLINE Packet8f padd<Packet8f>(const Packet8f& a, const Packet8f& b, uint8_t umask) {
809 __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF);
810 return _mm512_castps512_ps256(_mm512_maskz_add_ps(mask, _mm512_castps256_ps512(a), _mm512_castps256_ps512(b)));
811}
812#endif
813template <>
814EIGEN_STRONG_INLINE Packet4d padd<Packet4d>(const Packet4d& a, const Packet4d& b) {
815 return _mm256_add_pd(a, b);
816}
817template <>
818EIGEN_STRONG_INLINE Packet8i padd<Packet8i>(const Packet8i& a, const Packet8i& b) {
819#ifdef EIGEN_VECTORIZE_AVX2
820 return _mm256_add_epi32(a, b);
821#else
822 __m128i lo = _mm_add_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
823 __m128i hi = _mm_add_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
824 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
825#endif
826}
827template <>
828EIGEN_STRONG_INLINE Packet8ui padd<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
829#ifdef EIGEN_VECTORIZE_AVX2
830 return _mm256_add_epi32(a, b);
831#else
832 __m128i lo = _mm_add_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
833 __m128i hi = _mm_add_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
834 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
835#endif
836}
837
838template <>
839EIGEN_STRONG_INLINE Packet8f plset<Packet8f>(const float& a) {
840 return padd(pset1<Packet8f>(a), _mm256_set_ps(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
841}
842template <>
843EIGEN_STRONG_INLINE Packet4d plset<Packet4d>(const double& a) {
844 return padd(pset1<Packet4d>(a), _mm256_set_pd(3.0, 2.0, 1.0, 0.0));
845}
846template <>
847EIGEN_STRONG_INLINE Packet8i plset<Packet8i>(const int& a) {
848 return padd(pset1<Packet8i>(a), (Packet8i)_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0));
849}
850template <>
851EIGEN_STRONG_INLINE Packet8ui plset<Packet8ui>(const uint32_t& a) {
852 return padd(pset1<Packet8ui>(a), (Packet8ui)_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0));
853}
854
855template <>
856EIGEN_STRONG_INLINE Packet8f psub<Packet8f>(const Packet8f& a, const Packet8f& b) {
857 return _mm256_sub_ps(a, b);
858}
859template <>
860EIGEN_STRONG_INLINE Packet4d psub<Packet4d>(const Packet4d& a, const Packet4d& b) {
861 return _mm256_sub_pd(a, b);
862}
863template <>
864EIGEN_STRONG_INLINE Packet8i psub<Packet8i>(const Packet8i& a, const Packet8i& b) {
865#ifdef EIGEN_VECTORIZE_AVX2
866 return _mm256_sub_epi32(a, b);
867#else
868 __m128i lo = _mm_sub_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
869 __m128i hi = _mm_sub_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
870 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
871#endif
872}
873template <>
874EIGEN_STRONG_INLINE Packet8ui psub<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
875#ifdef EIGEN_VECTORIZE_AVX2
876 return _mm256_sub_epi32(a, b);
877#else
878 __m128i lo = _mm_sub_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
879 __m128i hi = _mm_sub_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
880 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
881#endif
882}
883
884template <>
885EIGEN_STRONG_INLINE Packet8f pnegate(const Packet8f& a) {
886 const Packet8f mask = _mm256_castsi256_ps(_mm256_set1_epi32(SIGN_MASK_I32));
887 return _mm256_xor_ps(a, mask);
888}
889template <>
890EIGEN_STRONG_INLINE Packet4d pnegate(const Packet4d& a) {
891 const Packet4d mask = _mm256_castsi256_pd(_mm256_set1_epi64x(SIGN_MASK_I64));
892 return _mm256_xor_pd(a, mask);
893}
894template <>
895EIGEN_STRONG_INLINE Packet8i pnegate(const Packet8i& a) {
896 return psub(pzero(a), a);
897}
898
899template <>
900EIGEN_STRONG_INLINE Packet8f pmul<Packet8f>(const Packet8f& a, const Packet8f& b) {
901 return _mm256_mul_ps(a, b);
902}
903template <>
904EIGEN_STRONG_INLINE Packet4d pmul<Packet4d>(const Packet4d& a, const Packet4d& b) {
905 return _mm256_mul_pd(a, b);
906}
907template <>
908EIGEN_STRONG_INLINE Packet8i pmul<Packet8i>(const Packet8i& a, const Packet8i& b) {
909#ifdef EIGEN_VECTORIZE_AVX2
910 return _mm256_mullo_epi32(a, b);
911#else
912 const __m128i lo = _mm_mullo_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
913 const __m128i hi = _mm_mullo_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
914 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
915#endif
916}
917template <>
918EIGEN_STRONG_INLINE Packet8ui pmul<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
919#ifdef EIGEN_VECTORIZE_AVX2
920 return _mm256_mullo_epi32(a, b);
921#else
922 const __m128i lo = _mm_mullo_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
923 const __m128i hi = _mm_mullo_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
924 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
925#endif
926}
927
928template <>
929EIGEN_STRONG_INLINE Packet8f pdiv<Packet8f>(const Packet8f& a, const Packet8f& b) {
930 return _mm256_div_ps(a, b);
931}
932template <>
933EIGEN_STRONG_INLINE Packet4d pdiv<Packet4d>(const Packet4d& a, const Packet4d& b) {
934 return _mm256_div_pd(a, b);
935}
936
937template <>
938EIGEN_STRONG_INLINE Packet8i pdiv<Packet8i>(const Packet8i& a, const Packet8i& b) {
939#ifdef EIGEN_VECTORIZE_AVX512
940 return _mm512_cvttpd_epi32(_mm512_div_pd(_mm512_cvtepi32_pd(a), _mm512_cvtepi32_pd(b)));
941#else
942 Packet4i lo = pdiv<Packet4i>(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
943 Packet4i hi = pdiv<Packet4i>(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
944 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1);
945#endif
946}
947
948#ifdef EIGEN_VECTORIZE_FMA
949template <>
950struct is_fused_madd<Packet8f> : true_type {};
951template <>
952EIGEN_STRONG_INLINE Packet8f pmadd(const Packet8f& a, const Packet8f& b, const Packet8f& c) {
953 return _mm256_fmadd_ps(a, b, c);
954}
955template <>
956struct is_fused_madd<Packet4d> : true_type {};
957template <>
958EIGEN_STRONG_INLINE Packet4d pmadd(const Packet4d& a, const Packet4d& b, const Packet4d& c) {
959 return _mm256_fmadd_pd(a, b, c);
960}
961
962template <>
963EIGEN_STRONG_INLINE Packet8f pmsub(const Packet8f& a, const Packet8f& b, const Packet8f& c) {
964 return _mm256_fmsub_ps(a, b, c);
965}
966
967template <>
968EIGEN_STRONG_INLINE Packet4d pmsub(const Packet4d& a, const Packet4d& b, const Packet4d& c) {
969 return _mm256_fmsub_pd(a, b, c);
970}
971
972template <>
973EIGEN_STRONG_INLINE Packet8f pnmadd(const Packet8f& a, const Packet8f& b, const Packet8f& c) {
974 return _mm256_fnmadd_ps(a, b, c);
975}
976
977template <>
978EIGEN_STRONG_INLINE Packet4d pnmadd(const Packet4d& a, const Packet4d& b, const Packet4d& c) {
979 return _mm256_fnmadd_pd(a, b, c);
980}
981
982template <>
983EIGEN_STRONG_INLINE Packet8f pnmsub(const Packet8f& a, const Packet8f& b, const Packet8f& c) {
984 return _mm256_fnmsub_ps(a, b, c);
985}
986
987template <>
988EIGEN_STRONG_INLINE Packet4d pnmsub(const Packet4d& a, const Packet4d& b, const Packet4d& c) {
989 return _mm256_fnmsub_pd(a, b, c);
990}
991
992#endif
993
994template <>
995EIGEN_STRONG_INLINE Packet8f pcmp_le(const Packet8f& a, const Packet8f& b) {
996 return _mm256_cmp_ps(a, b, _CMP_LE_OQ);
997}
998template <>
999EIGEN_STRONG_INLINE Packet8f pcmp_lt(const Packet8f& a, const Packet8f& b) {
1000 return _mm256_cmp_ps(a, b, _CMP_LT_OQ);
1001}
1002template <>
1003EIGEN_STRONG_INLINE Packet8f pcmp_lt_or_nan(const Packet8f& a, const Packet8f& b) {
1004 return _mm256_cmp_ps(a, b, _CMP_NGE_UQ);
1005}
1006template <>
1007EIGEN_STRONG_INLINE Packet8f pcmp_eq(const Packet8f& a, const Packet8f& b) {
1008 return _mm256_cmp_ps(a, b, _CMP_EQ_OQ);
1009}
1010template <>
1011EIGEN_STRONG_INLINE Packet8f pisnan(const Packet8f& a) {
1012 return _mm256_cmp_ps(a, a, _CMP_UNORD_Q);
1013}
1014
1015template <>
1016EIGEN_STRONG_INLINE Packet4d pcmp_le(const Packet4d& a, const Packet4d& b) {
1017 return _mm256_cmp_pd(a, b, _CMP_LE_OQ);
1018}
1019template <>
1020EIGEN_STRONG_INLINE Packet4d pcmp_lt(const Packet4d& a, const Packet4d& b) {
1021 return _mm256_cmp_pd(a, b, _CMP_LT_OQ);
1022}
1023template <>
1024EIGEN_STRONG_INLINE Packet4d pcmp_lt_or_nan(const Packet4d& a, const Packet4d& b) {
1025 return _mm256_cmp_pd(a, b, _CMP_NGE_UQ);
1026}
1027template <>
1028EIGEN_STRONG_INLINE Packet4d pcmp_eq(const Packet4d& a, const Packet4d& b) {
1029 return _mm256_cmp_pd(a, b, _CMP_EQ_OQ);
1030}
1031
1032template <>
1033EIGEN_STRONG_INLINE Packet8i pcmp_le(const Packet8i& a, const Packet8i& b) {
1034#ifdef EIGEN_VECTORIZE_AVX2
1035 return _mm256_xor_si256(_mm256_cmpgt_epi32(a, b), _mm256_set1_epi32(-1));
1036#else
1037 __m128i lo = _mm_cmpgt_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
1038 lo = _mm_xor_si128(lo, _mm_set1_epi32(-1));
1039 __m128i hi = _mm_cmpgt_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
1040 hi = _mm_xor_si128(hi, _mm_set1_epi32(-1));
1041 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1042#endif
1043}
1044template <>
1045EIGEN_STRONG_INLINE Packet8i pcmp_lt(const Packet8i& a, const Packet8i& b) {
1046#ifdef EIGEN_VECTORIZE_AVX2
1047 return _mm256_cmpgt_epi32(b, a);
1048#else
1049 __m128i lo = _mm_cmpgt_epi32(_mm256_extractf128_si256(b, 0), _mm256_extractf128_si256(a, 0));
1050 __m128i hi = _mm_cmpgt_epi32(_mm256_extractf128_si256(b, 1), _mm256_extractf128_si256(a, 1));
1051 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1052#endif
1053}
1054template <>
1055EIGEN_STRONG_INLINE Packet8i pcmp_eq(const Packet8i& a, const Packet8i& b) {
1056#ifdef EIGEN_VECTORIZE_AVX2
1057 return _mm256_cmpeq_epi32(a, b);
1058#else
1059 __m128i lo = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
1060 __m128i hi = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
1061 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1062#endif
1063}
1064template <>
1065EIGEN_STRONG_INLINE Packet8ui pcmp_eq(const Packet8ui& a, const Packet8ui& b) {
1066#ifdef EIGEN_VECTORIZE_AVX2
1067 return _mm256_cmpeq_epi32(a, b);
1068#else
1069 __m128i lo = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
1070 __m128i hi = _mm_cmpeq_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
1071 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1072#endif
1073}
1074
1075template <>
1076EIGEN_STRONG_INLINE Packet8f pmin<Packet8f>(const Packet8f& a, const Packet8f& b) {
1077#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
1078 // There appears to be a bug in GCC, by which the optimizer may flip
1079 // the argument order in calls to _mm_min_ps/_mm_max_ps, so we have to
1080 // resort to inline ASM here. This is supposed to be fixed in gcc6.3,
1081 // see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
1082 Packet8f res;
1083 asm("vminps %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
1084 return res;
1085#else
1086 // Arguments are swapped to match NaN propagation behavior of std::min.
1087 return _mm256_min_ps(b, a);
1088#endif
1089}
1090template <>
1091EIGEN_STRONG_INLINE Packet4d pmin<Packet4d>(const Packet4d& a, const Packet4d& b) {
1092#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
1093 // See pmin above
1094 Packet4d res;
1095 asm("vminpd %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
1096 return res;
1097#else
1098 // Arguments are swapped to match NaN propagation behavior of std::min.
1099 return _mm256_min_pd(b, a);
1100#endif
1101}
1102template <>
1103EIGEN_STRONG_INLINE Packet8i pmin<Packet8i>(const Packet8i& a, const Packet8i& b) {
1104#ifdef EIGEN_VECTORIZE_AVX2
1105 return _mm256_min_epi32(a, b);
1106#else
1107 __m128i lo = _mm_min_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
1108 __m128i hi = _mm_min_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
1109 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1110#endif
1111}
1112template <>
1113EIGEN_STRONG_INLINE Packet8ui pmin<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
1114#ifdef EIGEN_VECTORIZE_AVX2
1115 return _mm256_min_epu32(a, b);
1116#else
1117 __m128i lo = _mm_min_epu32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
1118 __m128i hi = _mm_min_epu32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
1119 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1120#endif
1121}
1122
1123template <>
1124EIGEN_STRONG_INLINE Packet8f pmax<Packet8f>(const Packet8f& a, const Packet8f& b) {
1125#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
1126 // See pmin above
1127 Packet8f res;
1128 asm("vmaxps %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
1129 return res;
1130#else
1131 // Arguments are swapped to match NaN propagation behavior of std::max.
1132 return _mm256_max_ps(b, a);
1133#endif
1134}
1135template <>
1136EIGEN_STRONG_INLINE Packet4d pmax<Packet4d>(const Packet4d& a, const Packet4d& b) {
1137#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
1138 // See pmin above
1139 Packet4d res;
1140 asm("vmaxpd %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
1141 return res;
1142#else
1143 // Arguments are swapped to match NaN propagation behavior of std::max.
1144 return _mm256_max_pd(b, a);
1145#endif
1146}
1147template <>
1148EIGEN_STRONG_INLINE Packet8i pmax<Packet8i>(const Packet8i& a, const Packet8i& b) {
1149#ifdef EIGEN_VECTORIZE_AVX2
1150 return _mm256_max_epi32(a, b);
1151#else
1152 __m128i lo = _mm_max_epi32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
1153 __m128i hi = _mm_max_epi32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
1154 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1155#endif
1156}
1157template <>
1158EIGEN_STRONG_INLINE Packet8ui pmax<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
1159#ifdef EIGEN_VECTORIZE_AVX2
1160 return _mm256_max_epu32(a, b);
1161#else
1162 __m128i lo = _mm_max_epu32(_mm256_extractf128_si256(a, 0), _mm256_extractf128_si256(b, 0));
1163 __m128i hi = _mm_max_epu32(_mm256_extractf128_si256(a, 1), _mm256_extractf128_si256(b, 1));
1164 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1165#endif
1166}
1167
1168#ifdef EIGEN_VECTORIZE_AVX2
1169template <>
1170EIGEN_STRONG_INLINE Packet8i psign(const Packet8i& a) {
1171 return _mm256_sign_epi32(_mm256_set1_epi32(1), a);
1172}
1173#endif
1174
1175// Add specializations for min/max with prescribed NaN propagation.
1176template <>
1177EIGEN_STRONG_INLINE Packet8f pmin<PropagateNumbers, Packet8f>(const Packet8f& a, const Packet8f& b) {
1178 return pminmax_propagate_numbers(a, b, pmin<Packet8f>);
1179}
1180template <>
1181EIGEN_STRONG_INLINE Packet4d pmin<PropagateNumbers, Packet4d>(const Packet4d& a, const Packet4d& b) {
1182 return pminmax_propagate_numbers(a, b, pmin<Packet4d>);
1183}
1184template <>
1185EIGEN_STRONG_INLINE Packet8f pmax<PropagateNumbers, Packet8f>(const Packet8f& a, const Packet8f& b) {
1186 return pminmax_propagate_numbers(a, b, pmax<Packet8f>);
1187}
1188template <>
1189EIGEN_STRONG_INLINE Packet4d pmax<PropagateNumbers, Packet4d>(const Packet4d& a, const Packet4d& b) {
1190 return pminmax_propagate_numbers(a, b, pmax<Packet4d>);
1191}
1192template <>
1193EIGEN_STRONG_INLINE Packet8f pmin<PropagateNaN, Packet8f>(const Packet8f& a, const Packet8f& b) {
1194 return pminmax_propagate_nan(a, b, pmin<Packet8f>);
1195}
1196template <>
1197EIGEN_STRONG_INLINE Packet4d pmin<PropagateNaN, Packet4d>(const Packet4d& a, const Packet4d& b) {
1198 return pminmax_propagate_nan(a, b, pmin<Packet4d>);
1199}
1200template <>
1201EIGEN_STRONG_INLINE Packet8f pmax<PropagateNaN, Packet8f>(const Packet8f& a, const Packet8f& b) {
1202 return pminmax_propagate_nan(a, b, pmax<Packet8f>);
1203}
1204template <>
1205EIGEN_STRONG_INLINE Packet4d pmax<PropagateNaN, Packet4d>(const Packet4d& a, const Packet4d& b) {
1206 return pminmax_propagate_nan(a, b, pmax<Packet4d>);
1207}
1208
1209template <>
1210EIGEN_STRONG_INLINE Packet8f print<Packet8f>(const Packet8f& a) {
1211 return _mm256_round_ps(a, _MM_FROUND_CUR_DIRECTION);
1212}
1213template <>
1214EIGEN_STRONG_INLINE Packet4d print<Packet4d>(const Packet4d& a) {
1215 return _mm256_round_pd(a, _MM_FROUND_CUR_DIRECTION);
1216}
1217
1218template <>
1219EIGEN_STRONG_INLINE Packet8f pceil<Packet8f>(const Packet8f& a) {
1220 return _mm256_ceil_ps(a);
1221}
1222template <>
1223EIGEN_STRONG_INLINE Packet4d pceil<Packet4d>(const Packet4d& a) {
1224 return _mm256_ceil_pd(a);
1225}
1226
1227template <>
1228EIGEN_STRONG_INLINE Packet8f pfloor<Packet8f>(const Packet8f& a) {
1229 return _mm256_floor_ps(a);
1230}
1231template <>
1232EIGEN_STRONG_INLINE Packet4d pfloor<Packet4d>(const Packet4d& a) {
1233 return _mm256_floor_pd(a);
1234}
1235
1236template <>
1237EIGEN_STRONG_INLINE Packet8f ptrunc<Packet8f>(const Packet8f& a) {
1238 return _mm256_round_ps(a, _MM_FROUND_TRUNC);
1239}
1240template <>
1241EIGEN_STRONG_INLINE Packet4d ptrunc<Packet4d>(const Packet4d& a) {
1242 return _mm256_round_pd(a, _MM_FROUND_TRUNC);
1243}
1244
1245template <>
1246EIGEN_STRONG_INLINE Packet8i ptrue<Packet8i>(const Packet8i& a) {
1247#ifdef EIGEN_VECTORIZE_AVX2
1248 // vpcmpeqd has lower latency than the more general vcmpps
1249 return _mm256_cmpeq_epi32(a, a);
1250#else
1251 const __m256 b = _mm256_castsi256_ps(a);
1252 return _mm256_castps_si256(_mm256_cmp_ps(b, b, _CMP_TRUE_UQ));
1253#endif
1254}
1255
1256template <>
1257EIGEN_STRONG_INLINE Packet8f ptrue<Packet8f>(const Packet8f& a) {
1258#ifdef EIGEN_VECTORIZE_AVX2
1259 // vpcmpeqd has lower latency than the more general vcmpps
1260 const __m256i b = _mm256_castps_si256(a);
1261 Packet8f r = _mm256_castsi256_ps(_mm256_cmpeq_epi32(b, b));
1262 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1263 return r;
1264#else
1265 Packet8f r = _mm256_cmp_ps(a, a, _CMP_TRUE_UQ);
1266 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1267 return r;
1268#endif
1269}
1270
1271template <>
1272EIGEN_STRONG_INLINE Packet4d ptrue<Packet4d>(const Packet4d& a) {
1273#ifdef EIGEN_VECTORIZE_AVX2
1274 // vpcmpeqq has lower latency than the more general vcmppd
1275 const __m256i b = _mm256_castpd_si256(a);
1276 Packet4d r = _mm256_castsi256_pd(_mm256_cmpeq_epi64(b, b));
1277 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1278 return r;
1279#else
1280 Packet4d r = _mm256_cmp_pd(a, a, _CMP_TRUE_UQ);
1281 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1282 return r;
1283#endif
1284}
1285
1286template <>
1287EIGEN_STRONG_INLINE Packet8f pand<Packet8f>(const Packet8f& a, const Packet8f& b) {
1288 return _mm256_and_ps(a, b);
1289}
1290template <>
1291EIGEN_STRONG_INLINE Packet4d pand<Packet4d>(const Packet4d& a, const Packet4d& b) {
1292 return _mm256_and_pd(a, b);
1293}
1294template <>
1295EIGEN_STRONG_INLINE Packet8i pand<Packet8i>(const Packet8i& a, const Packet8i& b) {
1296#ifdef EIGEN_VECTORIZE_AVX2
1297 return _mm256_and_si256(a, b);
1298#else
1299 return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
1300#endif
1301}
1302template <>
1303EIGEN_STRONG_INLINE Packet8ui pand<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
1304#ifdef EIGEN_VECTORIZE_AVX2
1305 return _mm256_and_si256(a, b);
1306#else
1307 return _mm256_castps_si256(_mm256_and_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
1308#endif
1309}
1310
1311template <>
1312EIGEN_STRONG_INLINE Packet8f por<Packet8f>(const Packet8f& a, const Packet8f& b) {
1313 return _mm256_or_ps(a, b);
1314}
1315template <>
1316EIGEN_STRONG_INLINE Packet4d por<Packet4d>(const Packet4d& a, const Packet4d& b) {
1317 return _mm256_or_pd(a, b);
1318}
1319template <>
1320EIGEN_STRONG_INLINE Packet8i por<Packet8i>(const Packet8i& a, const Packet8i& b) {
1321#ifdef EIGEN_VECTORIZE_AVX2
1322 return _mm256_or_si256(a, b);
1323#else
1324 return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
1325#endif
1326}
1327template <>
1328EIGEN_STRONG_INLINE Packet8ui por<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
1329#ifdef EIGEN_VECTORIZE_AVX2
1330 return _mm256_or_si256(a, b);
1331#else
1332 return _mm256_castps_si256(_mm256_or_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
1333#endif
1334}
1335
1336template <>
1337EIGEN_STRONG_INLINE Packet8f pxor<Packet8f>(const Packet8f& a, const Packet8f& b) {
1338 return _mm256_xor_ps(a, b);
1339}
1340template <>
1341EIGEN_STRONG_INLINE Packet4d pxor<Packet4d>(const Packet4d& a, const Packet4d& b) {
1342 return _mm256_xor_pd(a, b);
1343}
1344template <>
1345EIGEN_STRONG_INLINE Packet8i pxor<Packet8i>(const Packet8i& a, const Packet8i& b) {
1346#ifdef EIGEN_VECTORIZE_AVX2
1347 return _mm256_xor_si256(a, b);
1348#else
1349 return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
1350#endif
1351}
1352template <>
1353EIGEN_STRONG_INLINE Packet8ui pxor<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
1354#ifdef EIGEN_VECTORIZE_AVX2
1355 return _mm256_xor_si256(a, b);
1356#else
1357 return _mm256_castps_si256(_mm256_xor_ps(_mm256_castsi256_ps(a), _mm256_castsi256_ps(b)));
1358#endif
1359}
1360
1361template <>
1362EIGEN_STRONG_INLINE Packet8f pandnot<Packet8f>(const Packet8f& a, const Packet8f& b) {
1363 return _mm256_andnot_ps(b, a);
1364}
1365template <>
1366EIGEN_STRONG_INLINE Packet4d pandnot<Packet4d>(const Packet4d& a, const Packet4d& b) {
1367 return _mm256_andnot_pd(b, a);
1368}
1369template <>
1370EIGEN_STRONG_INLINE Packet8i pandnot<Packet8i>(const Packet8i& a, const Packet8i& b) {
1371#ifdef EIGEN_VECTORIZE_AVX2
1372 return _mm256_andnot_si256(b, a);
1373#else
1374 return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a)));
1375#endif
1376}
1377template <>
1378EIGEN_STRONG_INLINE Packet8ui pandnot<Packet8ui>(const Packet8ui& a, const Packet8ui& b) {
1379#ifdef EIGEN_VECTORIZE_AVX2
1380 return _mm256_andnot_si256(b, a);
1381#else
1382 return _mm256_castps_si256(_mm256_andnot_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a)));
1383#endif
1384}
1385
1386template <>
1387EIGEN_STRONG_INLINE Packet8ui pcmp_lt(const Packet8ui& a, const Packet8ui& b) {
1388 return pxor(pcmp_eq(a, pmax(a, b)), ptrue(a));
1389}
1390template <>
1391EIGEN_STRONG_INLINE Packet8ui pcmp_le(const Packet8ui& a, const Packet8ui& b) {
1392 return pcmp_eq(a, pmin(a, b));
1393}
1394
1395template <>
1396EIGEN_STRONG_INLINE Packet8f pround<Packet8f>(const Packet8f& a) {
1397 const Packet8f mask = pset1frombits<Packet8f>(static_cast<numext::uint32_t>(0x80000000u));
1398 const Packet8f prev0dot5 = pset1frombits<Packet8f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
1399 return _mm256_round_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
1400}
1401template <>
1402EIGEN_STRONG_INLINE Packet4d pround<Packet4d>(const Packet4d& a) {
1403 const Packet4d mask = pset1frombits<Packet4d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
1404 const Packet4d prev0dot5 = pset1frombits<Packet4d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
1405 return _mm256_round_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
1406}
1407
1408template <>
1409EIGEN_STRONG_INLINE Packet8f pselect<Packet8f>(const Packet8f& mask, const Packet8f& a, const Packet8f& b) {
1410 return _mm256_blendv_ps(b, a, mask);
1411}
1412template <>
1413EIGEN_STRONG_INLINE Packet8i pselect<Packet8i>(const Packet8i& mask, const Packet8i& a, const Packet8i& b) {
1414 return _mm256_castps_si256(
1415 _mm256_blendv_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a), _mm256_castsi256_ps(mask)));
1416}
1417template <>
1418EIGEN_STRONG_INLINE Packet8ui pselect<Packet8ui>(const Packet8ui& mask, const Packet8ui& a, const Packet8ui& b) {
1419 return _mm256_castps_si256(
1420 _mm256_blendv_ps(_mm256_castsi256_ps(b), _mm256_castsi256_ps(a), _mm256_castsi256_ps(mask)));
1421}
1422
1423template <>
1424EIGEN_STRONG_INLINE Packet4d pselect<Packet4d>(const Packet4d& mask, const Packet4d& a, const Packet4d& b) {
1425 return _mm256_blendv_pd(b, a, mask);
1426}
1427
1428template <int N>
1429EIGEN_STRONG_INLINE Packet8i parithmetic_shift_right(const Packet8i& a) {
1430#ifdef EIGEN_VECTORIZE_AVX2
1431 return _mm256_srai_epi32(a, N);
1432#else
1433 __m128i lo = _mm_srai_epi32(_mm256_extractf128_si256(a, 0), N);
1434 __m128i hi = _mm_srai_epi32(_mm256_extractf128_si256(a, 1), N);
1435 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1436#endif
1437}
1438
1439template <int N>
1440EIGEN_STRONG_INLINE Packet8i plogical_shift_right(const Packet8i& a) {
1441#ifdef EIGEN_VECTORIZE_AVX2
1442 return _mm256_srli_epi32(a, N);
1443#else
1444 __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(a, 0), N);
1445 __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(a, 1), N);
1446 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1447#endif
1448}
1449
1450template <int N>
1451EIGEN_STRONG_INLINE Packet8i plogical_shift_left(const Packet8i& a) {
1452#ifdef EIGEN_VECTORIZE_AVX2
1453 return _mm256_slli_epi32(a, N);
1454#else
1455 __m128i lo = _mm_slli_epi32(_mm256_extractf128_si256(a, 0), N);
1456 __m128i hi = _mm_slli_epi32(_mm256_extractf128_si256(a, 1), N);
1457 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1458#endif
1459}
1460
1461template <int N>
1462EIGEN_STRONG_INLINE Packet8ui parithmetic_shift_right(const Packet8ui& a) {
1463 return (Packet8ui)plogical_shift_right<N>((Packet8i)a);
1464}
1465template <int N>
1466EIGEN_STRONG_INLINE Packet8ui plogical_shift_right(const Packet8ui& a) {
1467 return (Packet8ui)plogical_shift_right<N>((Packet8i)a);
1468}
1469template <int N>
1470EIGEN_STRONG_INLINE Packet8ui plogical_shift_left(const Packet8ui& a) {
1471 return (Packet8ui)plogical_shift_left<N>((Packet8i)a);
1472}
1473
1474template <>
1475EIGEN_STRONG_INLINE Packet8f pload<Packet8f>(const float* from) {
1476 EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ps(from);
1477}
1478template <>
1479EIGEN_STRONG_INLINE Packet4d pload<Packet4d>(const double* from) {
1480 EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_pd(from);
1481}
1482template <>
1483EIGEN_STRONG_INLINE Packet8i pload<Packet8i>(const int* from) {
1484 EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1485}
1486template <>
1487EIGEN_STRONG_INLINE Packet8ui pload<Packet8ui>(const uint32_t* from) {
1488 EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1489}
1490
1491template <>
1492EIGEN_STRONG_INLINE Packet8f ploadu<Packet8f>(const float* from) {
1493 EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_ps(from);
1494}
1495template <>
1496EIGEN_STRONG_INLINE Packet4d ploadu<Packet4d>(const double* from) {
1497 EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_pd(from);
1498}
1499template <>
1500EIGEN_STRONG_INLINE Packet8i ploadu<Packet8i>(const int* from) {
1501 EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
1502}
1503template <>
1504EIGEN_STRONG_INLINE Packet8ui ploadu<Packet8ui>(const uint32_t* from) {
1505 EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
1506}
1507
1508template <>
1509EIGEN_STRONG_INLINE Packet8f ploadu<Packet8f>(const float* from, uint8_t umask) {
1510#ifdef EIGEN_VECTORIZE_AVX512
1511 __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF);
1512 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_castps512_ps256(_mm512_maskz_loadu_ps(mask, from));
1513#else
1514 Packet8i mask = _mm256_set1_epi8(static_cast<char>(umask));
1515 const Packet8i bit_mask =
1516 _mm256_set_epi32(0xffffff7f, 0xffffffbf, 0xffffffdf, 0xffffffef, 0xfffffff7, 0xfffffffb, 0xfffffffd, 0xfffffffe);
1517 mask = por<Packet8i>(mask, bit_mask);
1518 mask = pcmp_eq<Packet8i>(mask, _mm256_set1_epi32(0xffffffff));
1519 EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_maskload_ps(from, mask);
1520#endif
1521}
1522
1523// Loads 4 floats from memory a returns the packet {a0, a0, a1, a1, a2, a2, a3, a3}
1524template <>
1525EIGEN_STRONG_INLINE Packet8f ploaddup<Packet8f>(const float* from) {
1526 // TODO try to find a way to avoid the need of a temporary register
1527 // Packet8f tmp = _mm256_castps128_ps256(_mm_loadu_ps(from));
1528 // tmp = _mm256_insertf128_ps(tmp, _mm_movehl_ps(_mm256_castps256_ps128(tmp),_mm256_castps256_ps128(tmp)), 1);
1529 // return _mm256_unpacklo_ps(tmp,tmp);
1530
1531 // _mm256_insertf128_ps is very slow on Haswell, thus:
1532 Packet8f tmp = _mm256_broadcast_ps((const __m128*)(const void*)from);
1533 // mimic an "inplace" permutation of the lower 128bits using a blend
1534 tmp = _mm256_blend_ps(
1535 tmp, _mm256_castps128_ps256(_mm_permute_ps(_mm256_castps256_ps128(tmp), _MM_SHUFFLE(1, 0, 1, 0))), 15);
1536 // then we can perform a consistent permutation on the global register to get everything in shape:
1537 return _mm256_permute_ps(tmp, _MM_SHUFFLE(3, 3, 2, 2));
1538}
1539// Loads 2 doubles from memory a returns the packet {a0, a0, a1, a1}
1540template <>
1541EIGEN_STRONG_INLINE Packet4d ploaddup<Packet4d>(const double* from) {
1542 Packet4d tmp = _mm256_broadcast_pd((const __m128d*)(const void*)from);
1543 return _mm256_permute_pd(tmp, 3 << 2);
1544}
1545// Loads 4 integers from memory a returns the packet {a0, a0, a1, a1, a2, a2, a3, a3}
1546template <>
1547EIGEN_STRONG_INLINE Packet8i ploaddup<Packet8i>(const int* from) {
1548#ifdef EIGEN_VECTORIZE_AVX2
1549 const Packet8i a = _mm256_castsi128_si256(ploadu<Packet4i>(from));
1550 return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3));
1551#else
1552 __m256 tmp = _mm256_broadcast_ps((const __m128*)(const void*)from);
1553 // mimic an "inplace" permutation of the lower 128bits using a blend
1554 tmp = _mm256_blend_ps(
1555 tmp, _mm256_castps128_ps256(_mm_permute_ps(_mm256_castps256_ps128(tmp), _MM_SHUFFLE(1, 0, 1, 0))), 15);
1556 // then we can perform a consistent permutation on the global register to get everything in shape:
1557 return _mm256_castps_si256(_mm256_permute_ps(tmp, _MM_SHUFFLE(3, 3, 2, 2)));
1558#endif
1559}
1560template <>
1561EIGEN_STRONG_INLINE Packet8ui ploaddup<Packet8ui>(const uint32_t* from) {
1562#ifdef EIGEN_VECTORIZE_AVX2
1563 const Packet8ui a = _mm256_castsi128_si256(ploadu<Packet4ui>(from));
1564 return _mm256_permutevar8x32_epi32(a, _mm256_setr_epi32(0, 0, 1, 1, 2, 2, 3, 3));
1565#else
1566 __m256 tmp = _mm256_broadcast_ps((const __m128*)(const void*)from);
1567 // mimic an "inplace" permutation of the lower 128bits using a blend
1568 tmp = _mm256_blend_ps(
1569 tmp, _mm256_castps128_ps256(_mm_permute_ps(_mm256_castps256_ps128(tmp), _MM_SHUFFLE(1, 0, 1, 0))), 15);
1570 // then we can perform a consistent permutation on the global register to get
1571 // everything in shape:
1572 return _mm256_castps_si256(_mm256_permute_ps(tmp, _MM_SHUFFLE(3, 3, 2, 2)));
1573#endif
1574}
1575
1576// Loads 2 floats from memory a returns the packet {a0, a0, a0, a0, a1, a1, a1, a1}
1577template <>
1578EIGEN_STRONG_INLINE Packet8f ploadquad<Packet8f>(const float* from) {
1579 Packet8f tmp = _mm256_castps128_ps256(_mm_broadcast_ss(from));
1580 return _mm256_insertf128_ps(tmp, _mm_broadcast_ss(from + 1), 1);
1581}
1582template <>
1583EIGEN_STRONG_INLINE Packet8i ploadquad<Packet8i>(const int* from) {
1584 return _mm256_insertf128_si256(_mm256_set1_epi32(*from), _mm_set1_epi32(*(from + 1)), 1);
1585}
1586template <>
1587EIGEN_STRONG_INLINE Packet8ui ploadquad<Packet8ui>(const uint32_t* from) {
1588 return _mm256_insertf128_si256(_mm256_set1_epi32(*from), _mm_set1_epi32(*(from + 1)), 1);
1589}
1590
1591template <>
1592EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet8f& from) {
1593 EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ps(to, from);
1594}
1595template <>
1596EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet4d& from) {
1597 EIGEN_DEBUG_ALIGNED_STORE _mm256_store_pd(to, from);
1598}
1599template <>
1600EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet8i& from) {
1601 EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
1602}
1603template <>
1604EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet8ui& from) {
1605 EIGEN_DEBUG_ALIGNED_STORE _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
1606}
1607
1608template <>
1609EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet8f& from) {
1610 EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ps(to, from);
1611}
1612template <>
1613EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet4d& from) {
1614 EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_pd(to, from);
1615}
1616template <>
1617EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet8i& from) {
1618 EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
1619}
1620template <>
1621EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet8ui& from) {
1622 EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
1623}
1624
1625template <>
1626EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet8f& from, uint8_t umask) {
1627#ifdef EIGEN_VECTORIZE_AVX512
1628 __mmask16 mask = static_cast<__mmask16>(umask & 0x00FF);
1629 EIGEN_DEBUG_UNALIGNED_STORE _mm512_mask_storeu_ps(to, mask, _mm512_castps256_ps512(from));
1630#else
1631 Packet8i mask = _mm256_set1_epi8(static_cast<char>(umask));
1632 const Packet8i bit_mask =
1633 _mm256_set_epi32(0x7f7f7f7f, 0xbfbfbfbf, 0xdfdfdfdf, 0xefefefef, 0xf7f7f7f7, 0xfbfbfbfb, 0xfdfdfdfd, 0xfefefefe);
1634 mask = por<Packet8i>(mask, bit_mask);
1635 mask = pcmp_eq<Packet8i>(mask, _mm256_set1_epi32(0xffffffff));
1636#if EIGEN_COMP_MSVC
1637 // MSVC sometimes seems to use a bogus mask with maskstore.
1638 const __m256i ifrom = _mm256_castps_si256(from);
1639 EIGEN_DEBUG_UNALIGNED_STORE _mm_maskmoveu_si128(_mm256_extractf128_si256(ifrom, 0), _mm256_extractf128_si256(mask, 0),
1640 reinterpret_cast<char*>(to));
1641 EIGEN_DEBUG_UNALIGNED_STORE _mm_maskmoveu_si128(_mm256_extractf128_si256(ifrom, 1), _mm256_extractf128_si256(mask, 1),
1642 reinterpret_cast<char*>(to + 4));
1643#else
1644 EIGEN_DEBUG_UNALIGNED_STORE _mm256_maskstore_ps(to, mask, from);
1645#endif
1646#endif
1647}
1648
1649// NOTE: leverage _mm256_i32gather_ps and _mm256_i32gather_pd if AVX2 instructions are available
1650// NOTE: for the record the following seems to be slower: return _mm256_i32gather_ps(from, _mm256_set1_epi32(stride),
1651// 4);
1652template <>
1653EIGEN_DEVICE_FUNC inline Packet8f pgather<float, Packet8f>(const float* from, Index stride) {
1654 if (stride == 2) {
1655#ifdef EIGEN_VECTORIZE_AVX2
1656 // Both loads stay within from[0..14], including when from points to an imaginary component.
1657 const Packet8f low = _mm256_loadu_ps(from);
1658 const Packet8f high = _mm256_loadu_ps(from + 7);
1659 const Packet8f interleaved = _mm256_shuffle_ps(low, high, _MM_SHUFFLE(3, 1, 2, 0));
1660 return _mm256_castpd_ps(_mm256_permute4x64_pd(_mm256_castps_pd(interleaved), _MM_SHUFFLE(3, 1, 2, 0)));
1661#else
1662 const Packet4f low = pgather<float, Packet4f>(from, 2);
1663 const Packet4f high = pgather<float, Packet4f>(from + 8, 2);
1664 return _mm256_insertf128_ps(_mm256_castps128_ps256(low), high, 1);
1665#endif
1666 }
1667 return _mm256_set_ps(from[7 * stride], from[6 * stride], from[5 * stride], from[4 * stride], from[3 * stride],
1668 from[2 * stride], from[1 * stride], from[0 * stride]);
1669}
1670template <>
1671EIGEN_DEVICE_FUNC inline Packet4d pgather<double, Packet4d>(const double* from, Index stride) {
1672#ifdef EIGEN_VECTORIZE_AVX2
1673 if (stride == 2) {
1674 const Packet4d low = _mm256_loadu_pd(from);
1675 const Packet4d high = _mm256_loadu_pd(from + 3);
1676 const Packet4d interleaved = _mm256_shuffle_pd(low, high, 0xa);
1677 return _mm256_permute4x64_pd(interleaved, _MM_SHUFFLE(3, 1, 2, 0));
1678 }
1679#endif
1680 return _mm256_set_pd(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]);
1681}
1682template <>
1683EIGEN_DEVICE_FUNC inline Packet8i pgather<int, Packet8i>(const int* from, Index stride) {
1684 return _mm256_set_epi32(from[7 * stride], from[6 * stride], from[5 * stride], from[4 * stride], from[3 * stride],
1685 from[2 * stride], from[1 * stride], from[0 * stride]);
1686}
1687template <>
1688EIGEN_DEVICE_FUNC inline Packet8ui pgather<uint32_t, Packet8ui>(const uint32_t* from, Index stride) {
1689 return (Packet8ui)pgather<int, Packet8i>((int*)from, stride);
1690}
1691
1692template <>
1693EIGEN_DEVICE_FUNC inline void pscatter<float, Packet8f>(float* to, const Packet8f& from, Index stride) {
1694 __m128 low = _mm256_extractf128_ps(from, 0);
1695 to[stride * 0] = _mm_cvtss_f32(low);
1696 to[stride * 1] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 1));
1697 to[stride * 2] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 2));
1698 to[stride * 3] = _mm_cvtss_f32(_mm_shuffle_ps(low, low, 3));
1699
1700 __m128 high = _mm256_extractf128_ps(from, 1);
1701 to[stride * 4] = _mm_cvtss_f32(high);
1702 to[stride * 5] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 1));
1703 to[stride * 6] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 2));
1704 to[stride * 7] = _mm_cvtss_f32(_mm_shuffle_ps(high, high, 3));
1705}
1706template <>
1707EIGEN_DEVICE_FUNC inline void pscatter<double, Packet4d>(double* to, const Packet4d& from, Index stride) {
1708 __m128d low = _mm256_extractf128_pd(from, 0);
1709 to[stride * 0] = _mm_cvtsd_f64(low);
1710 to[stride * 1] = _mm_cvtsd_f64(_mm_shuffle_pd(low, low, 1));
1711 __m128d high = _mm256_extractf128_pd(from, 1);
1712 to[stride * 2] = _mm_cvtsd_f64(high);
1713 to[stride * 3] = _mm_cvtsd_f64(_mm_shuffle_pd(high, high, 1));
1714}
1715template <>
1716EIGEN_DEVICE_FUNC inline void pscatter<int, Packet8i>(int* to, const Packet8i& from, Index stride) {
1717 __m128i low = _mm256_extractf128_si256(from, 0);
1718 to[stride * 0] = _mm_extract_epi32(low, 0);
1719 to[stride * 1] = _mm_extract_epi32(low, 1);
1720 to[stride * 2] = _mm_extract_epi32(low, 2);
1721 to[stride * 3] = _mm_extract_epi32(low, 3);
1722
1723 __m128i high = _mm256_extractf128_si256(from, 1);
1724 to[stride * 4] = _mm_extract_epi32(high, 0);
1725 to[stride * 5] = _mm_extract_epi32(high, 1);
1726 to[stride * 6] = _mm_extract_epi32(high, 2);
1727 to[stride * 7] = _mm_extract_epi32(high, 3);
1728}
1729template <>
1730EIGEN_DEVICE_FUNC inline void pscatter<uint32_t, Packet8ui>(uint32_t* to, const Packet8ui& from, Index stride) {
1731 pscatter<int, Packet8i>((int*)to, (Packet8i)from, stride);
1732}
1733
1734template <>
1735EIGEN_STRONG_INLINE void pstore1<Packet8f>(float* to, const float& a) {
1736 Packet8f pa = pset1<Packet8f>(a);
1737 pstore(to, pa);
1738}
1739template <>
1740EIGEN_STRONG_INLINE void pstore1<Packet4d>(double* to, const double& a) {
1741 Packet4d pa = pset1<Packet4d>(a);
1742 pstore(to, pa);
1743}
1744template <>
1745EIGEN_STRONG_INLINE void pstore1<Packet8i>(int* to, const int& a) {
1746 Packet8i pa = pset1<Packet8i>(a);
1747 pstore(to, pa);
1748}
1749
1750#ifndef EIGEN_VECTORIZE_AVX512
1751template <>
1752EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1753 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1754}
1755template <>
1756EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1757 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1758}
1759template <>
1760EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
1761 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1762}
1763template <>
1764EIGEN_STRONG_INLINE void prefetch<uint32_t>(const uint32_t* addr) {
1765 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1766}
1767#endif
1768
1769template <>
1770EIGEN_STRONG_INLINE float pfirst<Packet8f>(const Packet8f& a) {
1771 return _mm_cvtss_f32(_mm256_castps256_ps128(a));
1772}
1773template <>
1774EIGEN_STRONG_INLINE double pfirst<Packet4d>(const Packet4d& a) {
1775 return _mm_cvtsd_f64(_mm256_castpd256_pd128(a));
1776}
1777template <>
1778EIGEN_STRONG_INLINE int pfirst<Packet8i>(const Packet8i& a) {
1779 return _mm_cvtsi128_si32(_mm256_castsi256_si128(a));
1780}
1781template <>
1782EIGEN_STRONG_INLINE uint32_t pfirst<Packet8ui>(const Packet8ui& a) {
1783 return numext::bit_cast<uint32_t>(_mm_cvtsi128_si32(_mm256_castsi256_si128(a)));
1784}
1785
1786template <>
1787EIGEN_STRONG_INLINE Packet8f preverse(const Packet8f& a) {
1788 __m256 tmp = _mm256_shuffle_ps(a, a, 0x1b);
1789 return _mm256_permute2f128_ps(tmp, tmp, 1);
1790}
1791template <>
1792EIGEN_STRONG_INLINE Packet4d preverse(const Packet4d& a) {
1793 __m256d tmp = _mm256_shuffle_pd(a, a, 5);
1794 return _mm256_permute2f128_pd(tmp, tmp, 1);
1795}
1796template <>
1797EIGEN_STRONG_INLINE Packet8i preverse(const Packet8i& a) {
1798 return _mm256_castps_si256(preverse(_mm256_castsi256_ps(a)));
1799}
1800template <>
1801EIGEN_STRONG_INLINE Packet8ui preverse(const Packet8ui& a) {
1802 return _mm256_castps_si256(preverse(_mm256_castsi256_ps(a)));
1803}
1804
1805#ifdef EIGEN_VECTORIZE_AVX2
1806template <>
1807EIGEN_STRONG_INLINE Packet4l preverse(const Packet4l& a) {
1808 return _mm256_castpd_si256(preverse(_mm256_castsi256_pd(a)));
1809}
1810template <>
1811EIGEN_STRONG_INLINE Packet4ul preverse(const Packet4ul& a) {
1812 return _mm256_castpd_si256(preverse(_mm256_castsi256_pd(a)));
1813}
1814#endif
1815
1816// pabs should be ok
1817template <>
1818EIGEN_STRONG_INLINE Packet8f pabs(const Packet8f& a) {
1819 const Packet8f mask = _mm256_castsi256_ps(_mm256_set1_epi32(0x7FFFFFFF));
1820 return _mm256_and_ps(a, mask);
1821}
1822template <>
1823EIGEN_STRONG_INLINE Packet4d pabs(const Packet4d& a) {
1824 const Packet4d mask = _mm256_castsi256_pd(_mm256_set1_epi64x(0x7FFFFFFFFFFFFFFF));
1825 return _mm256_and_pd(a, mask);
1826}
1827template <>
1828EIGEN_STRONG_INLINE Packet8i pabs(const Packet8i& a) {
1829#ifdef EIGEN_VECTORIZE_AVX2
1830 return _mm256_abs_epi32(a);
1831#else
1832 __m128i lo = _mm_abs_epi32(_mm256_extractf128_si256(a, 0));
1833 __m128i hi = _mm_abs_epi32(_mm256_extractf128_si256(a, 1));
1834 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), (hi), 1);
1835#endif
1836}
1837#ifndef EIGEN_VECTORIZE_AVX512FP16
1838template <>
1839EIGEN_STRONG_INLINE Packet8h psignbit(const Packet8h& a) {
1840 return _mm_cmpgt_epi16(_mm_setzero_si128(), a);
1841}
1842#endif // EIGEN_VECTORIZE_AVX512FP16
1843
1844template <>
1845EIGEN_STRONG_INLINE Packet8bf psignbit(const Packet8bf& a) {
1846 return _mm_cmpgt_epi16(_mm_setzero_si128(), a);
1847}
1848template <>
1849EIGEN_STRONG_INLINE Packet8f psignbit(const Packet8f& a) {
1850#ifdef EIGEN_VECTORIZE_AVX2
1851 return _mm256_castsi256_ps(_mm256_cmpgt_epi32(_mm256_setzero_si256(), _mm256_castps_si256(a)));
1852#else
1853 return _mm256_castsi256_ps(parithmetic_shift_right<31>(Packet8i(_mm256_castps_si256(a))));
1854#endif
1855}
1856#ifdef EIGEN_VECTORIZE_AVX2
1857template <>
1858EIGEN_STRONG_INLINE Packet4d psignbit(const Packet4d& a) {
1859 return _mm256_castsi256_pd(_mm256_cmpgt_epi64(_mm256_setzero_si256(), _mm256_castpd_si256(a)));
1860}
1861#endif
1862
1863template <>
1864EIGEN_STRONG_INLINE Packet8f pfrexp<Packet8f>(const Packet8f& a, Packet8f& exponent) {
1865 return pfrexp_generic(a, exponent);
1866}
1867
1868// Extract exponent without existence of Packet4l.
1869template <>
1870EIGEN_STRONG_INLINE Packet4d pfrexp_generic_get_biased_exponent(const Packet4d& a) {
1871 const Packet4d cst_exp_mask = pset1frombits<Packet4d>(static_cast<uint64_t>(0x7ff0000000000000ull));
1872 __m256i a_expo = _mm256_castpd_si256(pand(a, cst_exp_mask));
1873#ifdef EIGEN_VECTORIZE_AVX2
1874 a_expo = _mm256_srli_epi64(a_expo, 52);
1875 __m128i lo = _mm256_extractf128_si256(a_expo, 0);
1876 __m128i hi = _mm256_extractf128_si256(a_expo, 1);
1877#else
1878 __m128i lo = _mm256_extractf128_si256(a_expo, 0);
1879 __m128i hi = _mm256_extractf128_si256(a_expo, 1);
1880 lo = _mm_srli_epi64(lo, 52);
1881 hi = _mm_srli_epi64(hi, 52);
1882#endif
1883 Packet2d exponent_lo = _mm_cvtepi32_pd(vec4i_swizzle1(lo, 0, 2, 1, 3));
1884 Packet2d exponent_hi = _mm_cvtepi32_pd(vec4i_swizzle1(hi, 0, 2, 1, 3));
1885 Packet4d exponent = _mm256_insertf128_pd(_mm256_setzero_pd(), exponent_lo, 0);
1886 exponent = _mm256_insertf128_pd(exponent, exponent_hi, 1);
1887 return exponent;
1888}
1889
1890template <>
1891EIGEN_STRONG_INLINE Packet4d pfrexp<Packet4d>(const Packet4d& a, Packet4d& exponent) {
1892 return pfrexp_generic(a, exponent);
1893}
1894
1895#ifndef EIGEN_VECTORIZE_AVX2
1896namespace unary_pow {
1897// The exponent-bit operations the integer-exponent pow needs on Packet4d, likewise without Packet4l: AVX has no
1898// 256-bit integer shift or subtract, but SSE2's 128-bit ones, present on every x86-64 target, suffice. Split the
1899// bit pattern into its two 128-bit halves, operate on each with SSE2, and reassemble.
1900template <>
1901EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4d exponent_bits_shift_right<Packet4d>(const Packet4d& bits) {
1902 __m256i i = _mm256_castpd_si256(bits);
1903 __m128i lo = _mm_srli_epi64(_mm256_extractf128_si256(i, 0), 52);
1904 __m128i hi = _mm_srli_epi64(_mm256_extractf128_si256(i, 1), 52);
1905 return _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1));
1906}
1907template <>
1908EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet4d exponent_bits_sub<Packet4d>(const Packet4d& a_bits,
1909 const Packet4d& b_bits) {
1910 __m256i ai = _mm256_castpd_si256(a_bits);
1911 __m256i bi = _mm256_castpd_si256(b_bits);
1912 __m128i lo = _mm_sub_epi64(_mm256_extractf128_si256(ai, 0), _mm256_extractf128_si256(bi, 0));
1913 __m128i hi = _mm_sub_epi64(_mm256_extractf128_si256(ai, 1), _mm256_extractf128_si256(bi, 1));
1914 return _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1));
1915}
1916template <>
1917struct has_exponent_bit_ops<Packet4d> : true_type {};
1918} // namespace unary_pow
1919#endif
1920
1921template <>
1922EIGEN_STRONG_INLINE Packet8f pldexp<Packet8f>(const Packet8f& a, const Packet8f& exponent) {
1923 return pldexp_generic(a, exponent);
1924}
1925
1926// Build 2^k as Packet4d from a Packet4i holding the biased int32 exponent in
1927// each lane, shifted by Shift: 52 for the biased exponent, 51 for twice it.
1928// AVX2 has a single-instruction widen+shift path; AVX-only must split the
1929// 128-bit input into two halves, widen+shift each separately with SSE
1930// intrinsics, and reassemble with vinsertf128.
1931template <int Shift = 52>
1932EIGEN_STRONG_INLINE Packet4d pldexp_avx_pow2_from_biased(const Packet4i& biased) {
1933#ifdef EIGEN_VECTORIZE_AVX2
1934 return _mm256_castsi256_pd(_mm256_slli_epi64(_mm256_cvtepi32_epi64(biased), Shift));
1935#else
1936 __m128i lo = _mm_cvtepi32_epi64(biased); // SSE4.1: lower 2 int32 -> 2 int64
1937 __m128i hi = _mm_cvtepi32_epi64(_mm_unpackhi_epi64(biased, biased)); // upper 2 int32 -> 2 int64
1938 lo = _mm_slli_epi64(lo, Shift);
1939 hi = _mm_slli_epi64(hi, Shift);
1940 return _mm256_castsi256_pd(_mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1));
1941#endif
1942}
1943
1944template <>
1945EIGEN_STRONG_INLINE Packet4d pldexp<Packet4d>(const Packet4d& a, const Packet4d& exponent) {
1946 // Clamp exponent to [-2099, 2099]
1947 const Packet4d max_exponent = pset1<Packet4d>(2099.0);
1948 const Packet4i e = _mm256_cvttpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
1949
1950 // The single-rounding split of pldexp_generic.
1951 const Packet4i bias = pset1<Packet4i>(1023);
1952 const Packet4i b = pmin(pmax(e, pset1<Packet4i>(-1022)), pset1<Packet4i>(1022));
1953 const Packet4i t = pandnot(psub(e, b), pset1<Packet4i>(1)); // even
1954 const Packet4d c1 = pldexp_avx_pow2_from_biased<51>(padd(t, padd(bias, bias))); // 2^(t/2)
1955 const Packet4d c2 = pldexp_avx_pow2_from_biased(padd(psub(e, t), bias)); // 2^(e-t)
1956
1957 return pldexp_apply_factors(a, c1, c2); // a * 2^e
1958}
1959
1960template <>
1961EIGEN_STRONG_INLINE Packet4d pldexp_fast<Packet4d>(const Packet4d& a, const Packet4d& exponent) {
1962 // Clamp exponent to [-1024, 1024]
1963 const Packet4d min_exponent = pset1<Packet4d>(-1023.0);
1964 const Packet4d max_exponent = pset1<Packet4d>(1024.0);
1965 const Packet4i e = _mm256_cvtpd_epi32(pmin(pmax(exponent, min_exponent), max_exponent));
1966 const Packet4i bias = pset1<Packet4i>(1023);
1967
1968 return pmul(a, pldexp_avx_pow2_from_biased(padd(e, bias))); // a * 2^e
1969}
1970
1971template <>
1972EIGEN_STRONG_INLINE Packet4f predux_half<Packet8f>(const Packet8f& a) {
1973 return _mm_add_ps(_mm256_castps256_ps128(a), _mm256_extractf128_ps(a, 1));
1974}
1975template <>
1976EIGEN_STRONG_INLINE Packet4i predux_half<Packet8i>(const Packet8i& a) {
1977 return _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));
1978}
1979template <>
1980EIGEN_STRONG_INLINE Packet4ui predux_half<Packet8ui>(const Packet8ui& a) {
1981 return _mm_add_epi32(_mm256_castsi256_si128(a), _mm256_extractf128_si256(a, 1));
1982}
1983
1984EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8f, 8>& kernel) {
1985 __m256 T0 = _mm256_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1986 __m256 T1 = _mm256_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1987 __m256 T2 = _mm256_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1988 __m256 T3 = _mm256_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1989 __m256 T4 = _mm256_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1990 __m256 T5 = _mm256_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1991 __m256 T6 = _mm256_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1992 __m256 T7 = _mm256_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1993 __m256 S0 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1994 __m256 S1 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1995 __m256 S2 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1996 __m256 S3 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1997 __m256 S4 = _mm256_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1998 __m256 S5 = _mm256_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1999 __m256 S6 = _mm256_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
2000 __m256 S7 = _mm256_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
2001 kernel.packet[0] = _mm256_permute2f128_ps(S0, S4, 0x20);
2002 kernel.packet[1] = _mm256_permute2f128_ps(S1, S5, 0x20);
2003 kernel.packet[2] = _mm256_permute2f128_ps(S2, S6, 0x20);
2004 kernel.packet[3] = _mm256_permute2f128_ps(S3, S7, 0x20);
2005 kernel.packet[4] = _mm256_permute2f128_ps(S0, S4, 0x31);
2006 kernel.packet[5] = _mm256_permute2f128_ps(S1, S5, 0x31);
2007 kernel.packet[6] = _mm256_permute2f128_ps(S2, S6, 0x31);
2008 kernel.packet[7] = _mm256_permute2f128_ps(S3, S7, 0x31);
2009}
2010
2011EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8f, 4>& kernel) {
2012 __m256 T0 = _mm256_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
2013 __m256 T1 = _mm256_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
2014 __m256 T2 = _mm256_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
2015 __m256 T3 = _mm256_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
2016
2017 __m256 S0 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
2018 __m256 S1 = _mm256_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
2019 __m256 S2 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
2020 __m256 S3 = _mm256_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
2021
2022 kernel.packet[0] = _mm256_permute2f128_ps(S0, S1, 0x20);
2023 kernel.packet[1] = _mm256_permute2f128_ps(S2, S3, 0x20);
2024 kernel.packet[2] = _mm256_permute2f128_ps(S0, S1, 0x31);
2025 kernel.packet[3] = _mm256_permute2f128_ps(S2, S3, 0x31);
2026}
2027
2028#define MM256_SHUFFLE_EPI32(A, B, M) \
2029 _mm256_castps_si256(_mm256_shuffle_ps(_mm256_castsi256_ps(A), _mm256_castsi256_ps(B), M))
2030
2031#ifndef EIGEN_VECTORIZE_AVX2
2032#define MM256_UNPACKLO_EPI32(A, B) \
2033 _mm256_castps_si256(_mm256_unpacklo_ps(_mm256_castsi256_ps(A), _mm256_castsi256_ps(B)))
2034#define MM256_UNPACKHI_EPI32(A, B) \
2035 _mm256_castps_si256(_mm256_unpackhi_ps(_mm256_castsi256_ps(A), _mm256_castsi256_ps(B)))
2036#else
2037#define MM256_UNPACKLO_EPI32(A, B) _mm256_unpacklo_epi32(A, B)
2038#define MM256_UNPACKHI_EPI32(A, B) _mm256_unpackhi_epi32(A, B)
2039#endif
2040
2041EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8i, 8>& kernel) {
2042 __m256i T0 = MM256_UNPACKLO_EPI32(kernel.packet[0], kernel.packet[1]);
2043 __m256i T1 = MM256_UNPACKHI_EPI32(kernel.packet[0], kernel.packet[1]);
2044 __m256i T2 = MM256_UNPACKLO_EPI32(kernel.packet[2], kernel.packet[3]);
2045 __m256i T3 = MM256_UNPACKHI_EPI32(kernel.packet[2], kernel.packet[3]);
2046 __m256i T4 = MM256_UNPACKLO_EPI32(kernel.packet[4], kernel.packet[5]);
2047 __m256i T5 = MM256_UNPACKHI_EPI32(kernel.packet[4], kernel.packet[5]);
2048 __m256i T6 = MM256_UNPACKLO_EPI32(kernel.packet[6], kernel.packet[7]);
2049 __m256i T7 = MM256_UNPACKHI_EPI32(kernel.packet[6], kernel.packet[7]);
2050 __m256i S0 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
2051 __m256i S1 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
2052 __m256i S2 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
2053 __m256i S3 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
2054 __m256i S4 = MM256_SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
2055 __m256i S5 = MM256_SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
2056 __m256i S6 = MM256_SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
2057 __m256i S7 = MM256_SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
2058 kernel.packet[0] = _mm256_permute2f128_si256(S0, S4, 0x20);
2059 kernel.packet[1] = _mm256_permute2f128_si256(S1, S5, 0x20);
2060 kernel.packet[2] = _mm256_permute2f128_si256(S2, S6, 0x20);
2061 kernel.packet[3] = _mm256_permute2f128_si256(S3, S7, 0x20);
2062 kernel.packet[4] = _mm256_permute2f128_si256(S0, S4, 0x31);
2063 kernel.packet[5] = _mm256_permute2f128_si256(S1, S5, 0x31);
2064 kernel.packet[6] = _mm256_permute2f128_si256(S2, S6, 0x31);
2065 kernel.packet[7] = _mm256_permute2f128_si256(S3, S7, 0x31);
2066}
2067EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8ui, 8>& kernel) {
2068 ptranspose((PacketBlock<Packet8i, 8>&)kernel);
2069}
2070
2071EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8i, 4>& kernel) {
2072 __m256i T0 = MM256_UNPACKLO_EPI32(kernel.packet[0], kernel.packet[1]);
2073 __m256i T1 = MM256_UNPACKHI_EPI32(kernel.packet[0], kernel.packet[1]);
2074 __m256i T2 = MM256_UNPACKLO_EPI32(kernel.packet[2], kernel.packet[3]);
2075 __m256i T3 = MM256_UNPACKHI_EPI32(kernel.packet[2], kernel.packet[3]);
2076
2077 __m256i S0 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
2078 __m256i S1 = MM256_SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
2079 __m256i S2 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
2080 __m256i S3 = MM256_SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
2081
2082 kernel.packet[0] = _mm256_permute2f128_si256(S0, S1, 0x20);
2083 kernel.packet[1] = _mm256_permute2f128_si256(S2, S3, 0x20);
2084 kernel.packet[2] = _mm256_permute2f128_si256(S0, S1, 0x31);
2085 kernel.packet[3] = _mm256_permute2f128_si256(S2, S3, 0x31);
2086}
2087EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8ui, 4>& kernel) {
2088 ptranspose((PacketBlock<Packet8i, 4>&)kernel);
2089}
2090
2091EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet4d, 4>& kernel) {
2092 __m256d T0 = _mm256_shuffle_pd(kernel.packet[0], kernel.packet[1], 15);
2093 __m256d T1 = _mm256_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
2094 __m256d T2 = _mm256_shuffle_pd(kernel.packet[2], kernel.packet[3], 15);
2095 __m256d T3 = _mm256_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
2096
2097 kernel.packet[1] = _mm256_permute2f128_pd(T0, T2, 32);
2098 kernel.packet[3] = _mm256_permute2f128_pd(T0, T2, 49);
2099 kernel.packet[0] = _mm256_permute2f128_pd(T1, T3, 32);
2100 kernel.packet[2] = _mm256_permute2f128_pd(T1, T3, 49);
2101}
2102
2103// Packet math for Eigen::half
2104#ifndef EIGEN_VECTORIZE_AVX512FP16
2105template <>
2106struct unpacket_traits<Packet8h> {
2107 using type = Eigen::half;
2108 enum {
2109 size = 8,
2110 alignment = Aligned16,
2111 vectorizable = true,
2112 masked_load_available = false,
2113 masked_store_available = false
2114 };
2115 using half = Packet8h;
2116};
2117
2118template <>
2119EIGEN_STRONG_INLINE Packet8h pset1<Packet8h>(const Eigen::half& from) {
2120 return _mm_set1_epi16(numext::bit_cast<numext::uint16_t>(from));
2121}
2122
2123template <>
2124EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8h>(const Packet8h& from) {
2125 return numext::bit_cast<Eigen::half>(static_cast<numext::uint16_t>(_mm_extract_epi16(from, 0)));
2126}
2127
2128template <>
2129EIGEN_STRONG_INLINE Packet8h pload<Packet8h>(const Eigen::half* from) {
2130 return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
2131}
2132
2133template <>
2134EIGEN_STRONG_INLINE Packet8h ploadu<Packet8h>(const Eigen::half* from) {
2135 return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
2136}
2137
2138template <>
2139EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet8h& from) {
2140 _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
2141}
2142
2143template <>
2144EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet8h& from) {
2145 _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
2146}
2147
2148template <>
2149EIGEN_STRONG_INLINE Packet8h ploaddup<Packet8h>(const Eigen::half* from) {
2150 const numext::uint16_t a = numext::bit_cast<numext::uint16_t>(from[0]);
2151 const numext::uint16_t b = numext::bit_cast<numext::uint16_t>(from[1]);
2152 const numext::uint16_t c = numext::bit_cast<numext::uint16_t>(from[2]);
2153 const numext::uint16_t d = numext::bit_cast<numext::uint16_t>(from[3]);
2154 return _mm_set_epi16(d, d, c, c, b, b, a, a);
2155}
2156
2157template <>
2158EIGEN_STRONG_INLINE Packet8h ploadquad<Packet8h>(const Eigen::half* from) {
2159 const numext::uint16_t a = numext::bit_cast<numext::uint16_t>(from[0]);
2160 const numext::uint16_t b = numext::bit_cast<numext::uint16_t>(from[1]);
2161 return _mm_set_epi16(b, b, b, b, a, a, a, a);
2162}
2163
2164template <>
2165EIGEN_STRONG_INLINE Packet8h ptrue(const Packet8h& a) {
2166 return _mm_cmpeq_epi32(a, a);
2167}
2168
2169template <>
2170EIGEN_STRONG_INLINE Packet8h pabs(const Packet8h& a) {
2171 const __m128i sign_mask = _mm_set1_epi16(static_cast<short>(0x8000u));
2172 return _mm_andnot_si128(sign_mask, a);
2173}
2174
2175EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) {
2176#ifdef EIGEN_HAS_FP16_C
2177 return _mm256_cvtph_ps(a);
2178#else
2179 Eigen::internal::Packet8f pp = _mm256_castsi256_ps(
2180 _mm256_insertf128_si256(_mm256_castsi128_si256(half2floatsse(a)), half2floatsse(_mm_srli_si128(a, 8)), 1));
2181 return pp;
2182#endif
2183}
2184
2185EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) {
2186#ifdef EIGEN_HAS_FP16_C
2187 return _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT);
2188#else
2189 __m128i lo = float2half(_mm256_extractf128_ps(a, 0));
2190 __m128i hi = float2half(_mm256_extractf128_ps(a, 1));
2191 return _mm_packus_epi32(lo, hi);
2192#endif
2193}
2194
2195template <>
2196EIGEN_STRONG_INLINE Packet8h pmin<Packet8h>(const Packet8h& a, const Packet8h& b) {
2197 return float2half(pmin<Packet8f>(half2float(a), half2float(b)));
2198}
2199
2200template <>
2201EIGEN_STRONG_INLINE Packet8h pmax<Packet8h>(const Packet8h& a, const Packet8h& b) {
2202 return float2half(pmax<Packet8f>(half2float(a), half2float(b)));
2203}
2204
2205template <>
2206EIGEN_STRONG_INLINE Packet8h plset<Packet8h>(const half& a) {
2207 return float2half(plset<Packet8f>(static_cast<float>(a)));
2208}
2209
2210template <>
2211EIGEN_STRONG_INLINE Packet8h por(const Packet8h& a, const Packet8h& b) {
2212 // in some cases Packet4i is a wrapper around __m128i, so we either need to
2213 // cast to Packet4i to directly call the intrinsics as below:
2214 return _mm_or_si128(a, b);
2215}
2216template <>
2217EIGEN_STRONG_INLINE Packet8h pxor(const Packet8h& a, const Packet8h& b) {
2218 return _mm_xor_si128(a, b);
2219}
2220template <>
2221EIGEN_STRONG_INLINE Packet8h pand(const Packet8h& a, const Packet8h& b) {
2222 return _mm_and_si128(a, b);
2223}
2224template <>
2225EIGEN_STRONG_INLINE Packet8h pandnot(const Packet8h& a, const Packet8h& b) {
2226 return _mm_andnot_si128(b, a);
2227}
2228
2229template <>
2230EIGEN_STRONG_INLINE Packet8h pselect(const Packet8h& mask, const Packet8h& a, const Packet8h& b) {
2231 return _mm_blendv_epi8(b, a, mask);
2232}
2233
2234template <>
2235EIGEN_STRONG_INLINE Packet8h pround<Packet8h>(const Packet8h& a) {
2236 return float2half(pround<Packet8f>(half2float(a)));
2237}
2238
2239template <>
2240EIGEN_STRONG_INLINE Packet8h print<Packet8h>(const Packet8h& a) {
2241 return float2half(print<Packet8f>(half2float(a)));
2242}
2243
2244template <>
2245EIGEN_STRONG_INLINE Packet8h pceil<Packet8h>(const Packet8h& a) {
2246 return float2half(pceil<Packet8f>(half2float(a)));
2247}
2248
2249template <>
2250EIGEN_STRONG_INLINE Packet8h pfloor<Packet8h>(const Packet8h& a) {
2251 return float2half(pfloor<Packet8f>(half2float(a)));
2252}
2253
2254template <>
2255EIGEN_STRONG_INLINE Packet8h ptrunc<Packet8h>(const Packet8h& a) {
2256 return float2half(ptrunc<Packet8f>(half2float(a)));
2257}
2258
2259template <>
2260EIGEN_STRONG_INLINE Packet8h pisinf<Packet8h>(const Packet8h& a) {
2261 constexpr uint16_t kInf = ((1 << 5) - 1) << 10;
2262 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2263 return _mm_cmpeq_epi16(_mm_and_si128(a.m_val, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf));
2264}
2265
2266template <>
2267EIGEN_STRONG_INLINE Packet8h pisnan<Packet8h>(const Packet8h& a) {
2268 constexpr uint16_t kInf = ((1 << 5) - 1) << 10;
2269 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2270 return _mm_cmpgt_epi16(_mm_and_si128(a.m_val, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf));
2271}
2272
2273// convert the sign-magnitude representation to two's complement
2274EIGEN_STRONG_INLINE __m128i pmaptosigned(const __m128i& a) {
2275 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2276 // if 'a' has the sign bit set, clear the sign bit and negate the result as if it were an integer
2277 return _mm_sign_epi16(_mm_and_si128(a, _mm_set1_epi16(kAbsMask)), a);
2278}
2279
2280// return true if both `a` and `b` are not NaN
2281EIGEN_STRONG_INLINE Packet8h pisordered(const Packet8h& a, const Packet8h& b) {
2282 constexpr uint16_t kInf = ((1 << 5) - 1) << 10;
2283 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2284 __m128i abs_a = _mm_and_si128(a.m_val, _mm_set1_epi16(kAbsMask));
2285 __m128i abs_b = _mm_and_si128(b.m_val, _mm_set1_epi16(kAbsMask));
2286 // check if both `abs_a <= kInf` and `abs_b <= kInf` by checking if max(abs_a, abs_b) <= kInf
2287 // SSE has no `lesser or equal` instruction for integers, but comparing against kInf + 1 accomplishes the same goal
2288 return _mm_cmplt_epi16(_mm_max_epu16(abs_a, abs_b), _mm_set1_epi16(kInf + 1));
2289}
2290
2291template <>
2292EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a, const Packet8h& b) {
2293 __m128i isOrdered = pisordered(a, b);
2294 __m128i isEqual = _mm_cmpeq_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val));
2295 return _mm_and_si128(isOrdered, isEqual);
2296}
2297
2298template <>
2299EIGEN_STRONG_INLINE Packet8h pcmp_le(const Packet8h& a, const Packet8h& b) {
2300 __m128i isOrdered = pisordered(a, b);
2301 __m128i isGreater = _mm_cmpgt_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val));
2302 return _mm_andnot_si128(isGreater, isOrdered);
2303}
2304
2305template <>
2306EIGEN_STRONG_INLINE Packet8h pcmp_lt(const Packet8h& a, const Packet8h& b) {
2307 __m128i isOrdered = pisordered(a, b);
2308 __m128i isLess = _mm_cmplt_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val));
2309 return _mm_and_si128(isOrdered, isLess);
2310}
2311
2312template <>
2313EIGEN_STRONG_INLINE Packet8h pcmp_lt_or_nan(const Packet8h& a, const Packet8h& b) {
2314 __m128i isUnordered = por(pisnan(a), pisnan(b));
2315 __m128i isLess = _mm_cmplt_epi16(pmaptosigned(a.m_val), pmaptosigned(b.m_val));
2316 return _mm_or_si128(isUnordered, isLess);
2317}
2318
2319template <>
2320EIGEN_STRONG_INLINE Packet8h pnegate(const Packet8h& a) {
2321 Packet8h sign_mask = _mm_set1_epi16(static_cast<short>(0x8000u));
2322 return _mm_xor_si128(a, sign_mask);
2323}
2324
2325#ifndef EIGEN_VECTORIZE_AVX512FP16
2326template <>
2327EIGEN_STRONG_INLINE Packet8h padd<Packet8h>(const Packet8h& a, const Packet8h& b) {
2328 Packet8f af = half2float(a);
2329 Packet8f bf = half2float(b);
2330 Packet8f rf = padd(af, bf);
2331 return float2half(rf);
2332}
2333
2334template <>
2335EIGEN_STRONG_INLINE Packet8h psub<Packet8h>(const Packet8h& a, const Packet8h& b) {
2336 Packet8f af = half2float(a);
2337 Packet8f bf = half2float(b);
2338 Packet8f rf = psub(af, bf);
2339 return float2half(rf);
2340}
2341
2342template <>
2343EIGEN_STRONG_INLINE Packet8h pmul<Packet8h>(const Packet8h& a, const Packet8h& b) {
2344 Packet8f af = half2float(a);
2345 Packet8f bf = half2float(b);
2346 Packet8f rf = pmul(af, bf);
2347 return float2half(rf);
2348}
2349
2350template <>
2351EIGEN_STRONG_INLINE Packet8h pmadd<Packet8h>(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
2352 return float2half(pmadd(half2float(a), half2float(b), half2float(c)));
2353}
2354
2355template <>
2356EIGEN_STRONG_INLINE Packet8h pmsub<Packet8h>(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
2357 return float2half(pmsub(half2float(a), half2float(b), half2float(c)));
2358}
2359
2360template <>
2361EIGEN_STRONG_INLINE Packet8h pnmadd<Packet8h>(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
2362 return float2half(pnmadd(half2float(a), half2float(b), half2float(c)));
2363}
2364
2365template <>
2366EIGEN_STRONG_INLINE Packet8h pnmsub<Packet8h>(const Packet8h& a, const Packet8h& b, const Packet8h& c) {
2367 return float2half(pnmsub(half2float(a), half2float(b), half2float(c)));
2368}
2369
2370template <>
2371EIGEN_STRONG_INLINE Packet8h pdiv<Packet8h>(const Packet8h& a, const Packet8h& b) {
2372 Packet8f af = half2float(a);
2373 Packet8f bf = half2float(b);
2374 Packet8f rf = pdiv(af, bf);
2375 return float2half(rf);
2376}
2377#endif
2378
2379template <>
2380EIGEN_STRONG_INLINE Packet8h pgather<Eigen::half, Packet8h>(const Eigen::half* from, Index stride) {
2381 const numext::uint16_t s0 = numext::bit_cast<numext::uint16_t>(from[0 * stride]);
2382 const numext::uint16_t s1 = numext::bit_cast<numext::uint16_t>(from[1 * stride]);
2383 const numext::uint16_t s2 = numext::bit_cast<numext::uint16_t>(from[2 * stride]);
2384 const numext::uint16_t s3 = numext::bit_cast<numext::uint16_t>(from[3 * stride]);
2385 const numext::uint16_t s4 = numext::bit_cast<numext::uint16_t>(from[4 * stride]);
2386 const numext::uint16_t s5 = numext::bit_cast<numext::uint16_t>(from[5 * stride]);
2387 const numext::uint16_t s6 = numext::bit_cast<numext::uint16_t>(from[6 * stride]);
2388 const numext::uint16_t s7 = numext::bit_cast<numext::uint16_t>(from[7 * stride]);
2389 return _mm_set_epi16(s7, s6, s5, s4, s3, s2, s1, s0);
2390}
2391
2392template <>
2393EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index stride) {
2394 EIGEN_ALIGN32 Eigen::half aux[8];
2395 pstore(aux, from);
2396 to[stride * 0] = aux[0];
2397 to[stride * 1] = aux[1];
2398 to[stride * 2] = aux[2];
2399 to[stride * 3] = aux[3];
2400 to[stride * 4] = aux[4];
2401 to[stride * 5] = aux[5];
2402 to[stride * 6] = aux[6];
2403 to[stride * 7] = aux[7];
2404}
2405
2406template <>
2407EIGEN_STRONG_INLINE Packet8h preverse(const Packet8h& a) {
2408 __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
2409 return _mm_shuffle_epi8(a, m);
2410}
2411
2412EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8h, 8>& kernel) {
2413 __m128i a = kernel.packet[0];
2414 __m128i b = kernel.packet[1];
2415 __m128i c = kernel.packet[2];
2416 __m128i d = kernel.packet[3];
2417 __m128i e = kernel.packet[4];
2418 __m128i f = kernel.packet[5];
2419 __m128i g = kernel.packet[6];
2420 __m128i h = kernel.packet[7];
2421
2422 __m128i a03b03 = _mm_unpacklo_epi16(a, b);
2423 __m128i c03d03 = _mm_unpacklo_epi16(c, d);
2424 __m128i e03f03 = _mm_unpacklo_epi16(e, f);
2425 __m128i g03h03 = _mm_unpacklo_epi16(g, h);
2426 __m128i a47b47 = _mm_unpackhi_epi16(a, b);
2427 __m128i c47d47 = _mm_unpackhi_epi16(c, d);
2428 __m128i e47f47 = _mm_unpackhi_epi16(e, f);
2429 __m128i g47h47 = _mm_unpackhi_epi16(g, h);
2430
2431 __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03);
2432 __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03);
2433 __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03);
2434 __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03);
2435 __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47);
2436 __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47);
2437 __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47);
2438 __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47);
2439
2440 __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01);
2441 __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01);
2442 __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23);
2443 __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23);
2444 __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45);
2445 __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45);
2446 __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67);
2447 __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67);
2448
2449 kernel.packet[0] = a0b0c0d0e0f0g0h0;
2450 kernel.packet[1] = a1b1c1d1e1f1g1h1;
2451 kernel.packet[2] = a2b2c2d2e2f2g2h2;
2452 kernel.packet[3] = a3b3c3d3e3f3g3h3;
2453 kernel.packet[4] = a4b4c4d4e4f4g4h4;
2454 kernel.packet[5] = a5b5c5d5e5f5g5h5;
2455 kernel.packet[6] = a6b6c6d6e6f6g6h6;
2456 kernel.packet[7] = a7b7c7d7e7f7g7h7;
2457}
2458
2459EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8h, 4>& kernel) {
2460 EIGEN_ALIGN32 Eigen::half in[4][8];
2461 pstore<Eigen::half>(in[0], kernel.packet[0]);
2462 pstore<Eigen::half>(in[1], kernel.packet[1]);
2463 pstore<Eigen::half>(in[2], kernel.packet[2]);
2464 pstore<Eigen::half>(in[3], kernel.packet[3]);
2465
2466 EIGEN_ALIGN32 Eigen::half out[4][8];
2467
2468 for (int i = 0; i < 4; ++i) {
2469 for (int j = 0; j < 4; ++j) {
2470 out[i][j] = in[j][2 * i];
2471 }
2472 for (int j = 0; j < 4; ++j) {
2473 out[i][j + 4] = in[j][2 * i + 1];
2474 }
2475 }
2476
2477 kernel.packet[0] = pload<Packet8h>(out[0]);
2478 kernel.packet[1] = pload<Packet8h>(out[1]);
2479 kernel.packet[2] = pload<Packet8h>(out[2]);
2480 kernel.packet[3] = pload<Packet8h>(out[3]);
2481}
2482
2483#endif
2484
2485// BFloat16 implementation.
2486
2487EIGEN_STRONG_INLINE Packet8f Bf16ToF32(const Packet8bf& a) {
2488#ifdef EIGEN_VECTORIZE_AVX2
2489 __m256i extend = _mm256_cvtepu16_epi32(a);
2490 return _mm256_castsi256_ps(_mm256_slli_epi32(extend, 16));
2491#else
2492 __m128i lo = _mm_cvtepu16_epi32(a);
2493 __m128i hi = _mm_cvtepu16_epi32(_mm_srli_si128(a, 8));
2494 __m128i lo_shift = _mm_slli_epi32(lo, 16);
2495 __m128i hi_shift = _mm_slli_epi32(hi, 16);
2496 return _mm256_castsi256_ps(_mm256_insertf128_si256(_mm256_castsi128_si256(lo_shift), hi_shift, 1));
2497#endif
2498}
2499
2500// Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm.
2501EIGEN_STRONG_INLINE Packet8bf F32ToBf16(const Packet8f& a) {
2502 __m256i input = _mm256_castps_si256(a);
2503
2504#ifdef EIGEN_VECTORIZE_AVX2
2505 // uint32_t lsb = (input >> 16);
2506 __m256i t = _mm256_srli_epi32(input, 16);
2507 // uint32_t lsb = lsb & 1;
2508 t = _mm256_and_si256(t, _mm256_set1_epi32(1));
2509 // uint32_t rounding_bias = 0x7fff + lsb;
2510 t = _mm256_add_epi32(t, _mm256_set1_epi32(0x7fff));
2511 // input += rounding_bias;
2512 t = _mm256_add_epi32(t, input);
2513 // input = input >> 16;
2514 t = _mm256_srli_epi32(t, 16);
2515 // Check NaN before converting back to bf16
2516 __m256 mask = _mm256_cmp_ps(a, a, _CMP_ORD_Q);
2517 __m256i nan = _mm256_set1_epi32(0x7fc0);
2518 t = _mm256_blendv_epi8(nan, t, _mm256_castps_si256(mask));
2519 // output = numext::bit_cast<uint16_t>(input);
2520 return _mm_packus_epi32(_mm256_extractf128_si256(t, 0), _mm256_extractf128_si256(t, 1));
2521#else
2522 // uint32_t lsb = (input >> 16);
2523 __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(input, 0), 16);
2524 __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(input, 1), 16);
2525 // uint32_t lsb = lsb & 1;
2526 lo = _mm_and_si128(lo, _mm_set1_epi32(1));
2527 hi = _mm_and_si128(hi, _mm_set1_epi32(1));
2528 // uint32_t rounding_bias = 0x7fff + lsb;
2529 lo = _mm_add_epi32(lo, _mm_set1_epi32(0x7fff));
2530 hi = _mm_add_epi32(hi, _mm_set1_epi32(0x7fff));
2531 // input += rounding_bias;
2532 lo = _mm_add_epi32(lo, _mm256_extractf128_si256(input, 0));
2533 hi = _mm_add_epi32(hi, _mm256_extractf128_si256(input, 1));
2534 // input = input >> 16;
2535 lo = _mm_srli_epi32(lo, 16);
2536 hi = _mm_srli_epi32(hi, 16);
2537 // Check NaN before converting back to bf16
2538 __m256 mask = _mm256_cmp_ps(a, a, _CMP_ORD_Q);
2539 __m128i nan = _mm_set1_epi32(0x7fc0);
2540 lo = _mm_blendv_epi8(nan, lo, _mm_castps_si128(_mm256_castps256_ps128(mask)));
2541 hi = _mm_blendv_epi8(nan, hi, _mm_castps_si128(_mm256_extractf128_ps(mask, 1)));
2542 // output = numext::bit_cast<uint16_t>(input);
2543 return _mm_packus_epi32(lo, hi);
2544#endif
2545}
2546
2547template <>
2548EIGEN_STRONG_INLINE Packet8bf pset1<Packet8bf>(const bfloat16& from) {
2549 return _mm_set1_epi16(numext::bit_cast<numext::uint16_t>(from));
2550}
2551
2552template <>
2553EIGEN_STRONG_INLINE bfloat16 pfirst<Packet8bf>(const Packet8bf& from) {
2554 return numext::bit_cast<bfloat16>(static_cast<numext::uint16_t>(_mm_extract_epi16(from, 0)));
2555}
2556
2557template <>
2558EIGEN_STRONG_INLINE Packet8bf pload<Packet8bf>(const bfloat16* from) {
2559 return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
2560}
2561
2562template <>
2563EIGEN_STRONG_INLINE Packet8bf ploadu<Packet8bf>(const bfloat16* from) {
2564 return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
2565}
2566
2567template <>
2568EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet8bf& from) {
2569 _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
2570}
2571
2572template <>
2573EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet8bf& from) {
2574 _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
2575}
2576
2577template <>
2578EIGEN_STRONG_INLINE Packet8bf ploaddup<Packet8bf>(const bfloat16* from) {
2579 const numext::uint16_t a = numext::bit_cast<numext::uint16_t>(from[0]);
2580 const numext::uint16_t b = numext::bit_cast<numext::uint16_t>(from[1]);
2581 const numext::uint16_t c = numext::bit_cast<numext::uint16_t>(from[2]);
2582 const numext::uint16_t d = numext::bit_cast<numext::uint16_t>(from[3]);
2583 return _mm_set_epi16(d, d, c, c, b, b, a, a);
2584}
2585
2586template <>
2587EIGEN_STRONG_INLINE Packet8bf ploadquad<Packet8bf>(const bfloat16* from) {
2588 const numext::uint16_t a = numext::bit_cast<numext::uint16_t>(from[0]);
2589 const numext::uint16_t b = numext::bit_cast<numext::uint16_t>(from[1]);
2590 return _mm_set_epi16(b, b, b, b, a, a, a, a);
2591}
2592
2593// Discard the low 16 bits of each float. Only valid when every lane already holds an exact
2594// bfloat16 value, in which case this agrees with F32ToBf16 but skips its rounding and NaN
2595// canonicalization. pmin/pmax qualify: they return one of their operands bit for bit.
2596EIGEN_STRONG_INLINE Packet8bf F32ToBf16Truncate(const Packet8f& a) {
2597 __m256i input = _mm256_castps_si256(a);
2598#ifdef EIGEN_VECTORIZE_AVX2
2599 __m256i t = _mm256_srli_epi32(input, 16);
2600 return _mm_packus_epi32(_mm256_extractf128_si256(t, 0), _mm256_extractf128_si256(t, 1));
2601#else
2602 __m128i lo = _mm_srli_epi32(_mm256_extractf128_si256(input, 0), 16);
2603 __m128i hi = _mm_srli_epi32(_mm256_extractf128_si256(input, 1), 16);
2604 return _mm_packus_epi32(lo, hi);
2605#endif
2606}
2607
2608template <>
2609EIGEN_STRONG_INLINE Packet8bf ptrue(const Packet8bf& a) {
2610 return _mm_cmpeq_epi32(a, a);
2611}
2612
2613template <>
2614EIGEN_STRONG_INLINE Packet8bf pabs(const Packet8bf& a) {
2615 const __m128i sign_mask = _mm_set1_epi16(static_cast<short>(0x8000u));
2616 return _mm_andnot_si128(sign_mask, a);
2617}
2618
2619template <>
2620EIGEN_STRONG_INLINE Packet8bf pmin<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2621 return F32ToBf16Truncate(pmin<Packet8f>(Bf16ToF32(a), Bf16ToF32(b)));
2622}
2623
2624template <>
2625EIGEN_STRONG_INLINE Packet8bf pmax<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2626 return F32ToBf16Truncate(pmax<Packet8f>(Bf16ToF32(a), Bf16ToF32(b)));
2627}
2628
2629template <>
2630EIGEN_STRONG_INLINE Packet8bf plset<Packet8bf>(const bfloat16& a) {
2631 return F32ToBf16(plset<Packet8f>(static_cast<float>(a)));
2632}
2633
2634template <>
2635EIGEN_STRONG_INLINE Packet8bf por(const Packet8bf& a, const Packet8bf& b) {
2636 return _mm_or_si128(a, b);
2637}
2638template <>
2639EIGEN_STRONG_INLINE Packet8bf pxor(const Packet8bf& a, const Packet8bf& b) {
2640 return _mm_xor_si128(a, b);
2641}
2642template <>
2643EIGEN_STRONG_INLINE Packet8bf pand(const Packet8bf& a, const Packet8bf& b) {
2644 return _mm_and_si128(a, b);
2645}
2646template <>
2647EIGEN_STRONG_INLINE Packet8bf pandnot(const Packet8bf& a, const Packet8bf& b) {
2648 return _mm_andnot_si128(b, a);
2649}
2650
2651template <>
2652EIGEN_STRONG_INLINE Packet8bf pselect(const Packet8bf& mask, const Packet8bf& a, const Packet8bf& b) {
2653 return _mm_blendv_epi8(b, a, mask);
2654}
2655
2656template <>
2657EIGEN_STRONG_INLINE Packet8bf pround<Packet8bf>(const Packet8bf& a) {
2658 return F32ToBf16Truncate(pround<Packet8f>(Bf16ToF32(a)));
2659}
2660
2661template <>
2662EIGEN_STRONG_INLINE Packet8bf print<Packet8bf>(const Packet8bf& a) {
2663 return F32ToBf16Truncate(print<Packet8f>(Bf16ToF32(a)));
2664}
2665
2666template <>
2667EIGEN_STRONG_INLINE Packet8bf pceil<Packet8bf>(const Packet8bf& a) {
2668 return F32ToBf16Truncate(pceil<Packet8f>(Bf16ToF32(a)));
2669}
2670
2671template <>
2672EIGEN_STRONG_INLINE Packet8bf pfloor<Packet8bf>(const Packet8bf& a) {
2673 return F32ToBf16Truncate(pfloor<Packet8f>(Bf16ToF32(a)));
2674}
2675
2676template <>
2677EIGEN_STRONG_INLINE Packet8bf ptrunc<Packet8bf>(const Packet8bf& a) {
2678 return F32ToBf16Truncate(ptrunc<Packet8f>(Bf16ToF32(a)));
2679}
2680
2681template <>
2682EIGEN_STRONG_INLINE Packet8bf pcmp_eq(const Packet8bf& a, const Packet8bf& b) {
2683 return Pack16To8(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
2684}
2685
2686template <>
2687EIGEN_STRONG_INLINE Packet8bf pcmp_le(const Packet8bf& a, const Packet8bf& b) {
2688 return Pack16To8(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
2689}
2690
2691template <>
2692EIGEN_STRONG_INLINE Packet8bf pcmp_lt(const Packet8bf& a, const Packet8bf& b) {
2693 return Pack16To8(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
2694}
2695
2696template <>
2697EIGEN_STRONG_INLINE Packet8bf pcmp_lt_or_nan(const Packet8bf& a, const Packet8bf& b) {
2698 return Pack16To8(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
2699}
2700
2701// Classify on the raw bits, as the scalar isinf/isnan/isfinite do: |a| ==, >, < 0x7f80.
2702template <>
2703EIGEN_STRONG_INLINE Packet8bf pisinf<Packet8bf>(const Packet8bf& a) {
2704 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2705 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2706 return _mm_cmpeq_epi16(_mm_and_si128(a, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf));
2707}
2708
2709template <>
2710EIGEN_STRONG_INLINE Packet8bf pisnan<Packet8bf>(const Packet8bf& a) {
2711 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2712 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2713 return _mm_cmpgt_epi16(_mm_and_si128(a, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf));
2714}
2715
2716// Compare encoded lanes: widening bf16 subnormals to float loses them under DAZ/FZ.
2717template <>
2718EIGEN_STRONG_INLINE Packet8bf psign<Packet8bf>(const Packet8bf& a) {
2719 const __m128i magnitude = _mm_and_si128(a, _mm_set1_epi16(0x7fff));
2720 const __m128i is_nan = _mm_cmpgt_epi16(magnitude, _mm_set1_epi16(0x7f80));
2721 const __m128i keep = _mm_or_si128(is_nan, _mm_set1_epi16(static_cast<short>(0xbf80u)));
2722 return _mm_sign_epi16(_mm_and_si128(_mm_or_si128(a, _mm_set1_epi16(0x3f80)), keep), magnitude);
2723}
2724
2725template <>
2726EIGEN_STRONG_INLINE Packet8bf pisfinite<Packet8bf>(const Packet8bf& a) {
2727 constexpr uint16_t kInf = ((1 << 8) - 1) << 7;
2728 constexpr uint16_t kAbsMask = (1 << 15) - 1;
2729 return _mm_cmplt_epi16(_mm_and_si128(a, _mm_set1_epi16(kAbsMask)), _mm_set1_epi16(kInf));
2730}
2731
2732template <>
2733EIGEN_STRONG_INLINE Packet8bf pnegate(const Packet8bf& a) {
2734 Packet8bf sign_mask = _mm_set1_epi16(static_cast<short>(0x8000u));
2735 return _mm_xor_si128(a, sign_mask);
2736}
2737
2738template <>
2739EIGEN_STRONG_INLINE Packet8bf padd<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2740 return F32ToBf16(padd<Packet8f>(Bf16ToF32(a), Bf16ToF32(b)));
2741}
2742
2743template <>
2744EIGEN_STRONG_INLINE Packet8bf psub<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2745 return F32ToBf16(psub<Packet8f>(Bf16ToF32(a), Bf16ToF32(b)));
2746}
2747
2748template <>
2749EIGEN_STRONG_INLINE Packet8bf pmul<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2750 return F32ToBf16(pmul<Packet8f>(Bf16ToF32(a), Bf16ToF32(b)));
2751}
2752
2753template <>
2754EIGEN_STRONG_INLINE Packet8bf pmadd<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2755 return F32ToBf16(pmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2756}
2757
2758template <>
2759EIGEN_STRONG_INLINE Packet8bf pmsub<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2760 return F32ToBf16(pmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2761}
2762
2763template <>
2764EIGEN_STRONG_INLINE Packet8bf pnmadd<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2765 return F32ToBf16(pnmadd(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2766}
2767
2768template <>
2769EIGEN_STRONG_INLINE Packet8bf pnmsub<Packet8bf>(const Packet8bf& a, const Packet8bf& b, const Packet8bf& c) {
2770 return F32ToBf16(pnmsub(Bf16ToF32(a), Bf16ToF32(b), Bf16ToF32(c)));
2771}
2772
2773template <>
2774EIGEN_STRONG_INLINE Packet8bf pdiv<Packet8bf>(const Packet8bf& a, const Packet8bf& b) {
2775 return F32ToBf16(pdiv<Packet8f>(Bf16ToF32(a), Bf16ToF32(b)));
2776}
2777
2778template <>
2779EIGEN_STRONG_INLINE Packet8bf pgather<bfloat16, Packet8bf>(const bfloat16* from, Index stride) {
2780 const numext::uint16_t s0 = numext::bit_cast<numext::uint16_t>(from[0 * stride]);
2781 const numext::uint16_t s1 = numext::bit_cast<numext::uint16_t>(from[1 * stride]);
2782 const numext::uint16_t s2 = numext::bit_cast<numext::uint16_t>(from[2 * stride]);
2783 const numext::uint16_t s3 = numext::bit_cast<numext::uint16_t>(from[3 * stride]);
2784 const numext::uint16_t s4 = numext::bit_cast<numext::uint16_t>(from[4 * stride]);
2785 const numext::uint16_t s5 = numext::bit_cast<numext::uint16_t>(from[5 * stride]);
2786 const numext::uint16_t s6 = numext::bit_cast<numext::uint16_t>(from[6 * stride]);
2787 const numext::uint16_t s7 = numext::bit_cast<numext::uint16_t>(from[7 * stride]);
2788 return _mm_set_epi16(s7, s6, s5, s4, s3, s2, s1, s0);
2789}
2790
2791template <>
2792EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet8bf>(bfloat16* to, const Packet8bf& from, Index stride) {
2793 EIGEN_ALIGN32 bfloat16 aux[8];
2794 pstore(aux, from);
2795 to[stride * 0] = aux[0];
2796 to[stride * 1] = aux[1];
2797 to[stride * 2] = aux[2];
2798 to[stride * 3] = aux[3];
2799 to[stride * 4] = aux[4];
2800 to[stride * 5] = aux[5];
2801 to[stride * 6] = aux[6];
2802 to[stride * 7] = aux[7];
2803}
2804
2805template <>
2806EIGEN_STRONG_INLINE Packet8bf preverse(const Packet8bf& a) {
2807 __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
2808 return _mm_shuffle_epi8(a, m);
2809}
2810
2811EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8bf, 8>& kernel) {
2812 __m128i a = kernel.packet[0];
2813 __m128i b = kernel.packet[1];
2814 __m128i c = kernel.packet[2];
2815 __m128i d = kernel.packet[3];
2816 __m128i e = kernel.packet[4];
2817 __m128i f = kernel.packet[5];
2818 __m128i g = kernel.packet[6];
2819 __m128i h = kernel.packet[7];
2820
2821 __m128i a03b03 = _mm_unpacklo_epi16(a, b);
2822 __m128i c03d03 = _mm_unpacklo_epi16(c, d);
2823 __m128i e03f03 = _mm_unpacklo_epi16(e, f);
2824 __m128i g03h03 = _mm_unpacklo_epi16(g, h);
2825 __m128i a47b47 = _mm_unpackhi_epi16(a, b);
2826 __m128i c47d47 = _mm_unpackhi_epi16(c, d);
2827 __m128i e47f47 = _mm_unpackhi_epi16(e, f);
2828 __m128i g47h47 = _mm_unpackhi_epi16(g, h);
2829
2830 __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03);
2831 __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03);
2832 __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03);
2833 __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03);
2834 __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47);
2835 __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47);
2836 __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47);
2837 __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47);
2838
2839 kernel.packet[0] = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01);
2840 kernel.packet[1] = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01);
2841 kernel.packet[2] = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23);
2842 kernel.packet[3] = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23);
2843 kernel.packet[4] = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45);
2844 kernel.packet[5] = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45);
2845 kernel.packet[6] = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67);
2846 kernel.packet[7] = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67);
2847}
2848
2849EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8bf, 4>& kernel) {
2850 __m128i a = kernel.packet[0];
2851 __m128i b = kernel.packet[1];
2852 __m128i c = kernel.packet[2];
2853 __m128i d = kernel.packet[3];
2854
2855 __m128i ab_03 = _mm_unpacklo_epi16(a, b);
2856 __m128i cd_03 = _mm_unpacklo_epi16(c, d);
2857 __m128i ab_47 = _mm_unpackhi_epi16(a, b);
2858 __m128i cd_47 = _mm_unpackhi_epi16(c, d);
2859
2860 kernel.packet[0] = _mm_unpacklo_epi32(ab_03, cd_03);
2861 kernel.packet[1] = _mm_unpackhi_epi32(ab_03, cd_03);
2862 kernel.packet[2] = _mm_unpacklo_epi32(ab_47, cd_47);
2863 kernel.packet[3] = _mm_unpackhi_epi32(ab_47, cd_47);
2864}
2865
2866/*---------------- load/store segment support ----------------*/
2867
2868template <>
2869struct has_packet_segment<Packet4f> : std::true_type {};
2870
2871template <>
2872struct has_packet_segment<Packet8f> : std::true_type {};
2873
2874template <>
2875struct has_packet_segment<Packet4i> : std::true_type {};
2876
2877template <>
2878struct has_packet_segment<Packet8i> : std::true_type {};
2879
2880template <>
2881struct has_packet_segment<Packet4ui> : std::true_type {};
2882
2883template <>
2884struct has_packet_segment<Packet8ui> : std::true_type {};
2885
2886template <>
2887struct has_packet_segment<Packet2d> : std::true_type {};
2888
2889template <>
2890struct has_packet_segment<Packet4d> : std::true_type {};
2891
2892#ifdef EIGEN_VECTORIZE_AVX2
2893template <>
2894struct has_packet_segment<Packet2l> : std::true_type {};
2895
2896template <>
2897struct has_packet_segment<Packet4l> : std::true_type {};
2898
2899template <>
2900struct has_packet_segment<Packet4ul> : std::true_type {};
2901#endif
2902
2903// With AVX-512VL these packets use k-masked loads and stores instead; see AVX512/PacketMath.h.
2904#ifndef EIGEN_VECTORIZE_AVX512VL
2905
2906// returns a mask of 8-bit elements (at most 4) that are all 1's in the range [begin, begin + count) and 0 elsewhere.
2907inline __m128i segment_mask_4x8(Index begin, Index count) {
2908 eigen_assert(begin >= 0 && count >= 0 && begin + count <= 4);
2909 long long mask = 1;
2910 mask <<= CHAR_BIT * count;
2911 mask--;
2912 mask <<= CHAR_BIT * begin;
2913#if !EIGEN_ARCH_x86_64
2914 return _mm_loadl_epi64(reinterpret_cast<const __m128i*>(&mask));
2915#else
2916 return _mm_cvtsi64_si128(mask);
2917#endif
2918}
2919
2920// returns a mask of 8-bit elements (at most 8) that are all 1's in the range [begin, begin + count) and 0 elsewhere.
2921inline __m128i segment_mask_8x8(Index begin, Index count) {
2922 eigen_assert(begin >= 0 && count >= 0 && begin + count <= 8);
2923 unsigned long long mask = 1;
2924 // avoid UB when count == 8
2925 mask <<= (CHAR_BIT / 2) * count;
2926 mask <<= (CHAR_BIT / 2) * count;
2927 mask--;
2928 // begin == 8 only when count == 0, where mask == 0
2929 mask <<= CHAR_BIT * (begin & 7);
2930#if !EIGEN_ARCH_x86_64
2931 return _mm_loadl_epi64(reinterpret_cast<const __m128i*>(&mask));
2932#else
2933 return _mm_cvtsi64_si128(mask);
2934#endif
2935}
2936
2937// returns a mask of 32-bit elements (at most 4) that are all 1's in the range [begin, begin + count) and 0 elsewhere.
2938inline __m128i segment_mask_4x32(Index begin, Index count) {
2939 eigen_assert(begin >= 0 && count >= 0 && begin + count <= 4);
2940 return _mm_cvtepi8_epi32(segment_mask_4x8(begin, count));
2941}
2942
2943// returns a mask of 64-bit elements (at most 2) that are all 1's in the range [begin, begin + count) and 0 elsewhere.
2944inline __m128i segment_mask_2x64(Index begin, Index count) {
2945 eigen_assert(begin >= 0 && count >= 0 && begin + count <= 2);
2946 return _mm_cvtepi8_epi64(segment_mask_4x8(begin, count));
2947}
2948
2949// returns a mask of 32-bit elements (at most 8) that are all 1's in the range [begin, begin + count) and 0 elsewhere.
2950inline __m256i segment_mask_8x32(Index begin, Index count) {
2951 __m128i mask_epi8 = segment_mask_8x8(begin, count);
2952#ifdef EIGEN_VECTORIZE_AVX2
2953 __m256i mask_epi32 = _mm256_cvtepi8_epi32(mask_epi8);
2954#else
2955 __m128i mask_epi32_lo = _mm_cvtepi8_epi32(mask_epi8);
2956 __m128i mask_epi32_hi = _mm_cvtepi8_epi32(_mm_srli_epi64(mask_epi8, 32));
2957 __m256i mask_epi32 = _mm256_insertf128_si256(_mm256_castsi128_si256(mask_epi32_lo), mask_epi32_hi, 1);
2958#endif
2959 return mask_epi32;
2960}
2961
2962// returns a mask of 64-bit elements (at most 4) that are all 1's in the range [begin, begin + count) and 0 elsewhere.
2963inline __m256i segment_mask_4x64(Index begin, Index count) {
2964 __m128i mask_epi8 = segment_mask_4x8(begin, count);
2965#ifdef EIGEN_VECTORIZE_AVX2
2966 __m256i mask_epi64 = _mm256_cvtepi8_epi64(mask_epi8);
2967#else
2968 __m128i mask_epi64_lo = _mm_cvtepi8_epi64(mask_epi8);
2969 __m128i mask_epi64_hi = _mm_cvtepi8_epi64(_mm_srli_epi64(mask_epi8, 16));
2970 __m256i mask_epi64 = _mm256_insertf128_si256(_mm256_castsi128_si256(mask_epi64_lo), mask_epi64_hi, 1);
2971#endif
2972 return mask_epi64;
2973}
2974
2975/*---------------- float ----------------*/
2976
2977template <>
2978inline Packet4f ploaduSegment<Packet4f>(const float* from, Index begin, Index count) {
2979 return _mm_maskload_ps(from, segment_mask_4x32(begin, count));
2980}
2981
2982template <>
2983inline void pstoreuSegment<float, Packet4f>(float* to, const Packet4f& from, Index begin, Index count) {
2984 _mm_maskstore_ps(to, segment_mask_4x32(begin, count), from);
2985}
2986
2987template <>
2988inline Packet8f ploaduSegment<Packet8f>(const float* from, Index begin, Index count) {
2989 return _mm256_maskload_ps(from, segment_mask_8x32(begin, count));
2990}
2991
2992template <>
2993inline void pstoreuSegment<float, Packet8f>(float* to, const Packet8f& from, Index begin, Index count) {
2994 _mm256_maskstore_ps(to, segment_mask_8x32(begin, count), from);
2995}
2996
2997/*---------------- int32 ----------------*/
2998
2999#ifdef EIGEN_VECTORIZE_AVX2
3000
3001template <>
3002inline Packet4i ploaduSegment<Packet4i>(const int* from, Index begin, Index count) {
3003 return _mm_maskload_epi32(from, segment_mask_4x32(begin, count));
3004}
3005
3006template <>
3007inline void pstoreuSegment<int, Packet4i>(int* to, const Packet4i& from, Index begin, Index count) {
3008 _mm_maskstore_epi32(to, segment_mask_4x32(begin, count), from);
3009}
3010
3011template <>
3012inline Packet8i ploaduSegment<Packet8i>(const int* from, Index begin, Index count) {
3013 return _mm256_maskload_epi32(from, segment_mask_8x32(begin, count));
3014}
3015
3016template <>
3017inline void pstoreuSegment<int, Packet8i>(int* to, const Packet8i& from, Index begin, Index count) {
3018 _mm256_maskstore_epi32(to, segment_mask_8x32(begin, count), from);
3019}
3020
3021#else
3022
3023template <>
3024inline Packet4i ploaduSegment<Packet4i>(const int* from, Index begin, Index count) {
3025 return _mm_castps_si128(ploaduSegment<Packet4f>(reinterpret_cast<const float*>(from), begin, count));
3026}
3027
3028template <>
3029inline void pstoreuSegment<int, Packet4i>(int* to, const Packet4i& from, Index begin, Index count) {
3030 pstoreuSegment<float, Packet4f>(reinterpret_cast<float*>(to), _mm_castsi128_ps(from), begin, count);
3031}
3032
3033template <>
3034inline Packet8i ploaduSegment<Packet8i>(const int* from, Index begin, Index count) {
3035 return _mm256_castps_si256(ploaduSegment<Packet8f>(reinterpret_cast<const float*>(from), begin, count));
3036}
3037
3038template <>
3039inline void pstoreuSegment<int, Packet8i>(int* to, const Packet8i& from, Index begin, Index count) {
3040 pstoreuSegment<float, Packet8f>(reinterpret_cast<float*>(to), _mm256_castsi256_ps(from), begin, count);
3041}
3042
3043#endif
3044
3045/*---------------- uint32 ----------------*/
3046
3047template <>
3048inline Packet4ui ploaduSegment<Packet4ui>(const uint32_t* from, Index begin, Index count) {
3049 return Packet4ui(ploaduSegment<Packet4i>(reinterpret_cast<const int*>(from), begin, count));
3050}
3051
3052template <>
3053inline void pstoreuSegment<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from, Index begin, Index count) {
3054 pstoreuSegment<int, Packet4i>(reinterpret_cast<int*>(to), Packet4i(from), begin, count);
3055}
3056
3057template <>
3058inline Packet8ui ploaduSegment<Packet8ui>(const uint32_t* from, Index begin, Index count) {
3059 return Packet8ui(ploaduSegment<Packet8i>(reinterpret_cast<const int*>(from), begin, count));
3060}
3061
3062template <>
3063inline void pstoreuSegment<uint32_t, Packet8ui>(uint32_t* to, const Packet8ui& from, Index begin, Index count) {
3064 pstoreuSegment<int, Packet8i>(reinterpret_cast<int*>(to), Packet8i(from), begin, count);
3065}
3066
3067/*---------------- double ----------------*/
3068
3069template <>
3070inline Packet2d ploaduSegment<Packet2d>(const double* from, Index begin, Index count) {
3071 return _mm_maskload_pd(from, segment_mask_2x64(begin, count));
3072}
3073
3074template <>
3075inline void pstoreuSegment<double, Packet2d>(double* to, const Packet2d& from, Index begin, Index count) {
3076 _mm_maskstore_pd(to, segment_mask_2x64(begin, count), from);
3077}
3078
3079template <>
3080inline Packet4d ploaduSegment<Packet4d>(const double* from, Index begin, Index count) {
3081 return _mm256_maskload_pd(from, segment_mask_4x64(begin, count));
3082}
3083
3084template <>
3085inline void pstoreuSegment<double, Packet4d>(double* to, const Packet4d& from, Index begin, Index count) {
3086 _mm256_maskstore_pd(to, segment_mask_4x64(begin, count), from);
3087}
3088
3089#ifdef EIGEN_VECTORIZE_AVX2
3090
3091/*---------------- int64_t ----------------*/
3092
3093template <>
3094inline Packet2l ploaduSegment<Packet2l>(const int64_t* from, Index begin, Index count) {
3095 return _mm_maskload_epi64(reinterpret_cast<const long long*>(from), segment_mask_2x64(begin, count));
3096}
3097template <>
3098inline void pstoreuSegment<int64_t, Packet2l>(int64_t* to, const Packet2l& from, Index begin, Index count) {
3099 _mm_maskstore_epi64(reinterpret_cast<long long*>(to), segment_mask_2x64(begin, count), from);
3100}
3101template <>
3102inline Packet4l ploaduSegment<Packet4l>(const int64_t* from, Index begin, Index count) {
3103 return _mm256_maskload_epi64(reinterpret_cast<const long long*>(from), segment_mask_4x64(begin, count));
3104}
3105template <>
3106inline void pstoreuSegment<int64_t, Packet4l>(int64_t* to, const Packet4l& from, Index begin, Index count) {
3107 _mm256_maskstore_epi64(reinterpret_cast<long long*>(to), segment_mask_4x64(begin, count), from);
3108}
3109
3110/*---------------- uint64_t ----------------*/
3111
3112template <>
3113inline Packet4ul ploaduSegment<Packet4ul>(const uint64_t* from, Index begin, Index count) {
3114 return Packet4ul(ploaduSegment<Packet4l>(reinterpret_cast<const int64_t*>(from), begin, count));
3115}
3116template <>
3117inline void pstoreuSegment<uint64_t, Packet4ul>(uint64_t* to, const Packet4ul& from, Index begin, Index count) {
3118 pstoreuSegment<int64_t, Packet4l>(reinterpret_cast<int64_t*>(to), Packet4l(from), begin, count);
3119}
3120#endif
3121
3122#endif // EIGEN_VECTORIZE_AVX512VL
3123
3124/*---------------- end load/store segment support ----------------*/
3125
3126/*---------------- division by an invariant integer ----------------*/
3127
3128// pudiv_magic and psdiv_signed_magic (IntegerDivision.h); see the kernels in SSE/PacketMath.h.
3129#ifdef EIGEN_VECTORIZE_AVX2
3130// Per-lane shifts by broadcast counts: one uop, where a count in an xmm register costs two on Intel cores.
3131EIGEN_STRONG_INLINE __m256i pudiv_magic_epu32(const __m256i& a, uint32_t magic, uint32_t addend, int shift) {
3132 __m256i m = _mm256_set1_epi32(static_cast<int>(magic));
3133 __m256i c = _mm256_set1_epi64x(static_cast<long long>(addend));
3134 __m256i even = _mm256_shuffle_epi32(_mm256_add_epi64(_mm256_mul_epu32(a, m), c), 0xF5);
3135 __m256i odd = _mm256_add_epi64(_mm256_mul_epu32(_mm256_shuffle_epi32(a, 0xF5), m), c);
3136 return _mm256_srlv_epi32(_mm256_blend_epi32(even, odd, 0xAA), _mm256_set1_epi32(shift));
3137}
3138
3139EIGEN_STRONG_INLINE __m256i psdiv_signed_magic_epi32(const __m256i& a, uint32_t magic, int shift, int /*total_shift*/,
3140 const __m256i& divisor_sign) {
3141 __m256i m = _mm256_set1_epi32(static_cast<int>(magic));
3142 __m256i even = _mm256_shuffle_epi32(_mm256_mul_epi32(a, m), 0xF5);
3143 __m256i odd = _mm256_mul_epi32(_mm256_shuffle_epi32(a, 0xF5), m);
3144 __m256i t = _mm256_add_epi32(a, _mm256_blend_epi32(even, odd, 0xAA));
3145 __m256i q = _mm256_sub_epi32(_mm256_srav_epi32(t, _mm256_set1_epi32(shift)), _mm256_srai_epi32(a, 31));
3146 return _mm256_sub_epi32(_mm256_xor_si256(q, divisor_sign), divisor_sign);
3147}
3148#else
3149// AVX without AVX2 has no 256-bit integer arithmetic: divide the two 128-bit halves.
3150EIGEN_STRONG_INLINE __m256i pudiv_magic_epu32(const __m256i& a, uint32_t magic, uint32_t addend, int shift) {
3151 __m128i lo = pudiv_magic_epu32(_mm256_castsi256_si128(a), magic, addend, shift);
3152 __m128i hi = pudiv_magic_epu32(_mm256_extractf128_si256(a, 1), magic, addend, shift);
3153 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1);
3154}
3155
3156EIGEN_STRONG_INLINE __m256i psdiv_signed_magic_epi32(const __m256i& a, uint32_t magic, int shift, int total_shift,
3157 const __m256i& divisor_sign) {
3158 __m128i sign = _mm256_castsi256_si128(divisor_sign);
3159 __m128i lo = psdiv_signed_magic_epi32(_mm256_castsi256_si128(a), magic, shift, total_shift, sign);
3160 __m128i hi = psdiv_signed_magic_epi32(_mm256_extractf128_si256(a, 1), magic, shift, total_shift, sign);
3161 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1);
3162}
3163#endif
3164
3165template <>
3166EIGEN_STRONG_INLINE Packet8i psdiv_signed_magic<Packet8i>(const Packet8i& a, uint32_t magic, int shift, int total_shift,
3167 const Packet8i& divisor_sign) {
3168 return psdiv_signed_magic_epi32(a, magic, shift, total_shift, divisor_sign);
3169}
3170template <>
3171EIGEN_STRONG_INLINE Packet8ui pudiv_magic<Packet8ui>(const Packet8ui& a, uint32_t magic, uint32_t addend, int shift) {
3172 return pudiv_magic_epu32(a, magic, addend, shift);
3173}
3174
3175// The multiply-add form, as in SSE/PacketMath.h: x86 only.
3176#if defined(EIGEN_VECTORIZE_AVX2) && EIGEN_ARCH_i386_OR_x86_64
3177EIGEN_STRONG_INLINE __m256i pudiv_magic_epu64(const __m256i& a, uint64_t magic, uint64_t addend, int shift) {
3178 __m256i m_lo = _mm256_set1_epi64x(static_cast<long long>(magic));
3179 __m256i m_hi = _mm256_set1_epi64x(static_cast<long long>(magic >> 32));
3180 __m256i c_lo = _mm256_set1_epi64x(static_cast<long long>(addend & 0xffffffffu));
3181 __m256i c_hi = _mm256_set1_epi64x(static_cast<long long>(addend >> 32));
3182 __m256i a_hi = _mm256_srli_epi64(a, 32);
3183 __m256i u0 = _mm256_add_epi64(_mm256_mul_epu32(a, m_lo), c_lo);
3184 __m256i u = _mm256_add_epi64(_mm256_mul_epu32(a_hi, m_lo), _mm256_srli_epi64(u0, 32));
3185 __m256i w = _mm256_add_epi64(
3186 _mm256_add_epi64(_mm256_mul_epu32(a, m_hi), _mm256_blend_epi32(u, _mm256_setzero_si256(), 0xAA)), c_hi);
3187 __m256i t = _mm256_add_epi64(_mm256_add_epi64(_mm256_mul_epu32(a_hi, m_hi), _mm256_srli_epi64(u, 32)),
3188 _mm256_srli_epi64(w, 32));
3189 return _mm256_srlv_epi64(t, _mm256_set1_epi64x(shift));
3190}
3191
3192template <>
3193EIGEN_STRONG_INLINE Packet4l pudiv_magic<Packet4l>(const Packet4l& a, uint64_t magic, uint64_t addend, int shift) {
3194 return pudiv_magic_epu64(a, magic, addend, shift);
3195}
3196template <>
3197EIGEN_STRONG_INLINE Packet4ul pudiv_magic<Packet4ul>(const Packet4ul& a, uint64_t magic, uint64_t addend, int shift) {
3198 return pudiv_magic_epu64(a, magic, addend, shift);
3199}
3200#endif // EIGEN_VECTORIZE_AVX2 && EIGEN_ARCH_i386_OR_x86_64
3201
3202} // end namespace internal
3203
3204} // end namespace Eigen
3205
3206#endif // EIGEN_PACKET_MATH_AVX_H
@ Aligned32
Definition Constants.h:239
@ Aligned16
Definition Constants.h:238