Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_PACKET_MATH_SSE_H
12#define EIGEN_PACKET_MATH_SSE_H
13
14#include <cstdint>
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
23#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
24#endif
25
26#if !defined(EIGEN_VECTORIZE_AVX) && !defined(EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS)
27// 32 bits => 8 registers
28// 64 bits => 16 registers
29#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2 * sizeof(void*))
30#endif
31
32#ifdef EIGEN_VECTORIZE_FMA
33#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
34#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
35#endif
36#endif
37
38#if ((defined EIGEN_VECTORIZE_AVX) && (EIGEN_COMP_GNUC_STRICT || EIGEN_COMP_MINGW || EIGEN_COMP_LCC) && \
39 (__GXX_ABI_VERSION < 1004)) || \
40 EIGEN_OS_QNX
41// With GCC's default ABI version, a __m128 or __m256 are the same types and therefore we cannot
42// have overloads for both types without linking error.
43// One solution is to increase ABI version using -fabi-version=4 (or greater).
44// Otherwise, we workaround this inconvenience by wrapping 128bit types into the following helper
45// structure:
46using Packet4f = eigen_packet_wrapper<__m128>;
47using Packet2d = eigen_packet_wrapper<__m128d>;
48#else
49using Packet4f = __m128;
50using Packet2d = __m128d;
51#endif
52
53using Packet4i = eigen_packet_wrapper<__m128i, 0>;
54using Packet16b = eigen_packet_wrapper<__m128i, 1>;
55using Packet4ui = eigen_packet_wrapper<__m128i, 4>;
56using Packet2l = eigen_packet_wrapper<__m128i, 5>;
57
58template <>
59struct is_arithmetic<__m128> : std::true_type {};
60template <>
61struct is_arithmetic<__m128i> : std::true_type {};
62template <>
63struct is_arithmetic<__m128d> : std::true_type {};
64template <>
65struct is_arithmetic<Packet4i> : std::true_type {};
66template <>
67struct is_arithmetic<Packet2l> : std::true_type {};
68// Note that `Packet4ui` uses the underlying type `__m128i`, which is
69// interpreted as a vector of _signed_ `int32`s, which breaks some arithmetic
70// operations used in `GenericPacketMath.h`.
71template <>
72struct is_arithmetic<Packet4ui> : std::false_type {};
73template <>
74struct is_arithmetic<Packet16b> : std::true_type {};
75
76template <int p, int q, int r, int s>
77struct shuffle_mask {
78 enum { mask = (s) << 6 | (r) << 4 | (q) << 2 | (p) };
79};
80
81#define SIGN_MASK_I32 static_cast<int32_t>(0x80000000)
82
83// TODO: change the implementation of all swizzle* ops from macro to template,
84#define vec4f_swizzle1(v, p, q, r, s) \
85 Packet4f(_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(v), (shuffle_mask<p, q, r, s>::mask))))
86
87#define vec4i_swizzle1(v, p, q, r, s) Packet4i(_mm_shuffle_epi32(v, (shuffle_mask<p, q, r, s>::mask)))
88
89#define vec4ui_swizzle1(v, p, q, r, s) Packet4ui(vec4i_swizzle1(v, p, q, r, s))
90
91#define vec2d_swizzle1(v, p, q) \
92 Packet2d(_mm_castsi128_pd( \
93 _mm_shuffle_epi32(_mm_castpd_si128(v), (shuffle_mask<2 * p, 2 * p + 1, 2 * q, 2 * q + 1>::mask))))
94
95#define vec4f_swizzle2(a, b, p, q, r, s) Packet4f(_mm_shuffle_ps((a), (b), (shuffle_mask<p, q, r, s>::mask)))
96
97#define vec4i_swizzle2(a, b, p, q, r, s) \
98 Packet4i( \
99 _mm_castps_si128((_mm_shuffle_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b), (shuffle_mask<p, q, r, s>::mask)))))
100
101#define vec4ui_swizzle2(a, b, p, q, r, s) Packet4i(vec4i_swizzle2(a, b, p, q, r, s))
102
103EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) {
104 return Packet4f(_mm_movelh_ps(a, b));
105}
106EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) {
107 return Packet4f(_mm_movehl_ps(a, b));
108}
109EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) {
110 return Packet4f(_mm_unpacklo_ps(a, b));
111}
112EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) {
113 return Packet4f(_mm_unpackhi_ps(a, b));
114}
115#define vec4f_duplane(a, p) vec4f_swizzle2(a, a, p, p, p, p)
116
117#define vec2d_swizzle2(a, b, mask) Packet2d(_mm_shuffle_pd(a, b, mask))
118
119EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a, const Packet2d& b) {
120 return Packet2d(_mm_unpacklo_pd(a, b));
121}
122EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a, const Packet2d& b) {
123 return Packet2d(_mm_unpackhi_pd(a, b));
124}
125#define vec2d_duplane(a, p) vec2d_swizzle2(a, a, (p << 1) | p)
126
127#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = pset1<Packet4f>(X)
128
129#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) const Packet2d p2d_##NAME = pset1<Packet2d>(X)
130
131#define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME, X) const Packet4f p4f_##NAME = pset1frombits<Packet4f>(X)
132
133#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = pset1<Packet4i>(X)
134
135#define EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = pset1<Packet4ui>(X)
136
137// Work around lack of extract/cvt for epi64 when compiling for 32-bit.
138#if EIGEN_ARCH_x86_64
139EIGEN_ALWAYS_INLINE int64_t _mm_extract_epi64_0(const __m128i& a) { return _mm_cvtsi128_si64(a); }
140#ifdef EIGEN_VECTORIZE_SSE4_1
141EIGEN_ALWAYS_INLINE int64_t _mm_extract_epi64_1(const __m128i& a) { return _mm_extract_epi64(a, 1); }
142#else
143EIGEN_ALWAYS_INLINE int64_t _mm_extract_epi64_1(const __m128i& a) {
144 return _mm_cvtsi128_si64(_mm_castpd_si128(_mm_shuffle_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(a), 0x1)));
145}
146#endif
147#else
148// epi64 instructions are not available. The following seems to generate the same instructions
149// with -O2 in GCC/Clang.
150EIGEN_ALWAYS_INLINE int64_t _mm_extract_epi64_0(const __m128i& a) {
151 return numext::bit_cast<int64_t>(_mm_cvtsd_f64(_mm_castsi128_pd(a)));
152}
153EIGEN_ALWAYS_INLINE int64_t _mm_extract_epi64_1(const __m128i& a) {
154 return numext::bit_cast<int64_t>(_mm_cvtsd_f64(_mm_shuffle_pd(_mm_castsi128_pd(a), _mm_castsi128_pd(a), 0x1)));
155}
156#endif
157
158// Use the packet_traits defined in AVX/PacketMath.h instead if we're going
159// to leverage AVX instructions.
160#ifndef EIGEN_VECTORIZE_AVX
161template <>
162struct packet_traits<float> : default_packet_traits {
163 using type = Packet4f;
164 using half = Packet4f;
165 enum {
166 Vectorizable = 1,
167 AlignedOnScalar = 1,
168 size = 4,
169
170 HasCmp = 1,
171 HasDiv = 1,
172 HasReciprocal = EIGEN_FAST_MATH,
173 HasSin = EIGEN_FAST_MATH,
174 HasCos = EIGEN_FAST_MATH,
175 HasTan = EIGEN_FAST_MATH,
176 HasACos = 1,
177 HasASin = 1,
178 HasATan = 1,
179 HasATanh = 1,
180 HasSinh = 1,
181 HasCosh = 1,
182 HasASinh = 1,
183 HasACosh = 1,
184 HasLog = 1,
185 HasLog1p = 1,
186 HasLog10 = 1,
187 HasExpm1 = 1,
188 HasNdtri = 1,
189 HasExp = 1,
190 HasPow = 1,
191 HasBessel = 1,
192 HasSqrt = 1,
193 HasRsqrt = 1,
194 HasCbrt = 1,
195 HasTanh = EIGEN_FAST_MATH,
196 HasErf = EIGEN_FAST_MATH,
197 HasErfc = EIGEN_FAST_MATH,
198 HasSign = 0 // The manually vectorized version is slightly slower for SSE.
199 };
200};
201template <>
202struct packet_traits<double> : default_packet_traits {
203 using type = Packet2d;
204 using half = Packet2d;
205 enum {
206 Vectorizable = 1,
207 AlignedOnScalar = 1,
208 size = 2,
209
210 HasCmp = 1,
211 HasDiv = 1,
212 HasSin = EIGEN_FAST_MATH,
213 HasCos = EIGEN_FAST_MATH,
214 HasTan = EIGEN_FAST_MATH,
215 HasSinh = 1,
216 HasCosh = 1,
217 HasASinh = 1,
218 HasACosh = 1,
219 HasTanh = EIGEN_FAST_MATH,
220 HasErf = EIGEN_FAST_MATH,
221 HasErfc = EIGEN_FAST_MATH,
222 HasLog = 1,
223 HasLog10 = 1,
224 HasExp = 1,
225 HasLog1p = 1,
226 HasExpm1 = 1,
227 HasPow = 1,
228 HasSqrt = 1,
229 HasRsqrt = 1,
230 HasCbrt = 1,
231 HasATan = 1,
232 HasATanh = 1,
233 };
234};
235template <>
236struct packet_traits<int> : default_packet_traits {
237 using type = Packet4i;
238 using half = Packet4i;
239 enum {
240 Vectorizable = 1,
241 AlignedOnScalar = 1,
242 size = 4,
243
244 HasCmp = 1,
245 HasDiv = 1,
246 HasFastIntDiv = 1,
247 HasShift = 1,
248 };
249};
250template <>
251struct packet_traits<uint32_t> : default_packet_traits {
252 using type = Packet4ui;
253 using half = Packet4ui;
254 enum {
255 Vectorizable = 1,
256 AlignedOnScalar = 1,
257 size = 4,
258
259 HasNegate = 0,
260 HasCmp = 1,
261 HasFastIntDiv = 1,
262 HasShift = 1,
263 };
264};
265template <>
266struct packet_traits<int64_t> : default_packet_traits {
267 using type = Packet2l;
268 using half = Packet2l;
269 enum {
270 Vectorizable = 1,
271 AlignedOnScalar = 1,
272 size = 2,
273
274 HasCmp = 1,
275 HasShift = 1,
276 };
277};
278#endif
279template <>
280struct packet_traits<bool> : default_packet_traits {
281 using type = Packet16b;
282 using half = Packet16b;
283 enum {
284 Vectorizable = 1,
285 AlignedOnScalar = 1,
286 size = 16,
287
288 HasCmp = 1,
289 HasShift = 0,
290 HasAbs = 0,
291 HasMin = 0,
292 HasMax = 0,
293 HasConj = 0,
294 HasSqrt = 1,
295 HasNegate = 0,
296 HasSign = 0 // Don't try to vectorize psign<bool> = identity.
297 };
298};
299
300template <>
301struct unpacket_traits<Packet4f> {
302 using type = float;
303 using half = Packet4f;
304 using integer_packet = Packet4i;
305 enum {
306 size = 4,
307 alignment = Aligned16,
308 vectorizable = true,
309 masked_load_available = false,
310 masked_store_available = false
311 };
312};
313template <>
314struct unpacket_traits<Packet2d> {
315 using type = double;
316 using half = Packet2d;
317 using integer_packet = Packet2l;
318 enum {
319 size = 2,
320 alignment = Aligned16,
321 vectorizable = true,
322 masked_load_available = false,
323 masked_store_available = false
324 };
325};
326template <>
327struct unpacket_traits<Packet2l> {
328 using type = int64_t;
329 using half = Packet2l;
330 enum {
331 size = 2,
332 alignment = Aligned16,
333 vectorizable = true,
334 masked_load_available = false,
335 masked_store_available = false
336 };
337};
338template <>
339struct unpacket_traits<Packet4i> {
340 using type = int;
341 using half = Packet4i;
342 enum {
343 size = 4,
344 alignment = Aligned16,
345 vectorizable = true,
346 masked_load_available = false,
347 masked_store_available = false
348 };
349};
350template <>
351struct unpacket_traits<Packet4ui> {
352 using type = uint32_t;
353 using half = Packet4ui;
354 enum {
355 size = 4,
356 alignment = Aligned16,
357 vectorizable = true,
358 masked_load_available = false,
359 masked_store_available = false
360 };
361};
362template <>
363struct unpacket_traits<Packet16b> {
364 using type = bool;
365 using half = Packet16b;
366 enum {
367 size = 16,
368 alignment = Aligned16,
369 vectorizable = true,
370 masked_load_available = false,
371 masked_store_available = false
372 };
373};
374
375#ifndef EIGEN_VECTORIZE_AVX
376template <>
377struct scalar_div_cost<float, true> : std::integral_constant<int, 7> {};
378template <>
379struct scalar_div_cost<double, true> : std::integral_constant<int, 8> {};
380#endif
381
382template <>
383EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
384 return _mm_set_ps1(from);
385}
386template <>
387EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
388 return _mm_set1_pd(from);
389}
390template <>
391EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) {
392 return _mm_set1_epi64x(from);
393}
394template <>
395EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int& from) {
396 return _mm_set1_epi32(from);
397}
398template <>
399EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(const uint32_t& from) {
400 return _mm_set1_epi32(numext::bit_cast<int32_t>(from));
401}
402template <>
403EIGEN_STRONG_INLINE Packet16b pset1<Packet16b>(const bool& from) {
404 return _mm_set1_epi8(static_cast<char>(from));
405}
406
407template <>
408EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(unsigned int from) {
409 return _mm_castsi128_ps(pset1<Packet4i>(from));
410}
411template <>
412EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
413 return _mm_castsi128_pd(_mm_set1_epi64x(from));
414}
415
416template <>
417EIGEN_STRONG_INLINE Packet4f peven_mask(const Packet4f& /*a*/) {
418 Packet4f r = _mm_castsi128_ps(_mm_set_epi32(0, -1, 0, -1));
419 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
420 return r;
421}
422template <>
423EIGEN_STRONG_INLINE Packet2l peven_mask(const Packet2l& /*a*/) {
424 return _mm_set_epi32(0, 0, -1, -1);
425}
426template <>
427EIGEN_STRONG_INLINE Packet4i peven_mask(const Packet4i& /*a*/) {
428 return _mm_set_epi32(0, -1, 0, -1);
429}
430template <>
431EIGEN_STRONG_INLINE Packet4ui peven_mask(const Packet4ui& /*a*/) {
432 return _mm_set_epi32(0, -1, 0, -1);
433}
434template <>
435EIGEN_STRONG_INLINE Packet2d peven_mask(const Packet2d& /*a*/) {
436 Packet2d r = _mm_castsi128_pd(_mm_set_epi32(0, 0, -1, -1));
437 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
438 return r;
439}
440
441template <>
442EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /*a*/) {
443 return _mm_setzero_ps();
444}
445template <>
446EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /*a*/) {
447 return _mm_setzero_pd();
448}
449template <>
450EIGEN_STRONG_INLINE Packet2l pzero(const Packet2l& /*a*/) {
451 return _mm_setzero_si128();
452}
453template <>
454EIGEN_STRONG_INLINE Packet4i pzero(const Packet4i& /*a*/) {
455 return _mm_setzero_si128();
456}
457template <>
458EIGEN_STRONG_INLINE Packet4ui pzero(const Packet4ui& /*a*/) {
459 return _mm_setzero_si128();
460}
461
462// GCC generates a shufps instruction for _mm_set1_ps/_mm_load1_ps instead of the more efficient pshufd instruction.
463// However, using intrinsics for pset1 makes gcc to generate crappy code in some cases (see bug 203)
464// Using inline assembly is also not an option because then gcc fails to reorder properly the instructions.
465// Therefore, we introduced the pload1 functions to be used in product kernels for which bug 203 does not apply.
466// Also note that with AVX, we want it to generate a vbroadcastss.
467#if EIGEN_COMP_GNUC_STRICT && (!defined __AVX__)
468template <>
469EIGEN_STRONG_INLINE Packet4f pload1<Packet4f>(const float* from) {
470 return vec4f_swizzle1(_mm_load_ss(from), 0, 0, 0, 0);
471}
472#endif
473
474template <>
475EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
476 return _mm_add_ps(pset1<Packet4f>(a), _mm_set_ps(3, 2, 1, 0));
477}
478template <>
479EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
480 return _mm_add_pd(pset1<Packet2d>(a), _mm_set_pd(1, 0));
481}
482template <>
483EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(const int64_t& a) {
484 return _mm_add_epi64(pset1<Packet2l>(a), _mm_set_epi64x(1, 0));
485}
486template <>
487EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int& a) {
488 return _mm_add_epi32(pset1<Packet4i>(a), _mm_set_epi32(3, 2, 1, 0));
489}
490template <>
491EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(const uint32_t& a) {
492 return _mm_add_epi32(pset1<Packet4ui>(a), _mm_set_epi32(3, 2, 1, 0));
493}
494
495template <>
496EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
497 return _mm_add_ps(a, b);
498}
499template <>
500EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
501 return _mm_add_pd(a, b);
502}
503template <>
504EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) {
505 return _mm_add_epi64(a, b);
506}
507template <>
508EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
509 return _mm_add_epi32(a, b);
510}
511template <>
512EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
513 return _mm_add_epi32(a, b);
514}
515
516template <>
517EIGEN_STRONG_INLINE Packet16b padd<Packet16b>(const Packet16b& a, const Packet16b& b) {
518 return _mm_or_si128(a, b);
519}
520
521template <typename Packet>
522EIGEN_STRONG_INLINE Packet padds(const Packet& a, const Packet& b);
523template <>
524EIGEN_STRONG_INLINE Packet4f padds<Packet4f>(const Packet4f& a, const Packet4f& b) {
525 return _mm_add_ss(a, b);
526}
527template <>
528EIGEN_STRONG_INLINE Packet2d padds<Packet2d>(const Packet2d& a, const Packet2d& b) {
529 return _mm_add_sd(a, b);
530}
531
532template <>
533EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
534 return _mm_sub_ps(a, b);
535}
536template <>
537EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
538 return _mm_sub_pd(a, b);
539}
540template <>
541EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) {
542 return _mm_sub_epi64(a, b);
543}
544template <>
545EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
546 return _mm_sub_epi32(a, b);
547}
548template <>
549EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
550 return _mm_sub_epi32(a, b);
551}
552template <>
553EIGEN_STRONG_INLINE Packet16b psub<Packet16b>(const Packet16b& a, const Packet16b& b) {
554 return _mm_xor_si128(a, b);
555}
556
557template <>
558EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b);
559template <>
560EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(const Packet4f& a, const Packet4f& b) {
561#ifdef EIGEN_VECTORIZE_SSE3
562 return _mm_addsub_ps(a, b);
563#else
564 const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(SIGN_MASK_I32, 0x0, SIGN_MASK_I32, 0x0));
565 return padd(a, pxor(mask, b));
566#endif
567}
568
569template <>
570EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d&, const Packet2d&);
571template <>
572EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(const Packet2d& a, const Packet2d& b) {
573#ifdef EIGEN_VECTORIZE_SSE3
574 return _mm_addsub_pd(a, b);
575#else
576 const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0, SIGN_MASK_I32, 0x0, 0x0));
577 return padd(a, pxor(mask, b));
578#endif
579}
580
581template <>
582EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
583 const Packet4f mask = _mm_castsi128_ps(_mm_setr_epi32(SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32, SIGN_MASK_I32));
584 return _mm_xor_ps(a, mask);
585}
586template <>
587EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
588 const Packet2d mask = _mm_castsi128_pd(_mm_setr_epi32(0x0, SIGN_MASK_I32, 0x0, SIGN_MASK_I32));
589 return _mm_xor_pd(a, mask);
590}
591template <>
592EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) {
593 return psub(pzero(a), a);
594}
595
596template <>
597EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
598 return psub(pzero(a), a);
599}
600
601template <>
602EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
603 return _mm_mul_ps(a, b);
604}
605template <>
606EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
607 return _mm_mul_pd(a, b);
608}
609template <>
610EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(const Packet2l& a, const Packet2l& b) {
611 // 64-bit mul requires avx512, so do this with 32-bit multiplication
612 __m128i upper32_a = _mm_srli_epi64(a, 32);
613 __m128i upper32_b = _mm_srli_epi64(b, 32);
614
615 // upper * lower
616 __m128i mul1 = _mm_mul_epu32(upper32_a, b);
617 __m128i mul2 = _mm_mul_epu32(upper32_b, a);
618 // Gives us both upper*upper and lower*lower
619 __m128i mul3 = _mm_mul_epu32(a, b);
620
621 __m128i high = _mm_slli_epi64(_mm_add_epi64(mul1, mul2), 32);
622 return _mm_add_epi64(high, mul3);
623}
624template <>
625EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
626#ifdef EIGEN_VECTORIZE_SSE4_1
627 return _mm_mullo_epi32(a, b);
628#else
629 // this version is slightly faster than 4 scalar products
630 return vec4i_swizzle1(
631 vec4i_swizzle2(_mm_mul_epu32(a, b), _mm_mul_epu32(vec4i_swizzle1(a, 1, 0, 3, 2), vec4i_swizzle1(b, 1, 0, 3, 2)),
632 0, 2, 0, 2),
633 0, 2, 1, 3);
634#endif
635}
636template <>
637EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
638#ifdef EIGEN_VECTORIZE_SSE4_1
639 return _mm_mullo_epi32(a, b);
640#else
641 // this version is slightly faster than 4 scalar products
642 return vec4ui_swizzle1(
643 vec4ui_swizzle2(_mm_mul_epu32(a, b),
644 _mm_mul_epu32(vec4ui_swizzle1(a, 1, 0, 3, 2), vec4ui_swizzle1(b, 1, 0, 3, 2)), 0, 2, 0, 2),
645 0, 2, 1, 3);
646#endif
647}
648
649template <>
650EIGEN_STRONG_INLINE Packet16b pmul<Packet16b>(const Packet16b& a, const Packet16b& b) {
651 return _mm_and_si128(a, b);
652}
653
654template <>
655EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
656 return _mm_div_ps(a, b);
657}
658template <>
659EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
660 return _mm_div_pd(a, b);
661}
662
663template <>
664EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) {
665#ifdef EIGEN_VECTORIZE_AVX
666 return _mm256_cvttpd_epi32(_mm256_div_pd(_mm256_cvtepi32_pd(a), _mm256_cvtepi32_pd(b)));
667#else
668 __m128i q_lo = _mm_cvttpd_epi32(_mm_div_pd(_mm_cvtepi32_pd(a), _mm_cvtepi32_pd(b)));
669 __m128i q_hi = _mm_cvttpd_epi32(
670 _mm_div_pd(_mm_cvtepi32_pd(vec4i_swizzle1(a, 2, 3, 0, 1)), _mm_cvtepi32_pd(vec4i_swizzle1(b, 2, 3, 0, 1))));
671 return vec4i_swizzle1(_mm_unpacklo_epi32(q_lo, q_hi), 0, 2, 1, 3);
672#endif
673}
674
675#ifdef EIGEN_VECTORIZE_FMA
676template <>
677struct is_fused_madd<Packet4f> : true_type {};
678template <>
679EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
680 return _mm_fmadd_ps(a, b, c);
681}
682template <>
683struct is_fused_madd<Packet2d> : true_type {};
684template <>
685EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
686 return _mm_fmadd_pd(a, b, c);
687}
688template <>
689EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
690 return _mm_fmsub_ps(a, b, c);
691}
692template <>
693EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
694 return _mm_fmsub_pd(a, b, c);
695}
696template <>
697EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
698 return _mm_fnmadd_ps(a, b, c);
699}
700template <>
701EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
702 return _mm_fnmadd_pd(a, b, c);
703}
704template <>
705EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
706 return _mm_fnmsub_ps(a, b, c);
707}
708template <>
709EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
710 return _mm_fnmsub_pd(a, b, c);
711}
712
713template <typename Packet>
714EIGEN_STRONG_INLINE Packet pmadds(const Packet& a, const Packet& b, const Packet& c);
715template <>
716EIGEN_STRONG_INLINE Packet4f pmadds<Packet4f>(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
717 return _mm_fmadd_ss(a, b, c);
718}
719template <>
720EIGEN_STRONG_INLINE Packet2d pmadds<Packet2d>(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
721 return _mm_fmadd_sd(a, b, c);
722}
723#endif
724
725#ifdef EIGEN_VECTORIZE_SSE4_1
726template <>
727EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) {
728 return _mm_blendv_ps(b, a, mask);
729}
730
731template <>
732EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) {
733 return _mm_castpd_si128(_mm_blendv_pd(_mm_castsi128_pd(b), _mm_castsi128_pd(a), _mm_castsi128_pd(mask)));
734}
735
736template <>
737EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) {
738 return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(b), _mm_castsi128_ps(a), _mm_castsi128_ps(mask)));
739}
740
741template <>
742EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) {
743 return _mm_castps_si128(_mm_blendv_ps(_mm_castsi128_ps(b), _mm_castsi128_ps(a), _mm_castsi128_ps(mask)));
744}
745
746template <>
747EIGEN_STRONG_INLINE Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) {
748 return _mm_blendv_pd(b, a, mask);
749}
750#endif
751
752template <>
753EIGEN_STRONG_INLINE Packet2l ptrue<Packet2l>(const Packet2l& a) {
754 return _mm_cmpeq_epi32(a, a);
755}
756template <>
757EIGEN_STRONG_INLINE Packet4i ptrue<Packet4i>(const Packet4i& a) {
758 return _mm_cmpeq_epi32(a, a);
759}
760template <>
761EIGEN_STRONG_INLINE Packet16b ptrue<Packet16b>(const Packet16b& /*a*/) {
762 return pset1<Packet16b>(true);
763}
764template <>
765EIGEN_STRONG_INLINE Packet4f ptrue<Packet4f>(const Packet4f& a) {
766 Packet4i b = _mm_castps_si128(a);
767 Packet4f r = _mm_castsi128_ps(_mm_cmpeq_epi32(b, b));
768 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
769 return r;
770}
771template <>
772EIGEN_STRONG_INLINE Packet2d ptrue<Packet2d>(const Packet2d& a) {
773 Packet4i b = _mm_castpd_si128(a);
774 Packet2d r = _mm_castsi128_pd(_mm_cmpeq_epi32(b, b));
775 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
776 return r;
777}
778
779template <>
780EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
781 return _mm_and_ps(a, b);
782}
783template <>
784EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
785 return _mm_and_pd(a, b);
786}
787template <>
788EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) {
789 return _mm_and_si128(a, b);
790}
791template <>
792EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
793 return _mm_and_si128(a, b);
794}
795template <>
796EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
797 return _mm_and_si128(a, b);
798}
799template <>
800EIGEN_STRONG_INLINE Packet16b pand<Packet16b>(const Packet16b& a, const Packet16b& b) {
801 return _mm_and_si128(a, b);
802}
803
804template <>
805EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
806 return _mm_or_ps(a, b);
807}
808template <>
809EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
810 return _mm_or_pd(a, b);
811}
812template <>
813EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b) {
814 return _mm_or_si128(a, b);
815}
816template <>
817EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
818 return _mm_or_si128(a, b);
819}
820template <>
821EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
822 return _mm_or_si128(a, b);
823}
824template <>
825EIGEN_STRONG_INLINE Packet16b por<Packet16b>(const Packet16b& a, const Packet16b& b) {
826 return _mm_or_si128(a, b);
827}
828
829template <>
830EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
831 return _mm_xor_ps(a, b);
832}
833template <>
834EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
835 return _mm_xor_pd(a, b);
836}
837template <>
838EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b) {
839 return _mm_xor_si128(a, b);
840}
841template <>
842EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
843 return _mm_xor_si128(a, b);
844}
845template <>
846EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
847 return _mm_xor_si128(a, b);
848}
849template <>
850EIGEN_STRONG_INLINE Packet16b pxor<Packet16b>(const Packet16b& a, const Packet16b& b) {
851 return _mm_xor_si128(a, b);
852}
853
854template <>
855EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
856 return _mm_andnot_ps(b, a);
857}
858template <>
859EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
860 return _mm_andnot_pd(b, a);
861}
862template <>
863EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b) {
864 return _mm_andnot_si128(b, a);
865}
866template <>
867EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
868 return _mm_andnot_si128(b, a);
869}
870template <>
871EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
872 return _mm_andnot_si128(b, a);
873}
874template <>
875EIGEN_STRONG_INLINE Packet16b pandnot<Packet16b>(const Packet16b& a, const Packet16b& b) {
876 return _mm_andnot_si128(b, a);
877}
878template <>
879EIGEN_STRONG_INLINE Packet16b pcmp_lt(const Packet16b& a, const Packet16b& b) {
880 return _mm_andnot_si128(a, b);
881}
882template <>
883EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) {
884 return _mm_cmple_ps(a, b);
885}
886template <>
887EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) {
888 return _mm_cmplt_ps(a, b);
889}
890template <>
891EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) {
892 return _mm_cmpnge_ps(a, b);
893}
894template <>
895EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) {
896 return _mm_cmpeq_ps(a, b);
897}
898
899template <>
900EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b) {
901 return _mm_cmple_pd(a, b);
902}
903template <>
904EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b) {
905 return _mm_cmplt_pd(a, b);
906}
907template <>
908EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) {
909 return _mm_cmpnge_pd(a, b);
910}
911template <>
912EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) {
913 return _mm_cmpeq_pd(a, b);
914}
915template <>
916EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) {
917 return _mm_cmplt_epi32(a, b);
918}
919template <>
920EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) {
921 return _mm_cmpeq_epi32(a, b);
922}
923template <>
924EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) {
925#ifdef EIGEN_VECTORIZE_SSE4_1
926 return _mm_cmpeq_epi32(a, _mm_min_epi32(a, b));
927#else
928 return por(pcmp_lt(a, b), pcmp_eq(a, b));
929#endif
930}
931template <>
932EIGEN_STRONG_INLINE Packet2l pcmp_lt(const Packet2l& a, const Packet2l& b) {
933#ifdef EIGEN_VECTORIZE_SSE4_2
934 return _mm_cmpgt_epi64(b, a);
935#else
936 Packet4i eq = pcmp_eq<Packet4i>(Packet4i(a), Packet4i(b));
937 Packet2l hi_eq = Packet2l(_mm_shuffle_epi32(eq, (shuffle_mask<1, 1, 3, 3>::mask)));
938 // The low halves carry magnitude only and must be ordered as unsigned, while the high halves carry
939 // the sign. Biasing just the low lanes by 2^31 makes one signed 32-bit compare serve both.
940 const Packet4i kLowSignFlip = _mm_setr_epi32(SIGN_MASK_I32, 0x0, SIGN_MASK_I32, 0x0);
941 Packet4i lt = pcmp_lt<Packet4i>(pxor(Packet4i(a), kLowSignFlip), pxor(Packet4i(b), kLowSignFlip));
942 Packet2l hi_lt = Packet2l(_mm_shuffle_epi32(lt, (shuffle_mask<1, 1, 3, 3>::mask)));
943 Packet2l lo_lt = Packet2l(_mm_shuffle_epi32(lt, (shuffle_mask<0, 0, 2, 2>::mask)));
944 // return hi(a) < hi(b) || (hi(a) == hi(b) && lo(a) < lo(b))
945 return por(hi_lt, pand(hi_eq, lo_lt));
946#endif
947}
948template <>
949EIGEN_STRONG_INLINE Packet2l pcmp_eq(const Packet2l& a, const Packet2l& b) {
950#ifdef EIGEN_VECTORIZE_SSE4_1
951 return _mm_cmpeq_epi64(a, b);
952#else
953 Packet4i tmp = pcmp_eq<Packet4i>(Packet4i(a), Packet4i(b));
954 return Packet2l(pand<Packet4i>(tmp, _mm_shuffle_epi32(tmp, (shuffle_mask<1, 0, 3, 2>::mask))));
955#endif
956}
957template <>
958EIGEN_STRONG_INLINE Packet2l pcmp_le(const Packet2l& a, const Packet2l& b) {
959 return por(pcmp_lt(a, b), pcmp_eq(a, b));
960}
961template <>
962EIGEN_STRONG_INLINE Packet16b pcmp_eq(const Packet16b& a, const Packet16b& b) {
963 // Mask out invalid bool bits to avoid UB.
964 const Packet16b kBoolMask = pset1<Packet16b>(true);
965 return _mm_and_si128(_mm_cmpeq_epi8(a, b), kBoolMask);
966}
967template <>
968EIGEN_STRONG_INLINE Packet4ui pcmp_eq(const Packet4ui& a, const Packet4ui& b) {
969 return _mm_cmpeq_epi32(a, b);
970}
971
972template <>
973EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
974#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
975// There appears to be a bug in GCC, by which the optimizer may
976// flip the argument order in calls to _mm_min_ps, so we have to
977// resort to inline ASM here. This is supposed to be fixed in gcc6.3,
978// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
979#ifdef EIGEN_VECTORIZE_AVX
980 Packet4f res;
981 asm("vminps %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
982#else
983 Packet4f res = b;
984 asm("minps %[a], %[res]" : [res] "+x"(res) : [a] "x"(a));
985#endif
986 return res;
987#else
988 // Arguments are reversed to match NaN propagation behavior of std::min.
989 return _mm_min_ps(b, a);
990#endif
991}
992template <>
993EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
994#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
995// There appears to be a bug in GCC, by which the optimizer may
996// flip the argument order in calls to _mm_min_pd, so we have to
997// resort to inline ASM here. This is supposed to be fixed in gcc6.3,
998// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
999#ifdef EIGEN_VECTORIZE_AVX
1000 Packet2d res;
1001 asm("vminpd %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
1002#else
1003 Packet2d res = b;
1004 asm("minpd %[a], %[res]" : [res] "+x"(res) : [a] "x"(a));
1005#endif
1006 return res;
1007#else
1008 // Arguments are reversed to match NaN propagation behavior of std::min.
1009 return _mm_min_pd(b, a);
1010#endif
1011}
1012template <>
1013EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(const Packet2l& a, const Packet2l& b) {
1014 Packet2l a_lt_mask = pcmp_lt(a, b);
1015 return por(pandnot(b, a_lt_mask), pand(a, a_lt_mask));
1016}
1017template <>
1018EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
1019#ifdef EIGEN_VECTORIZE_SSE4_1
1020 return _mm_min_epi32(a, b);
1021#else
1022 // after some bench, this version *is* faster than a scalar implementation
1023 Packet4i mask = _mm_cmplt_epi32(a, b);
1024 return _mm_or_si128(_mm_and_si128(mask, a), _mm_andnot_si128(mask, b));
1025#endif
1026}
1027template <>
1028EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1029#ifdef EIGEN_VECTORIZE_SSE4_1
1030 return _mm_min_epu32(a, b);
1031#else
1032 return padd((Packet4ui)pmin((Packet4i)psub(a, pset1<Packet4ui>(0x80000000UL)),
1033 (Packet4i)psub(b, pset1<Packet4ui>(0x80000000UL))),
1034 pset1<Packet4ui>(0x80000000UL));
1035#endif
1036}
1037
1038template <>
1039EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
1040#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
1041// There appears to be a bug in GCC, by which the optimizer may
1042// flip the argument order in calls to _mm_max_ps, so we have to
1043// resort to inline ASM here. This is supposed to be fixed in gcc6.3,
1044// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
1045#ifdef EIGEN_VECTORIZE_AVX
1046 Packet4f res;
1047 asm("vmaxps %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
1048#else
1049 Packet4f res = b;
1050 asm("maxps %[a], %[res]" : [res] "+x"(res) : [a] "x"(a));
1051#endif
1052 return res;
1053#else
1054 // Arguments are reversed to match NaN propagation behavior of std::max.
1055 return _mm_max_ps(b, a);
1056#endif
1057}
1058template <>
1059EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
1060#if EIGEN_GNUC_STRICT_LESS_THAN(6, 3, 0)
1061// There appears to be a bug in GCC, by which the optimizer may
1062// flip the argument order in calls to _mm_max_pd, so we have to
1063// resort to inline ASM here. This is supposed to be fixed in gcc6.3,
1064// see also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=72867
1065#ifdef EIGEN_VECTORIZE_AVX
1066 Packet2d res;
1067 asm("vmaxpd %[a], %[b], %[res]" : [res] "=x"(res) : [a] "x"(a), [b] "x"(b));
1068#else
1069 Packet2d res = b;
1070 asm("maxpd %[a], %[res]" : [res] "+x"(res) : [a] "x"(a));
1071#endif
1072 return res;
1073#else
1074 // Arguments are reversed to match NaN propagation behavior of std::max.
1075 return _mm_max_pd(b, a);
1076#endif
1077}
1078template <>
1079EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(const Packet2l& a, const Packet2l& b) {
1080 Packet2l a_lt_mask = pcmp_lt(a, b);
1081 return por(pandnot(a, a_lt_mask), pand(b, a_lt_mask));
1082}
1083template <>
1084EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
1085#ifdef EIGEN_VECTORIZE_SSE4_1
1086 return _mm_max_epi32(a, b);
1087#else
1088 // after some bench, this version *is* faster than a scalar implementation
1089 Packet4i mask = _mm_cmpgt_epi32(a, b);
1090 return _mm_or_si128(_mm_and_si128(mask, a), _mm_andnot_si128(mask, b));
1091#endif
1092}
1093template <>
1094EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1095#ifdef EIGEN_VECTORIZE_SSE4_1
1096 return _mm_max_epu32(a, b);
1097#else
1098 return padd((Packet4ui)pmax((Packet4i)psub(a, pset1<Packet4ui>(0x80000000UL)),
1099 (Packet4i)psub(b, pset1<Packet4ui>(0x80000000UL))),
1100 pset1<Packet4ui>(0x80000000UL));
1101#endif
1102}
1103
1104template <>
1105EIGEN_STRONG_INLINE Packet4ui pcmp_lt(const Packet4ui& a, const Packet4ui& b) {
1106#ifdef EIGEN_VECTORIZE_SSE4_1
1107 return pxor(pcmp_eq(a, pmax(a, b)), ptrue(a));
1108#else
1109 return (Packet4ui)pcmp_lt((Packet4i)psub(a, pset1<Packet4ui>(0x80000000UL)),
1110 (Packet4i)psub(b, pset1<Packet4ui>(0x80000000UL)));
1111#endif
1112}
1113template <>
1114EIGEN_STRONG_INLINE Packet4ui pcmp_le(const Packet4ui& a, const Packet4ui& b) {
1115#ifdef EIGEN_VECTORIZE_SSE4_1
1116 return pcmp_eq(a, pmin(a, b));
1117#else
1118 return (Packet4ui)pcmp_le((Packet4i)psub(a, pset1<Packet4ui>(0x80000000UL)),
1119 (Packet4i)psub(b, pset1<Packet4ui>(0x80000000UL)));
1120#endif
1121}
1122
1123template <typename Packet, typename Op>
1124EIGEN_STRONG_INLINE Packet pminmax_propagate_numbers(const Packet& a, const Packet& b, Op op) {
1125 // In this implementation, we take advantage of the fact that pmin/pmax for SSE
1126 // always return a if either a or b is NaN.
1127 Packet not_nan_mask_a = pcmp_eq(a, a);
1128 Packet m = op(a, b);
1129 return pselect<Packet>(not_nan_mask_a, m, b);
1130}
1131
1132template <typename Packet, typename Op>
1133EIGEN_STRONG_INLINE Packet pminmax_propagate_nan(const Packet& a, const Packet& b, Op op) {
1134 // In this implementation, we take advantage of the fact that pmin/pmax for SSE
1135 // always return a if either a or b is NaN. Testing b rather than a is what supplies the
1136 // missing case, and it keeps op's operand order, hence its choice on a signed-zero tie.
1137 Packet not_nan_mask_b = pcmp_eq(b, b);
1138 Packet m = op(a, b);
1139 return pselect<Packet>(not_nan_mask_b, m, b);
1140}
1141
1142// Add specializations for min/max with prescribed NaN propagation.
1143template <>
1144EIGEN_STRONG_INLINE Packet4f pmin<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) {
1145 return pminmax_propagate_numbers(a, b, pmin<Packet4f>);
1146}
1147template <>
1148EIGEN_STRONG_INLINE Packet2d pmin<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) {
1149 return pminmax_propagate_numbers(a, b, pmin<Packet2d>);
1150}
1151template <>
1152EIGEN_STRONG_INLINE Packet4f pmax<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) {
1153 return pminmax_propagate_numbers(a, b, pmax<Packet4f>);
1154}
1155template <>
1156EIGEN_STRONG_INLINE Packet2d pmax<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) {
1157 return pminmax_propagate_numbers(a, b, pmax<Packet2d>);
1158}
1159template <>
1160EIGEN_STRONG_INLINE Packet4f pmin<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) {
1161 return pminmax_propagate_nan(a, b, pmin<Packet4f>);
1162}
1163template <>
1164EIGEN_STRONG_INLINE Packet2d pmin<PropagateNaN, Packet2d>(const Packet2d& a, const Packet2d& b) {
1165 return pminmax_propagate_nan(a, b, pmin<Packet2d>);
1166}
1167template <>
1168EIGEN_STRONG_INLINE Packet4f pmax<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) {
1169 return pminmax_propagate_nan(a, b, pmax<Packet4f>);
1170}
1171template <>
1172EIGEN_STRONG_INLINE Packet2d pmax<PropagateNaN, Packet2d>(const Packet2d& a, const Packet2d& b) {
1173 return pminmax_propagate_nan(a, b, pmax<Packet2d>);
1174}
1175
1176template <>
1177EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) {
1178 return _mm_castsi128_ps(_mm_srai_epi32(_mm_castps_si128(a), 31));
1179}
1180template <>
1181EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) {
1182 Packet4f tmp = psignbit<Packet4f>(_mm_castpd_ps(a));
1183#ifdef EIGEN_VECTORIZE_AVX
1184 return _mm_castps_pd(_mm_permute_ps(tmp, (shuffle_mask<1, 1, 3, 3>::mask)));
1185#else
1186 return _mm_castps_pd(_mm_shuffle_ps(tmp, tmp, (shuffle_mask<1, 1, 3, 3>::mask)));
1187#endif // EIGEN_VECTORIZE_AVX
1188}
1189template <>
1190EIGEN_STRONG_INLINE Packet4i psignbit(const Packet4i& a) {
1191 return _mm_srai_epi32(a, 31);
1192}
1193template <>
1194EIGEN_STRONG_INLINE Packet2l psignbit(const Packet2l& a) {
1195 Packet4i tmp = psignbit<Packet4i>(Packet4i(a));
1196 return Packet2l(_mm_shuffle_epi32(tmp, (shuffle_mask<1, 1, 3, 3>::mask)));
1197}
1198
1199template <int N>
1200EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(const Packet2l& a) {
1201 Packet2l signbit = psignbit(a);
1202 return por(_mm_slli_epi64(signbit, 64 - N), _mm_srli_epi64(a, N));
1203}
1204template <int N>
1205EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
1206 return _mm_srli_epi64(a, N);
1207}
1208template <int N>
1209EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
1210 return _mm_slli_epi64(a, N);
1211}
1212template <int N>
1213EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) {
1214 return _mm_srai_epi32(a, N);
1215}
1216template <int N>
1217EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) {
1218 return _mm_srli_epi32(a, N);
1219}
1220template <int N>
1221EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) {
1222 return _mm_slli_epi32(a, N);
1223}
1224template <int N>
1225EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(const Packet4ui& a) {
1226 return _mm_srli_epi32(a, N);
1227}
1228template <int N>
1229EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) {
1230 return _mm_srli_epi32(a, N);
1231}
1232template <int N>
1233EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) {
1234 return _mm_slli_epi32(a, N);
1235}
1236
1237template <>
1238EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) {
1239 const __m128i mask = _mm_setr_epi32(0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF);
1240 return _mm_castsi128_ps(_mm_and_si128(mask, _mm_castps_si128(a)));
1241}
1242template <>
1243EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) {
1244 const __m128i mask = _mm_setr_epi32(-1, 0x7FFFFFFF, -1, 0x7FFFFFFF);
1245 return _mm_castsi128_pd(_mm_and_si128(mask, _mm_castpd_si128(a)));
1246}
1247template <>
1248EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) {
1249 Packet2l signbit = psignbit(a);
1250 return _mm_sub_epi64(_mm_xor_si128(a, signbit), signbit);
1251}
1252template <>
1253EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) {
1254#ifdef EIGEN_VECTORIZE_SSSE3
1255 return _mm_abs_epi32(a);
1256#else
1257 Packet4i signbit = psignbit(a);
1258 return _mm_sub_epi32(_mm_xor_si128(a, signbit), signbit);
1259#endif
1260}
1261#ifdef EIGEN_VECTORIZE_SSE4_1
1262template <>
1263EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(const Packet4f& a) {
1264 // Unfortunately _mm_round_ps doesn't have a rounding mode to implement numext::round.
1265 const Packet4f mask = pset1frombits<Packet4f>(0x80000000u);
1266 const Packet4f prev0dot5 = pset1frombits<Packet4f>(0x3EFFFFFFu);
1267 return _mm_round_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
1268}
1269
1270template <>
1271EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(const Packet2d& a) {
1272 const Packet2d mask = _mm_castsi128_pd(_mm_set_epi64x(0x8000000000000000ull, 0x8000000000000000ull));
1273 const Packet2d prev0dot5 = _mm_castsi128_pd(_mm_set_epi64x(0x3FDFFFFFFFFFFFFFull, 0x3FDFFFFFFFFFFFFFull));
1274 return _mm_round_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
1275}
1276
1277template <>
1278EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
1279 return _mm_round_ps(a, _MM_FROUND_CUR_DIRECTION);
1280}
1281template <>
1282EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
1283 return _mm_round_pd(a, _MM_FROUND_CUR_DIRECTION);
1284}
1285
1286template <>
1287EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a) {
1288 return _mm_ceil_ps(a);
1289}
1290template <>
1291EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a) {
1292 return _mm_ceil_pd(a);
1293}
1294
1295template <>
1296EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a) {
1297 return _mm_floor_ps(a);
1298}
1299template <>
1300EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a) {
1301 return _mm_floor_pd(a);
1302}
1303
1304template <>
1305EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
1306 return _mm_round_ps(a, _MM_FROUND_TRUNC);
1307}
1308template <>
1309EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
1310 return _mm_round_pd(a, _MM_FROUND_TRUNC);
1311}
1312#endif
1313
1314template <>
1315EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
1316 EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ps(from);
1317}
1318template <>
1319EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
1320 EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_pd(from);
1321}
1322template <>
1323EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(const int64_t* from) {
1324 EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
1325}
1326template <>
1327EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int* from) {
1328 EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
1329}
1330template <>
1331EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(const uint32_t* from) {
1332 EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
1333}
1334template <>
1335EIGEN_STRONG_INLINE Packet16b pload<Packet16b>(const bool* from) {
1336 EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_si128(reinterpret_cast<const __m128i*>(from));
1337}
1338
1339template <>
1340EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
1341 EIGEN_DEBUG_UNALIGNED_LOAD
1342 return _mm_loadu_ps(from);
1343}
1344
1345template <>
1346EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
1347 EIGEN_DEBUG_UNALIGNED_LOAD
1348 return _mm_loadu_pd(from);
1349}
1350template <>
1351EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(const int64_t* from) {
1352 EIGEN_DEBUG_UNALIGNED_LOAD
1353 return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
1354}
1355template <>
1356EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int* from) {
1357 EIGEN_DEBUG_UNALIGNED_LOAD
1358 return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
1359}
1360template <>
1361EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(const uint32_t* from) {
1362 EIGEN_DEBUG_UNALIGNED_LOAD
1363 return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
1364}
1365template <>
1366EIGEN_STRONG_INLINE Packet16b ploadu<Packet16b>(const bool* from) {
1367 EIGEN_DEBUG_UNALIGNED_LOAD
1368 return _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
1369}
1370
1371EIGEN_STRONG_INLINE __m128i ploadu_si64(const void* from) {
1372#if EIGEN_GNUC_STRICT_LESS_THAN(9, 1, 0) || (EIGEN_COMP_MINGW && !EIGEN_COMP_CLANG && EIGEN_COMP_GNUC < 910)
1373 // GCC added _mm_loadu_si64 in 9.1. Copy through __m64 to avoid relying on _mm_loadl_epi64's type-punned load.
1374 EIGEN_USING_STD(memcpy);
1375 __m64 lo;
1376 memcpy(&lo, from, sizeof(lo));
1377 return _mm_set_epi64((__m64)0LL, lo);
1378#else
1379 return _mm_loadu_si64(from);
1380#endif
1381}
1382
1383// Load lower part of packet zero extending.
1384template <typename Packet>
1385EIGEN_STRONG_INLINE Packet ploadl(const typename unpacket_traits<Packet>::type* from);
1386template <>
1387EIGEN_STRONG_INLINE Packet4f ploadl<Packet4f>(const float* from) {
1388 EIGEN_DEBUG_UNALIGNED_LOAD return _mm_castsi128_ps(ploadu_si64(reinterpret_cast<const void*>(from)));
1389}
1390template <>
1391EIGEN_STRONG_INLINE Packet2d ploadl<Packet2d>(const double* from) {
1392 EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_sd(from);
1393}
1394
1395// Load scalar
1396template <typename Packet>
1397EIGEN_STRONG_INLINE Packet ploads(const typename unpacket_traits<Packet>::type* from);
1398template <>
1399EIGEN_STRONG_INLINE Packet4f ploads<Packet4f>(const float* from) {
1400 EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_ss(from);
1401}
1402template <>
1403EIGEN_STRONG_INLINE Packet2d ploads<Packet2d>(const double* from) {
1404 EIGEN_DEBUG_UNALIGNED_LOAD return _mm_load_sd(from);
1405}
1406
1407template <>
1408EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
1409 return vec4f_swizzle1(_mm_castsi128_ps(ploadu_si64(reinterpret_cast<const void*>(from))), 0, 0, 1, 1);
1410}
1411template <>
1412EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int* from) {
1413 Packet4i tmp;
1414 tmp = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(from));
1415 return vec4i_swizzle1(tmp, 0, 0, 1, 1);
1416}
1417template <>
1418EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(const uint32_t* from) {
1419 Packet4ui tmp;
1420 tmp = _mm_loadl_epi64(reinterpret_cast<const __m128i*>(from));
1421 return vec4ui_swizzle1(tmp, 0, 0, 1, 1);
1422}
1423
1424// Loads 8 bools from memory and returns the packet
1425// {b0, b0, b1, b1, b2, b2, b3, b3, b4, b4, b5, b5, b6, b6, b7, b7}
1426template <>
1427EIGEN_STRONG_INLINE Packet16b ploaddup<Packet16b>(const bool* from) {
1428 __m128i tmp = ploadu_si64(reinterpret_cast<const void*>(from));
1429 return _mm_unpacklo_epi8(tmp, tmp);
1430}
1431
1432// Loads 4 bools from memory and returns the packet
1433// {b0, b0, b0, b0, b1, b1, b1, b1, b2, b2, b2, b2, b3, b3, b3, b3}
1434template <>
1435EIGEN_STRONG_INLINE Packet16b ploadquad<Packet16b>(const bool* from) {
1436 EIGEN_USING_STD(memcpy);
1437 int val;
1438 memcpy(&val, from, sizeof(int));
1439 __m128i tmp = _mm_cvtsi32_si128(val);
1440 tmp = _mm_unpacklo_epi8(tmp, tmp);
1441 return _mm_unpacklo_epi16(tmp, tmp);
1442}
1443
1444template <>
1445EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
1446 EIGEN_DEBUG_ALIGNED_STORE _mm_store_ps(to, from);
1447}
1448template <>
1449EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
1450 EIGEN_DEBUG_ALIGNED_STORE _mm_store_pd(to, from);
1451}
1452template <>
1453EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet2l& from) {
1454 EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
1455}
1456template <>
1457EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet4i& from) {
1458 EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
1459}
1460template <>
1461EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet4ui& from) {
1462 EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
1463}
1464template <>
1465EIGEN_STRONG_INLINE void pstore<bool>(bool* to, const Packet16b& from) {
1466 EIGEN_DEBUG_ALIGNED_STORE _mm_store_si128(reinterpret_cast<__m128i*>(to), from);
1467}
1468
1469template <>
1470EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
1471 EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_pd(to, from);
1472}
1473template <>
1474EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
1475 EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ps(to, from);
1476}
1477template <>
1478EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet2l& from) {
1479 EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
1480}
1481template <>
1482EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet4i& from) {
1483 EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
1484}
1485template <>
1486EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet4ui& from) {
1487 EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
1488}
1489template <>
1490EIGEN_STRONG_INLINE void pstoreu<bool>(bool* to, const Packet16b& from) {
1491 EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from);
1492}
1493
1494template <typename Scalar, typename Packet>
1495EIGEN_STRONG_INLINE void pstorel(Scalar* to, const Packet& from);
1496template <>
1497EIGEN_STRONG_INLINE void pstorel(float* to, const Packet4f& from) {
1498 EIGEN_DEBUG_UNALIGNED_STORE _mm_storel_pi(reinterpret_cast<__m64*>(to), from);
1499}
1500template <>
1501EIGEN_STRONG_INLINE void pstorel(double* to, const Packet2d& from) {
1502 EIGEN_DEBUG_UNALIGNED_STORE _mm_storel_pd(to, from);
1503}
1504
1505template <typename Scalar, typename Packet>
1506EIGEN_STRONG_INLINE void pstores(Scalar* to, const Packet& from);
1507template <>
1508EIGEN_STRONG_INLINE void pstores(float* to, const Packet4f& from) {
1509 EIGEN_DEBUG_UNALIGNED_STORE _mm_store_ss(to, from);
1510}
1511template <>
1512EIGEN_STRONG_INLINE void pstores(double* to, const Packet2d& from) {
1513 EIGEN_DEBUG_UNALIGNED_STORE _mm_store_sd(to, from);
1514}
1515
1516template <>
1517EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
1518 return _mm_shuffle_ps(a, a, 0x1B);
1519}
1520template <>
1521EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
1522 return _mm_shuffle_pd(a, a, 0x1);
1523}
1524template <>
1525EIGEN_STRONG_INLINE Packet2l preverse(const Packet2l& a) {
1526 return _mm_castpd_si128(preverse(_mm_castsi128_pd(a)));
1527}
1528template <>
1529EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
1530 return _mm_shuffle_epi32(a, 0x1B);
1531}
1532template <>
1533EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a) {
1534 return _mm_shuffle_epi32(a, 0x1B);
1535}
1536template <>
1537EIGEN_STRONG_INLINE Packet16b preverse(const Packet16b& a) {
1538#ifdef EIGEN_VECTORIZE_SSSE3
1539 __m128i mask = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
1540 return _mm_shuffle_epi8(a, mask);
1541#else
1542 Packet16b tmp = _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3));
1543 tmp = _mm_shufflehi_epi16(_mm_shufflelo_epi16(tmp, _MM_SHUFFLE(2, 3, 0, 1)), _MM_SHUFFLE(2, 3, 0, 1));
1544 return _mm_or_si128(_mm_slli_epi16(tmp, 8), _mm_srli_epi16(tmp, 8));
1545#endif
1546}
1547
1548template <>
1549EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
1550 return _mm_cvtss_f32(a);
1551}
1552template <>
1553EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
1554 return _mm_cvtsd_f64(a);
1555}
1556template <>
1557EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(const Packet2l& a) {
1558 return _mm_extract_epi64_0(a);
1559}
1560template <>
1561EIGEN_STRONG_INLINE int pfirst<Packet4i>(const Packet4i& a) {
1562 return _mm_cvtsi128_si32(a);
1563}
1564template <>
1565EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(const Packet4ui& a) {
1566 return numext::bit_cast<uint32_t>(_mm_cvtsi128_si32(a));
1567}
1568template <>
1569EIGEN_STRONG_INLINE bool pfirst<Packet16b>(const Packet16b& a) {
1570 int x = _mm_cvtsi128_si32(a);
1571 return static_cast<bool>(x & 1);
1572}
1573
1574template <>
1575EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
1576 if (stride == 2) {
1577 // Overlap the loads so the second ends at the last gathered coefficient, from[6].
1578 return _mm_shuffle_ps(_mm_loadu_ps(from), _mm_loadu_ps(from + 3), _MM_SHUFFLE(3, 1, 2, 0));
1579 }
1580 return _mm_set_ps(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]);
1581}
1582template <>
1583EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
1584 return _mm_set_pd(from[1 * stride], from[0 * stride]);
1585}
1586template <>
1587EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(const int64_t* from, Index stride) {
1588 return _mm_set_epi64x(from[1 * stride], from[0 * stride]);
1589}
1590template <>
1591EIGEN_STRONG_INLINE Packet4i pgather<int, Packet4i>(const int* from, Index stride) {
1592 return _mm_set_epi32(from[3 * stride], from[2 * stride], from[1 * stride], from[0 * stride]);
1593}
1594template <>
1595EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(const uint32_t* from, Index stride) {
1596 return _mm_set_epi32(numext::bit_cast<int32_t>(from[3 * stride]), numext::bit_cast<int32_t>(from[2 * stride]),
1597 numext::bit_cast<int32_t>(from[1 * stride]), numext::bit_cast<int32_t>(from[0 * stride]));
1598}
1599
1600template <>
1601EIGEN_STRONG_INLINE Packet16b pgather<bool, Packet16b>(const bool* from, Index stride) {
1602 return _mm_set_epi8(from[15 * stride], from[14 * stride], from[13 * stride], from[12 * stride], from[11 * stride],
1603 from[10 * stride], from[9 * stride], from[8 * stride], from[7 * stride], from[6 * stride],
1604 from[5 * stride], from[4 * stride], from[3 * stride], from[2 * stride], from[1 * stride],
1605 from[0 * stride]);
1606}
1607
1608template <>
1609EIGEN_STRONG_INLINE void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
1610 to[stride * 0] = pfirst(from);
1611 to[stride * 1] = pfirst(Packet4f(_mm_shuffle_ps(from, from, 1)));
1612 to[stride * 2] = pfirst(Packet4f(_mm_shuffle_ps(from, from, 2)));
1613 to[stride * 3] = pfirst(Packet4f(_mm_shuffle_ps(from, from, 3)));
1614}
1615template <>
1616EIGEN_STRONG_INLINE void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
1617 to[stride * 0] = pfirst(from);
1618 to[stride * 1] = pfirst(preverse(from));
1619}
1620template <>
1621EIGEN_STRONG_INLINE void pscatter<int64_t, Packet2l>(int64_t* to, const Packet2l& from, Index stride) {
1622 to[stride * 0] = pfirst(from);
1623 to[stride * 1] = pfirst(preverse(from));
1624}
1625template <>
1626EIGEN_STRONG_INLINE void pscatter<int, Packet4i>(int* to, const Packet4i& from, Index stride) {
1627 to[stride * 0] = _mm_cvtsi128_si32(from);
1628 to[stride * 1] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1));
1629 to[stride * 2] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2));
1630 to[stride * 3] = _mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3));
1631}
1632template <>
1633EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from, Index stride) {
1634 to[stride * 0] = numext::bit_cast<uint32_t>(_mm_cvtsi128_si32(from));
1635 to[stride * 1] = numext::bit_cast<uint32_t>(_mm_cvtsi128_si32(_mm_shuffle_epi32(from, 1)));
1636 to[stride * 2] = numext::bit_cast<uint32_t>(_mm_cvtsi128_si32(_mm_shuffle_epi32(from, 2)));
1637 to[stride * 3] = numext::bit_cast<uint32_t>(_mm_cvtsi128_si32(_mm_shuffle_epi32(from, 3)));
1638}
1639template <>
1640EIGEN_STRONG_INLINE void pscatter<bool, Packet16b>(bool* to, const Packet16b& from, Index stride) {
1641 EIGEN_ALIGN16 bool tmp[16];
1642 pstore(tmp, from);
1643 to[stride * 0] = tmp[0];
1644 to[stride * 1] = tmp[1];
1645 to[stride * 2] = tmp[2];
1646 to[stride * 3] = tmp[3];
1647 to[stride * 4] = tmp[4];
1648 to[stride * 5] = tmp[5];
1649 to[stride * 6] = tmp[6];
1650 to[stride * 7] = tmp[7];
1651 to[stride * 8] = tmp[8];
1652 to[stride * 9] = tmp[9];
1653 to[stride * 10] = tmp[10];
1654 to[stride * 11] = tmp[11];
1655 to[stride * 12] = tmp[12];
1656 to[stride * 13] = tmp[13];
1657 to[stride * 14] = tmp[14];
1658 to[stride * 15] = tmp[15];
1659}
1660
1661// some compilers might be tempted to perform multiple moves instead of using a vector path.
1662template <>
1663EIGEN_STRONG_INLINE void pstore1<Packet4f>(float* to, const float& a) {
1664 Packet4f pa = _mm_set_ss(a);
1665 pstore(to, Packet4f(vec4f_swizzle1(pa, 0, 0, 0, 0)));
1666}
1667// some compilers might be tempted to perform multiple moves instead of using a vector path.
1668template <>
1669EIGEN_STRONG_INLINE void pstore1<Packet2d>(double* to, const double& a) {
1670 Packet2d pa = _mm_set_sd(a);
1671 pstore(to, Packet2d(vec2d_swizzle1(pa, 0, 0)));
1672}
1673
1674#if EIGEN_COMP_PGI && EIGEN_COMP_PGI < 1900
1675using SsePrefetchPtrType = const void*;
1676#else
1677using SsePrefetchPtrType = const char*;
1678#endif
1679
1680#ifndef EIGEN_VECTORIZE_AVX
1681template <>
1682EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1683 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1684}
1685template <>
1686EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1687 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1688}
1689template <>
1690EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) {
1691 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1692}
1693template <>
1694EIGEN_STRONG_INLINE void prefetch<int64_t>(const int64_t* addr) {
1695 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1696}
1697template <>
1698EIGEN_STRONG_INLINE void prefetch<uint32_t>(const uint32_t* addr) {
1699 _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0);
1700}
1701#endif
1702
1703template <>
1704EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) {
1705 return pfrexp_generic(a, exponent);
1706}
1707
1708// Extract exponent without existence of Packet2l.
1709template <>
1710EIGEN_STRONG_INLINE Packet2d pfrexp_generic_get_biased_exponent(const Packet2d& a) {
1711 const Packet2d cst_exp_mask = pset1frombits<Packet2d>(static_cast<uint64_t>(0x7ff0000000000000ull));
1712 __m128i a_expo = _mm_srli_epi64(_mm_castpd_si128(pand(a, cst_exp_mask)), 52);
1713 return _mm_cvtepi32_pd(vec4i_swizzle1(a_expo, 0, 2, 1, 3));
1714}
1715
1716template <>
1717EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent) {
1718 return pfrexp_generic(a, exponent);
1719}
1720
1721template <>
1722EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
1723 return pldexp_generic(a, exponent);
1724}
1725
1726// We specialize pldexp here, since the generic implementation uses Packet2l, which is not well
1727// supported by SSE, and has more range than is needed for exponents.
1728template <>
1729EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
1730 // The single-rounding split of pldexp_generic on the two low int32 lanes. Interleaving t and e - t puts each
1731 // biased pair in one 64-bit lane, t + 2 * bias low: shifting left by 51 drops the high half, and shifting right by
1732 // 32 first selects it.
1733 const Packet2d max_exponent = pset1<Packet2d>(2099.0);
1734 const Packet2d last_max = pset1<Packet2d>(1022.0);
1735 const Packet4i e = _mm_cvttpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
1736 const Packet4i b = _mm_cvttpd_epi32(pmin(pmax(exponent, pnegate(last_max)), last_max));
1737 const Packet4i t = pandnot(psub(e, b), pset1<Packet4i>(1)); // even
1738 const Packet4i biased =
1739 padd(Packet4i(_mm_unpacklo_epi32(t, psub(e, t))), Packet4i(_mm_set_epi32(1023, 2046, 1023, 2046)));
1740 const Packet2d c1 = _mm_castsi128_pd(_mm_slli_epi64(biased, 51)); // 2^(t/2)
1741 const Packet2d c2 = _mm_castsi128_pd(_mm_slli_epi64(_mm_srli_epi64(biased, 32), 52)); // 2^(e - t)
1742 return pldexp_apply_factors(a, c1, c2); // a * 2^e
1743}
1744
1745// We specialize pldexp here, since the generic implementation uses Packet2l, which is not well
1746// supported by SSE, and has more range than is needed for exponents.
1747template <>
1748EIGEN_STRONG_INLINE Packet2d pldexp_fast<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
1749 // Clamp exponent to [-1023, 1024]
1750 const Packet2d min_exponent = pset1<Packet2d>(-1023.0);
1751 const Packet2d max_exponent = pset1<Packet2d>(1024.0);
1752 const Packet2d e = pmin(pmax(exponent, min_exponent), max_exponent);
1753
1754 // Convert e to integer and swizzle to low-order bits.
1755 const Packet4i ei = vec4i_swizzle1(_mm_cvtpd_epi32(e), 0, 3, 1, 3);
1756
1757 // Compute 2^e multiply:
1758 const Packet4i bias = _mm_set_epi32(0, 1023, 0, 1023);
1759 const Packet2d c = _mm_castsi128_pd(_mm_slli_epi64(padd(ei, bias), 52)); // 2^e
1760 return pmul(a, c);
1761}
1762
1763// with AVX, the default implementations based on pload1 are faster
1764#ifndef __AVX__
1765template <>
1766EIGEN_STRONG_INLINE void pbroadcast4<Packet4f>(const float* a, Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
1767 a3 = pload<Packet4f>(a);
1768 a0 = vec4f_swizzle1(a3, 0, 0, 0, 0);
1769 a1 = vec4f_swizzle1(a3, 1, 1, 1, 1);
1770 a2 = vec4f_swizzle1(a3, 2, 2, 2, 2);
1771 a3 = vec4f_swizzle1(a3, 3, 3, 3, 3);
1772}
1773template <>
1774EIGEN_STRONG_INLINE void pbroadcast4<Packet2d>(const double* a, Packet2d& a0, Packet2d& a1, Packet2d& a2,
1775 Packet2d& a3) {
1776#ifdef EIGEN_VECTORIZE_SSE3
1777 a0 = _mm_loaddup_pd(a + 0);
1778 a1 = _mm_loaddup_pd(a + 1);
1779 a2 = _mm_loaddup_pd(a + 2);
1780 a3 = _mm_loaddup_pd(a + 3);
1781#else
1782 a1 = pload<Packet2d>(a);
1783 a0 = vec2d_swizzle1(a1, 0, 0);
1784 a1 = vec2d_swizzle1(a1, 1, 1);
1785 a3 = pload<Packet2d>(a + 2);
1786 a2 = vec2d_swizzle1(a3, 0, 0);
1787 a3 = vec2d_swizzle1(a3, 1, 1);
1788#endif
1789}
1790#endif
1791
1792EIGEN_STRONG_INLINE void punpackp(Packet4f* vecs) {
1793 vecs[1] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x55));
1794 vecs[2] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xAA));
1795 vecs[3] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0xFF));
1796 vecs[0] = _mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(vecs[0]), 0x00));
1797}
1798
1799EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
1800 _MM_TRANSPOSE4_PS(kernel.packet[0], kernel.packet[1], kernel.packet[2], kernel.packet[3]);
1801}
1802
1803EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
1804 __m128d tmp = _mm_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1805 kernel.packet[0] = _mm_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1806 kernel.packet[1] = tmp;
1807}
1808
1809EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2l, 2>& kernel) {
1810 __m128i tmp = _mm_unpackhi_epi64(kernel.packet[0], kernel.packet[1]);
1811 kernel.packet[0] = _mm_unpacklo_epi64(kernel.packet[0], kernel.packet[1]);
1812 kernel.packet[1] = tmp;
1813}
1814
1815EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
1816 __m128i T0 = _mm_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
1817 __m128i T1 = _mm_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
1818 __m128i T2 = _mm_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
1819 __m128i T3 = _mm_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
1820
1821 kernel.packet[0] = _mm_unpacklo_epi64(T0, T1);
1822 kernel.packet[1] = _mm_unpackhi_epi64(T0, T1);
1823 kernel.packet[2] = _mm_unpacklo_epi64(T2, T3);
1824 kernel.packet[3] = _mm_unpackhi_epi64(T2, T3);
1825}
1826EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
1827 ptranspose((PacketBlock<Packet4i, 4>&)kernel);
1828}
1829
1830EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16b, 4>& kernel) {
1831 __m128i T0 = _mm_unpacklo_epi8(kernel.packet[0], kernel.packet[1]);
1832 __m128i T1 = _mm_unpackhi_epi8(kernel.packet[0], kernel.packet[1]);
1833 __m128i T2 = _mm_unpacklo_epi8(kernel.packet[2], kernel.packet[3]);
1834 __m128i T3 = _mm_unpackhi_epi8(kernel.packet[2], kernel.packet[3]);
1835 kernel.packet[0] = _mm_unpacklo_epi16(T0, T2);
1836 kernel.packet[1] = _mm_unpackhi_epi16(T0, T2);
1837 kernel.packet[2] = _mm_unpacklo_epi16(T1, T3);
1838 kernel.packet[3] = _mm_unpackhi_epi16(T1, T3);
1839}
1840
1841EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16b, 16>& kernel) {
1842 // If we number the elements in the input thus:
1843 // kernel.packet[ 0] = {00, 01, 02, 03, 04, 05, 06, 07, 08, 09, 0a, 0b, 0c, 0d, 0e, 0f}
1844 // kernel.packet[ 1] = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 1a, 1b, 1c, 1d, 1e, 1f}
1845 // ...
1846 // kernel.packet[15] = {f0, f1, f2, f3, f4, f5, f6, f7, f8, f9, fa, fb, fc, fd, fe, ff},
1847 //
1848 // the desired output is:
1849 // kernel.packet[ 0] = {00, 10, 20, 30, 40, 50, 60, 70, 80, 90, a0, b0, c0, d0, e0, f0}
1850 // kernel.packet[ 1] = {01, 11, 21, 31, 41, 51, 61, 71, 81, 91, a1, b1, c1, d1, e1, f1}
1851 // ...
1852 // kernel.packet[15] = {0f, 1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, af, bf, cf, df, ef, ff},
1853 __m128i t0 =
1854 _mm_unpacklo_epi8(kernel.packet[0], kernel.packet[1]); // 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
1855 __m128i t1 =
1856 _mm_unpackhi_epi8(kernel.packet[0], kernel.packet[1]); // 08 18 09 19 0a 1a 0b 1b 0c 1c 0d 1d 0e 1e 0f 1f
1857 __m128i t2 =
1858 _mm_unpacklo_epi8(kernel.packet[2], kernel.packet[3]); // 20 30 21 31 22 32 ... 27 37
1859 __m128i t3 =
1860 _mm_unpackhi_epi8(kernel.packet[2], kernel.packet[3]); // 28 38 29 39 2a 3a ... 2f 3f
1861 __m128i t4 =
1862 _mm_unpacklo_epi8(kernel.packet[4], kernel.packet[5]); // 40 50 41 51 42 52 47 57
1863 __m128i t5 = _mm_unpackhi_epi8(kernel.packet[4], kernel.packet[5]); // 48 58 49 59 4a 5a
1864 __m128i t6 = _mm_unpacklo_epi8(kernel.packet[6], kernel.packet[7]);
1865 __m128i t7 = _mm_unpackhi_epi8(kernel.packet[6], kernel.packet[7]);
1866 __m128i t8 = _mm_unpacklo_epi8(kernel.packet[8], kernel.packet[9]);
1867 __m128i t9 = _mm_unpackhi_epi8(kernel.packet[8], kernel.packet[9]);
1868 __m128i ta = _mm_unpacklo_epi8(kernel.packet[10], kernel.packet[11]);
1869 __m128i tb = _mm_unpackhi_epi8(kernel.packet[10], kernel.packet[11]);
1870 __m128i tc = _mm_unpacklo_epi8(kernel.packet[12], kernel.packet[13]);
1871 __m128i td = _mm_unpackhi_epi8(kernel.packet[12], kernel.packet[13]);
1872 __m128i te = _mm_unpacklo_epi8(kernel.packet[14], kernel.packet[15]);
1873 __m128i tf = _mm_unpackhi_epi8(kernel.packet[14], kernel.packet[15]);
1874
1875 __m128i s0 = _mm_unpacklo_epi16(t0, t2); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
1876 __m128i s1 = _mm_unpackhi_epi16(t0, t2); // 04 14 24 34
1877 __m128i s2 = _mm_unpacklo_epi16(t1, t3); // 08 18 28 38 ...
1878 __m128i s3 = _mm_unpackhi_epi16(t1, t3); // 0c 1c 2c 3c ...
1879 __m128i s4 = _mm_unpacklo_epi16(t4, t6); // 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
1880 __m128i s5 = _mm_unpackhi_epi16(t4, t6); // 44 54 64 74 ...
1881 __m128i s6 = _mm_unpacklo_epi16(t5, t7);
1882 __m128i s7 = _mm_unpackhi_epi16(t5, t7);
1883 __m128i s8 = _mm_unpacklo_epi16(t8, ta);
1884 __m128i s9 = _mm_unpackhi_epi16(t8, ta);
1885 __m128i sa = _mm_unpacklo_epi16(t9, tb);
1886 __m128i sb = _mm_unpackhi_epi16(t9, tb);
1887 __m128i sc = _mm_unpacklo_epi16(tc, te);
1888 __m128i sd = _mm_unpackhi_epi16(tc, te);
1889 __m128i se = _mm_unpacklo_epi16(td, tf);
1890 __m128i sf = _mm_unpackhi_epi16(td, tf);
1891
1892 __m128i u0 = _mm_unpacklo_epi32(s0, s4); // 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
1893 __m128i u1 = _mm_unpackhi_epi32(s0, s4); // 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
1894 __m128i u2 = _mm_unpacklo_epi32(s1, s5);
1895 __m128i u3 = _mm_unpackhi_epi32(s1, s5);
1896 __m128i u4 = _mm_unpacklo_epi32(s2, s6);
1897 __m128i u5 = _mm_unpackhi_epi32(s2, s6);
1898 __m128i u6 = _mm_unpacklo_epi32(s3, s7);
1899 __m128i u7 = _mm_unpackhi_epi32(s3, s7);
1900 __m128i u8 = _mm_unpacklo_epi32(s8, sc);
1901 __m128i u9 = _mm_unpackhi_epi32(s8, sc);
1902 __m128i ua = _mm_unpacklo_epi32(s9, sd);
1903 __m128i ub = _mm_unpackhi_epi32(s9, sd);
1904 __m128i uc = _mm_unpacklo_epi32(sa, se);
1905 __m128i ud = _mm_unpackhi_epi32(sa, se);
1906 __m128i ue = _mm_unpacklo_epi32(sb, sf);
1907 __m128i uf = _mm_unpackhi_epi32(sb, sf);
1908
1909 kernel.packet[0] = _mm_unpacklo_epi64(u0, u8);
1910 kernel.packet[1] = _mm_unpackhi_epi64(u0, u8);
1911 kernel.packet[2] = _mm_unpacklo_epi64(u1, u9);
1912 kernel.packet[3] = _mm_unpackhi_epi64(u1, u9);
1913 kernel.packet[4] = _mm_unpacklo_epi64(u2, ua);
1914 kernel.packet[5] = _mm_unpackhi_epi64(u2, ua);
1915 kernel.packet[6] = _mm_unpacklo_epi64(u3, ub);
1916 kernel.packet[7] = _mm_unpackhi_epi64(u3, ub);
1917 kernel.packet[8] = _mm_unpacklo_epi64(u4, uc);
1918 kernel.packet[9] = _mm_unpackhi_epi64(u4, uc);
1919 kernel.packet[10] = _mm_unpacklo_epi64(u5, ud);
1920 kernel.packet[11] = _mm_unpackhi_epi64(u5, ud);
1921 kernel.packet[12] = _mm_unpacklo_epi64(u6, ue);
1922 kernel.packet[13] = _mm_unpackhi_epi64(u6, ue);
1923 kernel.packet[14] = _mm_unpacklo_epi64(u7, uf);
1924 kernel.packet[15] = _mm_unpackhi_epi64(u7, uf);
1925}
1926
1927// Scalar path for pmadd with FMA to ensure consistency with vectorized path.
1928#if defined(EIGEN_VECTORIZE_FMA)
1929template <>
1930EIGEN_STRONG_INLINE float pmadd(const float& a, const float& b, const float& c) {
1931 return numext::fma(a, b, c);
1932}
1933template <>
1934EIGEN_STRONG_INLINE double pmadd(const double& a, const double& b, const double& c) {
1935 return numext::fma(a, b, c);
1936}
1937template <>
1938EIGEN_STRONG_INLINE float pmsub(const float& a, const float& b, const float& c) {
1939 return numext::fma(a, b, -c);
1940}
1941template <>
1942EIGEN_STRONG_INLINE double pmsub(const double& a, const double& b, const double& c) {
1943 return numext::fma(a, b, -c);
1944}
1945template <>
1946EIGEN_STRONG_INLINE float pnmadd(const float& a, const float& b, const float& c) {
1947 return numext::fma(-a, b, c);
1948}
1949template <>
1950EIGEN_STRONG_INLINE double pnmadd(const double& a, const double& b, const double& c) {
1951 return numext::fma(-a, b, c);
1952}
1953template <>
1954EIGEN_STRONG_INLINE float pnmsub(const float& a, const float& b, const float& c) {
1955 return numext::fma(-a, b, -c);
1956}
1957template <>
1958EIGEN_STRONG_INLINE double pnmsub(const double& a, const double& b, const double& c) {
1959 return numext::fma(-a, b, -c);
1960}
1961#endif
1962
1963#ifdef EIGEN_VECTORIZE_SSE4_1
1964// Helpers for half->float and float->half conversions.
1965// Currently only used by the AVX code.
1966EIGEN_STRONG_INLINE __m128i half2floatsse(__m128i h) {
1967 __m128i input = _mm_cvtepu16_epi32(h);
1968
1969 // Direct vectorization of half_to_float, C parts in the comments.
1970 __m128i shifted_exp = _mm_set1_epi32(0x7c00 << 13);
1971 // o.u = (h.x & 0x7fff) << 13; // exponent/mantissa bits
1972 __m128i ou = _mm_slli_epi32(_mm_and_si128(input, _mm_set1_epi32(0x7fff)), 13);
1973 // exp = shifted_exp & o.u; // just the exponent
1974 __m128i exp = _mm_and_si128(ou, shifted_exp);
1975 // o.u += (127 - 15) << 23;
1976 ou = _mm_add_epi32(ou, _mm_set1_epi32((127 - 15) << 23));
1977
1978 // Inf/NaN?
1979 __m128i naninf_mask = _mm_cmpeq_epi32(exp, shifted_exp);
1980 // Inf/NaN adjust
1981 __m128i naninf_adj = _mm_and_si128(_mm_set1_epi32((128 - 16) << 23), naninf_mask);
1982 // extra exp adjust for Inf/NaN
1983 ou = _mm_add_epi32(ou, naninf_adj);
1984
1985 // Zero/Denormal?
1986 __m128i zeroden_mask = _mm_cmpeq_epi32(exp, _mm_setzero_si128());
1987 __m128i zeroden_adj = _mm_and_si128(zeroden_mask, _mm_set1_epi32(1 << 23));
1988 // o.u += 1 << 23;
1989 ou = _mm_add_epi32(ou, zeroden_adj);
1990 // magic.u = 113 << 23
1991 __m128i magic = _mm_and_si128(zeroden_mask, _mm_set1_epi32(113 << 23));
1992 // o.f -= magic.f
1993 ou = _mm_castps_si128(_mm_sub_ps(_mm_castsi128_ps(ou), _mm_castsi128_ps(magic)));
1994
1995 __m128i sign = _mm_slli_epi32(_mm_and_si128(input, _mm_set1_epi32(0x8000)), 16);
1996 // o.u |= (h.x & 0x8000) << 16; // sign bit
1997 ou = _mm_or_si128(ou, sign);
1998 // return o.f;
1999 // We are actually returning uint version, to make
2000 // _mm256_insertf128_si256 work.
2001 return ou;
2002}
2003
2004EIGEN_STRONG_INLINE __m128i float2half(__m128 f) {
2005 // unsigned int sign_mask = 0x80000000u;
2006 __m128i sign = _mm_set1_epi32(0x80000000u);
2007 // unsigned int sign = f.u & sign_mask;
2008 sign = _mm_and_si128(sign, _mm_castps_si128(f));
2009 // f.u ^= sign;
2010 f = _mm_xor_ps(f, _mm_castsi128_ps(sign));
2011
2012 __m128i fu = _mm_castps_si128(f);
2013
2014 __m128i f16max = _mm_set1_epi32((127 + 16) << 23);
2015 __m128i f32infty = _mm_set1_epi32(255 << 23);
2016 // if (f.u >= f16max.u) // result is Inf or NaN (all exponent bits set)
2017 // there is no _mm_cmpge_epi32, so use lt and swap operands
2018 __m128i infnan_mask = _mm_cmplt_epi32(f16max, _mm_castps_si128(f));
2019 __m128i inf_mask = _mm_cmpgt_epi32(_mm_castps_si128(f), f32infty);
2020 __m128i nan_mask = _mm_andnot_si128(inf_mask, infnan_mask);
2021 __m128i inf_value = _mm_and_si128(inf_mask, _mm_set1_epi32(0x7e00));
2022 __m128i nan_value = _mm_and_si128(nan_mask, _mm_set1_epi32(0x7c00));
2023 // o.x = (f.u > f32infty.u) ? 0x7e00 : 0x7c00; // NaN->qNaN and Inf->Inf
2024 __m128i naninf_value = _mm_or_si128(inf_value, nan_value);
2025
2026 __m128i denorm_magic = _mm_set1_epi32(((127 - 15) + (23 - 10) + 1) << 23);
2027 __m128i subnorm_mask = _mm_cmplt_epi32(_mm_castps_si128(f), _mm_set1_epi32(113 << 23));
2028 // f.f += denorm_magic.f;
2029 f = _mm_add_ps(f, _mm_castsi128_ps(denorm_magic));
2030 // f.u - denorm_magic.u
2031 __m128i o = _mm_sub_epi32(_mm_castps_si128(f), denorm_magic);
2032 o = _mm_and_si128(o, subnorm_mask);
2033 // Correct result for inf/nan/zero/subnormal, 0 otherwise
2034 o = _mm_or_si128(o, naninf_value);
2035
2036 __m128i mask = _mm_or_si128(infnan_mask, subnorm_mask);
2037 o = _mm_and_si128(o, mask);
2038
2039 // mant_odd = (f.u >> 13) & 1;
2040 __m128i mand_odd = _mm_and_si128(_mm_srli_epi32(fu, 13), _mm_set1_epi32(0x1));
2041 // f.u += 0xc8000fffU;
2042 fu = _mm_add_epi32(fu, _mm_set1_epi32(0xc8000fffU));
2043 // f.u += mant_odd;
2044 fu = _mm_add_epi32(fu, mand_odd);
2045 fu = _mm_andnot_si128(mask, fu);
2046 // f.u >> 13
2047 fu = _mm_srli_epi32(fu, 13);
2048 o = _mm_or_si128(fu, o);
2049
2050 // o.x |= static_cast<numext::uint16_t>(sign >> 16);
2051 o = _mm_or_si128(o, _mm_srli_epi32(sign, 16));
2052
2053 // 16 bit values
2054 return _mm_and_si128(o, _mm_set1_epi32(0xffff));
2055}
2056#endif
2057
2058/*---------------- division by an invariant integer ----------------*/
2059
2060// pudiv_magic (IntegerDivision.h) for 32-bit lanes. _mm_mul_epu32 multiplies the even lanes into 64-bit products.
2061EIGEN_STRONG_INLINE __m128i pudiv_magic_epu32(const __m128i& a, uint32_t magic, uint32_t addend, int shift) {
2062 __m128i m = _mm_set1_epi32(static_cast<int>(magic));
2063 __m128i c = _mm_set1_epi64x(static_cast<long long>(addend));
2064 __m128i even = _mm_srli_epi64(_mm_add_epi64(_mm_mul_epu32(a, m), c), 32);
2065 __m128i odd = _mm_add_epi64(_mm_mul_epu32(_mm_srli_epi64(a, 32), m), c);
2066#ifdef EIGEN_VECTORIZE_SSE4_1
2067 __m128i t = _mm_blend_epi16(even, odd, 0xCC);
2068#else
2069 __m128i t = _mm_or_si128(even, _mm_and_si128(odd, _mm_set_epi32(-1, 0, -1, 0)));
2070#endif
2071 return _mm_srl_epi32(t, _mm_cvtsi32_si128(shift));
2072}
2073
2074// 64-bit lanes: the high half of the 128-bit sum m n + c from four 32 x 32 -> 64-bit products, as in Hacker's Delight,
2075// 2nd ed., Fig. 8-2. Each partial sum adds at most two terms below 2^32 to a product <= (2^32 - 1)^2, which stays
2076// below 2^64. m and c are multiply-add constants, which int_div::params stores only where use_multiply_add<uint64_t>
2077// holds: on x86, not under Emscripten, which also defines __SSE2__.
2078#if EIGEN_ARCH_i386_OR_x86_64
2079EIGEN_STRONG_INLINE __m128i pudiv_magic_epu64(const __m128i& a, uint64_t magic, uint64_t addend, int shift) {
2080 static_assert(int_div::use_multiply_add<uint64_t>::value, "the 64-bit kernels take multiply-add constants");
2081 __m128i m_lo = _mm_set1_epi64x(static_cast<long long>(magic));
2082 __m128i m_hi = _mm_set1_epi64x(static_cast<long long>(magic >> 32));
2083 __m128i c_lo = _mm_set1_epi64x(static_cast<long long>(addend & 0xffffffffu));
2084 __m128i c_hi = _mm_set1_epi64x(static_cast<long long>(addend >> 32));
2085 __m128i a_hi = _mm_srli_epi64(a, 32);
2086 __m128i u0 = _mm_add_epi64(_mm_mul_epu32(a, m_lo), c_lo);
2087 __m128i u = _mm_add_epi64(_mm_mul_epu32(a_hi, m_lo), _mm_srli_epi64(u0, 32));
2088 __m128i w = _mm_add_epi64(_mm_add_epi64(_mm_mul_epu32(a, m_hi), _mm_and_si128(u, _mm_set1_epi64x(0xffffffff))), c_hi);
2089 __m128i t = _mm_add_epi64(_mm_add_epi64(_mm_mul_epu32(a_hi, m_hi), _mm_srli_epi64(u, 32)), _mm_srli_epi64(w, 32));
2090 return _mm_srl_epi64(t, _mm_cvtsi32_si128(shift));
2091}
2092
2093template <>
2094EIGEN_STRONG_INLINE Packet2l pudiv_magic<Packet2l>(const Packet2l& a, uint64_t magic, uint64_t addend, int shift) {
2095 return pudiv_magic_epu64(a, magic, addend, shift);
2096}
2097#endif // EIGEN_ARCH_i386_OR_x86_64
2098
2099// psdiv_signed_magic (IntegerDivision.h) for 32-bit lanes. Callers pass both shifts: the two kernels share one
2100// mangled name, so a program that links translation units built with and without SSE4.1 may run either.
2101#ifdef EIGEN_VECTORIZE_SSE4_1
2102// _mm_mul_epi32 gives the signed high half: mulhs(M - 2^32, n) + n, for the M of every divisor.
2103EIGEN_STRONG_INLINE __m128i psdiv_signed_magic_epi32(const __m128i& a, uint32_t magic, int shift, int /*total_shift*/,
2104 const __m128i& divisor_sign) {
2105 __m128i m = _mm_set1_epi32(static_cast<int>(magic));
2106 __m128i even = _mm_srli_epi64(_mm_mul_epi32(a, m), 32);
2107 __m128i odd = _mm_mul_epi32(_mm_srli_epi64(a, 32), m);
2108 __m128i t = _mm_add_epi32(a, _mm_blend_epi16(even, odd, 0xCC));
2109 __m128i q = _mm_sub_epi32(_mm_sra_epi32(t, _mm_cvtsi32_si128(shift)), _mm_srai_epi32(a, 31));
2110 return _mm_sub_epi32(_mm_xor_si128(q, divisor_sign), divisor_sign);
2111}
2112#else
2113// No signed 32 x 32 -> 64-bit product: multiply the magnitude, at most 2^31, and shift the whole product. The quotient
2114// fits in 32 bits, so the upper half of each shifted lane is 0.
2115EIGEN_STRONG_INLINE __m128i psdiv_signed_magic_epi32(const __m128i& a, uint32_t magic, int /*shift*/, int total_shift,
2116 const __m128i& divisor_sign) {
2117 __m128i m = _mm_set1_epi32(static_cast<int>(magic));
2118 __m128i count = _mm_cvtsi32_si128(total_shift);
2119 __m128i a_sign = _mm_srai_epi32(a, 31);
2120 __m128i abs_a = _mm_sub_epi32(_mm_xor_si128(a, a_sign), a_sign);
2121 __m128i even = _mm_srl_epi64(_mm_mul_epu32(abs_a, m), count);
2122 __m128i odd = _mm_srl_epi64(_mm_mul_epu32(_mm_srli_epi64(abs_a, 32), m), count);
2123 __m128i q = _mm_or_si128(even, _mm_slli_epi64(odd, 32));
2124 __m128i q_sign = _mm_xor_si128(a_sign, divisor_sign);
2125 return _mm_sub_epi32(_mm_xor_si128(q, q_sign), q_sign);
2126}
2127#endif
2128
2129template <>
2130EIGEN_STRONG_INLINE Packet4ui pudiv_magic<Packet4ui>(const Packet4ui& a, uint32_t magic, uint32_t addend, int shift) {
2131 return pudiv_magic_epu32(a, magic, addend, shift);
2132}
2133template <>
2134EIGEN_STRONG_INLINE Packet4i psdiv_signed_magic<Packet4i>(const Packet4i& a, uint32_t magic, int shift, int total_shift,
2135 const Packet4i& divisor_sign) {
2136 return psdiv_signed_magic_epi32(a, magic, shift, total_shift, divisor_sign);
2137}
2138
2139} // end namespace internal
2140
2141} // end namespace Eigen
2142
2143#if EIGEN_COMP_PGI && EIGEN_COMP_PGI < 1900
2144// PGI++ does not define the following intrinsics in C++ mode.
2145static inline __m128 _mm_castpd_ps(__m128d x) { return reinterpret_cast<__m128&>(x); }
2146static inline __m128i _mm_castpd_si128(__m128d x) { return reinterpret_cast<__m128i&>(x); }
2147static inline __m128d _mm_castps_pd(__m128 x) { return reinterpret_cast<__m128d&>(x); }
2148static inline __m128i _mm_castps_si128(__m128 x) { return reinterpret_cast<__m128i&>(x); }
2149static inline __m128 _mm_castsi128_ps(__m128i x) { return reinterpret_cast<__m128&>(x); }
2150static inline __m128d _mm_castsi128_pd(__m128i x) { return reinterpret_cast<__m128d&>(x); }
2151#endif
2152
2153#endif // EIGEN_PACKET_MATH_SSE_H
@ Aligned16
Definition Constants.h:238