11#ifndef EIGEN_TYPE_CASTING_AVX_H
12#define EIGEN_TYPE_CASTING_AVX_H
15#include "../../InternalHeaderCheck.h"
21#ifndef EIGEN_VECTORIZE_AVX512
23struct type_casting_traits<float, bool> : vectorized_type_casting_traits<float, bool> {};
25struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
28struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
30struct type_casting_traits<bool, half> : vectorized_type_casting_traits<bool, half> {};
32struct type_casting_traits<bool, bfloat16> : vectorized_type_casting_traits<bool, bfloat16> {};
35struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
37struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
40struct type_casting_traits<float, double> : vectorized_type_casting_traits<float, double> {};
42struct type_casting_traits<double, float> : vectorized_type_casting_traits<double, float> {};
45struct type_casting_traits<double, int> : vectorized_type_casting_traits<double, int> {};
47struct type_casting_traits<int, double> : vectorized_type_casting_traits<int, double> {};
50struct type_casting_traits<half, float> : vectorized_type_casting_traits<half, float> {};
52struct type_casting_traits<float, half> : vectorized_type_casting_traits<float, half> {};
55struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {};
57struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
59#ifdef EIGEN_VECTORIZE_AVX2
61struct type_casting_traits<double, int64_t> : vectorized_type_casting_traits<double, int64_t> {};
63struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int64_t, double> {};
68EIGEN_STRONG_INLINE __m256 _eigen_mm256_set_m128(__m128 hi, __m128 lo) {
69 return _mm256_insertf128_ps(_mm256_castps128_ps256(lo), hi, 1);
72EIGEN_STRONG_INLINE __m256d _eigen_mm256_set_m128d(__m128d hi, __m128d lo) {
73 return _mm256_insertf128_pd(_mm256_castpd128_pd256(lo), hi, 1);
76EIGEN_STRONG_INLINE __m256i _eigen_mm256_set_m128i(__m128i hi, __m128i lo) {
77 return _mm256_insertf128_si256(_mm256_castsi128_si256(lo), hi, 1);
81EIGEN_STRONG_INLINE Packet16b pcast<Packet8f, Packet16b>(
const Packet8f& a,
const Packet8f& b) {
82 __m256 nonzero_a = _mm256_cmp_ps(a, pzero(a), _CMP_NEQ_UQ);
83 __m256 nonzero_b = _mm256_cmp_ps(b, pzero(b), _CMP_NEQ_UQ);
84 constexpr char kFF =
'\255';
85#ifndef EIGEN_VECTORIZE_AVX2
86 __m128i shuffle_mask128_a_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0);
87 __m128i shuffle_mask128_a_hi = _mm_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF);
88 __m128i shuffle_mask128_b_lo = _mm_set_epi8(kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
89 __m128i shuffle_mask128_b_hi = _mm_set_epi8(12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
90 __m128i a_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 1), shuffle_mask128_a_hi);
91 __m128i a_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_a), 0), shuffle_mask128_a_lo);
92 __m128i b_hi = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 1), shuffle_mask128_b_hi);
93 __m128i b_lo = _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castps_si256(nonzero_b), 0), shuffle_mask128_b_lo);
94 __m128i merged = _mm_or_si128(_mm_or_si128(b_lo, b_hi), _mm_or_si128(a_lo, a_hi));
95 return _mm_and_si128(merged, _mm_set1_epi8(1));
97 __m256i a_shuffle_mask = _mm256_set_epi8(kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF,
98 kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, 12, 8, 4, 0);
99 __m256i b_shuffle_mask = _mm256_set_epi8(12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF,
100 kFF, kFF, kFF, 12, 8, 4, 0, kFF, kFF, kFF, kFF, kFF, kFF, kFF, kFF);
101 __m256i a_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_a), a_shuffle_mask);
102 __m256i b_shuff = _mm256_shuffle_epi8(_mm256_castps_si256(nonzero_b), b_shuffle_mask);
103 __m256i a_or_b = _mm256_or_si256(a_shuff, b_shuff);
104 __m256i merged = _mm256_or_si256(a_or_b, _mm256_castsi128_si256(_mm256_extractf128_si256(a_or_b, 1)));
105 return _mm256_castsi256_si128(_mm256_and_si256(merged, _mm256_set1_epi8(1)));
110EIGEN_STRONG_INLINE Packet8f pcast<Packet16b, Packet8f>(
const Packet16b& a) {
111 const __m256 cst_one = _mm256_set1_ps(1.0f);
112#ifdef EIGEN_VECTORIZE_AVX2
113 __m256i a_extended = _mm256_cvtepi8_epi32(a);
114 __m256i abcd_efgh = _mm256_cmpeq_epi32(a_extended, _mm256_setzero_si256());
116 __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
117 __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
118 __m128i aaaa_bbbb_cccc_dddd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
119 __m128i eeee_ffff_gggg_hhhh = _mm_unpackhi_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
120 __m256i abcd_efgh = _mm256_setr_m128i(aaaa_bbbb_cccc_dddd, eeee_ffff_gggg_hhhh);
122 __m256 result = _mm256_andnot_ps(_mm256_castsi256_ps(abcd_efgh), cst_one);
127EIGEN_STRONG_INLINE Packet8i pcast<Packet16b, Packet8i>(
const Packet16b& a) {
128#ifdef EIGEN_VECTORIZE_AVX2
129 __m256i a_extended = _mm256_cvtepi8_epi32(a);
130 __m256i abcd_efgh = _mm256_cmpeq_epi32(a_extended, _mm256_setzero_si256());
131 return _mm256_andnot_si256(abcd_efgh, _mm256_set1_epi32(1));
133 return _mm256_cvttps_epi32(pcast<Packet16b, Packet8f>(a));
138EIGEN_STRONG_INLINE Packet8i pcast<Packet8f, Packet8i>(
const Packet8f& a) {
139 return _mm256_cvttps_epi32(a);
143EIGEN_STRONG_INLINE Packet8i pcast<Packet4d, Packet8i>(
const Packet4d& a,
const Packet4d& b) {
144 return _eigen_mm256_set_m128i(_mm256_cvttpd_epi32(b), _mm256_cvttpd_epi32(a));
148EIGEN_STRONG_INLINE Packet4i pcast<Packet4d, Packet4i>(
const Packet4d& a) {
149 return _mm256_cvttpd_epi32(a);
153EIGEN_STRONG_INLINE Packet8f pcast<Packet8i, Packet8f>(
const Packet8i& a) {
154 return _mm256_cvtepi32_ps(a);
158EIGEN_STRONG_INLINE Packet8f pcast<Packet4d, Packet8f>(
const Packet4d& a,
const Packet4d& b) {
159 return _eigen_mm256_set_m128(_mm256_cvtpd_ps(b), _mm256_cvtpd_ps(a));
163EIGEN_STRONG_INLINE Packet4f pcast<Packet4d, Packet4f>(
const Packet4d& a) {
164 return _mm256_cvtpd_ps(a);
168EIGEN_STRONG_INLINE Packet4d pcast<Packet8i, Packet4d>(
const Packet8i& a) {
169 return _mm256_cvtepi32_pd(_mm256_castsi256_si128(a));
173EIGEN_STRONG_INLINE Packet4d pcast<Packet4i, Packet4d>(
const Packet4i& a) {
174 return _mm256_cvtepi32_pd(a);
178EIGEN_STRONG_INLINE Packet4d pcast<Packet8f, Packet4d>(
const Packet8f& a) {
179 return _mm256_cvtps_pd(_mm256_castps256_ps128(a));
183EIGEN_STRONG_INLINE Packet4d pcast<Packet4f, Packet4d>(
const Packet4f& a) {
184 return _mm256_cvtps_pd(a);
188EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i, Packet8f>(
const Packet8f& a) {
189 return _mm256_castps_si256(a);
193EIGEN_STRONG_INLINE Packet8f preinterpret<Packet8f, Packet8i>(
const Packet8i& a) {
194 return _mm256_castsi256_ps(a);
198EIGEN_STRONG_INLINE Packet8ui preinterpret<Packet8ui, Packet8i>(
const Packet8i& a) {
203EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i, Packet8ui>(
const Packet8ui& a) {
210EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet8f>(
const Packet8f& a) {
211 return _mm256_castps256_ps128(a);
215EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4d>(
const Packet4d& a) {
216 return _mm256_castpd256_pd128(a);
220EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet8i>(
const Packet8i& a) {
221 return _mm256_castsi256_si128(a);
225EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet8ui>(
const Packet8ui& a) {
226 return _mm256_castsi256_si128(a);
229#ifdef EIGEN_VECTORIZE_AVX2
231EIGEN_STRONG_INLINE Packet4l pcast<Packet4d, Packet4l>(
const Packet4d& a) {
232#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
233 return _mm256_cvttpd_epi64(a);
242 constexpr int kTotalBits =
sizeof(double) * CHAR_BIT, kMantissaBits = std::numeric_limits<double>::digits - 1,
243 kExponentBits = kTotalBits - kMantissaBits - 1, kBias = (1 << (kExponentBits - 1)) - 1;
245 const __m256i cst_one = _mm256_set1_epi64x(1);
246 const __m256i cst_total_bits = _mm256_set1_epi64x(kTotalBits);
247 const __m256i cst_bias = _mm256_set1_epi64x(kBias);
249 __m256i a_bits = _mm256_castpd_si256(a);
251 __m256i biased_e = _mm256_srli_epi64(_mm256_slli_epi64(a_bits, 1), kMantissaBits + 1);
252 __m256i e = _mm256_sub_epi64(biased_e, cst_bias);
255 __m256i shifted_mantissa = _mm256_slli_epi64(a_bits, kExponentBits + 1);
257 __m256i result_significand = _mm256_srlv_epi64(shifted_mantissa, _mm256_sub_epi64(cst_total_bits, e));
260 __m256i result_exponent = _mm256_sllv_epi64(cst_one, e);
262 __m256i result = _mm256_add_epi64(result_significand, result_exponent);
264 __m256i sign_mask = _mm256_cmpgt_epi64(_mm256_setzero_si256(), a_bits);
265 result = _mm256_sub_epi64(_mm256_xor_si256(result, sign_mask), sign_mask);
271EIGEN_STRONG_INLINE Packet4d pcast<Packet4l, Packet4d>(
const Packet4l& a) {
272#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
273 return _mm256_cvtepi64_pd(a);
277 return _mm256_set_pd(
static_cast<double>(aux[3]),
static_cast<double>(aux[2]),
static_cast<double>(aux[1]),
278 static_cast<double>(aux[0]));
283EIGEN_STRONG_INLINE Packet4d pcast<Packet2l, Packet4d>(
const Packet2l& a,
const Packet2l& b) {
284 return _eigen_mm256_set_m128d((pcast<Packet2l, Packet2d>(b)), (pcast<Packet2l, Packet2d>(a)));
288EIGEN_STRONG_INLINE Packet4ul preinterpret<Packet4ul, Packet4l>(
const Packet4l& a) {
293EIGEN_STRONG_INLINE Packet4l preinterpret<Packet4l, Packet4ul>(
const Packet4ul& a) {
298EIGEN_STRONG_INLINE Packet4l preinterpret<Packet4l, Packet4d>(
const Packet4d& a) {
299 return _mm256_castpd_si256(a);
303EIGEN_STRONG_INLINE Packet4d preinterpret<Packet4d, Packet4l>(
const Packet4l& a) {
304 return _mm256_castsi256_pd(a);
309EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet4l>(
const Packet4l& a) {
310 return _mm256_castsi256_si128(a);
314#ifndef EIGEN_VECTORIZE_AVX512FP16
316EIGEN_STRONG_INLINE Packet8f pcast<Packet8h, Packet8f>(
const Packet8h& a) {
317 return half2float(a);
321EIGEN_STRONG_INLINE Packet8h pcast<Packet8f, Packet8h>(
const Packet8f& a) {
322 return float2half(a);
326EIGEN_STRONG_INLINE Packet8h pcast<Packet16b, Packet8h>(
const Packet16b& a) {
327 __m128i eq_zero = _mm_cmpeq_epi8(a, _mm_setzero_si128());
328 __m128i mask16 = _mm_unpacklo_epi8(eq_zero, eq_zero);
329 return _mm_andnot_si128(mask16, _mm_set1_epi16(0x3c00));
334EIGEN_STRONG_INLINE Packet8f pcast<Packet8bf, Packet8f>(
const Packet8bf& a) {
339EIGEN_STRONG_INLINE Packet8bf pcast<Packet8f, Packet8bf>(
const Packet8f& a) {
344EIGEN_STRONG_INLINE Packet8bf pcast<Packet16b, Packet8bf>(
const Packet16b& a) {
345 __m128i eq_zero = _mm_cmpeq_epi8(a, _mm_setzero_si128());
346 __m128i mask16 = _mm_unpacklo_epi8(eq_zero, eq_zero);
347 return _mm_andnot_si128(mask16, _mm_set1_epi16(0x3f80));