11#ifndef EIGEN_TYPE_CASTING_AVX512_H
12#define EIGEN_TYPE_CASTING_AVX512_H
15#include "../../InternalHeaderCheck.h"
22struct type_casting_traits<float, bool> : vectorized_type_casting_traits<float, bool> {};
24struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
27struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
29struct type_casting_traits<bool, half> : vectorized_type_casting_traits<bool, half> {};
31struct type_casting_traits<bool, bfloat16> : vectorized_type_casting_traits<bool, bfloat16> {};
34struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
36struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
39struct type_casting_traits<float, double> : vectorized_type_casting_traits<float, double> {};
41struct type_casting_traits<double, float> : vectorized_type_casting_traits<double, float> {};
44struct type_casting_traits<double, int> : vectorized_type_casting_traits<double, int> {};
46struct type_casting_traits<int, double> : vectorized_type_casting_traits<int, double> {};
49struct type_casting_traits<double, int64_t> : vectorized_type_casting_traits<double, int64_t> {};
51struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int64_t, double> {};
54struct type_casting_traits<half, float> : vectorized_type_casting_traits<half, float> {};
56struct type_casting_traits<float, half> : vectorized_type_casting_traits<float, half> {};
59struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {};
61struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
63EIGEN_STRONG_INLINE __mmask16 _eigen_mm512_cmpneq_ps_mask(__m512 a, __m512 b) {
64 return _mm512_cmp_ps_mask(a, b, _CMP_NEQ_UQ);
68EIGEN_STRONG_INLINE Packet16b pcast<Packet16f, Packet16b>(
const Packet16f& a) {
69 __mmask16 mask = _eigen_mm512_cmpneq_ps_mask(a, pzero(a));
70 return _mm512_maskz_cvtepi32_epi8(mask, _mm512_set1_epi32(1));
74EIGEN_STRONG_INLINE Packet16f pcast<Packet16b, Packet16f>(
const Packet16b& a) {
75 return _mm512_cvtepi32_ps(_mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1)));
79EIGEN_STRONG_INLINE Packet16i pcast<Packet16b, Packet16i>(
const Packet16b& a) {
80 return _mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1));
84EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(
const Packet16f& a) {
85 return _mm512_cvttps_epi32(a);
89EIGEN_STRONG_INLINE Packet8d pcast<Packet16f, Packet8d>(
const Packet16f& a) {
90 return _mm512_cvtps_pd(_mm512_castps512_ps256(a));
94EIGEN_STRONG_INLINE Packet8d pcast<Packet8f, Packet8d>(
const Packet8f& a) {
95 return _mm512_cvtps_pd(a);
99EIGEN_STRONG_INLINE Packet8l pcast<Packet8d, Packet8l>(
const Packet8d& a) {
100#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
101 return _mm512_cvttpd_epi64(a);
103 constexpr int kTotalBits =
sizeof(double) * CHAR_BIT, kMantissaBits = std::numeric_limits<double>::digits - 1,
104 kExponentBits = kTotalBits - kMantissaBits - 1, kBias = (1 << (kExponentBits - 1)) - 1;
106 const __m512i cst_one = _mm512_set1_epi64(1);
107 const __m512i cst_total_bits = _mm512_set1_epi64(kTotalBits);
108 const __m512i cst_bias = _mm512_set1_epi64(kBias);
110 __m512i a_bits = _mm512_castpd_si512(a);
112 __m512i biased_e = _mm512_srli_epi64(_mm512_slli_epi64(a_bits, 1), kMantissaBits + 1);
113 __m512i e = _mm512_sub_epi64(biased_e, cst_bias);
116 __m512i shifted_mantissa = _mm512_slli_epi64(a_bits, kExponentBits + 1);
118 __m512i result_significand = _mm512_srlv_epi64(shifted_mantissa, _mm512_sub_epi64(cst_total_bits, e));
121 __m512i result_exponent = _mm512_sllv_epi64(cst_one, e);
123 __m512i result = _mm512_add_epi64(result_significand, result_exponent);
125 __mmask8 sign_mask = _mm512_cmplt_epi64_mask(a_bits, _mm512_setzero_si512());
126 result = _mm512_mask_sub_epi64(result, sign_mask, _mm512_setzero_si512(), result);
132EIGEN_STRONG_INLINE Packet16f pcast<Packet16i, Packet16f>(
const Packet16i& a) {
133 return _mm512_cvtepi32_ps(a);
137EIGEN_STRONG_INLINE Packet8d pcast<Packet16i, Packet8d>(
const Packet16i& a) {
138 return _mm512_cvtepi32_pd(_mm512_castsi512_si256(a));
142EIGEN_STRONG_INLINE Packet8d pcast<Packet8i, Packet8d>(
const Packet8i& a) {
143 return _mm512_cvtepi32_pd(a);
147EIGEN_STRONG_INLINE Packet8d pcast<Packet8l, Packet8d>(
const Packet8l& a) {
148#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
149 return _mm512_cvtepi64_pd(a);
151 EIGEN_ALIGN64 int64_t aux[8];
153 return _mm512_set_pd(
static_cast<double>(aux[7]),
static_cast<double>(aux[6]),
static_cast<double>(aux[5]),
154 static_cast<double>(aux[4]),
static_cast<double>(aux[3]),
static_cast<double>(aux[2]),
155 static_cast<double>(aux[1]),
static_cast<double>(aux[0]));
160EIGEN_STRONG_INLINE Packet16f pcast<Packet8d, Packet16f>(
const Packet8d& a,
const Packet8d& b) {
161 return cat256(_mm512_cvtpd_ps(a), _mm512_cvtpd_ps(b));
165EIGEN_STRONG_INLINE Packet16i pcast<Packet8d, Packet16i>(
const Packet8d& a,
const Packet8d& b) {
166 return cat256i(_mm512_cvttpd_epi32(a), _mm512_cvttpd_epi32(b));
170EIGEN_STRONG_INLINE Packet8i pcast<Packet8d, Packet8i>(
const Packet8d& a) {
171 return _mm512_cvttpd_epi32(a);
174EIGEN_STRONG_INLINE Packet8f pcast<Packet8d, Packet8f>(
const Packet8d& a) {
175 return _mm512_cvtpd_ps(a);
179EIGEN_STRONG_INLINE Packet16i preinterpret<Packet16i, Packet16f>(
const Packet16f& a) {
180 return _mm512_castps_si512(a);
184EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet16i>(
const Packet16i& a) {
185 return _mm512_castsi512_ps(a);
189EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet16f>(
const Packet16f& a) {
190 return _mm512_castps_pd(a);
194EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet8l>(
const Packet8l& a) {
195 return _mm512_castsi512_pd(a);
199EIGEN_STRONG_INLINE Packet8l preinterpret<Packet8l, Packet8d>(
const Packet8d& a) {
200 return _mm512_castpd_si512(a);
204EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet8d>(
const Packet8d& a) {
205 return _mm512_castpd_ps(a);
209EIGEN_STRONG_INLINE Packet8f preinterpret<Packet8f, Packet16f>(
const Packet16f& a) {
210 return _mm512_castps512_ps256(a);
214EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet16f>(
const Packet16f& a) {
215 return _mm512_castps512_ps128(a);
219EIGEN_STRONG_INLINE Packet4d preinterpret<Packet4d, Packet8d>(
const Packet8d& a) {
220 return _mm512_castpd512_pd256(a);
224EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet8d>(
const Packet8d& a) {
225 return _mm512_castpd512_pd128(a);
229EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet8f>(
const Packet8f& a) {
230 return _mm512_castps256_ps512(a);
234EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet4f>(
const Packet4f& a) {
235 return _mm512_castps128_ps512(a);
239EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet4d>(
const Packet4d& a) {
240 return _mm512_castpd256_pd512(a);
244EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet2d>(
const Packet2d& a) {
245 return _mm512_castpd128_pd512(a);
249EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i, Packet16i>(
const Packet16i& a) {
250 return _mm512_castsi512_si256(a);
253EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet16i>(
const Packet16i& a) {
254 return _mm512_castsi512_si128(a);
257#ifndef EIGEN_VECTORIZE_AVX512FP16
259EIGEN_STRONG_INLINE Packet8h preinterpret<Packet8h, Packet16h>(
const Packet16h& a) {
260 return _mm256_castsi256_si128(a);
264EIGEN_STRONG_INLINE Packet16f pcast<Packet16h, Packet16f>(
const Packet16h& a) {
265 return half2float(a);
269EIGEN_STRONG_INLINE Packet16h pcast<Packet16f, Packet16h>(
const Packet16f& a) {
270 return float2half(a);
274EIGEN_STRONG_INLINE Packet16h pcast<Packet16b, Packet16h>(
const Packet16b& a) {
275 __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256());
276 return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3c00));
282EIGEN_STRONG_INLINE Packet8bf preinterpret<Packet8bf, Packet16bf>(
const Packet16bf& a) {
283 return _mm256_castsi256_si128(a);
287EIGEN_STRONG_INLINE Packet16f pcast<Packet16bf, Packet16f>(
const Packet16bf& a) {
292EIGEN_STRONG_INLINE Packet16bf pcast<Packet16f, Packet16bf>(
const Packet16f& a) {
297EIGEN_STRONG_INLINE Packet16bf pcast<Packet16b, Packet16bf>(
const Packet16b& a) {
298 __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256());
299 return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3f80));