11#ifndef EIGEN_TYPE_CASTING_SSE_H
12#define EIGEN_TYPE_CASTING_SSE_H
15#include "../../InternalHeaderCheck.h"
21#ifndef EIGEN_VECTORIZE_AVX
23struct type_casting_traits<float, bool> : vectorized_type_casting_traits<float, bool> {};
25struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
28struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
31struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
33struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
36struct type_casting_traits<float, double> : vectorized_type_casting_traits<float, double> {};
38struct type_casting_traits<double, float> : vectorized_type_casting_traits<double, float> {};
41struct type_casting_traits<double, int> : vectorized_type_casting_traits<double, int> {};
43struct type_casting_traits<int, double> : vectorized_type_casting_traits<int, double> {};
45#ifndef EIGEN_VECTORIZE_AVX2
47struct type_casting_traits<double, int64_t> : vectorized_type_casting_traits<double, int64_t> {};
49struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int64_t, double> {};
54EIGEN_STRONG_INLINE Packet16b pcast<Packet4f, Packet16b>(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c,
56 __m128 zero = pzero(a);
57 __m128 nonzero_a = _mm_cmpneq_ps(a, zero);
58 __m128 nonzero_b = _mm_cmpneq_ps(b, zero);
59 __m128 nonzero_c = _mm_cmpneq_ps(c, zero);
60 __m128 nonzero_d = _mm_cmpneq_ps(d, zero);
61 __m128i ab_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_a), _mm_castps_si128(nonzero_b));
62 __m128i cd_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_c), _mm_castps_si128(nonzero_d));
63 __m128i merged = _mm_packs_epi16(ab_bytes, cd_bytes);
64 return _mm_and_si128(merged, _mm_set1_epi8(1));
68EIGEN_STRONG_INLINE Packet4f pcast<Packet16b, Packet4f>(
const Packet16b& a) {
69 const __m128 cst_one = _mm_set_ps1(1.0f);
70#ifdef EIGEN_VECTORIZE_SSE4_1
71 __m128i a_extended = _mm_cvtepi8_epi32(a);
72 __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128());
74 __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
75 __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
76 __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
78 __m128 result = _mm_andnot_ps(_mm_castsi128_ps(abcd), cst_one);
83EIGEN_STRONG_INLINE Packet4i pcast<Packet16b, Packet4i>(
const Packet16b& a) {
84#ifdef EIGEN_VECTORIZE_SSE4_1
85 __m128i a_extended = _mm_cvtepi8_epi32(a);
86 __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128());
88 __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
89 __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
90 __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
92 return _mm_andnot_si128(abcd, _mm_set1_epi32(1));
96EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(
const Packet4f& a) {
97 return _mm_cvttps_epi32(a);
101EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(
const Packet2d& a,
const Packet2d& b) {
102 return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(_mm_cvttpd_epi32(a)), _mm_castsi128_ps(_mm_cvttpd_epi32(b)),
103 (1 << 2) | (1 << 6)));
107EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(
const Packet2d& a) {
109 return _mm_set_epi64x(_mm_cvttsd_si64(preverse(a)), _mm_cvttsd_si64(a));
111 return _mm_set_epi64x(
static_cast<int64_t
>(pfirst(preverse(a))),
static_cast<int64_t
>(pfirst(a)));
116EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(
const Packet2l& a) {
117 EIGEN_ALIGN16 int64_t aux[2];
119 return _mm_set_pd(
static_cast<double>(aux[1]),
static_cast<double>(aux[0]));
123EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(
const Packet4i& a) {
124 return _mm_cvtepi32_ps(a);
128EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(
const Packet2d& a,
const Packet2d& b) {
129 return _mm_shuffle_ps(_mm_cvtpd_ps(a), _mm_cvtpd_ps(b), (1 << 2) | (1 << 6));
133EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(
const Packet4i& a) {
135 return _mm_cvtepi32_pd(a);
139EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(
const Packet4f& a) {
141 return _mm_cvtps_pd(a);
145EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4f>(
const Packet4f& a) {
146 return _mm_castps_pd(a);
150EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet2d>(
const Packet2d& a) {
151 return _mm_castpd_ps(a);
155EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(
const Packet4f& a) {
156 return _mm_castps_si128(a);
160EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(
const Packet4i& a) {
161 return _mm_castsi128_ps(a);
165EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(
const Packet4i& a) {
166 return _mm_castsi128_pd(a);
170EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(
const Packet2l& a) {
171 return _mm_castsi128_pd(a);
174EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(
const Packet2d& a) {
175 return _mm_castpd_si128(a);
179EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(
const Packet2d& a) {
180 return _mm_castpd_si128(a);
184EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(
const Packet4i& a) {
189EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(
const Packet4ui& a) {