Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TypeCasting.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2015 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TYPE_CASTING_SSE_H
12#define EIGEN_TYPE_CASTING_SSE_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21#ifndef EIGEN_VECTORIZE_AVX
22template <>
23struct type_casting_traits<float, bool> : vectorized_type_casting_traits<float, bool> {};
24template <>
25struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
26
27template <>
28struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
29
30template <>
31struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
32template <>
33struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
34
35template <>
36struct type_casting_traits<float, double> : vectorized_type_casting_traits<float, double> {};
37template <>
38struct type_casting_traits<double, float> : vectorized_type_casting_traits<double, float> {};
39
40template <>
41struct type_casting_traits<double, int> : vectorized_type_casting_traits<double, int> {};
42template <>
43struct type_casting_traits<int, double> : vectorized_type_casting_traits<int, double> {};
44
45#ifndef EIGEN_VECTORIZE_AVX2
46template <>
47struct type_casting_traits<double, int64_t> : vectorized_type_casting_traits<double, int64_t> {};
48template <>
49struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int64_t, double> {};
50#endif
51#endif
52
53template <>
54EIGEN_STRONG_INLINE Packet16b pcast<Packet4f, Packet16b>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
55 const Packet4f& d) {
56 __m128 zero = pzero(a);
57 __m128 nonzero_a = _mm_cmpneq_ps(a, zero);
58 __m128 nonzero_b = _mm_cmpneq_ps(b, zero);
59 __m128 nonzero_c = _mm_cmpneq_ps(c, zero);
60 __m128 nonzero_d = _mm_cmpneq_ps(d, zero);
61 __m128i ab_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_a), _mm_castps_si128(nonzero_b));
62 __m128i cd_bytes = _mm_packs_epi32(_mm_castps_si128(nonzero_c), _mm_castps_si128(nonzero_d));
63 __m128i merged = _mm_packs_epi16(ab_bytes, cd_bytes);
64 return _mm_and_si128(merged, _mm_set1_epi8(1));
65}
66
67template <>
68EIGEN_STRONG_INLINE Packet4f pcast<Packet16b, Packet4f>(const Packet16b& a) {
69 const __m128 cst_one = _mm_set_ps1(1.0f);
70#ifdef EIGEN_VECTORIZE_SSE4_1
71 __m128i a_extended = _mm_cvtepi8_epi32(a);
72 __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128());
73#else
74 __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
75 __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
76 __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
77#endif
78 __m128 result = _mm_andnot_ps(_mm_castsi128_ps(abcd), cst_one);
79 return result;
80}
81
82template <>
83EIGEN_STRONG_INLINE Packet4i pcast<Packet16b, Packet4i>(const Packet16b& a) {
84#ifdef EIGEN_VECTORIZE_SSE4_1
85 __m128i a_extended = _mm_cvtepi8_epi32(a);
86 __m128i abcd = _mm_cmpeq_epi32(a_extended, _mm_setzero_si128());
87#else
88 __m128i abcd_efhg_ijkl_mnop = _mm_cmpeq_epi8(a, _mm_setzero_si128());
89 __m128i aabb_ccdd_eeff_gghh = _mm_unpacklo_epi8(abcd_efhg_ijkl_mnop, abcd_efhg_ijkl_mnop);
90 __m128i abcd = _mm_unpacklo_epi8(aabb_ccdd_eeff_gghh, aabb_ccdd_eeff_gghh);
91#endif
92 return _mm_andnot_si128(abcd, _mm_set1_epi32(1));
93}
94
95template <>
96EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
97 return _mm_cvttps_epi32(a);
98}
99
100template <>
101EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
102 return _mm_castps_si128(_mm_shuffle_ps(_mm_castsi128_ps(_mm_cvttpd_epi32(a)), _mm_castsi128_ps(_mm_cvttpd_epi32(b)),
103 (1 << 2) | (1 << 6)));
104}
105
106template <>
107EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
108#if EIGEN_ARCH_x86_64
109 return _mm_set_epi64x(_mm_cvttsd_si64(preverse(a)), _mm_cvttsd_si64(a));
110#else
111 return _mm_set_epi64x(static_cast<int64_t>(pfirst(preverse(a))), static_cast<int64_t>(pfirst(a)));
112#endif
113}
114
115template <>
116EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
117 EIGEN_ALIGN16 int64_t aux[2];
118 pstore(aux, a);
119 return _mm_set_pd(static_cast<double>(aux[1]), static_cast<double>(aux[0]));
120}
121
122template <>
123EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
124 return _mm_cvtepi32_ps(a);
125}
126
127template <>
128EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
129 return _mm_shuffle_ps(_mm_cvtpd_ps(a), _mm_cvtpd_ps(b), (1 << 2) | (1 << 6));
130}
131
132template <>
133EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
134 // Simply discard the second half of the input
135 return _mm_cvtepi32_pd(a);
136}
137
138template <>
139EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
140 // Simply discard the second half of the input
141 return _mm_cvtps_pd(a);
142}
143
144template <>
145EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4f>(const Packet4f& a) {
146 return _mm_castps_pd(a);
147}
148
149template <>
150EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet2d>(const Packet2d& a) {
151 return _mm_castpd_ps(a);
152}
153
154template <>
155EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
156 return _mm_castps_si128(a);
157}
158
159template <>
160EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
161 return _mm_castsi128_ps(a);
162}
163
164template <>
165EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
166 return _mm_castsi128_pd(a);
167}
168
169template <>
170EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
171 return _mm_castsi128_pd(a);
172}
173template <>
174EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
175 return _mm_castpd_si128(a);
176}
177
178template <>
179EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
180 return _mm_castpd_si128(a);
181}
182
183template <>
184EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
185 return Packet4ui(a);
186}
187
188template <>
189EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
190 return Packet4i(a);
191}
192
193} // end namespace internal
194
195} // end namespace Eigen
196
197#endif // EIGEN_TYPE_CASTING_SSE_H