Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TypeCasting.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2019 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TYPE_CASTING_AVX512_H
12#define EIGEN_TYPE_CASTING_AVX512_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21template <>
22struct type_casting_traits<float, bool> : vectorized_type_casting_traits<float, bool> {};
23template <>
24struct type_casting_traits<bool, float> : vectorized_type_casting_traits<bool, float> {};
25
26template <>
27struct type_casting_traits<bool, int> : vectorized_type_casting_traits<bool, int> {};
28template <>
29struct type_casting_traits<bool, half> : vectorized_type_casting_traits<bool, half> {};
30template <>
31struct type_casting_traits<bool, bfloat16> : vectorized_type_casting_traits<bool, bfloat16> {};
32
33template <>
34struct type_casting_traits<float, int> : vectorized_type_casting_traits<float, int> {};
35template <>
36struct type_casting_traits<int, float> : vectorized_type_casting_traits<int, float> {};
37
38template <>
39struct type_casting_traits<float, double> : vectorized_type_casting_traits<float, double> {};
40template <>
41struct type_casting_traits<double, float> : vectorized_type_casting_traits<double, float> {};
42
43template <>
44struct type_casting_traits<double, int> : vectorized_type_casting_traits<double, int> {};
45template <>
46struct type_casting_traits<int, double> : vectorized_type_casting_traits<int, double> {};
47
48template <>
49struct type_casting_traits<double, int64_t> : vectorized_type_casting_traits<double, int64_t> {};
50template <>
51struct type_casting_traits<int64_t, double> : vectorized_type_casting_traits<int64_t, double> {};
52
53template <>
54struct type_casting_traits<half, float> : vectorized_type_casting_traits<half, float> {};
55template <>
56struct type_casting_traits<float, half> : vectorized_type_casting_traits<float, half> {};
57
58template <>
59struct type_casting_traits<bfloat16, float> : vectorized_type_casting_traits<bfloat16, float> {};
60template <>
61struct type_casting_traits<float, bfloat16> : vectorized_type_casting_traits<float, bfloat16> {};
62
63EIGEN_STRONG_INLINE __mmask16 _eigen_mm512_cmpneq_ps_mask(__m512 a, __m512 b) {
64 return _mm512_cmp_ps_mask(a, b, _CMP_NEQ_UQ);
65}
66
67template <>
68EIGEN_STRONG_INLINE Packet16b pcast<Packet16f, Packet16b>(const Packet16f& a) {
69 __mmask16 mask = _eigen_mm512_cmpneq_ps_mask(a, pzero(a));
70 return _mm512_maskz_cvtepi32_epi8(mask, _mm512_set1_epi32(1));
71}
72
73template <>
74EIGEN_STRONG_INLINE Packet16f pcast<Packet16b, Packet16f>(const Packet16b& a) {
75 return _mm512_cvtepi32_ps(_mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1)));
76}
77
78template <>
79EIGEN_STRONG_INLINE Packet16i pcast<Packet16b, Packet16i>(const Packet16b& a) {
80 return _mm512_and_si512(_mm512_cvtepi8_epi32(a), _mm512_set1_epi32(1));
81}
82
83template <>
84EIGEN_STRONG_INLINE Packet16i pcast<Packet16f, Packet16i>(const Packet16f& a) {
85 return _mm512_cvttps_epi32(a);
86}
87
88template <>
89EIGEN_STRONG_INLINE Packet8d pcast<Packet16f, Packet8d>(const Packet16f& a) {
90 return _mm512_cvtps_pd(_mm512_castps512_ps256(a));
91}
92
93template <>
94EIGEN_STRONG_INLINE Packet8d pcast<Packet8f, Packet8d>(const Packet8f& a) {
95 return _mm512_cvtps_pd(a);
96}
97
98template <>
99EIGEN_STRONG_INLINE Packet8l pcast<Packet8d, Packet8l>(const Packet8d& a) {
100#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
101 return _mm512_cvttpd_epi64(a);
102#else
103 constexpr int kTotalBits = sizeof(double) * CHAR_BIT, kMantissaBits = std::numeric_limits<double>::digits - 1,
104 kExponentBits = kTotalBits - kMantissaBits - 1, kBias = (1 << (kExponentBits - 1)) - 1;
105
106 const __m512i cst_one = _mm512_set1_epi64(1);
107 const __m512i cst_total_bits = _mm512_set1_epi64(kTotalBits);
108 const __m512i cst_bias = _mm512_set1_epi64(kBias);
109
110 __m512i a_bits = _mm512_castpd_si512(a);
111 // shift left by 1 to clear the sign bit, and shift right by kMantissaBits + 1 to recover biased exponent
112 __m512i biased_e = _mm512_srli_epi64(_mm512_slli_epi64(a_bits, 1), kMantissaBits + 1);
113 __m512i e = _mm512_sub_epi64(biased_e, cst_bias);
114
115 // shift to the left by kExponentBits + 1 to clear the sign and exponent bits
116 __m512i shifted_mantissa = _mm512_slli_epi64(a_bits, kExponentBits + 1);
117 // shift to the right by kTotalBits - e to convert the significand to an integer
118 __m512i result_significand = _mm512_srlv_epi64(shifted_mantissa, _mm512_sub_epi64(cst_total_bits, e));
119
120 // add the implied bit
121 __m512i result_exponent = _mm512_sllv_epi64(cst_one, e);
122 // e <= 0 is interpreted as a large positive shift (2's complement), which also conveniently results in zero
123 __m512i result = _mm512_add_epi64(result_significand, result_exponent);
124 // handle negative arguments
125 __mmask8 sign_mask = _mm512_cmplt_epi64_mask(a_bits, _mm512_setzero_si512());
126 result = _mm512_mask_sub_epi64(result, sign_mask, _mm512_setzero_si512(), result);
127 return result;
128#endif
129}
130
131template <>
132EIGEN_STRONG_INLINE Packet16f pcast<Packet16i, Packet16f>(const Packet16i& a) {
133 return _mm512_cvtepi32_ps(a);
134}
135
136template <>
137EIGEN_STRONG_INLINE Packet8d pcast<Packet16i, Packet8d>(const Packet16i& a) {
138 return _mm512_cvtepi32_pd(_mm512_castsi512_si256(a));
139}
140
141template <>
142EIGEN_STRONG_INLINE Packet8d pcast<Packet8i, Packet8d>(const Packet8i& a) {
143 return _mm512_cvtepi32_pd(a);
144}
145
146template <>
147EIGEN_STRONG_INLINE Packet8d pcast<Packet8l, Packet8d>(const Packet8l& a) {
148#if defined(EIGEN_VECTORIZE_AVX512DQ) && defined(EIGEN_VECTORIZE_AVX512VL)
149 return _mm512_cvtepi64_pd(a);
150#else
151 EIGEN_ALIGN64 int64_t aux[8];
152 pstore(aux, a);
153 return _mm512_set_pd(static_cast<double>(aux[7]), static_cast<double>(aux[6]), static_cast<double>(aux[5]),
154 static_cast<double>(aux[4]), static_cast<double>(aux[3]), static_cast<double>(aux[2]),
155 static_cast<double>(aux[1]), static_cast<double>(aux[0]));
156#endif
157}
158
159template <>
160EIGEN_STRONG_INLINE Packet16f pcast<Packet8d, Packet16f>(const Packet8d& a, const Packet8d& b) {
161 return cat256(_mm512_cvtpd_ps(a), _mm512_cvtpd_ps(b));
162}
163
164template <>
165EIGEN_STRONG_INLINE Packet16i pcast<Packet8d, Packet16i>(const Packet8d& a, const Packet8d& b) {
166 return cat256i(_mm512_cvttpd_epi32(a), _mm512_cvttpd_epi32(b));
167}
168
169template <>
170EIGEN_STRONG_INLINE Packet8i pcast<Packet8d, Packet8i>(const Packet8d& a) {
171 return _mm512_cvttpd_epi32(a);
172}
173template <>
174EIGEN_STRONG_INLINE Packet8f pcast<Packet8d, Packet8f>(const Packet8d& a) {
175 return _mm512_cvtpd_ps(a);
176}
177
178template <>
179EIGEN_STRONG_INLINE Packet16i preinterpret<Packet16i, Packet16f>(const Packet16f& a) {
180 return _mm512_castps_si512(a);
181}
182
183template <>
184EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet16i>(const Packet16i& a) {
185 return _mm512_castsi512_ps(a);
186}
187
188template <>
189EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet16f>(const Packet16f& a) {
190 return _mm512_castps_pd(a);
191}
192
193template <>
194EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet8l>(const Packet8l& a) {
195 return _mm512_castsi512_pd(a);
196}
197
198template <>
199EIGEN_STRONG_INLINE Packet8l preinterpret<Packet8l, Packet8d>(const Packet8d& a) {
200 return _mm512_castpd_si512(a);
201}
202
203template <>
204EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet8d>(const Packet8d& a) {
205 return _mm512_castpd_ps(a);
206}
207
208template <>
209EIGEN_STRONG_INLINE Packet8f preinterpret<Packet8f, Packet16f>(const Packet16f& a) {
210 return _mm512_castps512_ps256(a);
211}
212
213template <>
214EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet16f>(const Packet16f& a) {
215 return _mm512_castps512_ps128(a);
216}
217
218template <>
219EIGEN_STRONG_INLINE Packet4d preinterpret<Packet4d, Packet8d>(const Packet8d& a) {
220 return _mm512_castpd512_pd256(a);
221}
222
223template <>
224EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet8d>(const Packet8d& a) {
225 return _mm512_castpd512_pd128(a);
226}
227
228template <>
229EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet8f>(const Packet8f& a) {
230 return _mm512_castps256_ps512(a);
231}
232
233template <>
234EIGEN_STRONG_INLINE Packet16f preinterpret<Packet16f, Packet4f>(const Packet4f& a) {
235 return _mm512_castps128_ps512(a);
236}
237
238template <>
239EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet4d>(const Packet4d& a) {
240 return _mm512_castpd256_pd512(a);
241}
242
243template <>
244EIGEN_STRONG_INLINE Packet8d preinterpret<Packet8d, Packet2d>(const Packet2d& a) {
245 return _mm512_castpd128_pd512(a);
246}
247
248template <>
249EIGEN_STRONG_INLINE Packet8i preinterpret<Packet8i, Packet16i>(const Packet16i& a) {
250 return _mm512_castsi512_si256(a);
251}
252template <>
253EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet16i>(const Packet16i& a) {
254 return _mm512_castsi512_si128(a);
255}
256
257#ifndef EIGEN_VECTORIZE_AVX512FP16
258template <>
259EIGEN_STRONG_INLINE Packet8h preinterpret<Packet8h, Packet16h>(const Packet16h& a) {
260 return _mm256_castsi256_si128(a);
261}
262
263template <>
264EIGEN_STRONG_INLINE Packet16f pcast<Packet16h, Packet16f>(const Packet16h& a) {
265 return half2float(a);
266}
267
268template <>
269EIGEN_STRONG_INLINE Packet16h pcast<Packet16f, Packet16h>(const Packet16f& a) {
270 return float2half(a);
271}
272
273template <>
274EIGEN_STRONG_INLINE Packet16h pcast<Packet16b, Packet16h>(const Packet16b& a) {
275 __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256());
276 return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3c00));
277}
278
279#endif
280
281template <>
282EIGEN_STRONG_INLINE Packet8bf preinterpret<Packet8bf, Packet16bf>(const Packet16bf& a) {
283 return _mm256_castsi256_si128(a);
284}
285
286template <>
287EIGEN_STRONG_INLINE Packet16f pcast<Packet16bf, Packet16f>(const Packet16bf& a) {
288 return Bf16ToF32(a);
289}
290
291template <>
292EIGEN_STRONG_INLINE Packet16bf pcast<Packet16f, Packet16bf>(const Packet16f& a) {
293 return F32ToBf16(a);
294}
295
296template <>
297EIGEN_STRONG_INLINE Packet16bf pcast<Packet16b, Packet16bf>(const Packet16b& a) {
298 __m256i mask16 = _mm256_cmpeq_epi16(_mm256_cvtepi8_epi16(a), _mm256_setzero_si256());
299 return _mm256_andnot_si256(mask16, _mm256_set1_epi16(0x3f80));
300}
301
302} // end namespace internal
303
304} // end namespace Eigen
305
306#endif // EIGEN_TYPE_CASTING_AVX512_H