Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TypeCastingFP16.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2025 The Eigen Authors.
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TYPE_CASTING_FP16_AVX512_H
12#define EIGEN_TYPE_CASTING_FP16_AVX512_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17namespace Eigen {
18namespace internal {
19
20template <>
21EIGEN_STRONG_INLINE Packet32s preinterpret<Packet32s, Packet32h>(const Packet32h& a) {
22 return _mm512_castph_si512(a);
23}
24template <>
25EIGEN_STRONG_INLINE Packet16s preinterpret<Packet16s, Packet16h>(const Packet16h& a) {
26 return _mm256_castph_si256(a);
27}
28template <>
29EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8h>(const Packet8h& a) {
30 return _mm_castph_si128(a);
31}
32
33template <>
34EIGEN_STRONG_INLINE Packet32h preinterpret<Packet32h, Packet32s>(const Packet32s& a) {
35 return _mm512_castsi512_ph(a);
36}
37template <>
38EIGEN_STRONG_INLINE Packet16h preinterpret<Packet16h, Packet16s>(const Packet16s& a) {
39 return _mm256_castsi256_ph(a);
40}
41template <>
42EIGEN_STRONG_INLINE Packet8h preinterpret<Packet8h, Packet8s>(const Packet8s& a) {
43 return _mm_castsi128_ph(a);
44}
45
46template <>
47EIGEN_STRONG_INLINE Packet16f pcast<Packet16h, Packet16f>(const Packet16h& a) {
48 return half2float(a);
49}
50template <>
51EIGEN_STRONG_INLINE Packet8f pcast<Packet8h, Packet8f>(const Packet8h& a) {
52 return half2float(a);
53}
54
55template <>
56EIGEN_STRONG_INLINE Packet16h pcast<Packet16f, Packet16h>(const Packet16f& a) {
57 return float2half(a);
58}
59template <>
60EIGEN_STRONG_INLINE Packet8h pcast<Packet8f, Packet8h>(const Packet8f& a) {
61 return float2half(a);
62}
63
64template <>
65EIGEN_STRONG_INLINE Packet16f pcast<Packet32h, Packet16f>(const Packet32h& a) {
66 // Discard second-half of input.
67 Packet16h low = _mm256_castpd_ph(_mm512_extractf64x4_pd(_mm512_castph_pd(a), 0));
68 return _mm512_cvtxph_ps(low);
69}
70template <>
71EIGEN_STRONG_INLINE Packet8f pcast<Packet16h, Packet8f>(const Packet16h& a) {
72 // Discard second-half of input.
73 Packet8h low = _mm_castps_ph(_mm256_extractf32x4_ps(_mm256_castph_ps(a), 0));
74 return _mm256_cvtxph_ps(low);
75}
76template <>
77EIGEN_STRONG_INLINE Packet4f pcast<Packet8h, Packet4f>(const Packet8h& a) {
78 Packet8f full = _mm256_cvtxph_ps(a);
79 // Discard second-half of input.
80 return _mm256_extractf32x4_ps(full, 0);
81}
82
83template <>
84EIGEN_STRONG_INLINE Packet32h pcast<Packet16f, Packet32h>(const Packet16f& a, const Packet16f& b) {
85 return _mm512_castps_ph(cat256(_mm256_castph_ps(_mm512_cvtxps_ph(a)), _mm256_castph_ps(_mm512_cvtxps_ph(b))));
86}
87template <>
88EIGEN_STRONG_INLINE Packet16h pcast<Packet8f, Packet16h>(const Packet8f& a, const Packet8f& b) {
89 __m256 result = _mm256_castsi256_ps(_mm256_castsi128_si256(_mm_castph_si128(_mm256_cvtxps_ph(a))));
90 result = _mm256_insertf32x4(result, _mm_castph_ps(_mm256_cvtxps_ph(b)), 1);
91 return _mm256_castps_ph(result);
92}
93template <>
94EIGEN_STRONG_INLINE Packet8h pcast<Packet4f, Packet8h>(const Packet4f& a, const Packet4f& b) {
95 __m256 result = _mm256_castsi256_ps(_mm256_castsi128_si256(_mm_castps_si128(a)));
96 result = _mm256_insertf128_ps(result, b, 1);
97 return _mm256_cvtxps_ph(result);
98}
99
100template <>
101EIGEN_STRONG_INLINE Packet32s pcast<Packet32h, Packet32s>(const Packet32h& a) {
102 return _mm512_cvtph_epi16(a);
103}
104template <>
105EIGEN_STRONG_INLINE Packet16s pcast<Packet16h, Packet16s>(const Packet16h& a) {
106 return _mm256_cvtph_epi16(a);
107}
108template <>
109EIGEN_STRONG_INLINE Packet8s pcast<Packet8h, Packet8s>(const Packet8h& a) {
110 return _mm_cvtph_epi16(a);
111}
112
113template <>
114EIGEN_STRONG_INLINE Packet32h pcast<Packet32s, Packet32h>(const Packet32s& a) {
115 return _mm512_cvtepi16_ph(a);
116}
117template <>
118EIGEN_STRONG_INLINE Packet16h pcast<Packet16s, Packet16h>(const Packet16s& a) {
119 return _mm256_cvtepi16_ph(a);
120}
121template <>
122EIGEN_STRONG_INLINE Packet8h pcast<Packet8s, Packet8h>(const Packet8s& a) {
123 return _mm_cvtepi16_ph(a);
124}
125
126template <>
127EIGEN_STRONG_INLINE Packet32h pcast<Packet16b, Packet32h>(const Packet16b& a, const Packet16b& b) {
128 __m256i ab = _eigen_mm256_set_m128i(b, a);
129 __mmask32 mask = _mm256_cmpneq_epi8_mask(ab, _mm256_setzero_si256());
130 return _mm512_castsi512_ph(_mm512_maskz_set1_epi16(mask, 0x3c00));
131}
132template <>
133EIGEN_STRONG_INLINE Packet16h pcast<Packet16b, Packet16h>(const Packet16b& a) {
134 __mmask16 mask = _mm_cmpneq_epi8_mask(a, _mm_setzero_si128());
135 return _mm256_castsi256_ph(_mm256_maskz_set1_epi16(mask, 0x3c00));
136}
137template <>
138EIGEN_STRONG_INLINE Packet8h pcast<Packet16b, Packet8h>(const Packet16b& a) {
139 __mmask16 mask = _mm_cmpneq_epi8_mask(a, _mm_setzero_si128());
140 return _mm_castsi128_ph(_mm_maskz_set1_epi16(static_cast<__mmask8>(mask), 0x3c00));
141}
142
143} // namespace internal
144} // namespace Eigen
145
146#endif // EIGEN_TYPE_CASTING_FP16_AVX512_H