Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TypeCasting.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@gmail.com>
5// Copyright (C) 2020 Antonio Sanchez <cantonios@google.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_TYPE_CASTING_NEON_H
13#define EIGEN_TYPE_CASTING_NEON_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22//==============================================================================
23// preinterpret (truncation operations)
24//==============================================================================
25
26template <>
27EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet16c>(const Packet16c& a) {
28 return Packet8c(vget_low_s8(a));
29}
30template <>
31EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet8c>(const Packet8c& a) {
32 return Packet4c(vget_lane_s32(vreinterpret_s32_s8(a), 0));
33}
34template <>
35EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet16c>(const Packet16c& a) {
36 return preinterpret<Packet4c>(preinterpret<Packet8c>(a));
37}
38
39template <>
40EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet16uc>(const Packet16uc& a) {
41 return Packet8uc(vget_low_u8(a));
42}
43template <>
44EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet8uc>(const Packet8uc& a) {
45 return Packet4uc(vget_lane_u32(vreinterpret_u32_u8(a), 0));
46}
47template <>
48EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet16uc>(const Packet16uc& a) {
49 return preinterpret<Packet4uc>(preinterpret<Packet8uc>(a));
50}
51
52template <>
53EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet8s>(const Packet8s& a) {
54 return Packet4s(vget_low_s16(a));
55}
56
57template <>
58EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet8us>(const Packet8us& a) {
59 return Packet4us(vget_low_u16(a));
60}
61
62template <>
63EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet4i>(const Packet4i& a) {
64 return Packet2i(vget_low_s32(a));
65}
66template <>
67EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet4ui>(const Packet4ui& a) {
68 return Packet2ui(vget_low_u32(a));
69}
70
71template <>
72EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet4f>(const Packet4f& a) {
73 return Packet2f(vget_low_f32(a));
74}
75
76//==============================================================================
77// preinterpret
78//==============================================================================
79template <>
80EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2i>(const Packet2i& a) {
81 return Packet2f(vreinterpret_f32_s32(a));
82}
83template <>
84EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2ui>(const Packet2ui& a) {
85 return Packet2f(vreinterpret_f32_u32(a));
86}
87template <>
88EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
89 return Packet4f(vreinterpretq_f32_s32(a));
90}
91template <>
92EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4ui>(const Packet4ui& a) {
93 return Packet4f(vreinterpretq_f32_u32(a));
94}
95
96template <>
97EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet4uc>(const Packet4uc& a) {
98 return static_cast<Packet4c>(a);
99}
100template <>
101EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet8uc>(const Packet8uc& a) {
102 return Packet8c(vreinterpret_s8_u8(a));
103}
104template <>
105EIGEN_STRONG_INLINE Packet16c preinterpret<Packet16c, Packet16uc>(const Packet16uc& a) {
106 return Packet16c(vreinterpretq_s8_u8(a));
107}
108
109template <>
110EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet4c>(const Packet4c& a) {
111 return static_cast<Packet4uc>(a);
112}
113template <>
114EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet8c>(const Packet8c& a) {
115 return Packet8uc(vreinterpret_u8_s8(a));
116}
117template <>
118EIGEN_STRONG_INLINE Packet16uc preinterpret<Packet16uc, Packet16c>(const Packet16c& a) {
119 return Packet16uc(vreinterpretq_u8_s8(a));
120}
121
122template <>
123EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4us>(const Packet4us& a) {
124 return Packet4s(vreinterpret_s16_u16(a));
125}
126template <>
127EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8us>(const Packet8us& a) {
128 return Packet8s(vreinterpretq_s16_u16(a));
129}
130template <>
131EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4s>(const Packet4s& a) {
132 return Packet4us(vreinterpret_u16_s16(a));
133}
134template <>
135EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8s>(const Packet8s& a) {
136 return Packet8us(vreinterpretq_u16_s16(a));
137}
138
139template <>
140EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2f>(const Packet2f& a) {
141 return Packet2i(vreinterpret_s32_f32(a));
142}
143template <>
144EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2ui>(const Packet2ui& a) {
145 return Packet2i(vreinterpret_s32_u32(a));
146}
147template <>
148EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
149 return Packet4i(vreinterpretq_s32_f32(a));
150}
151template <>
152EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
153 return Packet4i(vreinterpretq_s32_u32(a));
154}
155
156template <>
157EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2f>(const Packet2f& a) {
158 return Packet2ui(vreinterpret_u32_f32(a));
159}
160template <>
161EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2i>(const Packet2i& a) {
162 return Packet2ui(vreinterpret_u32_s32(a));
163}
164template <>
165EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4f>(const Packet4f& a) {
166 return Packet4ui(vreinterpretq_u32_f32(a));
167}
168template <>
169EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
170 return Packet4ui(vreinterpretq_u32_s32(a));
171}
172
173template <>
174EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2ul>(const Packet2ul& a) {
175 return Packet2l(vreinterpretq_s64_u64(a));
176}
177template <>
178EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2l>(const Packet2l& a) {
179 return Packet2ul(vreinterpretq_u64_s64(a));
180}
181
182#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
183template <>
184EIGEN_STRONG_INLINE Packet4hf preinterpret<Packet4hf, Packet4s>(const Packet4s& a) {
185 return Packet4hf(vreinterpret_f16_s16(a));
186}
187template <>
188EIGEN_STRONG_INLINE Packet4hf preinterpret<Packet4hf, Packet4us>(const Packet4us& a) {
189 return Packet4hf(vreinterpret_f16_u16(a));
190}
191template <>
192EIGEN_STRONG_INLINE Packet8hf preinterpret<Packet8hf, Packet8s>(const Packet8s& a) {
193 return Packet8hf(vreinterpretq_f16_s16(a));
194}
195template <>
196EIGEN_STRONG_INLINE Packet8hf preinterpret<Packet8hf, Packet8us>(const Packet8us& a) {
197 return Packet8hf(vreinterpretq_f16_u16(a));
198}
199
200template <>
201EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4hf>(const Packet4hf& a) {
202 return Packet4s(vreinterpret_s16_f16(a));
203}
204template <>
205EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8hf>(const Packet8hf& a) {
206 return Packet8s(vreinterpretq_s16_f16(a));
207}
208template <>
209EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4hf>(const Packet4hf& a) {
210 return Packet4us(vreinterpret_u16_f16(a));
211}
212template <>
213EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8hf>(const Packet8hf& a) {
214 return Packet8us(vreinterpretq_u16_f16(a));
215}
216#endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
217
218//==============================================================================
219// pcast, SrcType = float
220//==============================================================================
221
222template <>
223struct type_casting_traits<float, numext::int64_t> {
224 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
225};
226template <>
227struct type_casting_traits<float, numext::uint64_t> {
228 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
229};
230// If float64 exists, first convert to that to keep as much precision as possible.
231#if EIGEN_ARCH_ARM64
232template <>
233EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
234 // Discard second half of input.
235 return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a)));
236}
237template <>
238EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
239 return vcvtq_s64_f64(vcvt_f64_f32(a));
240}
241template <>
242EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
243 // Discard second half of input.
244 return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a)));
245}
246template <>
247EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
248 return vcvtq_u64_f64(vcvt_f64_f32(a));
249}
250#else
251// A 32-bit conversion would saturate at 2^31 (2^32), so convert each lane in scalar.
252template <>
253EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
254 return vcombine_s64(vdup_n_s64(static_cast<int64_t>(vget_lane_f32(a, 0))),
255 vdup_n_s64(static_cast<int64_t>(vget_lane_f32(a, 1))));
256}
257template <>
258EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
259 // Discard second half of input.
260 return pcast<Packet2f, Packet2l>(vget_low_f32(a));
261}
262template <>
263EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
264 return vcombine_u64(vdup_n_u64(static_cast<uint64_t>(vget_lane_f32(a, 0))),
265 vdup_n_u64(static_cast<uint64_t>(vget_lane_f32(a, 1))));
266}
267template <>
268EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
269 // Discard second half of input.
270 return pcast<Packet2f, Packet2ul>(vget_low_f32(a));
271}
272#endif // EIGEN_ARCH_ARM64
273
274#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
275template <>
276struct type_casting_traits<float, half> {
277 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
278};
279
280template <>
281EIGEN_STRONG_INLINE Packet8hf pcast<Packet4f, Packet8hf>(const Packet4f& a, const Packet4f& b) {
282 return vcombine_f16(vcvt_f16_f32(a), vcvt_f16_f32(b));
283}
284
285template <>
286EIGEN_STRONG_INLINE Packet4hf pcast<Packet4f, Packet4hf>(const Packet4f& a) {
287 return vcvt_f16_f32(a);
288}
289
290template <>
291EIGEN_STRONG_INLINE Packet4hf pcast<Packet2f, Packet4hf>(const Packet2f& a, const Packet2f& b) {
292 return vcvt_f16_f32(vcombine_f32(a, b));
293}
294#endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
295
296template <>
297struct type_casting_traits<float, numext::int32_t> {
298 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
299};
300template <>
301EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
302 return vcvtq_s32_f32(a);
303}
304template <>
305EIGEN_STRONG_INLINE Packet2i pcast<Packet2f, Packet2i>(const Packet2f& a) {
306 return vcvt_s32_f32(a);
307}
308
309template <>
310struct type_casting_traits<float, numext::uint32_t> {
311 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
312};
313template <>
314EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
315 return vcvtq_u32_f32(a);
316}
317template <>
318EIGEN_STRONG_INLINE Packet2ui pcast<Packet2f, Packet2ui>(const Packet2f& a) {
319 return vcvt_u32_f32(a);
320}
321
322template <>
323struct type_casting_traits<float, numext::int16_t> {
324 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
325};
326template <>
327EIGEN_STRONG_INLINE Packet8s pcast<Packet4f, Packet8s>(const Packet4f& a, const Packet4f& b) {
328 return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b)));
329}
330template <>
331EIGEN_STRONG_INLINE Packet4s pcast<Packet4f, Packet4s>(const Packet4f& a) {
332 return vmovn_s32(vcvtq_s32_f32(a));
333}
334template <>
335EIGEN_STRONG_INLINE Packet4s pcast<Packet2f, Packet4s>(const Packet2f& a, const Packet2f& b) {
336 return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
337}
338
339template <>
340struct type_casting_traits<float, numext::uint16_t> {
341 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
342};
343template <>
344EIGEN_STRONG_INLINE Packet8us pcast<Packet4f, Packet8us>(const Packet4f& a, const Packet4f& b) {
345 return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b)));
346}
347template <>
348EIGEN_STRONG_INLINE Packet4us pcast<Packet4f, Packet4us>(const Packet4f& a) {
349 return vmovn_u32(vcvtq_u32_f32(a));
350}
351template <>
352EIGEN_STRONG_INLINE Packet4us pcast<Packet2f, Packet4us>(const Packet2f& a, const Packet2f& b) {
353 return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
354}
355
356template <>
357struct type_casting_traits<float, numext::int8_t> {
358 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
359};
360template <>
361EIGEN_STRONG_INLINE Packet16c pcast<Packet4f, Packet16c>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
362 const Packet4f& d) {
363 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
364 const int16x8_t cd_s16 = pcast<Packet4f, Packet8s>(c, d);
365 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
366}
367template <>
368EIGEN_STRONG_INLINE Packet8c pcast<Packet4f, Packet8c>(const Packet4f& a, const Packet4f& b) {
369 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
370 return vmovn_s16(ab_s16);
371}
372template <>
373EIGEN_STRONG_INLINE Packet8c pcast<Packet2f, Packet8c>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
374 const Packet2f& d) {
375 const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
376 const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
377 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
378}
379template <>
380EIGEN_STRONG_INLINE Packet4c pcast<Packet4f, Packet4c>(const Packet4f& a) {
381 const int32x4_t a_s32x4 = vcvtq_s32_f32(a);
382 const int16x4_t a_s16x4 = vmovn_s32(a_s32x4);
383 const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
384 const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
385 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
386}
387
388template <>
389struct type_casting_traits<float, numext::uint8_t> {
390 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
391};
392template <>
393EIGEN_STRONG_INLINE Packet16uc pcast<Packet4f, Packet16uc>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
394 const Packet4f& d) {
395 return preinterpret<Packet16uc>(pcast<Packet4f, Packet16c>(a, b, c, d));
396}
397template <>
398EIGEN_STRONG_INLINE Packet8uc pcast<Packet4f, Packet8uc>(const Packet4f& a, const Packet4f& b) {
399 return preinterpret<Packet8uc>(pcast<Packet4f, Packet8c>(a, b));
400}
401template <>
402EIGEN_STRONG_INLINE Packet8uc pcast<Packet2f, Packet8uc>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
403 const Packet2f& d) {
404 return preinterpret<Packet8uc>(pcast<Packet2f, Packet8c>(a, b, c, d));
405}
406template <>
407EIGEN_STRONG_INLINE Packet4uc pcast<Packet4f, Packet4uc>(const Packet4f& a) {
408 return static_cast<Packet4uc>(pcast<Packet4f, Packet4c>(a));
409}
410
411#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
412//==============================================================================
413// pcast, SrcType = half
414//==============================================================================
415template <>
416struct type_casting_traits<half, float> {
417 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
418};
419
420template <>
421EIGEN_STRONG_INLINE Packet4f pcast<Packet8hf, Packet4f>(const Packet8hf& a) {
422 return vcvt_f32_f16(vget_low_f16(a));
423}
424
425template <>
426EIGEN_STRONG_INLINE Packet4f pcast<Packet4hf, Packet4f>(const Packet4hf& a) {
427 return vcvt_f32_f16(a);
428}
429#endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
430
431//==============================================================================
432// pcast, SrcType = int8_t
433//==============================================================================
434template <>
435struct type_casting_traits<numext::int8_t, float> {
436 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
437};
438template <>
439EIGEN_STRONG_INLINE Packet4f pcast<Packet16c, Packet4f>(const Packet16c& a) {
440 // Discard all but first 4 bytes.
441 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))));
442}
443template <>
444EIGEN_STRONG_INLINE Packet4f pcast<Packet4c, Packet4f>(const Packet4c& a) {
445 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s32(vdup_n_s32(a))))));
446}
447template <>
448EIGEN_STRONG_INLINE Packet2f pcast<Packet8c, Packet2f>(const Packet8c& a) {
449 // Discard all but first 2 bytes.
450 return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
451}
452
453template <>
454struct type_casting_traits<numext::int8_t, numext::int64_t> {
455 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
456};
457template <>
458EIGEN_STRONG_INLINE Packet2l pcast<Packet16c, Packet2l>(const Packet16c& a) {
459 // Discard all but first two bytes.
460 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))));
461}
462
463template <>
464struct type_casting_traits<numext::int8_t, numext::uint64_t> {
465 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
466};
467template <>
468EIGEN_STRONG_INLINE Packet2ul pcast<Packet16c, Packet2ul>(const Packet16c& a) {
469 return preinterpret<Packet2ul>(pcast<Packet16c, Packet2l>(a));
470}
471
472template <>
473struct type_casting_traits<numext::int8_t, numext::int32_t> {
474 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
475};
476template <>
477EIGEN_STRONG_INLINE Packet4i pcast<Packet16c, Packet4i>(const Packet16c& a) {
478 // Discard all but first 4 bytes.
479 return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))));
480}
481template <>
482EIGEN_STRONG_INLINE Packet4i pcast<Packet8c, Packet4i>(const Packet8c& a) {
483 return vmovl_s16(vget_low_s16(vmovl_s8(a)));
484}
485template <>
486EIGEN_STRONG_INLINE Packet4i pcast<Packet4c, Packet4i>(const Packet4c& a) {
487 return pcast<Packet8c, Packet4i>(vreinterpret_s8_s32(vdup_n_s32(a)));
488}
489template <>
490EIGEN_STRONG_INLINE Packet2i pcast<Packet8c, Packet2i>(const Packet8c& a) {
491 // Discard all but first 2 bytes.
492 return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
493}
494
495template <>
496struct type_casting_traits<numext::int8_t, numext::uint32_t> {
497 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
498};
499template <>
500EIGEN_STRONG_INLINE Packet4ui pcast<Packet16c, Packet4ui>(const Packet16c& a) {
501 return preinterpret<Packet4ui>(pcast<Packet16c, Packet4i>(a));
502}
503template <>
504EIGEN_STRONG_INLINE Packet2ui pcast<Packet8c, Packet2ui>(const Packet8c& a) {
505 return preinterpret<Packet2ui>(pcast<Packet8c, Packet2i>(a));
506}
507template <>
508EIGEN_STRONG_INLINE Packet4ui pcast<Packet4c, Packet4ui>(const Packet4c& a) {
509 return preinterpret<Packet4ui>(pcast<Packet4c, Packet4i>(a));
510}
511
512template <>
513struct type_casting_traits<numext::int8_t, numext::int16_t> {
514 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
515};
516template <>
517EIGEN_STRONG_INLINE Packet8s pcast<Packet16c, Packet8s>(const Packet16c& a) {
518 // Discard second half of input.
519 return vmovl_s8(vget_low_s8(a));
520}
521template <>
522EIGEN_STRONG_INLINE Packet8s pcast<Packet8c, Packet8s>(const Packet8c& a) {
523 return vmovl_s8(a);
524}
525template <>
526EIGEN_STRONG_INLINE Packet4s pcast<Packet8c, Packet4s>(const Packet8c& a) {
527 // Discard second half of input.
528 return vget_low_s16(vmovl_s8(a));
529}
530template <>
531EIGEN_STRONG_INLINE Packet4s pcast<Packet4c, Packet4s>(const Packet4c& a) {
532 return pcast<Packet8c, Packet4s>(vreinterpret_s8_s32(vdup_n_s32(a)));
533}
534
535template <>
536struct type_casting_traits<numext::int8_t, numext::uint16_t> {
537 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
538};
539template <>
540EIGEN_STRONG_INLINE Packet8us pcast<Packet16c, Packet8us>(const Packet16c& a) {
541 return preinterpret<Packet8us>(pcast<Packet16c, Packet8s>(a));
542}
543template <>
544EIGEN_STRONG_INLINE Packet8us pcast<Packet8c, Packet8us>(const Packet8c& a) {
545 return preinterpret<Packet8us>(pcast<Packet8c, Packet8s>(a));
546}
547template <>
548EIGEN_STRONG_INLINE Packet4us pcast<Packet8c, Packet4us>(const Packet8c& a) {
549 return preinterpret<Packet4us>(pcast<Packet8c, Packet4s>(a));
550}
551template <>
552EIGEN_STRONG_INLINE Packet4us pcast<Packet4c, Packet4us>(const Packet4c& a) {
553 return preinterpret<Packet4us>(pcast<Packet4c, Packet4s>(a));
554}
555
556//==============================================================================
557// pcast, SrcType = uint8_t
558//==============================================================================
559template <>
560struct type_casting_traits<numext::uint8_t, float> {
561 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
562};
563template <>
564EIGEN_STRONG_INLINE Packet4f pcast<Packet16uc, Packet4f>(const Packet16uc& a) {
565 // Discard all but first 4 bytes.
566 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))));
567}
568template <>
569EIGEN_STRONG_INLINE Packet4f pcast<Packet4uc, Packet4f>(const Packet4uc& a) {
570 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))))));
571}
572template <>
573EIGEN_STRONG_INLINE Packet2f pcast<Packet8uc, Packet2f>(const Packet8uc& a) {
574 // Discard all but first 2 bytes.
575 return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
576}
577
578template <>
579struct type_casting_traits<numext::uint8_t, numext::uint64_t> {
580 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
581};
582template <>
583EIGEN_STRONG_INLINE Packet2ul pcast<Packet16uc, Packet2ul>(const Packet16uc& a) {
584 // Discard all but first two bytes.
585 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))));
586}
587
588template <>
589struct type_casting_traits<numext::uint8_t, numext::int64_t> {
590 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
591};
592template <>
593EIGEN_STRONG_INLINE Packet2l pcast<Packet16uc, Packet2l>(const Packet16uc& a) {
594 return preinterpret<Packet2l>(pcast<Packet16uc, Packet2ul>(a));
595}
596
597template <>
598struct type_casting_traits<numext::uint8_t, numext::uint32_t> {
599 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
600};
601template <>
602EIGEN_STRONG_INLINE Packet4ui pcast<Packet16uc, Packet4ui>(const Packet16uc& a) {
603 // Discard all but first 4 bytes.
604 return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))));
605}
606template <>
607EIGEN_STRONG_INLINE Packet4ui pcast<Packet8uc, Packet4ui>(const Packet8uc& a) {
608 return vmovl_u16(vget_low_u16(vmovl_u8(a)));
609}
610template <>
611EIGEN_STRONG_INLINE Packet2ui pcast<Packet8uc, Packet2ui>(const Packet8uc& a) {
612 // Discard all but first 2 bytes.
613 return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
614}
615template <>
616EIGEN_STRONG_INLINE Packet4ui pcast<Packet4uc, Packet4ui>(const Packet4uc& a) {
617 return pcast<Packet8uc, Packet4ui>(vreinterpret_u8_u32(vdup_n_u32(a)));
618}
619
620template <>
621struct type_casting_traits<numext::uint8_t, numext::int32_t> {
622 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
623};
624template <>
625EIGEN_STRONG_INLINE Packet4i pcast<Packet16uc, Packet4i>(const Packet16uc& a) {
626 return preinterpret<Packet4i>(pcast<Packet16uc, Packet4ui>(a));
627}
628template <>
629EIGEN_STRONG_INLINE Packet2i pcast<Packet8uc, Packet2i>(const Packet8uc& a) {
630 return preinterpret<Packet2i>(pcast<Packet8uc, Packet2ui>(a));
631}
632template <>
633EIGEN_STRONG_INLINE Packet4i pcast<Packet4uc, Packet4i>(const Packet4uc& a) {
634 return preinterpret<Packet4i>(pcast<Packet4uc, Packet4ui>(a));
635}
636
637template <>
638struct type_casting_traits<numext::uint8_t, numext::uint16_t> {
639 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
640};
641template <>
642EIGEN_STRONG_INLINE Packet8us pcast<Packet16uc, Packet8us>(const Packet16uc& a) {
643 // Discard second half of input.
644 return vmovl_u8(vget_low_u8(a));
645}
646template <>
647EIGEN_STRONG_INLINE Packet8us pcast<Packet8uc, Packet8us>(const Packet8uc& a) {
648 return vmovl_u8(a);
649}
650template <>
651EIGEN_STRONG_INLINE Packet4us pcast<Packet4uc, Packet4us>(const Packet4uc& a) {
652 return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))));
653}
654
655template <>
656struct type_casting_traits<numext::uint8_t, numext::int16_t> {
657 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
658};
659template <>
660EIGEN_STRONG_INLINE Packet8s pcast<Packet16uc, Packet8s>(const Packet16uc& a) {
661 return preinterpret<Packet8s>(pcast<Packet16uc, Packet8us>(a));
662}
663template <>
664EIGEN_STRONG_INLINE Packet8s pcast<Packet8uc, Packet8s>(const Packet8uc& a) {
665 return preinterpret<Packet8s>(pcast<Packet8uc, Packet8us>(a));
666}
667template <>
668EIGEN_STRONG_INLINE Packet4s pcast<Packet4uc, Packet4s>(const Packet4uc& a) {
669 return preinterpret<Packet4s>(pcast<Packet4uc, Packet4us>(a));
670}
671
672//==============================================================================
673// pcast, SrcType = int16_t
674//==============================================================================
675template <>
676struct type_casting_traits<numext::int16_t, float> {
677 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
678};
679template <>
680EIGEN_STRONG_INLINE Packet4f pcast<Packet8s, Packet4f>(const Packet8s& a) {
681 // Discard second half of input.
682 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a)));
683}
684template <>
685EIGEN_STRONG_INLINE Packet4f pcast<Packet4s, Packet4f>(const Packet4s& a) {
686 return vcvtq_f32_s32(vmovl_s16(a));
687}
688template <>
689EIGEN_STRONG_INLINE Packet2f pcast<Packet4s, Packet2f>(const Packet4s& a) {
690 // Discard second half of input.
691 return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
692}
693
694template <>
695struct type_casting_traits<numext::int16_t, numext::int64_t> {
696 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
697};
698template <>
699EIGEN_STRONG_INLINE Packet2l pcast<Packet8s, Packet2l>(const Packet8s& a) {
700 // Discard all but first two values.
701 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a))));
702}
703
704template <>
705struct type_casting_traits<numext::int16_t, numext::uint64_t> {
706 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
707};
708template <>
709EIGEN_STRONG_INLINE Packet2ul pcast<Packet8s, Packet2ul>(const Packet8s& a) {
710 return preinterpret<Packet2ul>(pcast<Packet8s, Packet2l>(a));
711}
712
713template <>
714struct type_casting_traits<numext::int16_t, numext::int32_t> {
715 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
716};
717template <>
718EIGEN_STRONG_INLINE Packet4i pcast<Packet8s, Packet4i>(const Packet8s& a) {
719 // Discard second half of input.
720 return vmovl_s16(vget_low_s16(a));
721}
722template <>
723EIGEN_STRONG_INLINE Packet4i pcast<Packet4s, Packet4i>(const Packet4s& a) {
724 return vmovl_s16(a);
725}
726template <>
727EIGEN_STRONG_INLINE Packet2i pcast<Packet4s, Packet2i>(const Packet4s& a) {
728 // Discard second half of input.
729 return vget_low_s32(vmovl_s16(a));
730}
731
732template <>
733struct type_casting_traits<numext::int16_t, numext::uint32_t> {
734 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
735};
736template <>
737EIGEN_STRONG_INLINE Packet4ui pcast<Packet8s, Packet4ui>(const Packet8s& a) {
738 return preinterpret<Packet4ui>(pcast<Packet8s, Packet4i>(a));
739}
740template <>
741EIGEN_STRONG_INLINE Packet4ui pcast<Packet4s, Packet4ui>(const Packet4s& a) {
742 return preinterpret<Packet4ui>(pcast<Packet4s, Packet4i>(a));
743}
744template <>
745EIGEN_STRONG_INLINE Packet2ui pcast<Packet4s, Packet2ui>(const Packet4s& a) {
746 return preinterpret<Packet2ui>(pcast<Packet4s, Packet2i>(a));
747}
748
749template <>
750struct type_casting_traits<numext::int16_t, numext::int8_t> {
751 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
752};
753template <>
754EIGEN_STRONG_INLINE Packet16c pcast<Packet8s, Packet16c>(const Packet8s& a, const Packet8s& b) {
755 return vcombine_s8(vmovn_s16(a), vmovn_s16(b));
756}
757template <>
758EIGEN_STRONG_INLINE Packet8c pcast<Packet8s, Packet8c>(const Packet8s& a) {
759 return vmovn_s16(a);
760}
761template <>
762EIGEN_STRONG_INLINE Packet8c pcast<Packet4s, Packet8c>(const Packet4s& a, const Packet4s& b) {
763 return vmovn_s16(vcombine_s16(a, b));
764}
765template <>
766EIGEN_STRONG_INLINE Packet4c pcast<Packet4s, Packet4c>(const Packet4s& a) {
767 const int8x8_t aa_s8x8 = pcast<Packet4s, Packet8c>(a, a);
768 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
769}
770
771template <>
772struct type_casting_traits<numext::int16_t, numext::uint8_t> {
773 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
774};
775template <>
776EIGEN_STRONG_INLINE Packet16uc pcast<Packet8s, Packet16uc>(const Packet8s& a, const Packet8s& b) {
777 return preinterpret<Packet16uc>(pcast<Packet8s, Packet16c>(a, b));
778}
779template <>
780EIGEN_STRONG_INLINE Packet8uc pcast<Packet8s, Packet8uc>(const Packet8s& a) {
781 return preinterpret<Packet8uc>(pcast<Packet8s, Packet8c>(a));
782}
783template <>
784EIGEN_STRONG_INLINE Packet8uc pcast<Packet4s, Packet8uc>(const Packet4s& a, const Packet4s& b) {
785 return preinterpret<Packet8uc>(pcast<Packet4s, Packet8c>(a, b));
786}
787template <>
788EIGEN_STRONG_INLINE Packet4uc pcast<Packet4s, Packet4uc>(const Packet4s& a) {
789 return static_cast<Packet4uc>(pcast<Packet4s, Packet4c>(a));
790}
791
792//==============================================================================
793// pcast, SrcType = uint16_t
794//==============================================================================
795template <>
796struct type_casting_traits<numext::uint16_t, float> {
797 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
798};
799template <>
800EIGEN_STRONG_INLINE Packet4f pcast<Packet8us, Packet4f>(const Packet8us& a) {
801 // Discard second half of input.
802 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a)));
803}
804template <>
805EIGEN_STRONG_INLINE Packet4f pcast<Packet4us, Packet4f>(const Packet4us& a) {
806 return vcvtq_f32_u32(vmovl_u16(a));
807}
808template <>
809EIGEN_STRONG_INLINE Packet2f pcast<Packet4us, Packet2f>(const Packet4us& a) {
810 // Discard second half of input.
811 return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
812}
813
814template <>
815struct type_casting_traits<numext::uint16_t, numext::uint64_t> {
816 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
817};
818template <>
819EIGEN_STRONG_INLINE Packet2ul pcast<Packet8us, Packet2ul>(const Packet8us& a) {
820 // Discard all but first two values.
821 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a))));
822}
823
824template <>
825struct type_casting_traits<numext::uint16_t, numext::int64_t> {
826 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
827};
828template <>
829EIGEN_STRONG_INLINE Packet2l pcast<Packet8us, Packet2l>(const Packet8us& a) {
830 return preinterpret<Packet2l>(pcast<Packet8us, Packet2ul>(a));
831}
832
833template <>
834struct type_casting_traits<numext::uint16_t, numext::uint32_t> {
835 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
836};
837template <>
838EIGEN_STRONG_INLINE Packet4ui pcast<Packet8us, Packet4ui>(const Packet8us& a) {
839 // Discard second half of input.
840 return vmovl_u16(vget_low_u16(a));
841}
842template <>
843EIGEN_STRONG_INLINE Packet4ui pcast<Packet4us, Packet4ui>(const Packet4us& a) {
844 return vmovl_u16(a);
845}
846template <>
847EIGEN_STRONG_INLINE Packet2ui pcast<Packet4us, Packet2ui>(const Packet4us& a) {
848 // Discard second half of input.
849 return vget_low_u32(vmovl_u16(a));
850}
851
852template <>
853struct type_casting_traits<numext::uint16_t, numext::int32_t> {
854 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
855};
856template <>
857EIGEN_STRONG_INLINE Packet4i pcast<Packet8us, Packet4i>(const Packet8us& a) {
858 return preinterpret<Packet4i>(pcast<Packet8us, Packet4ui>(a));
859}
860template <>
861EIGEN_STRONG_INLINE Packet4i pcast<Packet4us, Packet4i>(const Packet4us& a) {
862 return preinterpret<Packet4i>(pcast<Packet4us, Packet4ui>(a));
863}
864template <>
865EIGEN_STRONG_INLINE Packet2i pcast<Packet4us, Packet2i>(const Packet4us& a) {
866 return preinterpret<Packet2i>(pcast<Packet4us, Packet2ui>(a));
867}
868
869template <>
870struct type_casting_traits<numext::uint16_t, numext::uint8_t> {
871 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
872};
873template <>
874EIGEN_STRONG_INLINE Packet16uc pcast<Packet8us, Packet16uc>(const Packet8us& a, const Packet8us& b) {
875 return vcombine_u8(vmovn_u16(a), vmovn_u16(b));
876}
877template <>
878EIGEN_STRONG_INLINE Packet8uc pcast<Packet8us, Packet8uc>(const Packet8us& a) {
879 return vmovn_u16(a);
880}
881template <>
882EIGEN_STRONG_INLINE Packet8uc pcast<Packet4us, Packet8uc>(const Packet4us& a, const Packet4us& b) {
883 return vmovn_u16(vcombine_u16(a, b));
884}
885template <>
886EIGEN_STRONG_INLINE Packet4uc pcast<Packet4us, Packet4uc>(const Packet4us& a) {
887 uint8x8_t aa_u8x8 = pcast<Packet4us, Packet8uc>(a, a);
888 return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
889}
890
891template <>
892struct type_casting_traits<numext::uint16_t, numext::int8_t> {
893 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
894};
895template <>
896EIGEN_STRONG_INLINE Packet16c pcast<Packet8us, Packet16c>(const Packet8us& a, const Packet8us& b) {
897 return preinterpret<Packet16c>(pcast<Packet8us, Packet16uc>(a, b));
898}
899template <>
900EIGEN_STRONG_INLINE Packet8c pcast<Packet8us, Packet8c>(const Packet8us& a) {
901 return preinterpret<Packet8c>(pcast<Packet8us, Packet8uc>(a));
902}
903template <>
904EIGEN_STRONG_INLINE Packet8c pcast<Packet4us, Packet8c>(const Packet4us& a, const Packet4us& b) {
905 return preinterpret<Packet8c>(pcast<Packet4us, Packet8uc>(a, b));
906}
907template <>
908EIGEN_STRONG_INLINE Packet4c pcast<Packet4us, Packet4c>(const Packet4us& a) {
909 return static_cast<Packet4c>(pcast<Packet4us, Packet4uc>(a));
910}
911
912//==============================================================================
913// pcast, SrcType = int32_t
914//==============================================================================
915template <>
916struct type_casting_traits<numext::int32_t, float> {
917 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
918};
919template <>
920EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
921 return vcvtq_f32_s32(a);
922}
923template <>
924EIGEN_STRONG_INLINE Packet2f pcast<Packet2i, Packet2f>(const Packet2i& a) {
925 return vcvt_f32_s32(a);
926}
927
928template <>
929struct type_casting_traits<numext::int32_t, numext::int64_t> {
930 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
931};
932template <>
933EIGEN_STRONG_INLINE Packet2l pcast<Packet4i, Packet2l>(const Packet4i& a) {
934 // Discard second half of input.
935 return vmovl_s32(vget_low_s32(a));
936}
937template <>
938EIGEN_STRONG_INLINE Packet2l pcast<Packet2i, Packet2l>(const Packet2i& a) {
939 return vmovl_s32(a);
940}
941
942template <>
943struct type_casting_traits<numext::int32_t, numext::uint64_t> {
944 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
945};
946template <>
947EIGEN_STRONG_INLINE Packet2ul pcast<Packet4i, Packet2ul>(const Packet4i& a) {
948 return preinterpret<Packet2ul>(pcast<Packet4i, Packet2l>(a));
949}
950template <>
951EIGEN_STRONG_INLINE Packet2ul pcast<Packet2i, Packet2ul>(const Packet2i& a) {
952 return preinterpret<Packet2ul>(pcast<Packet2i, Packet2l>(a));
953}
954
955template <>
956struct type_casting_traits<numext::int32_t, numext::int16_t> {
957 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
958};
959template <>
960EIGEN_STRONG_INLINE Packet8s pcast<Packet4i, Packet8s>(const Packet4i& a, const Packet4i& b) {
961 return vcombine_s16(vmovn_s32(a), vmovn_s32(b));
962}
963template <>
964EIGEN_STRONG_INLINE Packet4s pcast<Packet4i, Packet4s>(const Packet4i& a) {
965 return vmovn_s32(a);
966}
967template <>
968EIGEN_STRONG_INLINE Packet4s pcast<Packet2i, Packet4s>(const Packet2i& a, const Packet2i& b) {
969 return vmovn_s32(vcombine_s32(a, b));
970}
971
972template <>
973struct type_casting_traits<numext::int32_t, numext::uint16_t> {
974 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
975};
976template <>
977EIGEN_STRONG_INLINE Packet8us pcast<Packet4i, Packet8us>(const Packet4i& a, const Packet4i& b) {
978 return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b)));
979}
980template <>
981EIGEN_STRONG_INLINE Packet4us pcast<Packet4i, Packet4us>(const Packet4i& a) {
982 return vmovn_u32(vreinterpretq_u32_s32(a));
983}
984template <>
985EIGEN_STRONG_INLINE Packet4us pcast<Packet2i, Packet4us>(const Packet2i& a, const Packet2i& b) {
986 return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
987}
988
989template <>
990struct type_casting_traits<numext::int32_t, numext::int8_t> {
991 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
992};
993template <>
994EIGEN_STRONG_INLINE Packet16c pcast<Packet4i, Packet16c>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
995 const Packet4i& d) {
996 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
997 const int16x8_t cd_s16 = pcast<Packet4i, Packet8s>(c, d);
998 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
999}
1000template <>
1001EIGEN_STRONG_INLINE Packet8c pcast<Packet4i, Packet8c>(const Packet4i& a, const Packet4i& b) {
1002 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
1003 return vmovn_s16(ab_s16);
1004}
1005template <>
1006EIGEN_STRONG_INLINE Packet8c pcast<Packet2i, Packet8c>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
1007 const Packet2i& d) {
1008 const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
1009 const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
1010 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
1011}
1012template <>
1013EIGEN_STRONG_INLINE Packet4c pcast<Packet4i, Packet4c>(const Packet4i& a) {
1014 const int16x4_t a_s16x4 = vmovn_s32(a);
1015 const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
1016 const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
1017 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
1018}
1019
1020template <>
1021struct type_casting_traits<numext::int32_t, numext::uint8_t> {
1022 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1023};
1024template <>
1025EIGEN_STRONG_INLINE Packet16uc pcast<Packet4i, Packet16uc>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
1026 const Packet4i& d) {
1027 return preinterpret<Packet16uc>(pcast<Packet4i, Packet16c>(a, b, c, d));
1028}
1029template <>
1030EIGEN_STRONG_INLINE Packet8uc pcast<Packet4i, Packet8uc>(const Packet4i& a, const Packet4i& b) {
1031 return preinterpret<Packet8uc>(pcast<Packet4i, Packet8c>(a, b));
1032}
1033template <>
1034EIGEN_STRONG_INLINE Packet8uc pcast<Packet2i, Packet8uc>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
1035 const Packet2i& d) {
1036 return preinterpret<Packet8uc>(pcast<Packet2i, Packet8c>(a, b, c, d));
1037}
1038template <>
1039EIGEN_STRONG_INLINE Packet4uc pcast<Packet4i, Packet4uc>(const Packet4i& a) {
1040 return static_cast<Packet4uc>(pcast<Packet4i, Packet4c>(a));
1041}
1042
1043//==============================================================================
1044// pcast, SrcType = uint32_t
1045//==============================================================================
1046template <>
1047struct type_casting_traits<numext::uint32_t, float> {
1048 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1049};
1050template <>
1051EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
1052 return vcvtq_f32_u32(a);
1053}
1054template <>
1055EIGEN_STRONG_INLINE Packet2f pcast<Packet2ui, Packet2f>(const Packet2ui& a) {
1056 return vcvt_f32_u32(a);
1057}
1058
1059template <>
1060struct type_casting_traits<numext::uint32_t, numext::uint64_t> {
1061 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1062};
1063template <>
1064EIGEN_STRONG_INLINE Packet2ul pcast<Packet4ui, Packet2ul>(const Packet4ui& a) {
1065 // Discard second half of input.
1066 return vmovl_u32(vget_low_u32(a));
1067}
1068template <>
1069EIGEN_STRONG_INLINE Packet2ul pcast<Packet2ui, Packet2ul>(const Packet2ui& a) {
1070 return vmovl_u32(a);
1071}
1072
1073template <>
1074struct type_casting_traits<numext::uint32_t, numext::int64_t> {
1075 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1076};
1077template <>
1078EIGEN_STRONG_INLINE Packet2l pcast<Packet4ui, Packet2l>(const Packet4ui& a) {
1079 return preinterpret<Packet2l>(pcast<Packet4ui, Packet2ul>(a));
1080}
1081template <>
1082EIGEN_STRONG_INLINE Packet2l pcast<Packet2ui, Packet2l>(const Packet2ui& a) {
1083 return preinterpret<Packet2l>(pcast<Packet2ui, Packet2ul>(a));
1084}
1085
1086template <>
1087struct type_casting_traits<numext::uint32_t, numext::uint16_t> {
1088 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1089};
1090template <>
1091EIGEN_STRONG_INLINE Packet8us pcast<Packet4ui, Packet8us>(const Packet4ui& a, const Packet4ui& b) {
1092 return vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1093}
1094template <>
1095EIGEN_STRONG_INLINE Packet4us pcast<Packet2ui, Packet4us>(const Packet2ui& a, const Packet2ui& b) {
1096 return vmovn_u32(vcombine_u32(a, b));
1097}
1098template <>
1099EIGEN_STRONG_INLINE Packet4us pcast<Packet4ui, Packet4us>(const Packet4ui& a) {
1100 return vmovn_u32(a);
1101}
1102
1103template <>
1104struct type_casting_traits<numext::uint32_t, numext::int16_t> {
1105 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1106};
1107template <>
1108EIGEN_STRONG_INLINE Packet8s pcast<Packet4ui, Packet8s>(const Packet4ui& a, const Packet4ui& b) {
1109 return preinterpret<Packet8s>(pcast<Packet4ui, Packet8us>(a, b));
1110}
1111template <>
1112EIGEN_STRONG_INLINE Packet4s pcast<Packet2ui, Packet4s>(const Packet2ui& a, const Packet2ui& b) {
1113 return preinterpret<Packet4s>(pcast<Packet2ui, Packet4us>(a, b));
1114}
1115template <>
1116EIGEN_STRONG_INLINE Packet4s pcast<Packet4ui, Packet4s>(const Packet4ui& a) {
1117 return preinterpret<Packet4s>(pcast<Packet4ui, Packet4us>(a));
1118}
1119
1120template <>
1121struct type_casting_traits<numext::uint32_t, numext::uint8_t> {
1122 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1123};
1124template <>
1125EIGEN_STRONG_INLINE Packet16uc pcast<Packet4ui, Packet16uc>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1126 const Packet4ui& d) {
1127 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1128 const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d));
1129 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
1130}
1131template <>
1132EIGEN_STRONG_INLINE Packet8uc pcast<Packet4ui, Packet8uc>(const Packet4ui& a, const Packet4ui& b) {
1133 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1134 return vmovn_u16(ab_u16);
1135}
1136template <>
1137EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ui, Packet8uc>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1138 const Packet2ui& d) {
1139 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
1140 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
1141 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
1142}
1143template <>
1144EIGEN_STRONG_INLINE Packet4uc pcast<Packet4ui, Packet4uc>(const Packet4ui& a) {
1145 const uint16x4_t a_u16x4 = vmovn_u32(a);
1146 const uint16x8_t aa_u16x8 = vcombine_u16(a_u16x4, a_u16x4);
1147 const uint8x8_t aa_u8x8 = vmovn_u16(aa_u16x8);
1148 return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
1149}
1150
1151template <>
1152struct type_casting_traits<numext::uint32_t, numext::int8_t> {
1153 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1154};
1155template <>
1156EIGEN_STRONG_INLINE Packet16c pcast<Packet4ui, Packet16c>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1157 const Packet4ui& d) {
1158 return preinterpret<Packet16c>(pcast<Packet4ui, Packet16uc>(a, b, c, d));
1159}
1160template <>
1161EIGEN_STRONG_INLINE Packet8c pcast<Packet4ui, Packet8c>(const Packet4ui& a, const Packet4ui& b) {
1162 return preinterpret<Packet8c>(pcast<Packet4ui, Packet8uc>(a, b));
1163}
1164template <>
1165EIGEN_STRONG_INLINE Packet8c pcast<Packet2ui, Packet8c>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1166 const Packet2ui& d) {
1167 return preinterpret<Packet8c>(pcast<Packet2ui, Packet8uc>(a, b, c, d));
1168}
1169template <>
1170EIGEN_STRONG_INLINE Packet4c pcast<Packet4ui, Packet4c>(const Packet4ui& a) {
1171 return static_cast<Packet4c>(pcast<Packet4ui, Packet4uc>(a));
1172}
1173
1174//==============================================================================
1175// pcast, SrcType = int64_t
1176//==============================================================================
1177template <>
1178struct type_casting_traits<numext::int64_t, float> {
1179 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1180};
1181
1182#if EIGEN_ARCH_ARM64
1183// int64 -> double -> float rounds twice, which can differ from rounding once; Clang before 21 compiles scalar
1184// int64 -> float conversions of adjacent lanes that way too. Rounding x to odd on the multiples of 2^11 where
1185// |x| >= 2^53 makes the conversion to double exact and leaves the rounding to float unchanged.
1186EIGEN_STRONG_INLINE Packet2d int64_to_double_round_to_odd(const Packet2l& a) {
1187 const int64x2_t low = vdupq_n_s64(0x7ff);
1188 const int64x2_t odd =
1189 vorrq_s64(vbicq_s64(a, low), vreinterpretq_s64_u64(vandq_u64(vtstq_s64(a, low), vdupq_n_u64(0x800))));
1190 return vcvtq_f64_s64(vbslq_s64(vcgeq_s64(vabsq_s64(a), vdupq_n_s64(int64_t(1) << 53)), odd, a));
1191}
1192template <>
1193EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
1194 return vcombine_f32(vcvt_f32_f64(int64_to_double_round_to_odd(a)), vcvt_f32_f64(int64_to_double_round_to_odd(b)));
1195}
1196template <>
1197EIGEN_STRONG_INLINE Packet2f pcast<Packet2l, Packet2f>(const Packet2l& a) {
1198 return vcvt_f32_f64(int64_to_double_round_to_odd(a));
1199}
1200#else
1201template <>
1202EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
1203 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2l>::alignment) int64_t lvals[4];
1204 pstore(lvals, a);
1205 pstore(lvals + 2, b);
1206 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet4f>::alignment)
1207 float fvals[4] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1]), static_cast<float>(lvals[2]),
1208 static_cast<float>(lvals[3])};
1209 return pload<Packet4f>(fvals);
1210}
1211
1212template <>
1213EIGEN_STRONG_INLINE Packet2f pcast<Packet2l, Packet2f>(const Packet2l& a) {
1214 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2l>::alignment) int64_t lvals[2];
1215 pstore(lvals, a);
1216 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2f>::alignment)
1217 float fvals[2] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1])};
1218 return pload<Packet2f>(fvals);
1219}
1220#endif
1221
1222template <>
1223struct type_casting_traits<numext::int64_t, numext::int32_t> {
1224 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1225};
1226template <>
1227EIGEN_STRONG_INLINE Packet4i pcast<Packet2l, Packet4i>(const Packet2l& a, const Packet2l& b) {
1228 return vcombine_s32(vmovn_s64(a), vmovn_s64(b));
1229}
1230template <>
1231EIGEN_STRONG_INLINE Packet2i pcast<Packet2l, Packet2i>(const Packet2l& a) {
1232 return vmovn_s64(a);
1233}
1234
1235template <>
1236struct type_casting_traits<numext::int64_t, numext::uint32_t> {
1237 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1238};
1239template <>
1240EIGEN_STRONG_INLINE Packet4ui pcast<Packet2l, Packet4ui>(const Packet2l& a, const Packet2l& b) {
1241 return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b)));
1242}
1243template <>
1244EIGEN_STRONG_INLINE Packet2ui pcast<Packet2l, Packet2ui>(const Packet2l& a) {
1245 return vmovn_u64(vreinterpretq_u64_s64(a));
1246}
1247
1248template <>
1249struct type_casting_traits<numext::int64_t, numext::int16_t> {
1250 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1251};
1252template <>
1253EIGEN_STRONG_INLINE Packet8s pcast<Packet2l, Packet8s>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1254 const Packet2l& d) {
1255 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1256 const int32x4_t cd_s32 = pcast<Packet2l, Packet4i>(c, d);
1257 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1258}
1259template <>
1260EIGEN_STRONG_INLINE Packet4s pcast<Packet2l, Packet4s>(const Packet2l& a, const Packet2l& b) {
1261 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1262 return vmovn_s32(ab_s32);
1263}
1264
1265template <>
1266struct type_casting_traits<numext::int64_t, numext::uint16_t> {
1267 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1268};
1269template <>
1270EIGEN_STRONG_INLINE Packet8us pcast<Packet2l, Packet8us>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1271 const Packet2l& d) {
1272 return preinterpret<Packet8us>(pcast<Packet2l, Packet8s>(a, b, c, d));
1273}
1274template <>
1275EIGEN_STRONG_INLINE Packet4us pcast<Packet2l, Packet4us>(const Packet2l& a, const Packet2l& b) {
1276 return preinterpret<Packet4us>(pcast<Packet2l, Packet4s>(a, b));
1277}
1278
1279template <>
1280struct type_casting_traits<numext::int64_t, numext::int8_t> {
1281 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1282};
1283template <>
1284EIGEN_STRONG_INLINE Packet16c pcast<Packet2l, Packet16c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1285 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1286 const Packet2l& g, const Packet2l& h) {
1287 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1288 const int16x8_t efgh_s16 = pcast<Packet2l, Packet8s>(e, f, g, h);
1289 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1290}
1291template <>
1292EIGEN_STRONG_INLINE Packet8c pcast<Packet2l, Packet8c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1293 const Packet2l& d) {
1294 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1295 return vmovn_s16(abcd_s16);
1296}
1297template <>
1298EIGEN_STRONG_INLINE Packet4c pcast<Packet2l, Packet4c>(const Packet2l& a, const Packet2l& b) {
1299 const int16x4_t ab_s16 = pcast<Packet2l, Packet4s>(a, b);
1300 const int16x8_t abab_s16 = vcombine_s16(ab_s16, ab_s16);
1301 const int8x8_t abab_s8 = vmovn_s16(abab_s16);
1302 return vget_lane_s32(vreinterpret_s32_s8(abab_s8), 0);
1303}
1304
1305template <>
1306struct type_casting_traits<numext::int64_t, numext::uint8_t> {
1307 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1308};
1309template <>
1310EIGEN_STRONG_INLINE Packet16uc pcast<Packet2l, Packet16uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1311 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1312 const Packet2l& g, const Packet2l& h) {
1313 const uint16x8_t abcd_u16 = pcast<Packet2l, Packet8us>(a, b, c, d);
1314 const uint16x8_t efgh_u16 = pcast<Packet2l, Packet8us>(e, f, g, h);
1315 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1316}
1317template <>
1318EIGEN_STRONG_INLINE Packet8uc pcast<Packet2l, Packet8uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1319 const Packet2l& d) {
1320 return preinterpret<Packet8uc>(pcast<Packet2l, Packet8c>(a, b, c, d));
1321}
1322template <>
1323EIGEN_STRONG_INLINE Packet4uc pcast<Packet2l, Packet4uc>(const Packet2l& a, const Packet2l& b) {
1324 return static_cast<Packet4uc>(pcast<Packet2l, Packet4c>(a, b));
1325}
1326
1327//==============================================================================
1328// pcast, SrcType = uint64_t
1329//==============================================================================
1330template <>
1331struct type_casting_traits<numext::uint64_t, float> {
1332 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1333};
1334#if EIGEN_ARCH_ARM64
1335EIGEN_STRONG_INLINE Packet2d uint64_to_double_round_to_odd(const Packet2ul& a) {
1336 const uint64x2_t low = vdupq_n_u64(0x7ff);
1337 const uint64x2_t odd = vorrq_u64(vbicq_u64(a, low), vandq_u64(vtstq_u64(a, low), vdupq_n_u64(0x800)));
1338 return vcvtq_f64_u64(vbslq_u64(vcgeq_u64(a, vdupq_n_u64(uint64_t(1) << 53)), odd, a));
1339}
1340template <>
1341EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
1342 return vcombine_f32(vcvt_f32_f64(uint64_to_double_round_to_odd(a)), vcvt_f32_f64(uint64_to_double_round_to_odd(b)));
1343}
1344template <>
1345EIGEN_STRONG_INLINE Packet2f pcast<Packet2ul, Packet2f>(const Packet2ul& a) {
1346 return vcvt_f32_f64(uint64_to_double_round_to_odd(a));
1347}
1348#else
1349template <>
1350EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
1351 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2ul>::alignment) uint64_t uvals[4];
1352 pstore(uvals, a);
1353 pstore(uvals + 2, b);
1354 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet4f>::alignment)
1355 float fvals[4] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1]), static_cast<float>(uvals[2]),
1356 static_cast<float>(uvals[3])};
1357 return pload<Packet4f>(fvals);
1358}
1359template <>
1360EIGEN_STRONG_INLINE Packet2f pcast<Packet2ul, Packet2f>(const Packet2ul& a) {
1361 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2ul>::alignment) uint64_t uvals[2];
1362 pstore(uvals, a);
1363 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2f>::alignment)
1364 float fvals[2] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1])};
1365 return pload<Packet2f>(fvals);
1366}
1367#endif
1368
1369template <>
1370struct type_casting_traits<numext::uint64_t, numext::uint32_t> {
1371 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1372};
1373template <>
1374EIGEN_STRONG_INLINE Packet4ui pcast<Packet2ul, Packet4ui>(const Packet2ul& a, const Packet2ul& b) {
1375 return vcombine_u32(vmovn_u64(a), vmovn_u64(b));
1376}
1377template <>
1378EIGEN_STRONG_INLINE Packet2ui pcast<Packet2ul, Packet2ui>(const Packet2ul& a) {
1379 return vmovn_u64(a);
1380}
1381
1382template <>
1383struct type_casting_traits<numext::uint64_t, numext::int32_t> {
1384 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1385};
1386template <>
1387EIGEN_STRONG_INLINE Packet4i pcast<Packet2ul, Packet4i>(const Packet2ul& a, const Packet2ul& b) {
1388 return preinterpret<Packet4i>(pcast<Packet2ul, Packet4ui>(a, b));
1389}
1390template <>
1391EIGEN_STRONG_INLINE Packet2i pcast<Packet2ul, Packet2i>(const Packet2ul& a) {
1392 return preinterpret<Packet2i>(pcast<Packet2ul, Packet2ui>(a));
1393}
1394
1395template <>
1396struct type_casting_traits<numext::uint64_t, numext::uint16_t> {
1397 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1398};
1399template <>
1400EIGEN_STRONG_INLINE Packet8us pcast<Packet2ul, Packet8us>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1401 const Packet2ul& d) {
1402 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1403 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d)));
1404 return vcombine_u16(ab_u16, cd_u16);
1405}
1406template <>
1407EIGEN_STRONG_INLINE Packet4us pcast<Packet2ul, Packet4us>(const Packet2ul& a, const Packet2ul& b) {
1408 return vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1409}
1410
1411template <>
1412struct type_casting_traits<numext::uint64_t, numext::int16_t> {
1413 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1414};
1415template <>
1416EIGEN_STRONG_INLINE Packet8s pcast<Packet2ul, Packet8s>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1417 const Packet2ul& d) {
1418 return preinterpret<Packet8s>(pcast<Packet2ul, Packet8us>(a, b, c, d));
1419}
1420template <>
1421EIGEN_STRONG_INLINE Packet4s pcast<Packet2ul, Packet4s>(const Packet2ul& a, const Packet2ul& b) {
1422 return preinterpret<Packet4s>(pcast<Packet2ul, Packet4us>(a, b));
1423}
1424
1425template <>
1426struct type_casting_traits<numext::uint64_t, numext::uint8_t> {
1427 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1428};
1429template <>
1430EIGEN_STRONG_INLINE Packet16uc pcast<Packet2ul, Packet16uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1431 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1432 const Packet2ul& g, const Packet2ul& h) {
1433 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1434 const uint16x8_t efgh_u16 = pcast<Packet2ul, Packet8us>(e, f, g, h);
1435 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1436}
1437template <>
1438EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ul, Packet8uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1439 const Packet2ul& d) {
1440 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1441 return vmovn_u16(abcd_u16);
1442}
1443template <>
1444EIGEN_STRONG_INLINE Packet4uc pcast<Packet2ul, Packet4uc>(const Packet2ul& a, const Packet2ul& b) {
1445 const uint16x4_t ab_u16 = pcast<Packet2ul, Packet4us>(a, b);
1446 const uint16x8_t abab_u16 = vcombine_u16(ab_u16, ab_u16);
1447 const uint8x8_t abab_u8 = vmovn_u16(abab_u16);
1448 return vget_lane_u32(vreinterpret_u32_u8(abab_u8), 0);
1449}
1450
1451template <>
1452struct type_casting_traits<numext::uint64_t, numext::int8_t> {
1453 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1454};
1455template <>
1456EIGEN_STRONG_INLINE Packet16c pcast<Packet2ul, Packet16c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1457 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1458 const Packet2ul& g, const Packet2ul& h) {
1459 return preinterpret<Packet16c>(pcast<Packet2ul, Packet16uc>(a, b, c, d, e, f, g, h));
1460}
1461template <>
1462EIGEN_STRONG_INLINE Packet8c pcast<Packet2ul, Packet8c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1463 const Packet2ul& d) {
1464 return preinterpret<Packet8c>(pcast<Packet2ul, Packet8uc>(a, b, c, d));
1465}
1466template <>
1467EIGEN_STRONG_INLINE Packet4c pcast<Packet2ul, Packet4c>(const Packet2ul& a, const Packet2ul& b) {
1468 return static_cast<Packet4c>(pcast<Packet2ul, Packet4uc>(a, b));
1469}
1470
1471#if EIGEN_ARCH_ARM64
1472
1473//==============================================================================
1474// pcast/preinterpret, Double
1475//==============================================================================
1476
1477template <>
1478EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
1479 return Packet2d(vreinterpretq_f64_s64(a));
1480}
1481template <>
1482EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2ul>(const Packet2ul& a) {
1483 return Packet2d(vreinterpretq_f64_u64(a));
1484}
1485template <>
1486EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
1487 return Packet2l(vreinterpretq_s64_f64(a));
1488}
1489template <>
1490EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2d>(const Packet2d& a) {
1491 return Packet2ul(vreinterpretq_u64_f64(a));
1492}
1493template <>
1494EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
1495 return Packet2d(vreinterpretq_f64_s32(a));
1496}
1497template <>
1498EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
1499 return Packet4i(vreinterpretq_s32_f64(a));
1500}
1501
1502template <>
1503struct type_casting_traits<double, float> {
1504 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1505};
1506template <>
1507EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
1508 return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b));
1509}
1510template <>
1511EIGEN_STRONG_INLINE Packet2f pcast<Packet2d, Packet2f>(const Packet2d& a) {
1512 return vcvt_f32_f64(a);
1513}
1514
1515template <>
1516struct type_casting_traits<double, numext::int64_t> {
1517 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1518};
1519template <>
1520EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
1521 return vcvtq_s64_f64(a);
1522}
1523
1524template <>
1525struct type_casting_traits<double, numext::uint64_t> {
1526 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1527};
1528template <>
1529EIGEN_STRONG_INLINE Packet2ul pcast<Packet2d, Packet2ul>(const Packet2d& a) {
1530 return vcvtq_u64_f64(a);
1531}
1532
1533template <>
1534struct type_casting_traits<double, numext::int32_t> {
1535 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1536};
1537template <>
1538EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
1539 return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b)));
1540}
1541template <>
1542EIGEN_STRONG_INLINE Packet2i pcast<Packet2d, Packet2i>(const Packet2d& a) {
1543 return vmovn_s64(vcvtq_s64_f64(a));
1544}
1545
1546template <>
1547struct type_casting_traits<double, numext::uint32_t> {
1548 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1549};
1550template <>
1551EIGEN_STRONG_INLINE Packet4ui pcast<Packet2d, Packet4ui>(const Packet2d& a, const Packet2d& b) {
1552 return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b)));
1553}
1554template <>
1555EIGEN_STRONG_INLINE Packet2ui pcast<Packet2d, Packet2ui>(const Packet2d& a) {
1556 return vmovn_u64(vcvtq_u64_f64(a));
1557}
1558
1559template <>
1560struct type_casting_traits<double, numext::int16_t> {
1561 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1562};
1563template <>
1564EIGEN_STRONG_INLINE Packet8s pcast<Packet2d, Packet8s>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1565 const Packet2d& d) {
1566 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1567 const int32x4_t cd_s32 = pcast<Packet2d, Packet4i>(c, d);
1568 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1569}
1570template <>
1571EIGEN_STRONG_INLINE Packet4s pcast<Packet2d, Packet4s>(const Packet2d& a, const Packet2d& b) {
1572 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1573 return vmovn_s32(ab_s32);
1574}
1575
1576template <>
1577struct type_casting_traits<double, numext::uint16_t> {
1578 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1579};
1580template <>
1581EIGEN_STRONG_INLINE Packet8us pcast<Packet2d, Packet8us>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1582 const Packet2d& d) {
1583 return preinterpret<Packet8us>(pcast<Packet2d, Packet8s>(a, b, c, d));
1584}
1585template <>
1586EIGEN_STRONG_INLINE Packet4us pcast<Packet2d, Packet4us>(const Packet2d& a, const Packet2d& b) {
1587 return preinterpret<Packet4us>(pcast<Packet2d, Packet4s>(a, b));
1588}
1589
1590template <>
1591struct type_casting_traits<double, numext::int8_t> {
1592 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1593};
1594template <>
1595EIGEN_STRONG_INLINE Packet16c pcast<Packet2d, Packet16c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1596 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1597 const Packet2d& g, const Packet2d& h) {
1598 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1599 const int16x8_t efgh_s16 = pcast<Packet2d, Packet8s>(e, f, g, h);
1600 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1601}
1602template <>
1603EIGEN_STRONG_INLINE Packet8c pcast<Packet2d, Packet8c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1604 const Packet2d& d) {
1605 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1606 return vmovn_s16(abcd_s16);
1607}
1608template <>
1609EIGEN_STRONG_INLINE Packet4c pcast<Packet2d, Packet4c>(const Packet2d& a, const Packet2d& b) {
1610 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1611 return pcast<Packet4i, Packet4c>(ab_s32);
1612}
1613
1614template <>
1615struct type_casting_traits<double, numext::uint8_t> {
1616 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1617};
1618template <>
1619EIGEN_STRONG_INLINE Packet16uc pcast<Packet2d, Packet16uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1620 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1621 const Packet2d& g, const Packet2d& h) {
1622 const uint16x8_t abcd_u16 = pcast<Packet2d, Packet8us>(a, b, c, d);
1623 const uint16x8_t efgh_u16 = pcast<Packet2d, Packet8us>(e, f, g, h);
1624 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1625}
1626template <>
1627EIGEN_STRONG_INLINE Packet8uc pcast<Packet2d, Packet8uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1628 const Packet2d& d) {
1629 return preinterpret<Packet8uc>(pcast<Packet2d, Packet8c>(a, b, c, d));
1630}
1631template <>
1632EIGEN_STRONG_INLINE Packet4uc pcast<Packet2d, Packet4uc>(const Packet2d& a, const Packet2d& b) {
1633 return static_cast<Packet4uc>(pcast<Packet2d, Packet4c>(a, b));
1634}
1635
1636template <>
1637struct type_casting_traits<float, double> {
1638 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1639};
1640template <>
1641EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
1642 // Discard second-half of input.
1643 return vcvt_f64_f32(vget_low_f32(a));
1644}
1645template <>
1646EIGEN_STRONG_INLINE Packet2d pcast<Packet2f, Packet2d>(const Packet2f& a) {
1647 return vcvt_f64_f32(a);
1648}
1649
1650template <>
1651struct type_casting_traits<numext::int8_t, double> {
1652 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1653};
1654template <>
1655EIGEN_STRONG_INLINE Packet2d pcast<Packet16c, Packet2d>(const Packet16c& a) {
1656 // Discard all but first two values.
1657 // MSVC defines most intrinsics as macros, so we need to do this in two lines for portability.
1658 Packet2f tmp = pcast<Packet8c, Packet2f>(vget_low_s8(a));
1659 return vcvt_f64_f32(tmp);
1660}
1661
1662template <>
1663struct type_casting_traits<numext::uint8_t, double> {
1664 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1665};
1666template <>
1667EIGEN_STRONG_INLINE Packet2d pcast<Packet16uc, Packet2d>(const Packet16uc& a) {
1668 // Discard all but first two values.
1669 Packet2f tmp = pcast<Packet8uc, Packet2f>(vget_low_u8(a));
1670 return vcvt_f64_f32(tmp);
1671}
1672
1673template <>
1674struct type_casting_traits<numext::int16_t, double> {
1675 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1676};
1677template <>
1678EIGEN_STRONG_INLINE Packet2d pcast<Packet8s, Packet2d>(const Packet8s& a) {
1679 // Discard all but first two values.
1680 Packet2f tmp = pcast<Packet4s, Packet2f>(vget_low_s16(a));
1681 return vcvt_f64_f32(tmp);
1682}
1683
1684template <>
1685struct type_casting_traits<numext::uint16_t, double> {
1686 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1687};
1688template <>
1689EIGEN_STRONG_INLINE Packet2d pcast<Packet8us, Packet2d>(const Packet8us& a) {
1690 // Discard all but first two values.
1691 Packet2f tmp = pcast<Packet4us, Packet2f>(vget_low_u16(a));
1692 return vcvt_f64_f32(tmp);
1693}
1694
1695template <>
1696struct type_casting_traits<numext::int32_t, double> {
1697 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1698};
1699template <>
1700EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
1701 // Discard second half of input.
1702 return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a)));
1703}
1704template <>
1705EIGEN_STRONG_INLINE Packet2d pcast<Packet2i, Packet2d>(const Packet2i& a) {
1706 return vcvtq_f64_s64(vmovl_s32(a));
1707}
1708
1709template <>
1710struct type_casting_traits<numext::uint32_t, double> {
1711 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1712};
1713template <>
1714EIGEN_STRONG_INLINE Packet2d pcast<Packet4ui, Packet2d>(const Packet4ui& a) {
1715 // Discard second half of input.
1716 return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a)));
1717}
1718template <>
1719EIGEN_STRONG_INLINE Packet2d pcast<Packet2ui, Packet2d>(const Packet2ui& a) {
1720 return vcvtq_f64_u64(vmovl_u32(a));
1721}
1722
1723template <>
1724struct type_casting_traits<numext::int64_t, double> {
1725 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1726};
1727template <>
1728EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
1729 return vcvtq_f64_s64(a);
1730}
1731
1732template <>
1733struct type_casting_traits<numext::uint64_t, double> {
1734 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1735};
1736template <>
1737EIGEN_STRONG_INLINE Packet2d pcast<Packet2ul, Packet2d>(const Packet2ul& a) {
1738 return vcvtq_f64_u64(a);
1739}
1740
1741#endif // EIGEN_ARCH_ARM64
1742
1743} // end namespace internal
1744
1745} // end namespace Eigen
1746
1747#endif // EIGEN_TYPE_CASTING_NEON_H