Eigen  5.0.1
 
Loading...
Searching...
No Matches
TypeCasting.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@gmail.com>
5// Copyright (C) 2020 Antonio Sanchez <cantonios@google.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_TYPE_CASTING_NEON_H
13#define EIGEN_TYPE_CASTING_NEON_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22//==============================================================================
23// preinterpret (truncation operations)
24//==============================================================================
25
26template <>
27EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet16c>(const Packet16c& a) {
28 return Packet8c(vget_low_s8(a));
29}
30template <>
31EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet8c>(const Packet8c& a) {
32 return Packet4c(vget_lane_s32(vreinterpret_s32_s8(a), 0));
33}
34template <>
35EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet16c>(const Packet16c& a) {
36 return preinterpret<Packet4c>(preinterpret<Packet8c>(a));
37}
38
39template <>
40EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet16uc>(const Packet16uc& a) {
41 return Packet8uc(vget_low_u8(a));
42}
43template <>
44EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet8uc>(const Packet8uc& a) {
45 return Packet4uc(vget_lane_u32(vreinterpret_u32_u8(a), 0));
46}
47template <>
48EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet16uc>(const Packet16uc& a) {
49 return preinterpret<Packet4uc>(preinterpret<Packet8uc>(a));
50}
51
52template <>
53EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet8s>(const Packet8s& a) {
54 return Packet4s(vget_low_s16(a));
55}
56
57template <>
58EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet8us>(const Packet8us& a) {
59 return Packet4us(vget_low_u16(a));
60}
61
62template <>
63EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet4i>(const Packet4i& a) {
64 return Packet2i(vget_low_s32(a));
65}
66template <>
67EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet4ui>(const Packet4ui& a) {
68 return Packet2ui(vget_low_u32(a));
69}
70
71template <>
72EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet4f>(const Packet4f& a) {
73 return Packet2f(vget_low_f32(a));
74}
75
76//==============================================================================
77// preinterpret
78//==============================================================================
79template <>
80EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2i>(const Packet2i& a) {
81 return Packet2f(vreinterpret_f32_s32(a));
82}
83template <>
84EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2ui>(const Packet2ui& a) {
85 return Packet2f(vreinterpret_f32_u32(a));
86}
87template <>
88EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
89 return Packet4f(vreinterpretq_f32_s32(a));
90}
91template <>
92EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4ui>(const Packet4ui& a) {
93 return Packet4f(vreinterpretq_f32_u32(a));
94}
95
96template <>
97EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet4uc>(const Packet4uc& a) {
98 return static_cast<Packet4c>(a);
99}
100template <>
101EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet8uc>(const Packet8uc& a) {
102 return Packet8c(vreinterpret_s8_u8(a));
103}
104template <>
105EIGEN_STRONG_INLINE Packet16c preinterpret<Packet16c, Packet16uc>(const Packet16uc& a) {
106 return Packet16c(vreinterpretq_s8_u8(a));
107}
108
109template <>
110EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet4c>(const Packet4c& a) {
111 return static_cast<Packet4uc>(a);
112}
113template <>
114EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet8c>(const Packet8c& a) {
115 return Packet8uc(vreinterpret_u8_s8(a));
116}
117template <>
118EIGEN_STRONG_INLINE Packet16uc preinterpret<Packet16uc, Packet16c>(const Packet16c& a) {
119 return Packet16uc(vreinterpretq_u8_s8(a));
120}
121
122template <>
123EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4us>(const Packet4us& a) {
124 return Packet4s(vreinterpret_s16_u16(a));
125}
126template <>
127EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8us>(const Packet8us& a) {
128 return Packet8s(vreinterpretq_s16_u16(a));
129}
130template <>
131EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4s>(const Packet4s& a) {
132 return Packet4us(vreinterpret_u16_s16(a));
133}
134template <>
135EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8s>(const Packet8s& a) {
136 return Packet8us(vreinterpretq_u16_s16(a));
137}
138
139template <>
140EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2f>(const Packet2f& a) {
141 return Packet2i(vreinterpret_s32_f32(a));
142}
143template <>
144EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2ui>(const Packet2ui& a) {
145 return Packet2i(vreinterpret_s32_u32(a));
146}
147template <>
148EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
149 return Packet4i(vreinterpretq_s32_f32(a));
150}
151template <>
152EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
153 return Packet4i(vreinterpretq_s32_u32(a));
154}
155
156template <>
157EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2f>(const Packet2f& a) {
158 return Packet2ui(vreinterpret_u32_f32(a));
159}
160template <>
161EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2i>(const Packet2i& a) {
162 return Packet2ui(vreinterpret_u32_s32(a));
163}
164template <>
165EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4f>(const Packet4f& a) {
166 return Packet4ui(vreinterpretq_u32_f32(a));
167}
168template <>
169EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
170 return Packet4ui(vreinterpretq_u32_s32(a));
171}
172
173template <>
174EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2ul>(const Packet2ul& a) {
175 return Packet2l(vreinterpretq_s64_u64(a));
176}
177template <>
178EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2l>(const Packet2l& a) {
179 return Packet2ul(vreinterpretq_u64_s64(a));
180}
181
182#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
183template <>
184EIGEN_STRONG_INLINE Packet4hf preinterpret<Packet4hf, Packet4s>(const Packet4s& a) {
185 return Packet4hf(vreinterpret_f16_s16(a));
186}
187template <>
188EIGEN_STRONG_INLINE Packet4hf preinterpret<Packet4hf, Packet4us>(const Packet4us& a) {
189 return Packet4hf(vreinterpret_f16_u16(a));
190}
191template <>
192EIGEN_STRONG_INLINE Packet8hf preinterpret<Packet8hf, Packet8s>(const Packet8s& a) {
193 return Packet8hf(vreinterpretq_f16_s16(a));
194}
195template <>
196EIGEN_STRONG_INLINE Packet8hf preinterpret<Packet8hf, Packet8us>(const Packet8us& a) {
197 return Packet8hf(vreinterpretq_f16_u16(a));
198}
199
200template <>
201EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4hf>(const Packet4hf& a) {
202 return Packet4s(vreinterpret_s16_f16(a));
203}
204template <>
205EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8hf>(const Packet8hf& a) {
206 return Packet8s(vreinterpretq_s16_f16(a));
207}
208template <>
209EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4hf>(const Packet4hf& a) {
210 return Packet4us(vreinterpret_u16_f16(a));
211}
212template <>
213EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8hf>(const Packet8hf& a) {
214 return Packet8us(vreinterpretq_u16_f16(a));
215}
216#endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
217
218//==============================================================================
219// pcast, SrcType = float
220//==============================================================================
221
222template <>
223struct type_casting_traits<float, numext::int64_t> {
224 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
225};
226template <>
227struct type_casting_traits<float, numext::uint64_t> {
228 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
229};
230// If float64 exists, first convert to that to keep as much precision as possible.
231#if EIGEN_ARCH_ARM64
232template <>
233EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
234 // Discard second half of input.
235 return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a)));
236}
237template <>
238EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
239 return vcvtq_s64_f64(vcvt_f64_f32(a));
240}
241template <>
242EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
243 // Discard second half of input.
244 return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a)));
245}
246template <>
247EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
248 return vcvtq_u64_f64(vcvt_f64_f32(a));
249}
250#else
251template <>
252EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
253 // Discard second half of input.
254 return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a)));
255}
256template <>
257EIGEN_STRONG_INLINE Packet2l pcast<Packet2f, Packet2l>(const Packet2f& a) {
258 return vmovl_s32(vcvt_s32_f32(a));
259}
260template <>
261EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
262 // Discard second half of input.
263 return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a)));
264}
265template <>
266EIGEN_STRONG_INLINE Packet2ul pcast<Packet2f, Packet2ul>(const Packet2f& a) {
267 return vmovl_u32(vcvt_u32_f32(a));
268}
269#endif // EIGEN_ARCH_ARM64
270
271#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
272template <>
273struct type_casting_traits<float, half> {
274 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
275};
276
277template <>
278EIGEN_STRONG_INLINE Packet8hf pcast<Packet4f, Packet8hf>(const Packet4f& a, const Packet4f& b) {
279 return vcombine_f16(vcvt_f16_f32(a), vcvt_f16_f32(b));
280}
281
282template <>
283EIGEN_STRONG_INLINE Packet4hf pcast<Packet4f, Packet4hf>(const Packet4f& a) {
284 return vcvt_f16_f32(a);
285}
286
287template <>
288EIGEN_STRONG_INLINE Packet4hf pcast<Packet2f, Packet4hf>(const Packet2f& a, const Packet2f& b) {
289 return vcvt_f16_f32(vcombine_f32(a, b));
290}
291#endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
292
293template <>
294struct type_casting_traits<float, numext::int32_t> {
295 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
296};
297template <>
298EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
299 return vcvtq_s32_f32(a);
300}
301template <>
302EIGEN_STRONG_INLINE Packet2i pcast<Packet2f, Packet2i>(const Packet2f& a) {
303 return vcvt_s32_f32(a);
304}
305
306template <>
307struct type_casting_traits<float, numext::uint32_t> {
308 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
309};
310template <>
311EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
312 return vcvtq_u32_f32(a);
313}
314template <>
315EIGEN_STRONG_INLINE Packet2ui pcast<Packet2f, Packet2ui>(const Packet2f& a) {
316 return vcvt_u32_f32(a);
317}
318
319template <>
320struct type_casting_traits<float, numext::int16_t> {
321 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
322};
323template <>
324EIGEN_STRONG_INLINE Packet8s pcast<Packet4f, Packet8s>(const Packet4f& a, const Packet4f& b) {
325 return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b)));
326}
327template <>
328EIGEN_STRONG_INLINE Packet4s pcast<Packet4f, Packet4s>(const Packet4f& a) {
329 return vmovn_s32(vcvtq_s32_f32(a));
330}
331template <>
332EIGEN_STRONG_INLINE Packet4s pcast<Packet2f, Packet4s>(const Packet2f& a, const Packet2f& b) {
333 return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
334}
335
336template <>
337struct type_casting_traits<float, numext::uint16_t> {
338 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
339};
340template <>
341EIGEN_STRONG_INLINE Packet8us pcast<Packet4f, Packet8us>(const Packet4f& a, const Packet4f& b) {
342 return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b)));
343}
344template <>
345EIGEN_STRONG_INLINE Packet4us pcast<Packet4f, Packet4us>(const Packet4f& a) {
346 return vmovn_u32(vcvtq_u32_f32(a));
347}
348template <>
349EIGEN_STRONG_INLINE Packet4us pcast<Packet2f, Packet4us>(const Packet2f& a, const Packet2f& b) {
350 return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
351}
352
353template <>
354struct type_casting_traits<float, numext::int8_t> {
355 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
356};
357template <>
358EIGEN_STRONG_INLINE Packet16c pcast<Packet4f, Packet16c>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
359 const Packet4f& d) {
360 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
361 const int16x8_t cd_s16 = pcast<Packet4f, Packet8s>(c, d);
362 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
363}
364template <>
365EIGEN_STRONG_INLINE Packet8c pcast<Packet4f, Packet8c>(const Packet4f& a, const Packet4f& b) {
366 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
367 return vmovn_s16(ab_s16);
368}
369template <>
370EIGEN_STRONG_INLINE Packet8c pcast<Packet2f, Packet8c>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
371 const Packet2f& d) {
372 const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
373 const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
374 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
375}
376template <>
377EIGEN_STRONG_INLINE Packet4c pcast<Packet4f, Packet4c>(const Packet4f& a) {
378 const int32x4_t a_s32x4 = vcvtq_s32_f32(a);
379 const int16x4_t a_s16x4 = vmovn_s32(a_s32x4);
380 const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
381 const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
382 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
383}
384
385template <>
386struct type_casting_traits<float, numext::uint8_t> {
387 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
388};
389template <>
390EIGEN_STRONG_INLINE Packet16uc pcast<Packet4f, Packet16uc>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
391 const Packet4f& d) {
392 return preinterpret<Packet16uc>(pcast<Packet4f, Packet16c>(a, b, c, d));
393}
394template <>
395EIGEN_STRONG_INLINE Packet8uc pcast<Packet4f, Packet8uc>(const Packet4f& a, const Packet4f& b) {
396 return preinterpret<Packet8uc>(pcast<Packet4f, Packet8c>(a, b));
397}
398template <>
399EIGEN_STRONG_INLINE Packet8uc pcast<Packet2f, Packet8uc>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
400 const Packet2f& d) {
401 return preinterpret<Packet8uc>(pcast<Packet2f, Packet8c>(a, b, c, d));
402}
403template <>
404EIGEN_STRONG_INLINE Packet4uc pcast<Packet4f, Packet4uc>(const Packet4f& a) {
405 return static_cast<Packet4uc>(pcast<Packet4f, Packet4c>(a));
406}
407
408#if EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
409//==============================================================================
410// pcast, SrcType = half
411//==============================================================================
412template <>
413struct type_casting_traits<half, float> {
414 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
415};
416
417template <>
418EIGEN_STRONG_INLINE Packet4f pcast<Packet8hf, Packet4f>(const Packet8hf& a) {
419 return vcvt_f32_f16(vget_low_f16(a));
420}
421
422template <>
423EIGEN_STRONG_INLINE Packet4f pcast<Packet4hf, Packet4f>(const Packet4hf& a) {
424 return vcvt_f32_f16(a);
425}
426#endif // EIGEN_ARCH_ARM64 && EIGEN_HAS_ARM64_FP16
427
428//==============================================================================
429// pcast, SrcType = int8_t
430//==============================================================================
431template <>
432struct type_casting_traits<numext::int8_t, float> {
433 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
434};
435template <>
436EIGEN_STRONG_INLINE Packet4f pcast<Packet16c, Packet4f>(const Packet16c& a) {
437 // Discard all but first 4 bytes.
438 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))));
439}
440template <>
441EIGEN_STRONG_INLINE Packet4f pcast<Packet4c, Packet4f>(const Packet4c& a) {
442 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vreinterpret_s8_s32(vdup_n_s32(a))))));
443}
444template <>
445EIGEN_STRONG_INLINE Packet2f pcast<Packet8c, Packet2f>(const Packet8c& a) {
446 // Discard all but first 2 bytes.
447 return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
448}
449
450template <>
451struct type_casting_traits<numext::int8_t, numext::int64_t> {
452 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
453};
454template <>
455EIGEN_STRONG_INLINE Packet2l pcast<Packet16c, Packet2l>(const Packet16c& a) {
456 // Discard all but first two bytes.
457 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))));
458}
459
460template <>
461struct type_casting_traits<numext::int8_t, numext::uint64_t> {
462 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
463};
464template <>
465EIGEN_STRONG_INLINE Packet2ul pcast<Packet16c, Packet2ul>(const Packet16c& a) {
466 return preinterpret<Packet2ul>(pcast<Packet16c, Packet2l>(a));
467}
468
469template <>
470struct type_casting_traits<numext::int8_t, numext::int32_t> {
471 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
472};
473template <>
474EIGEN_STRONG_INLINE Packet4i pcast<Packet16c, Packet4i>(const Packet16c& a) {
475 // Discard all but first 4 bytes.
476 return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))));
477}
478template <>
479EIGEN_STRONG_INLINE Packet4i pcast<Packet8c, Packet4i>(const Packet8c& a) {
480 return vmovl_s16(vget_low_s16(vmovl_s8(a)));
481}
482template <>
483EIGEN_STRONG_INLINE Packet4i pcast<Packet4c, Packet4i>(const Packet4c& a) {
484 return pcast<Packet8c, Packet4i>(vreinterpret_s8_s32(vdup_n_s32(a)));
485}
486template <>
487EIGEN_STRONG_INLINE Packet2i pcast<Packet8c, Packet2i>(const Packet8c& a) {
488 // Discard all but first 2 bytes.
489 return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
490}
491
492template <>
493struct type_casting_traits<numext::int8_t, numext::uint32_t> {
494 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
495};
496template <>
497EIGEN_STRONG_INLINE Packet4ui pcast<Packet16c, Packet4ui>(const Packet16c& a) {
498 return preinterpret<Packet4ui>(pcast<Packet16c, Packet4i>(a));
499}
500template <>
501EIGEN_STRONG_INLINE Packet2ui pcast<Packet8c, Packet2ui>(const Packet8c& a) {
502 return preinterpret<Packet2ui>(pcast<Packet8c, Packet2i>(a));
503}
504template <>
505EIGEN_STRONG_INLINE Packet4ui pcast<Packet4c, Packet4ui>(const Packet4c& a) {
506 return preinterpret<Packet4ui>(pcast<Packet4c, Packet4i>(a));
507}
508
509template <>
510struct type_casting_traits<numext::int8_t, numext::int16_t> {
511 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
512};
513template <>
514EIGEN_STRONG_INLINE Packet8s pcast<Packet16c, Packet8s>(const Packet16c& a) {
515 // Discard second half of input.
516 return vmovl_s8(vget_low_s8(a));
517}
518template <>
519EIGEN_STRONG_INLINE Packet8s pcast<Packet8c, Packet8s>(const Packet8c& a) {
520 return vmovl_s8(a);
521}
522template <>
523EIGEN_STRONG_INLINE Packet4s pcast<Packet8c, Packet4s>(const Packet8c& a) {
524 // Discard second half of input.
525 return vget_low_s16(vmovl_s8(a));
526}
527template <>
528EIGEN_STRONG_INLINE Packet4s pcast<Packet4c, Packet4s>(const Packet4c& a) {
529 return pcast<Packet8c, Packet4s>(vreinterpret_s8_s32(vdup_n_s32(a)));
530}
531
532template <>
533struct type_casting_traits<numext::int8_t, numext::uint16_t> {
534 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
535};
536template <>
537EIGEN_STRONG_INLINE Packet8us pcast<Packet16c, Packet8us>(const Packet16c& a) {
538 return preinterpret<Packet8us>(pcast<Packet16c, Packet8s>(a));
539}
540template <>
541EIGEN_STRONG_INLINE Packet8us pcast<Packet8c, Packet8us>(const Packet8c& a) {
542 return preinterpret<Packet8us>(pcast<Packet8c, Packet8s>(a));
543}
544template <>
545EIGEN_STRONG_INLINE Packet4us pcast<Packet8c, Packet4us>(const Packet8c& a) {
546 return preinterpret<Packet4us>(pcast<Packet8c, Packet4s>(a));
547}
548template <>
549EIGEN_STRONG_INLINE Packet4us pcast<Packet4c, Packet4us>(const Packet4c& a) {
550 return preinterpret<Packet4us>(pcast<Packet4c, Packet4s>(a));
551}
552
553//==============================================================================
554// pcast, SrcType = uint8_t
555//==============================================================================
556template <>
557struct type_casting_traits<numext::uint8_t, float> {
558 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
559};
560template <>
561EIGEN_STRONG_INLINE Packet4f pcast<Packet16uc, Packet4f>(const Packet16uc& a) {
562 // Discard all but first 4 bytes.
563 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))));
564}
565template <>
566EIGEN_STRONG_INLINE Packet4f pcast<Packet4uc, Packet4f>(const Packet4uc& a) {
567 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))))));
568}
569template <>
570EIGEN_STRONG_INLINE Packet2f pcast<Packet8uc, Packet2f>(const Packet8uc& a) {
571 // Discard all but first 2 bytes.
572 return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
573}
574
575template <>
576struct type_casting_traits<numext::uint8_t, numext::uint64_t> {
577 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
578};
579template <>
580EIGEN_STRONG_INLINE Packet2ul pcast<Packet16uc, Packet2ul>(const Packet16uc& a) {
581 // Discard all but first two bytes.
582 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))));
583}
584
585template <>
586struct type_casting_traits<numext::uint8_t, numext::int64_t> {
587 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
588};
589template <>
590EIGEN_STRONG_INLINE Packet2l pcast<Packet16uc, Packet2l>(const Packet16uc& a) {
591 return preinterpret<Packet2l>(pcast<Packet16uc, Packet2ul>(a));
592}
593
594template <>
595struct type_casting_traits<numext::uint8_t, numext::uint32_t> {
596 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
597};
598template <>
599EIGEN_STRONG_INLINE Packet4ui pcast<Packet16uc, Packet4ui>(const Packet16uc& a) {
600 // Discard all but first 4 bytes.
601 return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))));
602}
603template <>
604EIGEN_STRONG_INLINE Packet4ui pcast<Packet8uc, Packet4ui>(const Packet8uc& a) {
605 return vmovl_u16(vget_low_u16(vmovl_u8(a)));
606}
607template <>
608EIGEN_STRONG_INLINE Packet2ui pcast<Packet8uc, Packet2ui>(const Packet8uc& a) {
609 // Discard all but first 2 bytes.
610 return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
611}
612template <>
613EIGEN_STRONG_INLINE Packet4ui pcast<Packet4uc, Packet4ui>(const Packet4uc& a) {
614 return pcast<Packet8uc, Packet4ui>(vreinterpret_u8_u32(vdup_n_u32(a)));
615}
616
617template <>
618struct type_casting_traits<numext::uint8_t, numext::int32_t> {
619 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
620};
621template <>
622EIGEN_STRONG_INLINE Packet4i pcast<Packet16uc, Packet4i>(const Packet16uc& a) {
623 return preinterpret<Packet4i>(pcast<Packet16uc, Packet4ui>(a));
624}
625template <>
626EIGEN_STRONG_INLINE Packet2i pcast<Packet8uc, Packet2i>(const Packet8uc& a) {
627 return preinterpret<Packet2i>(pcast<Packet8uc, Packet2ui>(a));
628}
629template <>
630EIGEN_STRONG_INLINE Packet4i pcast<Packet4uc, Packet4i>(const Packet4uc& a) {
631 return preinterpret<Packet4i>(pcast<Packet4uc, Packet4ui>(a));
632}
633
634template <>
635struct type_casting_traits<numext::uint8_t, numext::uint16_t> {
636 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
637};
638template <>
639EIGEN_STRONG_INLINE Packet8us pcast<Packet16uc, Packet8us>(const Packet16uc& a) {
640 // Discard second half of input.
641 return vmovl_u8(vget_low_u8(a));
642}
643template <>
644EIGEN_STRONG_INLINE Packet8us pcast<Packet8uc, Packet8us>(const Packet8uc& a) {
645 return vmovl_u8(a);
646}
647template <>
648EIGEN_STRONG_INLINE Packet4us pcast<Packet4uc, Packet4us>(const Packet4uc& a) {
649 return vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vdup_n_u32(a))));
650}
651
652template <>
653struct type_casting_traits<numext::uint8_t, numext::int16_t> {
654 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
655};
656template <>
657EIGEN_STRONG_INLINE Packet8s pcast<Packet16uc, Packet8s>(const Packet16uc& a) {
658 return preinterpret<Packet8s>(pcast<Packet16uc, Packet8us>(a));
659}
660template <>
661EIGEN_STRONG_INLINE Packet8s pcast<Packet8uc, Packet8s>(const Packet8uc& a) {
662 return preinterpret<Packet8s>(pcast<Packet8uc, Packet8us>(a));
663}
664template <>
665EIGEN_STRONG_INLINE Packet4s pcast<Packet4uc, Packet4s>(const Packet4uc& a) {
666 return preinterpret<Packet4s>(pcast<Packet4uc, Packet4us>(a));
667}
668
669//==============================================================================
670// pcast, SrcType = int16_t
671//==============================================================================
672template <>
673struct type_casting_traits<numext::int16_t, float> {
674 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
675};
676template <>
677EIGEN_STRONG_INLINE Packet4f pcast<Packet8s, Packet4f>(const Packet8s& a) {
678 // Discard second half of input.
679 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a)));
680}
681template <>
682EIGEN_STRONG_INLINE Packet4f pcast<Packet4s, Packet4f>(const Packet4s& a) {
683 return vcvtq_f32_s32(vmovl_s16(a));
684}
685template <>
686EIGEN_STRONG_INLINE Packet2f pcast<Packet4s, Packet2f>(const Packet4s& a) {
687 // Discard second half of input.
688 return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
689}
690
691template <>
692struct type_casting_traits<numext::int16_t, numext::int64_t> {
693 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
694};
695template <>
696EIGEN_STRONG_INLINE Packet2l pcast<Packet8s, Packet2l>(const Packet8s& a) {
697 // Discard all but first two values.
698 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a))));
699}
700
701template <>
702struct type_casting_traits<numext::int16_t, numext::uint64_t> {
703 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
704};
705template <>
706EIGEN_STRONG_INLINE Packet2ul pcast<Packet8s, Packet2ul>(const Packet8s& a) {
707 return preinterpret<Packet2ul>(pcast<Packet8s, Packet2l>(a));
708}
709
710template <>
711struct type_casting_traits<numext::int16_t, numext::int32_t> {
712 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
713};
714template <>
715EIGEN_STRONG_INLINE Packet4i pcast<Packet8s, Packet4i>(const Packet8s& a) {
716 // Discard second half of input.
717 return vmovl_s16(vget_low_s16(a));
718}
719template <>
720EIGEN_STRONG_INLINE Packet4i pcast<Packet4s, Packet4i>(const Packet4s& a) {
721 return vmovl_s16(a);
722}
723template <>
724EIGEN_STRONG_INLINE Packet2i pcast<Packet4s, Packet2i>(const Packet4s& a) {
725 // Discard second half of input.
726 return vget_low_s32(vmovl_s16(a));
727}
728
729template <>
730struct type_casting_traits<numext::int16_t, numext::uint32_t> {
731 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
732};
733template <>
734EIGEN_STRONG_INLINE Packet4ui pcast<Packet8s, Packet4ui>(const Packet8s& a) {
735 return preinterpret<Packet4ui>(pcast<Packet8s, Packet4i>(a));
736}
737template <>
738EIGEN_STRONG_INLINE Packet4ui pcast<Packet4s, Packet4ui>(const Packet4s& a) {
739 return preinterpret<Packet4ui>(pcast<Packet4s, Packet4i>(a));
740}
741template <>
742EIGEN_STRONG_INLINE Packet2ui pcast<Packet4s, Packet2ui>(const Packet4s& a) {
743 return preinterpret<Packet2ui>(pcast<Packet4s, Packet2i>(a));
744}
745
746template <>
747struct type_casting_traits<numext::int16_t, numext::int8_t> {
748 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
749};
750template <>
751EIGEN_STRONG_INLINE Packet16c pcast<Packet8s, Packet16c>(const Packet8s& a, const Packet8s& b) {
752 return vcombine_s8(vmovn_s16(a), vmovn_s16(b));
753}
754template <>
755EIGEN_STRONG_INLINE Packet8c pcast<Packet8s, Packet8c>(const Packet8s& a) {
756 return vmovn_s16(a);
757}
758template <>
759EIGEN_STRONG_INLINE Packet8c pcast<Packet4s, Packet8c>(const Packet4s& a, const Packet4s& b) {
760 return vmovn_s16(vcombine_s16(a, b));
761}
762template <>
763EIGEN_STRONG_INLINE Packet4c pcast<Packet4s, Packet4c>(const Packet4s& a) {
764 const int8x8_t aa_s8x8 = pcast<Packet4s, Packet8c>(a, a);
765 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
766}
767
768template <>
769struct type_casting_traits<numext::int16_t, numext::uint8_t> {
770 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
771};
772template <>
773EIGEN_STRONG_INLINE Packet16uc pcast<Packet8s, Packet16uc>(const Packet8s& a, const Packet8s& b) {
774 return preinterpret<Packet16uc>(pcast<Packet8s, Packet16c>(a, b));
775}
776template <>
777EIGEN_STRONG_INLINE Packet8uc pcast<Packet8s, Packet8uc>(const Packet8s& a) {
778 return preinterpret<Packet8uc>(pcast<Packet8s, Packet8c>(a));
779}
780template <>
781EIGEN_STRONG_INLINE Packet8uc pcast<Packet4s, Packet8uc>(const Packet4s& a, const Packet4s& b) {
782 return preinterpret<Packet8uc>(pcast<Packet4s, Packet8c>(a, b));
783}
784template <>
785EIGEN_STRONG_INLINE Packet4uc pcast<Packet4s, Packet4uc>(const Packet4s& a) {
786 return static_cast<Packet4uc>(pcast<Packet4s, Packet4c>(a));
787}
788
789//==============================================================================
790// pcast, SrcType = uint16_t
791//==============================================================================
792template <>
793struct type_casting_traits<numext::uint16_t, float> {
794 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
795};
796template <>
797EIGEN_STRONG_INLINE Packet4f pcast<Packet8us, Packet4f>(const Packet8us& a) {
798 // Discard second half of input.
799 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a)));
800}
801template <>
802EIGEN_STRONG_INLINE Packet4f pcast<Packet4us, Packet4f>(const Packet4us& a) {
803 return vcvtq_f32_u32(vmovl_u16(a));
804}
805template <>
806EIGEN_STRONG_INLINE Packet2f pcast<Packet4us, Packet2f>(const Packet4us& a) {
807 // Discard second half of input.
808 return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
809}
810
811template <>
812struct type_casting_traits<numext::uint16_t, numext::uint64_t> {
813 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
814};
815template <>
816EIGEN_STRONG_INLINE Packet2ul pcast<Packet8us, Packet2ul>(const Packet8us& a) {
817 // Discard all but first two values.
818 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a))));
819}
820
821template <>
822struct type_casting_traits<numext::uint16_t, numext::int64_t> {
823 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
824};
825template <>
826EIGEN_STRONG_INLINE Packet2l pcast<Packet8us, Packet2l>(const Packet8us& a) {
827 return preinterpret<Packet2l>(pcast<Packet8us, Packet2ul>(a));
828}
829
830template <>
831struct type_casting_traits<numext::uint16_t, numext::uint32_t> {
832 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
833};
834template <>
835EIGEN_STRONG_INLINE Packet4ui pcast<Packet8us, Packet4ui>(const Packet8us& a) {
836 // Discard second half of input.
837 return vmovl_u16(vget_low_u16(a));
838}
839template <>
840EIGEN_STRONG_INLINE Packet4ui pcast<Packet4us, Packet4ui>(const Packet4us& a) {
841 return vmovl_u16(a);
842}
843template <>
844EIGEN_STRONG_INLINE Packet2ui pcast<Packet4us, Packet2ui>(const Packet4us& a) {
845 // Discard second half of input.
846 return vget_low_u32(vmovl_u16(a));
847}
848
849template <>
850struct type_casting_traits<numext::uint16_t, numext::int32_t> {
851 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
852};
853template <>
854EIGEN_STRONG_INLINE Packet4i pcast<Packet8us, Packet4i>(const Packet8us& a) {
855 return preinterpret<Packet4i>(pcast<Packet8us, Packet4ui>(a));
856}
857template <>
858EIGEN_STRONG_INLINE Packet4i pcast<Packet4us, Packet4i>(const Packet4us& a) {
859 return preinterpret<Packet4i>(pcast<Packet4us, Packet4ui>(a));
860}
861template <>
862EIGEN_STRONG_INLINE Packet2i pcast<Packet4us, Packet2i>(const Packet4us& a) {
863 return preinterpret<Packet2i>(pcast<Packet4us, Packet2ui>(a));
864}
865
866template <>
867struct type_casting_traits<numext::uint16_t, numext::uint8_t> {
868 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
869};
870template <>
871EIGEN_STRONG_INLINE Packet16uc pcast<Packet8us, Packet16uc>(const Packet8us& a, const Packet8us& b) {
872 return vcombine_u8(vmovn_u16(a), vmovn_u16(b));
873}
874template <>
875EIGEN_STRONG_INLINE Packet8uc pcast<Packet8us, Packet8uc>(const Packet8us& a) {
876 return vmovn_u16(a);
877}
878template <>
879EIGEN_STRONG_INLINE Packet8uc pcast<Packet4us, Packet8uc>(const Packet4us& a, const Packet4us& b) {
880 return vmovn_u16(vcombine_u16(a, b));
881}
882template <>
883EIGEN_STRONG_INLINE Packet4uc pcast<Packet4us, Packet4uc>(const Packet4us& a) {
884 uint8x8_t aa_u8x8 = pcast<Packet4us, Packet8uc>(a, a);
885 return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
886}
887
888template <>
889struct type_casting_traits<numext::uint16_t, numext::int8_t> {
890 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
891};
892template <>
893EIGEN_STRONG_INLINE Packet16c pcast<Packet8us, Packet16c>(const Packet8us& a, const Packet8us& b) {
894 return preinterpret<Packet16c>(pcast<Packet8us, Packet16uc>(a, b));
895}
896template <>
897EIGEN_STRONG_INLINE Packet8c pcast<Packet8us, Packet8c>(const Packet8us& a) {
898 return preinterpret<Packet8c>(pcast<Packet8us, Packet8uc>(a));
899}
900template <>
901EIGEN_STRONG_INLINE Packet8c pcast<Packet4us, Packet8c>(const Packet4us& a, const Packet4us& b) {
902 return preinterpret<Packet8c>(pcast<Packet4us, Packet8uc>(a, b));
903}
904template <>
905EIGEN_STRONG_INLINE Packet4c pcast<Packet4us, Packet4c>(const Packet4us& a) {
906 return static_cast<Packet4c>(pcast<Packet4us, Packet4uc>(a));
907}
908
909//==============================================================================
910// pcast, SrcType = int32_t
911//==============================================================================
912template <>
913struct type_casting_traits<numext::int32_t, float> {
914 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
915};
916template <>
917EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
918 return vcvtq_f32_s32(a);
919}
920template <>
921EIGEN_STRONG_INLINE Packet2f pcast<Packet2i, Packet2f>(const Packet2i& a) {
922 return vcvt_f32_s32(a);
923}
924
925template <>
926struct type_casting_traits<numext::int32_t, numext::int64_t> {
927 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
928};
929template <>
930EIGEN_STRONG_INLINE Packet2l pcast<Packet4i, Packet2l>(const Packet4i& a) {
931 // Discard second half of input.
932 return vmovl_s32(vget_low_s32(a));
933}
934template <>
935EIGEN_STRONG_INLINE Packet2l pcast<Packet2i, Packet2l>(const Packet2i& a) {
936 return vmovl_s32(a);
937}
938
939template <>
940struct type_casting_traits<numext::int32_t, numext::uint64_t> {
941 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
942};
943template <>
944EIGEN_STRONG_INLINE Packet2ul pcast<Packet4i, Packet2ul>(const Packet4i& a) {
945 return preinterpret<Packet2ul>(pcast<Packet4i, Packet2l>(a));
946}
947template <>
948EIGEN_STRONG_INLINE Packet2ul pcast<Packet2i, Packet2ul>(const Packet2i& a) {
949 return preinterpret<Packet2ul>(pcast<Packet2i, Packet2l>(a));
950}
951
952template <>
953struct type_casting_traits<numext::int32_t, numext::int16_t> {
954 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
955};
956template <>
957EIGEN_STRONG_INLINE Packet8s pcast<Packet4i, Packet8s>(const Packet4i& a, const Packet4i& b) {
958 return vcombine_s16(vmovn_s32(a), vmovn_s32(b));
959}
960template <>
961EIGEN_STRONG_INLINE Packet4s pcast<Packet4i, Packet4s>(const Packet4i& a) {
962 return vmovn_s32(a);
963}
964template <>
965EIGEN_STRONG_INLINE Packet4s pcast<Packet2i, Packet4s>(const Packet2i& a, const Packet2i& b) {
966 return vmovn_s32(vcombine_s32(a, b));
967}
968
969template <>
970struct type_casting_traits<numext::int32_t, numext::uint16_t> {
971 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
972};
973template <>
974EIGEN_STRONG_INLINE Packet8us pcast<Packet4i, Packet8us>(const Packet4i& a, const Packet4i& b) {
975 return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b)));
976}
977template <>
978EIGEN_STRONG_INLINE Packet4us pcast<Packet4i, Packet4us>(const Packet4i& a) {
979 return vmovn_u32(vreinterpretq_u32_s32(a));
980}
981template <>
982EIGEN_STRONG_INLINE Packet4us pcast<Packet2i, Packet4us>(const Packet2i& a, const Packet2i& b) {
983 return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
984}
985
986template <>
987struct type_casting_traits<numext::int32_t, numext::int8_t> {
988 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
989};
990template <>
991EIGEN_STRONG_INLINE Packet16c pcast<Packet4i, Packet16c>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
992 const Packet4i& d) {
993 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
994 const int16x8_t cd_s16 = pcast<Packet4i, Packet8s>(c, d);
995 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
996}
997template <>
998EIGEN_STRONG_INLINE Packet8c pcast<Packet4i, Packet8c>(const Packet4i& a, const Packet4i& b) {
999 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
1000 return vmovn_s16(ab_s16);
1001}
1002template <>
1003EIGEN_STRONG_INLINE Packet8c pcast<Packet2i, Packet8c>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
1004 const Packet2i& d) {
1005 const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
1006 const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
1007 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
1008}
1009template <>
1010EIGEN_STRONG_INLINE Packet4c pcast<Packet4i, Packet4c>(const Packet4i& a) {
1011 const int16x4_t a_s16x4 = vmovn_s32(a);
1012 const int16x8_t aa_s16x8 = vcombine_s16(a_s16x4, a_s16x4);
1013 const int8x8_t aa_s8x8 = vmovn_s16(aa_s16x8);
1014 return vget_lane_s32(vreinterpret_s32_s8(aa_s8x8), 0);
1015}
1016
1017template <>
1018struct type_casting_traits<numext::int32_t, numext::uint8_t> {
1019 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1020};
1021template <>
1022EIGEN_STRONG_INLINE Packet16uc pcast<Packet4i, Packet16uc>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
1023 const Packet4i& d) {
1024 return preinterpret<Packet16uc>(pcast<Packet4i, Packet16c>(a, b, c, d));
1025}
1026template <>
1027EIGEN_STRONG_INLINE Packet8uc pcast<Packet4i, Packet8uc>(const Packet4i& a, const Packet4i& b) {
1028 return preinterpret<Packet8uc>(pcast<Packet4i, Packet8c>(a, b));
1029}
1030template <>
1031EIGEN_STRONG_INLINE Packet8uc pcast<Packet2i, Packet8uc>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
1032 const Packet2i& d) {
1033 return preinterpret<Packet8uc>(pcast<Packet2i, Packet8c>(a, b, c, d));
1034}
1035template <>
1036EIGEN_STRONG_INLINE Packet4uc pcast<Packet4i, Packet4uc>(const Packet4i& a) {
1037 return static_cast<Packet4uc>(pcast<Packet4i, Packet4c>(a));
1038}
1039
1040//==============================================================================
1041// pcast, SrcType = uint32_t
1042//==============================================================================
1043template <>
1044struct type_casting_traits<numext::uint32_t, float> {
1045 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1046};
1047template <>
1048EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
1049 return vcvtq_f32_u32(a);
1050}
1051template <>
1052EIGEN_STRONG_INLINE Packet2f pcast<Packet2ui, Packet2f>(const Packet2ui& a) {
1053 return vcvt_f32_u32(a);
1054}
1055
1056template <>
1057struct type_casting_traits<numext::uint32_t, numext::uint64_t> {
1058 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1059};
1060template <>
1061EIGEN_STRONG_INLINE Packet2ul pcast<Packet4ui, Packet2ul>(const Packet4ui& a) {
1062 // Discard second half of input.
1063 return vmovl_u32(vget_low_u32(a));
1064}
1065template <>
1066EIGEN_STRONG_INLINE Packet2ul pcast<Packet2ui, Packet2ul>(const Packet2ui& a) {
1067 return vmovl_u32(a);
1068}
1069
1070template <>
1071struct type_casting_traits<numext::uint32_t, numext::int64_t> {
1072 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1073};
1074template <>
1075EIGEN_STRONG_INLINE Packet2l pcast<Packet4ui, Packet2l>(const Packet4ui& a) {
1076 return preinterpret<Packet2l>(pcast<Packet4ui, Packet2ul>(a));
1077}
1078template <>
1079EIGEN_STRONG_INLINE Packet2l pcast<Packet2ui, Packet2l>(const Packet2ui& a) {
1080 return preinterpret<Packet2l>(pcast<Packet2ui, Packet2ul>(a));
1081}
1082
1083template <>
1084struct type_casting_traits<numext::uint32_t, numext::uint16_t> {
1085 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1086};
1087template <>
1088EIGEN_STRONG_INLINE Packet8us pcast<Packet4ui, Packet8us>(const Packet4ui& a, const Packet4ui& b) {
1089 return vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1090}
1091template <>
1092EIGEN_STRONG_INLINE Packet4us pcast<Packet2ui, Packet4us>(const Packet2ui& a, const Packet2ui& b) {
1093 return vmovn_u32(vcombine_u32(a, b));
1094}
1095template <>
1096EIGEN_STRONG_INLINE Packet4us pcast<Packet4ui, Packet4us>(const Packet4ui& a) {
1097 return vmovn_u32(a);
1098}
1099
1100template <>
1101struct type_casting_traits<numext::uint32_t, numext::int16_t> {
1102 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1103};
1104template <>
1105EIGEN_STRONG_INLINE Packet8s pcast<Packet4ui, Packet8s>(const Packet4ui& a, const Packet4ui& b) {
1106 return preinterpret<Packet8s>(pcast<Packet4ui, Packet8us>(a, b));
1107}
1108template <>
1109EIGEN_STRONG_INLINE Packet4s pcast<Packet2ui, Packet4s>(const Packet2ui& a, const Packet2ui& b) {
1110 return preinterpret<Packet4s>(pcast<Packet2ui, Packet4us>(a, b));
1111}
1112template <>
1113EIGEN_STRONG_INLINE Packet4s pcast<Packet4ui, Packet4s>(const Packet4ui& a) {
1114 return preinterpret<Packet4s>(pcast<Packet4ui, Packet4us>(a));
1115}
1116
1117template <>
1118struct type_casting_traits<numext::uint32_t, numext::uint8_t> {
1119 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1120};
1121template <>
1122EIGEN_STRONG_INLINE Packet16uc pcast<Packet4ui, Packet16uc>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1123 const Packet4ui& d) {
1124 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1125 const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d));
1126 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
1127}
1128template <>
1129EIGEN_STRONG_INLINE Packet8uc pcast<Packet4ui, Packet8uc>(const Packet4ui& a, const Packet4ui& b) {
1130 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
1131 return vmovn_u16(ab_u16);
1132}
1133template <>
1134EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ui, Packet8uc>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1135 const Packet2ui& d) {
1136 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
1137 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
1138 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
1139}
1140template <>
1141EIGEN_STRONG_INLINE Packet4uc pcast<Packet4ui, Packet4uc>(const Packet4ui& a) {
1142 const uint16x4_t a_u16x4 = vmovn_u32(a);
1143 const uint16x8_t aa_u16x8 = vcombine_u16(a_u16x4, a_u16x4);
1144 const uint8x8_t aa_u8x8 = vmovn_u16(aa_u16x8);
1145 return vget_lane_u32(vreinterpret_u32_u8(aa_u8x8), 0);
1146}
1147
1148template <>
1149struct type_casting_traits<numext::uint32_t, numext::int8_t> {
1150 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1151};
1152template <>
1153EIGEN_STRONG_INLINE Packet16c pcast<Packet4ui, Packet16c>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
1154 const Packet4ui& d) {
1155 return preinterpret<Packet16c>(pcast<Packet4ui, Packet16uc>(a, b, c, d));
1156}
1157template <>
1158EIGEN_STRONG_INLINE Packet8c pcast<Packet4ui, Packet8c>(const Packet4ui& a, const Packet4ui& b) {
1159 return preinterpret<Packet8c>(pcast<Packet4ui, Packet8uc>(a, b));
1160}
1161template <>
1162EIGEN_STRONG_INLINE Packet8c pcast<Packet2ui, Packet8c>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
1163 const Packet2ui& d) {
1164 return preinterpret<Packet8c>(pcast<Packet2ui, Packet8uc>(a, b, c, d));
1165}
1166template <>
1167EIGEN_STRONG_INLINE Packet4c pcast<Packet4ui, Packet4c>(const Packet4ui& a) {
1168 return static_cast<Packet4c>(pcast<Packet4ui, Packet4uc>(a));
1169}
1170
1171//==============================================================================
1172// pcast, SrcType = int64_t
1173//==============================================================================
1174template <>
1175struct type_casting_traits<numext::int64_t, float> {
1176 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1177};
1178
1179template <>
1180EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
1181#if EIGEN_ARCH_ARM64
1182 return vcombine_f32(vcvt_f32_f64(vcvtq_f64_s64(a)), vcvt_f32_f64(vcvtq_f64_s64(b)));
1183#else
1184 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2l>::alignment) int64_t lvals[4];
1185 pstore(lvals, a);
1186 pstore(lvals + 2, b);
1187 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet4f>::alignment)
1188 float fvals[4] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1]), static_cast<float>(lvals[2]),
1189 static_cast<float>(lvals[3])};
1190 return pload<Packet4f>(fvals);
1191#endif
1192}
1193
1194template <>
1195EIGEN_STRONG_INLINE Packet2f pcast<Packet2l, Packet2f>(const Packet2l& a) {
1196#if EIGEN_ARCH_ARM64
1197 return vcvt_f32_f64(vcvtq_f64_s64(a));
1198#else
1199 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2l>::alignment) int64_t lvals[2];
1200 pstore(lvals, a);
1201 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2f>::alignment)
1202 float fvals[2] = {static_cast<float>(lvals[0]), static_cast<float>(lvals[1])};
1203 return pload<Packet2f>(fvals);
1204#endif
1205}
1206
1207template <>
1208struct type_casting_traits<numext::int64_t, numext::int32_t> {
1209 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1210};
1211template <>
1212EIGEN_STRONG_INLINE Packet4i pcast<Packet2l, Packet4i>(const Packet2l& a, const Packet2l& b) {
1213 return vcombine_s32(vmovn_s64(a), vmovn_s64(b));
1214}
1215template <>
1216EIGEN_STRONG_INLINE Packet2i pcast<Packet2l, Packet2i>(const Packet2l& a) {
1217 return vmovn_s64(a);
1218}
1219
1220template <>
1221struct type_casting_traits<numext::int64_t, numext::uint32_t> {
1222 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1223};
1224template <>
1225EIGEN_STRONG_INLINE Packet4ui pcast<Packet2l, Packet4ui>(const Packet2l& a, const Packet2l& b) {
1226 return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b)));
1227}
1228template <>
1229EIGEN_STRONG_INLINE Packet2ui pcast<Packet2l, Packet2ui>(const Packet2l& a) {
1230 return vmovn_u64(vreinterpretq_u64_s64(a));
1231}
1232
1233template <>
1234struct type_casting_traits<numext::int64_t, numext::int16_t> {
1235 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1236};
1237template <>
1238EIGEN_STRONG_INLINE Packet8s pcast<Packet2l, Packet8s>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1239 const Packet2l& d) {
1240 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1241 const int32x4_t cd_s32 = pcast<Packet2l, Packet4i>(c, d);
1242 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1243}
1244template <>
1245EIGEN_STRONG_INLINE Packet4s pcast<Packet2l, Packet4s>(const Packet2l& a, const Packet2l& b) {
1246 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1247 return vmovn_s32(ab_s32);
1248}
1249
1250template <>
1251struct type_casting_traits<numext::int64_t, numext::uint16_t> {
1252 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1253};
1254template <>
1255EIGEN_STRONG_INLINE Packet8us pcast<Packet2l, Packet8us>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1256 const Packet2l& d) {
1257 return preinterpret<Packet8us>(pcast<Packet2l, Packet8s>(a, b, c, d));
1258}
1259template <>
1260EIGEN_STRONG_INLINE Packet4us pcast<Packet2l, Packet4us>(const Packet2l& a, const Packet2l& b) {
1261 return preinterpret<Packet4us>(pcast<Packet2l, Packet4s>(a, b));
1262}
1263
1264template <>
1265struct type_casting_traits<numext::int64_t, numext::int8_t> {
1266 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1267};
1268template <>
1269EIGEN_STRONG_INLINE Packet16c pcast<Packet2l, Packet16c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1270 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1271 const Packet2l& g, const Packet2l& h) {
1272 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1273 const int16x8_t efgh_s16 = pcast<Packet2l, Packet8s>(e, f, g, h);
1274 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1275}
1276template <>
1277EIGEN_STRONG_INLINE Packet8c pcast<Packet2l, Packet8c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1278 const Packet2l& d) {
1279 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1280 return vmovn_s16(abcd_s16);
1281}
1282template <>
1283EIGEN_STRONG_INLINE Packet4c pcast<Packet2l, Packet4c>(const Packet2l& a, const Packet2l& b) {
1284 const int16x4_t ab_s16 = pcast<Packet2l, Packet4s>(a, b);
1285 const int16x8_t abab_s16 = vcombine_s16(ab_s16, ab_s16);
1286 const int8x8_t abab_s8 = vmovn_s16(abab_s16);
1287 return vget_lane_s32(vreinterpret_s32_s8(abab_s8), 0);
1288}
1289
1290template <>
1291struct type_casting_traits<numext::int64_t, numext::uint8_t> {
1292 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1293};
1294template <>
1295EIGEN_STRONG_INLINE Packet16uc pcast<Packet2l, Packet16uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1296 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1297 const Packet2l& g, const Packet2l& h) {
1298 const uint16x8_t abcd_u16 = pcast<Packet2l, Packet8us>(a, b, c, d);
1299 const uint16x8_t efgh_u16 = pcast<Packet2l, Packet8us>(e, f, g, h);
1300 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1301}
1302template <>
1303EIGEN_STRONG_INLINE Packet8uc pcast<Packet2l, Packet8uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1304 const Packet2l& d) {
1305 return preinterpret<Packet8uc>(pcast<Packet2l, Packet8c>(a, b, c, d));
1306}
1307template <>
1308EIGEN_STRONG_INLINE Packet4uc pcast<Packet2l, Packet4uc>(const Packet2l& a, const Packet2l& b) {
1309 return static_cast<Packet4uc>(pcast<Packet2l, Packet4c>(a, b));
1310}
1311
1312//==============================================================================
1313// pcast, SrcType = uint64_t
1314//==============================================================================
1315template <>
1316struct type_casting_traits<numext::uint64_t, float> {
1317 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1318};
1319template <>
1320EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
1321#if EIGEN_ARCH_ARM64
1322 return vcombine_f32(vcvt_f32_f64(vcvtq_f64_u64(a)), vcvt_f32_f64(vcvtq_f64_u64(b)));
1323#else
1324 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2ul>::alignment) uint64_t uvals[4];
1325 pstore(uvals, a);
1326 pstore(uvals + 2, b);
1327 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet4f>::alignment)
1328 float fvals[4] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1]), static_cast<float>(uvals[2]),
1329 static_cast<float>(uvals[3])};
1330 return pload<Packet4f>(fvals);
1331#endif
1332}
1333template <>
1334EIGEN_STRONG_INLINE Packet2f pcast<Packet2ul, Packet2f>(const Packet2ul& a) {
1335#if EIGEN_ARCH_ARM64
1336 return vcvt_f32_f64(vcvtq_f64_u64(a));
1337#else
1338 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2ul>::alignment) uint64_t uvals[2];
1339 pstore(uvals, a);
1340 EIGEN_ALIGN_TO_BOUNDARY(unpacket_traits<Packet2f>::alignment)
1341 float fvals[2] = {static_cast<float>(uvals[0]), static_cast<float>(uvals[1])};
1342 return pload<Packet2f>(fvals);
1343#endif
1344}
1345
1346template <>
1347struct type_casting_traits<numext::uint64_t, numext::uint32_t> {
1348 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1349};
1350template <>
1351EIGEN_STRONG_INLINE Packet4ui pcast<Packet2ul, Packet4ui>(const Packet2ul& a, const Packet2ul& b) {
1352 return vcombine_u32(vmovn_u64(a), vmovn_u64(b));
1353}
1354template <>
1355EIGEN_STRONG_INLINE Packet2ui pcast<Packet2ul, Packet2ui>(const Packet2ul& a) {
1356 return vmovn_u64(a);
1357}
1358
1359template <>
1360struct type_casting_traits<numext::uint64_t, numext::int32_t> {
1361 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1362};
1363template <>
1364EIGEN_STRONG_INLINE Packet4i pcast<Packet2ul, Packet4i>(const Packet2ul& a, const Packet2ul& b) {
1365 return preinterpret<Packet4i>(pcast<Packet2ul, Packet4ui>(a, b));
1366}
1367template <>
1368EIGEN_STRONG_INLINE Packet2i pcast<Packet2ul, Packet2i>(const Packet2ul& a) {
1369 return preinterpret<Packet2i>(pcast<Packet2ul, Packet2ui>(a));
1370}
1371
1372template <>
1373struct type_casting_traits<numext::uint64_t, numext::uint16_t> {
1374 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1375};
1376template <>
1377EIGEN_STRONG_INLINE Packet8us pcast<Packet2ul, Packet8us>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1378 const Packet2ul& d) {
1379 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1380 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d)));
1381 return vcombine_u16(ab_u16, cd_u16);
1382}
1383template <>
1384EIGEN_STRONG_INLINE Packet4us pcast<Packet2ul, Packet4us>(const Packet2ul& a, const Packet2ul& b) {
1385 return vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1386}
1387
1388template <>
1389struct type_casting_traits<numext::uint64_t, numext::int16_t> {
1390 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1391};
1392template <>
1393EIGEN_STRONG_INLINE Packet8s pcast<Packet2ul, Packet8s>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1394 const Packet2ul& d) {
1395 return preinterpret<Packet8s>(pcast<Packet2ul, Packet8us>(a, b, c, d));
1396}
1397template <>
1398EIGEN_STRONG_INLINE Packet4s pcast<Packet2ul, Packet4s>(const Packet2ul& a, const Packet2ul& b) {
1399 return preinterpret<Packet4s>(pcast<Packet2ul, Packet4us>(a, b));
1400}
1401
1402template <>
1403struct type_casting_traits<numext::uint64_t, numext::uint8_t> {
1404 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1405};
1406template <>
1407EIGEN_STRONG_INLINE Packet16uc pcast<Packet2ul, Packet16uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1408 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1409 const Packet2ul& g, const Packet2ul& h) {
1410 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1411 const uint16x8_t efgh_u16 = pcast<Packet2ul, Packet8us>(e, f, g, h);
1412 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1413}
1414template <>
1415EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ul, Packet8uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1416 const Packet2ul& d) {
1417 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1418 return vmovn_u16(abcd_u16);
1419}
1420template <>
1421EIGEN_STRONG_INLINE Packet4uc pcast<Packet2ul, Packet4uc>(const Packet2ul& a, const Packet2ul& b) {
1422 const uint16x4_t ab_u16 = pcast<Packet2ul, Packet4us>(a, b);
1423 const uint16x8_t abab_u16 = vcombine_u16(ab_u16, ab_u16);
1424 const uint8x8_t abab_u8 = vmovn_u16(abab_u16);
1425 return vget_lane_u32(vreinterpret_u32_u8(abab_u8), 0);
1426}
1427
1428template <>
1429struct type_casting_traits<numext::uint64_t, numext::int8_t> {
1430 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1431};
1432template <>
1433EIGEN_STRONG_INLINE Packet16c pcast<Packet2ul, Packet16c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1434 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1435 const Packet2ul& g, const Packet2ul& h) {
1436 return preinterpret<Packet16c>(pcast<Packet2ul, Packet16uc>(a, b, c, d, e, f, g, h));
1437}
1438template <>
1439EIGEN_STRONG_INLINE Packet8c pcast<Packet2ul, Packet8c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1440 const Packet2ul& d) {
1441 return preinterpret<Packet8c>(pcast<Packet2ul, Packet8uc>(a, b, c, d));
1442}
1443template <>
1444EIGEN_STRONG_INLINE Packet4c pcast<Packet2ul, Packet4c>(const Packet2ul& a, const Packet2ul& b) {
1445 return static_cast<Packet4c>(pcast<Packet2ul, Packet4uc>(a, b));
1446}
1447
1448#if EIGEN_ARCH_ARM64
1449
1450//==============================================================================
1451// pcast/preinterpret, Double
1452//==============================================================================
1453
1454template <>
1455EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
1456 return Packet2d(vreinterpretq_f64_s64(a));
1457}
1458template <>
1459EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2ul>(const Packet2ul& a) {
1460 return Packet2d(vreinterpretq_f64_u64(a));
1461}
1462template <>
1463EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
1464 return Packet2l(vreinterpretq_s64_f64(a));
1465}
1466template <>
1467EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2d>(const Packet2d& a) {
1468 return Packet2ul(vreinterpretq_u64_f64(a));
1469}
1470template <>
1471EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
1472 return Packet2d(vreinterpretq_f64_s32(a));
1473}
1474template <>
1475EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
1476 return Packet4i(vreinterpretq_s32_f64(a));
1477}
1478
1479template <>
1480struct type_casting_traits<double, float> {
1481 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1482};
1483template <>
1484EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
1485 return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b));
1486}
1487template <>
1488EIGEN_STRONG_INLINE Packet2f pcast<Packet2d, Packet2f>(const Packet2d& a) {
1489 return vcvt_f32_f64(a);
1490}
1491
1492template <>
1493struct type_casting_traits<double, numext::int64_t> {
1494 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1495};
1496template <>
1497EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
1498 return vcvtq_s64_f64(a);
1499}
1500
1501template <>
1502struct type_casting_traits<double, numext::uint64_t> {
1503 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1504};
1505template <>
1506EIGEN_STRONG_INLINE Packet2ul pcast<Packet2d, Packet2ul>(const Packet2d& a) {
1507 return vcvtq_u64_f64(a);
1508}
1509
1510template <>
1511struct type_casting_traits<double, numext::int32_t> {
1512 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1513};
1514template <>
1515EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
1516 return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b)));
1517}
1518template <>
1519EIGEN_STRONG_INLINE Packet2i pcast<Packet2d, Packet2i>(const Packet2d& a) {
1520 return vmovn_s64(vcvtq_s64_f64(a));
1521}
1522
1523template <>
1524struct type_casting_traits<double, numext::uint32_t> {
1525 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1526};
1527template <>
1528EIGEN_STRONG_INLINE Packet4ui pcast<Packet2d, Packet4ui>(const Packet2d& a, const Packet2d& b) {
1529 return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b)));
1530}
1531template <>
1532EIGEN_STRONG_INLINE Packet2ui pcast<Packet2d, Packet2ui>(const Packet2d& a) {
1533 return vmovn_u64(vcvtq_u64_f64(a));
1534}
1535
1536template <>
1537struct type_casting_traits<double, numext::int16_t> {
1538 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1539};
1540template <>
1541EIGEN_STRONG_INLINE Packet8s pcast<Packet2d, Packet8s>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1542 const Packet2d& d) {
1543 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1544 const int32x4_t cd_s32 = pcast<Packet2d, Packet4i>(c, d);
1545 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1546}
1547template <>
1548EIGEN_STRONG_INLINE Packet4s pcast<Packet2d, Packet4s>(const Packet2d& a, const Packet2d& b) {
1549 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1550 return vmovn_s32(ab_s32);
1551}
1552
1553template <>
1554struct type_casting_traits<double, numext::uint16_t> {
1555 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1556};
1557template <>
1558EIGEN_STRONG_INLINE Packet8us pcast<Packet2d, Packet8us>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1559 const Packet2d& d) {
1560 return preinterpret<Packet8us>(pcast<Packet2d, Packet8s>(a, b, c, d));
1561}
1562template <>
1563EIGEN_STRONG_INLINE Packet4us pcast<Packet2d, Packet4us>(const Packet2d& a, const Packet2d& b) {
1564 return preinterpret<Packet4us>(pcast<Packet2d, Packet4s>(a, b));
1565}
1566
1567template <>
1568struct type_casting_traits<double, numext::int8_t> {
1569 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1570};
1571template <>
1572EIGEN_STRONG_INLINE Packet16c pcast<Packet2d, Packet16c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1573 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1574 const Packet2d& g, const Packet2d& h) {
1575 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1576 const int16x8_t efgh_s16 = pcast<Packet2d, Packet8s>(e, f, g, h);
1577 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1578}
1579template <>
1580EIGEN_STRONG_INLINE Packet8c pcast<Packet2d, Packet8c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1581 const Packet2d& d) {
1582 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1583 return vmovn_s16(abcd_s16);
1584}
1585template <>
1586EIGEN_STRONG_INLINE Packet4c pcast<Packet2d, Packet4c>(const Packet2d& a, const Packet2d& b) {
1587 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1588 return pcast<Packet4i, Packet4c>(ab_s32);
1589}
1590
1591template <>
1592struct type_casting_traits<double, numext::uint8_t> {
1593 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1594};
1595template <>
1596EIGEN_STRONG_INLINE Packet16uc pcast<Packet2d, Packet16uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1597 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1598 const Packet2d& g, const Packet2d& h) {
1599 const uint16x8_t abcd_u16 = pcast<Packet2d, Packet8us>(a, b, c, d);
1600 const uint16x8_t efgh_u16 = pcast<Packet2d, Packet8us>(e, f, g, h);
1601 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1602}
1603template <>
1604EIGEN_STRONG_INLINE Packet8uc pcast<Packet2d, Packet8uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1605 const Packet2d& d) {
1606 return preinterpret<Packet8uc>(pcast<Packet2d, Packet8c>(a, b, c, d));
1607}
1608template <>
1609EIGEN_STRONG_INLINE Packet4uc pcast<Packet2d, Packet4uc>(const Packet2d& a, const Packet2d& b) {
1610 return static_cast<Packet4uc>(pcast<Packet2d, Packet4c>(a, b));
1611}
1612
1613template <>
1614struct type_casting_traits<float, double> {
1615 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1616};
1617template <>
1618EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
1619 // Discard second-half of input.
1620 return vcvt_f64_f32(vget_low_f32(a));
1621}
1622template <>
1623EIGEN_STRONG_INLINE Packet2d pcast<Packet2f, Packet2d>(const Packet2f& a) {
1624 return vcvt_f64_f32(a);
1625}
1626
1627template <>
1628struct type_casting_traits<numext::int8_t, double> {
1629 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1630};
1631template <>
1632EIGEN_STRONG_INLINE Packet2d pcast<Packet16c, Packet2d>(const Packet16c& a) {
1633 // Discard all but first two values.
1634 // MSVC defines most intrinsics as macros, so we need to do this in two lines for portability.
1635 Packet2f tmp = pcast<Packet8c, Packet2f>(vget_low_s8(a));
1636 return vcvt_f64_f32(tmp);
1637}
1638
1639template <>
1640struct type_casting_traits<numext::uint8_t, double> {
1641 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1642};
1643template <>
1644EIGEN_STRONG_INLINE Packet2d pcast<Packet16uc, Packet2d>(const Packet16uc& a) {
1645 // Discard all but first two values.
1646 Packet2f tmp = pcast<Packet8uc, Packet2f>(vget_low_u8(a));
1647 return vcvt_f64_f32(tmp);
1648}
1649
1650template <>
1651struct type_casting_traits<numext::int16_t, double> {
1652 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1653};
1654template <>
1655EIGEN_STRONG_INLINE Packet2d pcast<Packet8s, Packet2d>(const Packet8s& a) {
1656 // Discard all but first two values.
1657 Packet2f tmp = pcast<Packet4s, Packet2f>(vget_low_s16(a));
1658 return vcvt_f64_f32(tmp);
1659}
1660
1661template <>
1662struct type_casting_traits<numext::uint16_t, double> {
1663 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1664};
1665template <>
1666EIGEN_STRONG_INLINE Packet2d pcast<Packet8us, Packet2d>(const Packet8us& a) {
1667 // Discard all but first two values.
1668 Packet2f tmp = pcast<Packet4us, Packet2f>(vget_low_u16(a));
1669 return vcvt_f64_f32(tmp);
1670}
1671
1672template <>
1673struct type_casting_traits<numext::int32_t, double> {
1674 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1675};
1676template <>
1677EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
1678 // Discard second half of input.
1679 return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a)));
1680}
1681template <>
1682EIGEN_STRONG_INLINE Packet2d pcast<Packet2i, Packet2d>(const Packet2i& a) {
1683 return vcvtq_f64_s64(vmovl_s32(a));
1684}
1685
1686template <>
1687struct type_casting_traits<numext::uint32_t, double> {
1688 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1689};
1690template <>
1691EIGEN_STRONG_INLINE Packet2d pcast<Packet4ui, Packet2d>(const Packet4ui& a) {
1692 // Discard second half of input.
1693 return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a)));
1694}
1695template <>
1696EIGEN_STRONG_INLINE Packet2d pcast<Packet2ui, Packet2d>(const Packet2ui& a) {
1697 return vcvtq_f64_u64(vmovl_u32(a));
1698}
1699
1700template <>
1701struct type_casting_traits<numext::int64_t, double> {
1702 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1703};
1704template <>
1705EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
1706 return vcvtq_f64_s64(a);
1707}
1708
1709template <>
1710struct type_casting_traits<numext::uint64_t, double> {
1711 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1712};
1713template <>
1714EIGEN_STRONG_INLINE Packet2d pcast<Packet2ul, Packet2d>(const Packet2ul& a) {
1715 return vcvtq_f64_u64(a);
1716}
1717
1718#endif // EIGEN_ARCH_ARM64
1719
1720} // end namespace internal
1721
1722} // end namespace Eigen
1723
1724#endif // EIGEN_TYPE_CASTING_NEON_H