Eigen  3.4.1 (git rev 28ded8800c26864e537852658428ab44c8399e87)
 
Loading...
Searching...
No Matches
TypeCasting.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2018 Rasmus Munk Larsen <rmlarsen@google.com>
5// Copyright (C) 2020 Antonio Sanchez <cantonios@google.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10
11#ifndef EIGEN_TYPE_CASTING_NEON_H
12#define EIGEN_TYPE_CASTING_NEON_H
13
14namespace Eigen {
15
16namespace internal {
17
18//==============================================================================
19// preinterpret
20//==============================================================================
21template <>
22EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2i>(const Packet2i& a) {
23 return Packet2f(vreinterpret_f32_s32(a));
24}
25template <>
26EIGEN_STRONG_INLINE Packet2f preinterpret<Packet2f, Packet2ui>(const Packet2ui& a) {
27 return Packet2f(vreinterpret_f32_u32(a));
28}
29template <>
30EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4i>(const Packet4i& a) {
31 return Packet4f(vreinterpretq_f32_s32(a));
32}
33template <>
34EIGEN_STRONG_INLINE Packet4f preinterpret<Packet4f, Packet4ui>(const Packet4ui& a) {
35 return Packet4f(vreinterpretq_f32_u32(a));
36}
37
38template <>
39EIGEN_STRONG_INLINE Packet4c preinterpret<Packet4c, Packet4uc>(const Packet4uc& a) {
40 return static_cast<Packet4c>(a);
41}
42template <>
43EIGEN_STRONG_INLINE Packet8c preinterpret<Packet8c, Packet8uc>(const Packet8uc& a) {
44 return Packet8c(vreinterpret_s8_u8(a));
45}
46template <>
47EIGEN_STRONG_INLINE Packet16c preinterpret<Packet16c, Packet16uc>(const Packet16uc& a) {
48 return Packet16c(vreinterpretq_s8_u8(a));
49}
50
51template <>
52EIGEN_STRONG_INLINE Packet4uc preinterpret<Packet4uc, Packet4c>(const Packet4c& a) {
53 return static_cast<Packet4uc>(a);
54}
55template <>
56EIGEN_STRONG_INLINE Packet8uc preinterpret<Packet8uc, Packet8c>(const Packet8c& a) {
57 return Packet8uc(vreinterpret_u8_s8(a));
58}
59template <>
60EIGEN_STRONG_INLINE Packet16uc preinterpret<Packet16uc, Packet16c>(const Packet16c& a) {
61 return Packet16uc(vreinterpretq_u8_s8(a));
62}
63
64template <>
65EIGEN_STRONG_INLINE Packet4s preinterpret<Packet4s, Packet4us>(const Packet4us& a) {
66 return Packet4s(vreinterpret_s16_u16(a));
67}
68template <>
69EIGEN_STRONG_INLINE Packet8s preinterpret<Packet8s, Packet8us>(const Packet8us& a) {
70 return Packet8s(vreinterpretq_s16_u16(a));
71}
72
73template <>
74EIGEN_STRONG_INLINE Packet4us preinterpret<Packet4us, Packet4s>(const Packet4s& a) {
75 return Packet4us(vreinterpret_u16_s16(a));
76}
77template <>
78EIGEN_STRONG_INLINE Packet8us preinterpret<Packet8us, Packet8s>(const Packet8s& a) {
79 return Packet8us(vreinterpretq_u16_s16(a));
80}
81
82template <>
83EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2f>(const Packet2f& a) {
84 return Packet2i(vreinterpret_s32_f32(a));
85}
86template <>
87EIGEN_STRONG_INLINE Packet2i preinterpret<Packet2i, Packet2ui>(const Packet2ui& a) {
88 return Packet2i(vreinterpret_s32_u32(a));
89}
90template <>
91EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4f>(const Packet4f& a) {
92 return Packet4i(vreinterpretq_s32_f32(a));
93}
94template <>
95EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet4ui>(const Packet4ui& a) {
96 return Packet4i(vreinterpretq_s32_u32(a));
97}
98
99template <>
100EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2f>(const Packet2f& a) {
101 return Packet2ui(vreinterpret_u32_f32(a));
102}
103template <>
104EIGEN_STRONG_INLINE Packet2ui preinterpret<Packet2ui, Packet2i>(const Packet2i& a) {
105 return Packet2ui(vreinterpret_u32_s32(a));
106}
107template <>
108EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4f>(const Packet4f& a) {
109 return Packet4ui(vreinterpretq_u32_f32(a));
110}
111template <>
112EIGEN_STRONG_INLINE Packet4ui preinterpret<Packet4ui, Packet4i>(const Packet4i& a) {
113 return Packet4ui(vreinterpretq_u32_s32(a));
114}
115
116template <>
117EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2ul>(const Packet2ul& a) {
118 return Packet2l(vreinterpretq_s64_u64(a));
119}
120template <>
121EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2l>(const Packet2l& a) {
122 return Packet2ul(vreinterpretq_u64_s64(a));
123}
124
125//==============================================================================
126// pcast, SrcType = float
127//==============================================================================
128template <>
129struct type_casting_traits<float, float> {
130 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
131};
132template <>
133EIGEN_STRONG_INLINE Packet4f pcast<Packet4f, Packet4f>(const Packet4f& a) {
134 return a;
135}
136template <>
137EIGEN_STRONG_INLINE Packet2f pcast<Packet2f, Packet2f>(const Packet2f& a) {
138 return a;
139}
140
141template <>
142struct type_casting_traits<float, numext::int64_t> {
143 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
144};
145template <>
146struct type_casting_traits<float, numext::uint64_t> {
147 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
148};
149// If float64 exists, first convert to that to keep as much precision as possible.
150#if EIGEN_ARCH_ARM64
151template <>
152EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
153 // Discard second half of input.
154 return vcvtq_s64_f64(vcvt_f64_f32(vget_low_f32(a)));
155}
156template <>
157EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
158 // Discard second half of input.
159 return vcvtq_u64_f64(vcvt_f64_f32(vget_low_f32(a)));
160}
161#else
162template <>
163EIGEN_STRONG_INLINE Packet2l pcast<Packet4f, Packet2l>(const Packet4f& a) {
164 // Discard second half of input.
165 return vmovl_s32(vget_low_s32(vcvtq_s32_f32(a)));
166}
167template <>
168EIGEN_STRONG_INLINE Packet2ul pcast<Packet4f, Packet2ul>(const Packet4f& a) {
169 // Discard second half of input.
170 return vmovl_u32(vget_low_u32(vcvtq_u32_f32(a)));
171}
172#endif // EIGEN_ARCH_ARM64
173
174template <>
175struct type_casting_traits<float, numext::int32_t> {
176 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
177};
178template <>
179EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(const Packet4f& a) {
180 return vcvtq_s32_f32(a);
181}
182template <>
183EIGEN_STRONG_INLINE Packet2i pcast<Packet2f, Packet2i>(const Packet2f& a) {
184 return vcvt_s32_f32(a);
185}
186
187template <>
188struct type_casting_traits<float, numext::uint32_t> {
189 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
190};
191template <>
192EIGEN_STRONG_INLINE Packet4ui pcast<Packet4f, Packet4ui>(const Packet4f& a) {
193 return vcvtq_u32_f32(a);
194}
195template <>
196EIGEN_STRONG_INLINE Packet2ui pcast<Packet2f, Packet2ui>(const Packet2f& a) {
197 return vcvt_u32_f32(a);
198}
199
200template <>
201struct type_casting_traits<float, numext::int16_t> {
202 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
203};
204template <>
205EIGEN_STRONG_INLINE Packet8s pcast<Packet4f, Packet8s>(const Packet4f& a, const Packet4f& b) {
206 return vcombine_s16(vmovn_s32(vcvtq_s32_f32(a)), vmovn_s32(vcvtq_s32_f32(b)));
207}
208template <>
209EIGEN_STRONG_INLINE Packet4s pcast<Packet2f, Packet4s>(const Packet2f& a, const Packet2f& b) {
210 return vmovn_s32(vcombine_s32(vcvt_s32_f32(a), vcvt_s32_f32(b)));
211}
212
213template <>
214struct type_casting_traits<float, numext::uint16_t> {
215 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
216};
217template <>
218EIGEN_STRONG_INLINE Packet8us pcast<Packet4f, Packet8us>(const Packet4f& a, const Packet4f& b) {
219 return vcombine_u16(vmovn_u32(vcvtq_u32_f32(a)), vmovn_u32(vcvtq_u32_f32(b)));
220}
221template <>
222EIGEN_STRONG_INLINE Packet4us pcast<Packet2f, Packet4us>(const Packet2f& a, const Packet2f& b) {
223 return vmovn_u32(vcombine_u32(vcvt_u32_f32(a), vcvt_u32_f32(b)));
224}
225
226template <>
227struct type_casting_traits<float, numext::int8_t> {
228 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
229};
230template <>
231EIGEN_STRONG_INLINE Packet16c pcast<Packet4f, Packet16c>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
232 const Packet4f& d) {
233 const int16x8_t ab_s16 = pcast<Packet4f, Packet8s>(a, b);
234 const int16x8_t cd_s16 = pcast<Packet4f, Packet8s>(c, d);
235 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
236}
237template <>
238EIGEN_STRONG_INLINE Packet8c pcast<Packet2f, Packet8c>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
239 const Packet2f& d) {
240 const int16x4_t ab_s16 = pcast<Packet2f, Packet4s>(a, b);
241 const int16x4_t cd_s16 = pcast<Packet2f, Packet4s>(c, d);
242 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
243}
244
245template <>
246struct type_casting_traits<float, numext::uint8_t> {
247 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
248};
249template <>
250EIGEN_STRONG_INLINE Packet16uc pcast<Packet4f, Packet16uc>(const Packet4f& a, const Packet4f& b, const Packet4f& c,
251 const Packet4f& d) {
252 const uint16x8_t ab_u16 = pcast<Packet4f, Packet8us>(a, b);
253 const uint16x8_t cd_u16 = pcast<Packet4f, Packet8us>(c, d);
254 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
255}
256template <>
257EIGEN_STRONG_INLINE Packet8uc pcast<Packet2f, Packet8uc>(const Packet2f& a, const Packet2f& b, const Packet2f& c,
258 const Packet2f& d) {
259 const uint16x4_t ab_u16 = pcast<Packet2f, Packet4us>(a, b);
260 const uint16x4_t cd_u16 = pcast<Packet2f, Packet4us>(c, d);
261 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
262}
263
264//==============================================================================
265// pcast, SrcType = int8_t
266//==============================================================================
267template <>
268struct type_casting_traits<numext::int8_t, float> {
269 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
270};
271template <>
272EIGEN_STRONG_INLINE Packet4f pcast<Packet16c, Packet4f>(const Packet16c& a) {
273 // Discard all but first 4 bytes.
274 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a)))));
275}
276template <>
277EIGEN_STRONG_INLINE Packet2f pcast<Packet8c, Packet2f>(const Packet8c& a) {
278 // Discard all but first 2 bytes.
279 return vcvt_f32_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a)))));
280}
281
282template <>
283struct type_casting_traits<numext::int8_t, numext::int64_t> {
284 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
285};
286template <>
287EIGEN_STRONG_INLINE Packet2l pcast<Packet16c, Packet2l>(const Packet16c& a) {
288 // Discard all but first two bytes.
289 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))))));
290}
291
292template <>
293struct type_casting_traits<numext::int8_t, numext::uint64_t> {
294 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
295};
296template <>
297EIGEN_STRONG_INLINE Packet2ul pcast<Packet16c, Packet2ul>(const Packet16c& a) {
298 return preinterpret<Packet2ul>(pcast<Packet16c, Packet2l>(a));
299}
300
301template <>
302struct type_casting_traits<numext::int8_t, numext::int32_t> {
303 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
304};
305template <>
306EIGEN_STRONG_INLINE Packet4i pcast<Packet16c, Packet4i>(const Packet16c& a) {
307 // Discard all but first 4 bytes.
308 return vmovl_s16(vget_low_s16(vmovl_s8(vget_low_s8(a))));
309}
310template <>
311EIGEN_STRONG_INLINE Packet2i pcast<Packet8c, Packet2i>(const Packet8c& a) {
312 // Discard all but first 2 bytes.
313 return vget_low_s32(vmovl_s16(vget_low_s16(vmovl_s8(a))));
314}
315
316template <>
317struct type_casting_traits<numext::int8_t, numext::uint32_t> {
318 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
319};
320template <>
321EIGEN_STRONG_INLINE Packet4ui pcast<Packet16c, Packet4ui>(const Packet16c& a) {
322 return preinterpret<Packet4ui>(pcast<Packet16c, Packet4i>(a));
323}
324template <>
325EIGEN_STRONG_INLINE Packet2ui pcast<Packet8c, Packet2ui>(const Packet8c& a) {
326 return preinterpret<Packet2ui>(pcast<Packet8c, Packet2i>(a));
327}
328
329template <>
330struct type_casting_traits<numext::int8_t, numext::int16_t> {
331 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
332};
333template <>
334EIGEN_STRONG_INLINE Packet8s pcast<Packet16c, Packet8s>(const Packet16c& a) {
335 // Discard second half of input.
336 return vmovl_s8(vget_low_s8(a));
337}
338template <>
339EIGEN_STRONG_INLINE Packet4s pcast<Packet8c, Packet4s>(const Packet8c& a) {
340 // Discard second half of input.
341 return vget_low_s16(vmovl_s8(a));
342}
343
344template <>
345struct type_casting_traits<numext::int8_t, numext::uint16_t> {
346 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
347};
348template <>
349EIGEN_STRONG_INLINE Packet8us pcast<Packet16c, Packet8us>(const Packet16c& a) {
350 return preinterpret<Packet8us>(pcast<Packet16c, Packet8s>(a));
351}
352template <>
353EIGEN_STRONG_INLINE Packet4us pcast<Packet8c, Packet4us>(const Packet8c& a) {
354 return preinterpret<Packet4us>(pcast<Packet8c, Packet4s>(a));
355}
356
357template <>
358struct type_casting_traits<numext::int8_t, numext::int8_t> {
359 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
360};
361template <>
362EIGEN_STRONG_INLINE Packet16c pcast<Packet16c, Packet16c>(const Packet16c& a) {
363 return a;
364}
365template <>
366EIGEN_STRONG_INLINE Packet8c pcast<Packet8c, Packet8c>(const Packet8c& a) {
367 return a;
368}
369template <>
370EIGEN_STRONG_INLINE Packet4c pcast<Packet4c, Packet4c>(const Packet4c& a) {
371 return a;
372}
373
374template <>
375struct type_casting_traits<numext::int8_t, numext::uint8_t> {
376 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
377};
378template <>
379EIGEN_STRONG_INLINE Packet16uc pcast<Packet16c, Packet16uc>(const Packet16c& a) {
380 return preinterpret<Packet16uc>(a);
381}
382template <>
383EIGEN_STRONG_INLINE Packet8uc pcast<Packet8c, Packet8uc>(const Packet8c& a) {
384 return preinterpret<Packet8uc>(a);
385}
386template <>
387EIGEN_STRONG_INLINE Packet4uc pcast<Packet4c, Packet4uc>(const Packet4c& a) {
388 return static_cast<Packet4uc>(a);
389}
390
391//==============================================================================
392// pcast, SrcType = uint8_t
393//==============================================================================
394template <>
395struct type_casting_traits<numext::uint8_t, float> {
396 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
397};
398template <>
399EIGEN_STRONG_INLINE Packet4f pcast<Packet16uc, Packet4f>(const Packet16uc& a) {
400 // Discard all but first 4 bytes.
401 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a)))));
402}
403template <>
404EIGEN_STRONG_INLINE Packet2f pcast<Packet8uc, Packet2f>(const Packet8uc& a) {
405 // Discard all but first 2 bytes.
406 return vcvt_f32_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a)))));
407}
408
409template <>
410struct type_casting_traits<numext::uint8_t, numext::uint64_t> {
411 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
412};
413template <>
414EIGEN_STRONG_INLINE Packet2ul pcast<Packet16uc, Packet2ul>(const Packet16uc& a) {
415 // Discard all but first two bytes.
416 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))))));
417}
418
419template <>
420struct type_casting_traits<numext::uint8_t, numext::int64_t> {
421 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
422};
423template <>
424EIGEN_STRONG_INLINE Packet2l pcast<Packet16uc, Packet2l>(const Packet16uc& a) {
425 return preinterpret<Packet2l>(pcast<Packet16uc, Packet2ul>(a));
426}
427
428template <>
429struct type_casting_traits<numext::uint8_t, numext::uint32_t> {
430 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
431};
432template <>
433EIGEN_STRONG_INLINE Packet4ui pcast<Packet16uc, Packet4ui>(const Packet16uc& a) {
434 // Discard all but first 4 bytes.
435 return vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(a))));
436}
437template <>
438EIGEN_STRONG_INLINE Packet2ui pcast<Packet8uc, Packet2ui>(const Packet8uc& a) {
439 // Discard all but first 2 bytes.
440 return vget_low_u32(vmovl_u16(vget_low_u16(vmovl_u8(a))));
441}
442
443template <>
444struct type_casting_traits<numext::uint8_t, numext::int32_t> {
445 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
446};
447template <>
448EIGEN_STRONG_INLINE Packet4i pcast<Packet16uc, Packet4i>(const Packet16uc& a) {
449 return preinterpret<Packet4i>(pcast<Packet16uc, Packet4ui>(a));
450}
451template <>
452EIGEN_STRONG_INLINE Packet2i pcast<Packet8uc, Packet2i>(const Packet8uc& a) {
453 return preinterpret<Packet2i>(pcast<Packet8uc, Packet2ui>(a));
454}
455
456template <>
457struct type_casting_traits<numext::uint8_t, numext::uint16_t> {
458 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
459};
460template <>
461EIGEN_STRONG_INLINE Packet8us pcast<Packet16uc, Packet8us>(const Packet16uc& a) {
462 // Discard second half of input.
463 return vmovl_u8(vget_low_u8(a));
464}
465template <>
466EIGEN_STRONG_INLINE Packet4us pcast<Packet8uc, Packet4us>(const Packet8uc& a) {
467 // Discard second half of input.
468 return vget_low_u16(vmovl_u8(a));
469}
470
471template <>
472struct type_casting_traits<numext::uint8_t, numext::int16_t> {
473 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
474};
475template <>
476EIGEN_STRONG_INLINE Packet8s pcast<Packet16uc, Packet8s>(const Packet16uc& a) {
477 return preinterpret<Packet8s>(pcast<Packet16uc, Packet8us>(a));
478}
479template <>
480EIGEN_STRONG_INLINE Packet4s pcast<Packet8uc, Packet4s>(const Packet8uc& a) {
481 return preinterpret<Packet4s>(pcast<Packet8uc, Packet4us>(a));
482}
483
484template <>
485struct type_casting_traits<numext::uint8_t, numext::uint8_t> {
486 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
487};
488template <>
489EIGEN_STRONG_INLINE Packet16uc pcast<Packet16uc, Packet16uc>(const Packet16uc& a) {
490 return a;
491}
492template <>
493EIGEN_STRONG_INLINE Packet8uc pcast<Packet8uc, Packet8uc>(const Packet8uc& a) {
494 return a;
495}
496template <>
497EIGEN_STRONG_INLINE Packet4uc pcast<Packet4uc, Packet4uc>(const Packet4uc& a) {
498 return a;
499}
500
501template <>
502struct type_casting_traits<numext::uint8_t, numext::int8_t> {
503 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
504};
505template <>
506EIGEN_STRONG_INLINE Packet16c pcast<Packet16uc, Packet16c>(const Packet16uc& a) {
507 return preinterpret<Packet16c>(a);
508}
509template <>
510EIGEN_STRONG_INLINE Packet8c pcast<Packet8uc, Packet8c>(const Packet8uc& a) {
511 return preinterpret<Packet8c>(a);
512}
513template <>
514EIGEN_STRONG_INLINE Packet4c pcast<Packet4uc, Packet4c>(const Packet4uc& a) {
515 return static_cast<Packet4c>(a);
516}
517
518//==============================================================================
519// pcast, SrcType = int16_t
520//==============================================================================
521template <>
522struct type_casting_traits<numext::int16_t, float> {
523 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
524};
525template <>
526EIGEN_STRONG_INLINE Packet4f pcast<Packet8s, Packet4f>(const Packet8s& a) {
527 // Discard second half of input.
528 return vcvtq_f32_s32(vmovl_s16(vget_low_s16(a)));
529}
530template <>
531EIGEN_STRONG_INLINE Packet2f pcast<Packet4s, Packet2f>(const Packet4s& a) {
532 // Discard second half of input.
533 return vcvt_f32_s32(vget_low_s32(vmovl_s16(a)));
534}
535
536template <>
537struct type_casting_traits<numext::int16_t, numext::int64_t> {
538 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
539};
540template <>
541EIGEN_STRONG_INLINE Packet2l pcast<Packet8s, Packet2l>(const Packet8s& a) {
542 // Discard all but first two values.
543 return vmovl_s32(vget_low_s32(vmovl_s16(vget_low_s16(a))));
544}
545
546template <>
547struct type_casting_traits<numext::int16_t, numext::uint64_t> {
548 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
549};
550template <>
551EIGEN_STRONG_INLINE Packet2ul pcast<Packet8s, Packet2ul>(const Packet8s& a) {
552 return preinterpret<Packet2ul>(pcast<Packet8s, Packet2l>(a));
553}
554
555template <>
556struct type_casting_traits<numext::int16_t, numext::int32_t> {
557 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
558};
559template <>
560EIGEN_STRONG_INLINE Packet4i pcast<Packet8s, Packet4i>(const Packet8s& a) {
561 // Discard second half of input.
562 return vmovl_s16(vget_low_s16(a));
563}
564template <>
565EIGEN_STRONG_INLINE Packet2i pcast<Packet4s, Packet2i>(const Packet4s& a) {
566 // Discard second half of input.
567 return vget_low_s32(vmovl_s16(a));
568}
569
570template <>
571struct type_casting_traits<numext::int16_t, numext::uint32_t> {
572 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
573};
574template <>
575EIGEN_STRONG_INLINE Packet4ui pcast<Packet8s, Packet4ui>(const Packet8s& a) {
576 return preinterpret<Packet4ui>(pcast<Packet8s, Packet4i>(a));
577}
578template <>
579EIGEN_STRONG_INLINE Packet2ui pcast<Packet4s, Packet2ui>(const Packet4s& a) {
580 return preinterpret<Packet2ui>(pcast<Packet4s, Packet2i>(a));
581}
582
583template <>
584struct type_casting_traits<numext::int16_t, numext::int16_t> {
585 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
586};
587template <>
588EIGEN_STRONG_INLINE Packet8s pcast<Packet8s, Packet8s>(const Packet8s& a) {
589 return a;
590}
591template <>
592EIGEN_STRONG_INLINE Packet4s pcast<Packet4s, Packet4s>(const Packet4s& a) {
593 return a;
594}
595
596template <>
597struct type_casting_traits<numext::int16_t, numext::uint16_t> {
598 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
599};
600template <>
601EIGEN_STRONG_INLINE Packet8us pcast<Packet8s, Packet8us>(const Packet8s& a) {
602 return preinterpret<Packet8us>(a);
603}
604template <>
605EIGEN_STRONG_INLINE Packet4us pcast<Packet4s, Packet4us>(const Packet4s& a) {
606 return preinterpret<Packet4us>(a);
607}
608
609template <>
610struct type_casting_traits<numext::int16_t, numext::int8_t> {
611 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
612};
613template <>
614EIGEN_STRONG_INLINE Packet16c pcast<Packet8s, Packet16c>(const Packet8s& a, const Packet8s& b) {
615 return vcombine_s8(vmovn_s16(a), vmovn_s16(b));
616}
617template <>
618EIGEN_STRONG_INLINE Packet8c pcast<Packet4s, Packet8c>(const Packet4s& a, const Packet4s& b) {
619 return vmovn_s16(vcombine_s16(a, b));
620}
621
622template <>
623struct type_casting_traits<numext::int16_t, numext::uint8_t> {
624 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
625};
626template <>
627EIGEN_STRONG_INLINE Packet16uc pcast<Packet8s, Packet16uc>(const Packet8s& a, const Packet8s& b) {
628 return vcombine_u8(vmovn_u16(vreinterpretq_u16_s16(a)), vmovn_u16(vreinterpretq_u16_s16(b)));
629}
630template <>
631EIGEN_STRONG_INLINE Packet8uc pcast<Packet4s, Packet8uc>(const Packet4s& a, const Packet4s& b) {
632 return vmovn_u16(vcombine_u16(vreinterpret_u16_s16(a), vreinterpret_u16_s16(b)));
633}
634
635//==============================================================================
636// pcast, SrcType = uint16_t
637//==============================================================================
638template <>
639struct type_casting_traits<numext::uint16_t, float> {
640 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
641};
642template <>
643EIGEN_STRONG_INLINE Packet4f pcast<Packet8us, Packet4f>(const Packet8us& a) {
644 // Discard second half of input.
645 return vcvtq_f32_u32(vmovl_u16(vget_low_u16(a)));
646}
647template <>
648EIGEN_STRONG_INLINE Packet2f pcast<Packet4us, Packet2f>(const Packet4us& a) {
649 // Discard second half of input.
650 return vcvt_f32_u32(vget_low_u32(vmovl_u16(a)));
651}
652
653template <>
654struct type_casting_traits<numext::uint16_t, numext::uint64_t> {
655 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
656};
657template <>
658EIGEN_STRONG_INLINE Packet2ul pcast<Packet8us, Packet2ul>(const Packet8us& a) {
659 // Discard all but first two values.
660 return vmovl_u32(vget_low_u32(vmovl_u16(vget_low_u16(a))));
661}
662
663template <>
664struct type_casting_traits<numext::uint16_t, numext::int64_t> {
665 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
666};
667template <>
668EIGEN_STRONG_INLINE Packet2l pcast<Packet8us, Packet2l>(const Packet8us& a) {
669 return preinterpret<Packet2l>(pcast<Packet8us, Packet2ul>(a));
670}
671
672template <>
673struct type_casting_traits<numext::uint16_t, numext::uint32_t> {
674 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
675};
676template <>
677EIGEN_STRONG_INLINE Packet4ui pcast<Packet8us, Packet4ui>(const Packet8us& a) {
678 // Discard second half of input.
679 return vmovl_u16(vget_low_u16(a));
680}
681template <>
682EIGEN_STRONG_INLINE Packet2ui pcast<Packet4us, Packet2ui>(const Packet4us& a) {
683 // Discard second half of input.
684 return vget_low_u32(vmovl_u16(a));
685}
686
687template <>
688struct type_casting_traits<numext::uint16_t, numext::int32_t> {
689 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
690};
691template <>
692EIGEN_STRONG_INLINE Packet4i pcast<Packet8us, Packet4i>(const Packet8us& a) {
693 return preinterpret<Packet4i>(pcast<Packet8us, Packet4ui>(a));
694}
695template <>
696EIGEN_STRONG_INLINE Packet2i pcast<Packet4us, Packet2i>(const Packet4us& a) {
697 return preinterpret<Packet2i>(pcast<Packet4us, Packet2ui>(a));
698}
699
700template <>
701struct type_casting_traits<numext::uint16_t, numext::uint16_t> {
702 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
703};
704template <>
705EIGEN_STRONG_INLINE Packet8us pcast<Packet8us, Packet8us>(const Packet8us& a) {
706 return a;
707}
708template <>
709EIGEN_STRONG_INLINE Packet4us pcast<Packet4us, Packet4us>(const Packet4us& a) {
710 return a;
711}
712
713template <>
714struct type_casting_traits<numext::uint16_t, numext::int16_t> {
715 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
716};
717template <>
718EIGEN_STRONG_INLINE Packet8s pcast<Packet8us, Packet8s>(const Packet8us& a) {
719 return preinterpret<Packet8s>(a);
720}
721template <>
722EIGEN_STRONG_INLINE Packet4s pcast<Packet4us, Packet4s>(const Packet4us& a) {
723 return preinterpret<Packet4s>(a);
724}
725
726template <>
727struct type_casting_traits<numext::uint16_t, numext::uint8_t> {
728 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
729};
730template <>
731EIGEN_STRONG_INLINE Packet16uc pcast<Packet8us, Packet16uc>(const Packet8us& a, const Packet8us& b) {
732 return vcombine_u8(vmovn_u16(a), vmovn_u16(b));
733}
734template <>
735EIGEN_STRONG_INLINE Packet8uc pcast<Packet4us, Packet8uc>(const Packet4us& a, const Packet4us& b) {
736 return vmovn_u16(vcombine_u16(a, b));
737}
738
739template <>
740struct type_casting_traits<numext::uint16_t, numext::int8_t> {
741 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
742};
743template <>
744EIGEN_STRONG_INLINE Packet16c pcast<Packet8us, Packet16c>(const Packet8us& a, const Packet8us& b) {
745 return preinterpret<Packet16c>(pcast<Packet8us, Packet16uc>(a, b));
746}
747template <>
748EIGEN_STRONG_INLINE Packet8c pcast<Packet4us, Packet8c>(const Packet4us& a, const Packet4us& b) {
749 return preinterpret<Packet8c>(pcast<Packet4us, Packet8uc>(a, b));
750}
751
752//==============================================================================
753// pcast, SrcType = int32_t
754//==============================================================================
755template <>
756struct type_casting_traits<numext::int32_t, float> {
757 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
758};
759template <>
760EIGEN_STRONG_INLINE Packet4f pcast<Packet4i, Packet4f>(const Packet4i& a) {
761 return vcvtq_f32_s32(a);
762}
763template <>
764EIGEN_STRONG_INLINE Packet2f pcast<Packet2i, Packet2f>(const Packet2i& a) {
765 return vcvt_f32_s32(a);
766}
767
768template <>
769struct type_casting_traits<numext::int32_t, numext::int64_t> {
770 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
771};
772template <>
773EIGEN_STRONG_INLINE Packet2l pcast<Packet4i, Packet2l>(const Packet4i& a) {
774 // Discard second half of input.
775 return vmovl_s32(vget_low_s32(a));
776}
777
778template <>
779struct type_casting_traits<numext::int32_t, numext::uint64_t> {
780 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
781};
782template <>
783EIGEN_STRONG_INLINE Packet2ul pcast<Packet4i, Packet2ul>(const Packet4i& a) {
784 return preinterpret<Packet2ul>(pcast<Packet4i, Packet2l>(a));
785}
786
787template <>
788struct type_casting_traits<numext::int32_t, numext::int32_t> {
789 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
790};
791template <>
792EIGEN_STRONG_INLINE Packet4i pcast<Packet4i, Packet4i>(const Packet4i& a) {
793 return a;
794}
795template <>
796EIGEN_STRONG_INLINE Packet2i pcast<Packet2i, Packet2i>(const Packet2i& a) {
797 return a;
798}
799
800template <>
801struct type_casting_traits<numext::int32_t, numext::uint32_t> {
802 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
803};
804template <>
805EIGEN_STRONG_INLINE Packet4ui pcast<Packet4i, Packet4ui>(const Packet4i& a) {
806 return preinterpret<Packet4ui>(a);
807}
808template <>
809EIGEN_STRONG_INLINE Packet2ui pcast<Packet2i, Packet2ui>(const Packet2i& a) {
810 return preinterpret<Packet2ui>(a);
811}
812
813template <>
814struct type_casting_traits<numext::int32_t, numext::int16_t> {
815 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
816};
817template <>
818EIGEN_STRONG_INLINE Packet8s pcast<Packet4i, Packet8s>(const Packet4i& a, const Packet4i& b) {
819 return vcombine_s16(vmovn_s32(a), vmovn_s32(b));
820}
821template <>
822EIGEN_STRONG_INLINE Packet4s pcast<Packet2i, Packet4s>(const Packet2i& a, const Packet2i& b) {
823 return vmovn_s32(vcombine_s32(a, b));
824}
825
826template <>
827struct type_casting_traits<numext::int32_t, numext::uint16_t> {
828 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
829};
830template <>
831EIGEN_STRONG_INLINE Packet8us pcast<Packet4i, Packet8us>(const Packet4i& a, const Packet4i& b) {
832 return vcombine_u16(vmovn_u32(vreinterpretq_u32_s32(a)), vmovn_u32(vreinterpretq_u32_s32(b)));
833}
834template <>
835EIGEN_STRONG_INLINE Packet4us pcast<Packet2i, Packet4us>(const Packet2i& a, const Packet2i& b) {
836 return vmovn_u32(vreinterpretq_u32_s32(vcombine_s32(a, b)));
837}
838
839template <>
840struct type_casting_traits<numext::int32_t, numext::int8_t> {
841 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
842};
843template <>
844EIGEN_STRONG_INLINE Packet16c pcast<Packet4i, Packet16c>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
845 const Packet4i& d) {
846 const int16x8_t ab_s16 = pcast<Packet4i, Packet8s>(a, b);
847 const int16x8_t cd_s16 = pcast<Packet4i, Packet8s>(c, d);
848 return vcombine_s8(vmovn_s16(ab_s16), vmovn_s16(cd_s16));
849}
850template <>
851EIGEN_STRONG_INLINE Packet8c pcast<Packet2i, Packet8c>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
852 const Packet2i& d) {
853 const int16x4_t ab_s16 = vmovn_s32(vcombine_s32(a, b));
854 const int16x4_t cd_s16 = vmovn_s32(vcombine_s32(c, d));
855 return vmovn_s16(vcombine_s16(ab_s16, cd_s16));
856}
857
858template <>
859struct type_casting_traits<numext::int32_t, numext::uint8_t> {
860 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
861};
862template <>
863EIGEN_STRONG_INLINE Packet16uc pcast<Packet4i, Packet16uc>(const Packet4i& a, const Packet4i& b, const Packet4i& c,
864 const Packet4i& d) {
865 const uint16x8_t ab_u16 = pcast<Packet4i, Packet8us>(a, b);
866 const uint16x8_t cd_u16 = pcast<Packet4i, Packet8us>(c, d);
867 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
868}
869template <>
870EIGEN_STRONG_INLINE Packet8uc pcast<Packet2i, Packet8uc>(const Packet2i& a, const Packet2i& b, const Packet2i& c,
871 const Packet2i& d) {
872 const uint16x4_t ab_u16 = pcast<Packet2i, Packet4us>(a, b);
873 const uint16x4_t cd_u16 = pcast<Packet2i, Packet4us>(c, d);
874 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
875}
876
877//==============================================================================
878// pcast, SrcType = uint32_t
879//==============================================================================
880template <>
881struct type_casting_traits<numext::uint32_t, float> {
882 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
883};
884template <>
885EIGEN_STRONG_INLINE Packet4f pcast<Packet4ui, Packet4f>(const Packet4ui& a) {
886 return vcvtq_f32_u32(a);
887}
888template <>
889EIGEN_STRONG_INLINE Packet2f pcast<Packet2ui, Packet2f>(const Packet2ui& a) {
890 return vcvt_f32_u32(a);
891}
892
893template <>
894struct type_casting_traits<numext::uint32_t, numext::uint64_t> {
895 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
896};
897template <>
898EIGEN_STRONG_INLINE Packet2ul pcast<Packet4ui, Packet2ul>(const Packet4ui& a) {
899 // Discard second half of input.
900 return vmovl_u32(vget_low_u32(a));
901}
902
903template <>
904struct type_casting_traits<numext::uint32_t, numext::int64_t> {
905 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
906};
907template <>
908EIGEN_STRONG_INLINE Packet2l pcast<Packet4ui, Packet2l>(const Packet4ui& a) {
909 return preinterpret<Packet2l>(pcast<Packet4ui, Packet2ul>(a));
910}
911
912template <>
913struct type_casting_traits<numext::uint32_t, numext::uint32_t> {
914 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
915};
916template <>
917EIGEN_STRONG_INLINE Packet4ui pcast<Packet4ui, Packet4ui>(const Packet4ui& a) {
918 return a;
919}
920template <>
921EIGEN_STRONG_INLINE Packet2ui pcast<Packet2ui, Packet2ui>(const Packet2ui& a) {
922 return a;
923}
924
925template <>
926struct type_casting_traits<numext::uint32_t, numext::int32_t> {
927 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
928};
929template <>
930EIGEN_STRONG_INLINE Packet4i pcast<Packet4ui, Packet4i>(const Packet4ui& a) {
931 return preinterpret<Packet4i>(a);
932}
933template <>
934EIGEN_STRONG_INLINE Packet2i pcast<Packet2ui, Packet2i>(const Packet2ui& a) {
935 return preinterpret<Packet2i>(a);
936}
937
938template <>
939struct type_casting_traits<numext::uint32_t, numext::uint16_t> {
940 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
941};
942template <>
943EIGEN_STRONG_INLINE Packet8us pcast<Packet4ui, Packet8us>(const Packet4ui& a, const Packet4ui& b) {
944 return vcombine_u16(vmovn_u32(a), vmovn_u32(b));
945}
946template <>
947EIGEN_STRONG_INLINE Packet4us pcast<Packet2ui, Packet4us>(const Packet2ui& a, const Packet2ui& b) {
948 return vmovn_u32(vcombine_u32(a, b));
949}
950
951template <>
952struct type_casting_traits<numext::uint32_t, numext::int16_t> {
953 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
954};
955template <>
956EIGEN_STRONG_INLINE Packet8s pcast<Packet4ui, Packet8s>(const Packet4ui& a, const Packet4ui& b) {
957 return preinterpret<Packet8s>(pcast<Packet4ui, Packet8us>(a, b));
958}
959template <>
960EIGEN_STRONG_INLINE Packet4s pcast<Packet2ui, Packet4s>(const Packet2ui& a, const Packet2ui& b) {
961 return preinterpret<Packet4s>(pcast<Packet2ui, Packet4us>(a, b));
962}
963
964template <>
965struct type_casting_traits<numext::uint32_t, numext::uint8_t> {
966 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
967};
968template <>
969EIGEN_STRONG_INLINE Packet16uc pcast<Packet4ui, Packet16uc>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
970 const Packet4ui& d) {
971 const uint16x8_t ab_u16 = vcombine_u16(vmovn_u32(a), vmovn_u32(b));
972 const uint16x8_t cd_u16 = vcombine_u16(vmovn_u32(c), vmovn_u32(d));
973 return vcombine_u8(vmovn_u16(ab_u16), vmovn_u16(cd_u16));
974}
975template <>
976EIGEN_STRONG_INLINE Packet8uc pcast<Packet2ui, Packet8uc>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
977 const Packet2ui& d) {
978 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(a, b));
979 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(c, d));
980 return vmovn_u16(vcombine_u16(ab_u16, cd_u16));
981}
982
983template <>
984struct type_casting_traits<numext::uint32_t, numext::int8_t> {
985 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
986};
987template <>
988EIGEN_STRONG_INLINE Packet16c pcast<Packet4ui, Packet16c>(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c,
989 const Packet4ui& d) {
990 return preinterpret<Packet16c>(pcast<Packet4ui, Packet16uc>(a, b, c, d));
991}
992template <>
993EIGEN_STRONG_INLINE Packet8c pcast<Packet2ui, Packet8c>(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c,
994 const Packet2ui& d) {
995 return preinterpret<Packet8c>(pcast<Packet2ui, Packet8uc>(a, b, c, d));
996}
997
998//==============================================================================
999// pcast, SrcType = int64_t
1000//==============================================================================
1001template <>
1002struct type_casting_traits<numext::int64_t, float> {
1003 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1004};
1005template <>
1006EIGEN_STRONG_INLINE Packet4f pcast<Packet2l, Packet4f>(const Packet2l& a, const Packet2l& b) {
1007 return vcvtq_f32_s32(vcombine_s32(vmovn_s64(a), vmovn_s64(b)));
1008}
1009
1010template <>
1011struct type_casting_traits<numext::int64_t, numext::int64_t> {
1012 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1013};
1014template <>
1015EIGEN_STRONG_INLINE Packet2l pcast<Packet2l, Packet2l>(const Packet2l& a) {
1016 return a;
1017}
1018
1019template <>
1020struct type_casting_traits<numext::int64_t, numext::uint64_t> {
1021 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1022};
1023template <>
1024EIGEN_STRONG_INLINE Packet2ul pcast<Packet2l, Packet2ul>(const Packet2l& a) {
1025 return preinterpret<Packet2ul>(a);
1026}
1027
1028template <>
1029struct type_casting_traits<numext::int64_t, numext::int32_t> {
1030 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1031};
1032template <>
1033EIGEN_STRONG_INLINE Packet4i pcast<Packet2l, Packet4i>(const Packet2l& a, const Packet2l& b) {
1034 return vcombine_s32(vmovn_s64(a), vmovn_s64(b));
1035}
1036
1037template <>
1038struct type_casting_traits<numext::int64_t, numext::uint32_t> {
1039 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1040};
1041template <>
1042EIGEN_STRONG_INLINE Packet4ui pcast<Packet2l, Packet4ui>(const Packet2l& a, const Packet2l& b) {
1043 return vcombine_u32(vmovn_u64(vreinterpretq_u64_s64(a)), vmovn_u64(vreinterpretq_u64_s64(b)));
1044}
1045
1046template <>
1047struct type_casting_traits<numext::int64_t, numext::int16_t> {
1048 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1049};
1050template <>
1051EIGEN_STRONG_INLINE Packet8s pcast<Packet2l, Packet8s>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1052 const Packet2l& d) {
1053 const int32x4_t ab_s32 = pcast<Packet2l, Packet4i>(a, b);
1054 const int32x4_t cd_s32 = pcast<Packet2l, Packet4i>(c, d);
1055 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1056}
1057
1058template <>
1059struct type_casting_traits<numext::int64_t, numext::uint16_t> {
1060 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1061};
1062template <>
1063EIGEN_STRONG_INLINE Packet8us pcast<Packet2l, Packet8us>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1064 const Packet2l& d) {
1065 const uint32x4_t ab_u32 = pcast<Packet2l, Packet4ui>(a, b);
1066 const uint32x4_t cd_u32 = pcast<Packet2l, Packet4ui>(c, d);
1067 return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32));
1068}
1069
1070template <>
1071struct type_casting_traits<numext::int64_t, numext::int8_t> {
1072 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1073};
1074template <>
1075EIGEN_STRONG_INLINE Packet16c pcast<Packet2l, Packet16c>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1076 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1077 const Packet2l& g, const Packet2l& h) {
1078 const int16x8_t abcd_s16 = pcast<Packet2l, Packet8s>(a, b, c, d);
1079 const int16x8_t efgh_s16 = pcast<Packet2l, Packet8s>(e, f, g, h);
1080 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1081}
1082
1083template <>
1084struct type_casting_traits<numext::int64_t, numext::uint8_t> {
1085 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1086};
1087template <>
1088EIGEN_STRONG_INLINE Packet16uc pcast<Packet2l, Packet16uc>(const Packet2l& a, const Packet2l& b, const Packet2l& c,
1089 const Packet2l& d, const Packet2l& e, const Packet2l& f,
1090 const Packet2l& g, const Packet2l& h) {
1091 const uint16x8_t abcd_u16 = pcast<Packet2l, Packet8us>(a, b, c, d);
1092 const uint16x8_t efgh_u16 = pcast<Packet2l, Packet8us>(e, f, g, h);
1093 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1094}
1095
1096//==============================================================================
1097// pcast, SrcType = uint64_t
1098//==============================================================================
1099template <>
1100struct type_casting_traits<numext::uint64_t, float> {
1101 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1102};
1103template <>
1104EIGEN_STRONG_INLINE Packet4f pcast<Packet2ul, Packet4f>(const Packet2ul& a, const Packet2ul& b) {
1105 return vcvtq_f32_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1106}
1107
1108template <>
1109struct type_casting_traits<numext::uint64_t, numext::uint64_t> {
1110 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1111};
1112template <>
1113EIGEN_STRONG_INLINE Packet2ul pcast<Packet2ul, Packet2ul>(const Packet2ul& a) {
1114 return a;
1115}
1116
1117template <>
1118struct type_casting_traits<numext::uint64_t, numext::int64_t> {
1119 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1120};
1121template <>
1122EIGEN_STRONG_INLINE Packet2l pcast<Packet2ul, Packet2l>(const Packet2ul& a) {
1123 return preinterpret<Packet2l>(a);
1124}
1125
1126template <>
1127struct type_casting_traits<numext::uint64_t, numext::uint32_t> {
1128 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1129};
1130template <>
1131EIGEN_STRONG_INLINE Packet4ui pcast<Packet2ul, Packet4ui>(const Packet2ul& a, const Packet2ul& b) {
1132 return vcombine_u32(vmovn_u64(a), vmovn_u64(b));
1133}
1134
1135template <>
1136struct type_casting_traits<numext::uint64_t, numext::int32_t> {
1137 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1138};
1139template <>
1140EIGEN_STRONG_INLINE Packet4i pcast<Packet2ul, Packet4i>(const Packet2ul& a, const Packet2ul& b) {
1141 return preinterpret<Packet4i>(pcast<Packet2ul, Packet4ui>(a, b));
1142}
1143
1144template <>
1145struct type_casting_traits<numext::uint64_t, numext::uint16_t> {
1146 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1147};
1148template <>
1149EIGEN_STRONG_INLINE Packet8us pcast<Packet2ul, Packet8us>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1150 const Packet2ul& d) {
1151 const uint16x4_t ab_u16 = vmovn_u32(vcombine_u32(vmovn_u64(a), vmovn_u64(b)));
1152 const uint16x4_t cd_u16 = vmovn_u32(vcombine_u32(vmovn_u64(c), vmovn_u64(d)));
1153 return vcombine_u16(ab_u16, cd_u16);
1154}
1155
1156template <>
1157struct type_casting_traits<numext::uint64_t, numext::int16_t> {
1158 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1159};
1160template <>
1161EIGEN_STRONG_INLINE Packet8s pcast<Packet2ul, Packet8s>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1162 const Packet2ul& d) {
1163 return preinterpret<Packet8s>(pcast<Packet2ul, Packet8us>(a, b, c, d));
1164}
1165
1166template <>
1167struct type_casting_traits<numext::uint64_t, numext::uint8_t> {
1168 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1169};
1170template <>
1171EIGEN_STRONG_INLINE Packet16uc pcast<Packet2ul, Packet16uc>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1172 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1173 const Packet2ul& g, const Packet2ul& h) {
1174 const uint16x8_t abcd_u16 = pcast<Packet2ul, Packet8us>(a, b, c, d);
1175 const uint16x8_t efgh_u16 = pcast<Packet2ul, Packet8us>(e, f, g, h);
1176 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1177}
1178
1179template <>
1180struct type_casting_traits<numext::uint64_t, numext::int8_t> {
1181 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1182};
1183template <>
1184EIGEN_STRONG_INLINE Packet16c pcast<Packet2ul, Packet16c>(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c,
1185 const Packet2ul& d, const Packet2ul& e, const Packet2ul& f,
1186 const Packet2ul& g, const Packet2ul& h) {
1187 return preinterpret<Packet16c>(pcast<Packet2ul, Packet16uc>(a, b, c, d, e, f, g, h));
1188}
1189
1190#if EIGEN_ARCH_ARM64
1191
1192//==============================================================================
1193// pcast/preinterpret, Double
1194//==============================================================================
1195
1196template <>
1197EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2l>(const Packet2l& a) {
1198 return Packet2d(vreinterpretq_f64_s64(a));
1199}
1200template <>
1201EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet2ul>(const Packet2ul& a) {
1202 return Packet2d(vreinterpretq_f64_u64(a));
1203}
1204template <>
1205EIGEN_STRONG_INLINE Packet2l preinterpret<Packet2l, Packet2d>(const Packet2d& a) {
1206 return Packet2l(vreinterpretq_s64_f64(a));
1207}
1208template <>
1209EIGEN_STRONG_INLINE Packet2ul preinterpret<Packet2ul, Packet2d>(const Packet2d& a) {
1210 return Packet2ul(vreinterpretq_u64_f64(a));
1211}
1212template <>
1213EIGEN_STRONG_INLINE Packet2d preinterpret<Packet2d, Packet4i>(const Packet4i& a) {
1214 return Packet2d(vreinterpretq_f64_s32(a));
1215}
1216template <>
1217EIGEN_STRONG_INLINE Packet4i preinterpret<Packet4i, Packet2d>(const Packet2d& a) {
1218 return Packet4i(vreinterpretq_s32_f64(a));
1219}
1220
1221template <>
1222struct type_casting_traits<double, double> {
1223 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1224};
1225template <>
1226EIGEN_STRONG_INLINE Packet2d pcast<Packet2d, Packet2d>(const Packet2d& a) {
1227 return a;
1228}
1229
1230template <>
1231struct type_casting_traits<double, float> {
1232 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1233};
1234template <>
1235EIGEN_STRONG_INLINE Packet4f pcast<Packet2d, Packet4f>(const Packet2d& a, const Packet2d& b) {
1236 return vcombine_f32(vcvt_f32_f64(a), vcvt_f32_f64(b));
1237}
1238
1239template <>
1240struct type_casting_traits<double, numext::int64_t> {
1241 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1242};
1243template <>
1244EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(const Packet2d& a) {
1245 return vcvtq_s64_f64(a);
1246}
1247
1248template <>
1249struct type_casting_traits<double, numext::uint64_t> {
1250 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1251};
1252template <>
1253EIGEN_STRONG_INLINE Packet2ul pcast<Packet2d, Packet2ul>(const Packet2d& a) {
1254 return vcvtq_u64_f64(a);
1255}
1256
1257template <>
1258struct type_casting_traits<double, numext::int32_t> {
1259 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1260};
1261template <>
1262EIGEN_STRONG_INLINE Packet4i pcast<Packet2d, Packet4i>(const Packet2d& a, const Packet2d& b) {
1263 return vcombine_s32(vmovn_s64(vcvtq_s64_f64(a)), vmovn_s64(vcvtq_s64_f64(b)));
1264}
1265
1266template <>
1267struct type_casting_traits<double, numext::uint32_t> {
1268 enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 };
1269};
1270template <>
1271EIGEN_STRONG_INLINE Packet4ui pcast<Packet2d, Packet4ui>(const Packet2d& a, const Packet2d& b) {
1272 return vcombine_u32(vmovn_u64(vcvtq_u64_f64(a)), vmovn_u64(vcvtq_u64_f64(b)));
1273}
1274
1275template <>
1276struct type_casting_traits<double, numext::int16_t> {
1277 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1278};
1279template <>
1280EIGEN_STRONG_INLINE Packet8s pcast<Packet2d, Packet8s>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1281 const Packet2d& d) {
1282 const int32x4_t ab_s32 = pcast<Packet2d, Packet4i>(a, b);
1283 const int32x4_t cd_s32 = pcast<Packet2d, Packet4i>(c, d);
1284 return vcombine_s16(vmovn_s32(ab_s32), vmovn_s32(cd_s32));
1285}
1286
1287template <>
1288struct type_casting_traits<double, numext::uint16_t> {
1289 enum { VectorizedCast = 1, SrcCoeffRatio = 4, TgtCoeffRatio = 1 };
1290};
1291template <>
1292EIGEN_STRONG_INLINE Packet8us pcast<Packet2d, Packet8us>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1293 const Packet2d& d) {
1294 const uint32x4_t ab_u32 = pcast<Packet2d, Packet4ui>(a, b);
1295 const uint32x4_t cd_u32 = pcast<Packet2d, Packet4ui>(c, d);
1296 return vcombine_u16(vmovn_u32(ab_u32), vmovn_u32(cd_u32));
1297}
1298
1299template <>
1300struct type_casting_traits<double, numext::int8_t> {
1301 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1302};
1303template <>
1304EIGEN_STRONG_INLINE Packet16c pcast<Packet2d, Packet16c>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1305 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1306 const Packet2d& g, const Packet2d& h) {
1307 const int16x8_t abcd_s16 = pcast<Packet2d, Packet8s>(a, b, c, d);
1308 const int16x8_t efgh_s16 = pcast<Packet2d, Packet8s>(e, f, g, h);
1309 return vcombine_s8(vmovn_s16(abcd_s16), vmovn_s16(efgh_s16));
1310}
1311
1312template <>
1313struct type_casting_traits<double, numext::uint8_t> {
1314 enum { VectorizedCast = 1, SrcCoeffRatio = 8, TgtCoeffRatio = 1 };
1315};
1316template <>
1317EIGEN_STRONG_INLINE Packet16uc pcast<Packet2d, Packet16uc>(const Packet2d& a, const Packet2d& b, const Packet2d& c,
1318 const Packet2d& d, const Packet2d& e, const Packet2d& f,
1319 const Packet2d& g, const Packet2d& h) {
1320 const uint16x8_t abcd_u16 = pcast<Packet2d, Packet8us>(a, b, c, d);
1321 const uint16x8_t efgh_u16 = pcast<Packet2d, Packet8us>(e, f, g, h);
1322 return vcombine_u8(vmovn_u16(abcd_u16), vmovn_u16(efgh_u16));
1323}
1324
1325template <>
1326struct type_casting_traits<float, double> {
1327 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1328};
1329template <>
1330EIGEN_STRONG_INLINE Packet2d pcast<Packet4f, Packet2d>(const Packet4f& a) {
1331 // Discard second-half of input.
1332 return vcvt_f64_f32(vget_low_f32(a));
1333}
1334
1335template <>
1336struct type_casting_traits<numext::int8_t, double> {
1337 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1338};
1339template <>
1340EIGEN_STRONG_INLINE Packet2d pcast<Packet16c, Packet2d>(const Packet16c& a) {
1341 // Discard all but first two values.
1342 // MSVC defines most intrinsics as macros, so we need to do this in two lines for portability.
1343 Packet2f tmp = pcast<Packet8c, Packet2f>(vget_low_s8(a));
1344 return vcvt_f64_f32(tmp);
1345}
1346
1347template <>
1348struct type_casting_traits<numext::uint8_t, double> {
1349 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 8 };
1350};
1351template <>
1352EIGEN_STRONG_INLINE Packet2d pcast<Packet16uc, Packet2d>(const Packet16uc& a) {
1353 // Discard all but first two values.
1354 Packet2f tmp = pcast<Packet8uc, Packet2f>(vget_low_u8(a));
1355 return vcvt_f64_f32(tmp);
1356}
1357
1358template <>
1359struct type_casting_traits<numext::int16_t, double> {
1360 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1361};
1362template <>
1363EIGEN_STRONG_INLINE Packet2d pcast<Packet8s, Packet2d>(const Packet8s& a) {
1364 // Discard all but first two values.
1365 Packet2f tmp = pcast<Packet4s, Packet2f>(vget_low_s16(a));
1366 return vcvt_f64_f32(tmp);
1367}
1368
1369template <>
1370struct type_casting_traits<numext::uint16_t, double> {
1371 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 4 };
1372};
1373template <>
1374EIGEN_STRONG_INLINE Packet2d pcast<Packet8us, Packet2d>(const Packet8us& a) {
1375 // Discard all but first two values.
1376 Packet2f tmp = pcast<Packet4us, Packet2f>(vget_low_u16(a));
1377 return vcvt_f64_f32(tmp);
1378}
1379
1380template <>
1381struct type_casting_traits<numext::int32_t, double> {
1382 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1383};
1384template <>
1385EIGEN_STRONG_INLINE Packet2d pcast<Packet4i, Packet2d>(const Packet4i& a) {
1386 // Discard second half of input.
1387 return vcvtq_f64_s64(vmovl_s32(vget_low_s32(a)));
1388}
1389
1390template <>
1391struct type_casting_traits<numext::uint32_t, double> {
1392 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 };
1393};
1394template <>
1395EIGEN_STRONG_INLINE Packet2d pcast<Packet4ui, Packet2d>(const Packet4ui& a) {
1396 // Discard second half of input.
1397 return vcvtq_f64_u64(vmovl_u32(vget_low_u32(a)));
1398}
1399
1400template <>
1401struct type_casting_traits<numext::int64_t, double> {
1402 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1403};
1404template <>
1405EIGEN_STRONG_INLINE Packet2d pcast<Packet2l, Packet2d>(const Packet2l& a) {
1406 return vcvtq_f64_s64(a);
1407}
1408
1409template <>
1410struct type_casting_traits<numext::uint64_t, double> {
1411 enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 };
1412};
1413template <>
1414EIGEN_STRONG_INLINE Packet2d pcast<Packet2ul, Packet2d>(const Packet2ul& a) {
1415 return vcvtq_f64_u64(a);
1416}
1417
1418#endif // EIGEN_ARCH_ARM64
1419
1420} // end namespace internal
1421
1422} // end namespace Eigen
1423
1424#endif // EIGEN_TYPE_CASTING_NEON_H
Eigen
Namespace containing all symbols from the Eigen library.
Definition B01_Experimental.dox:1