Eigen  5.0.1
 
Loading...
Searching...
No Matches
PacketMath.h
1// SPDX-FileCopyrightText: The Eigen Authors
2// SPDX-License-Identifier: MPL-2.0
3
4#ifndef EIGEN_HVX_PACKET_MATH_H
5#define EIGEN_HVX_PACKET_MATH_H
6
7// Only support 128B HVX now.
8// Floating-point operations are supported only since V68.
9#if defined __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68
10
11// All the floating-point operations do not support IEEE standard.
12// From HVX document:
13// There is no concept of infinity or NaN. QFloat saturates to maximum
14// exponent with maximum positive or minimum negative significand.
15
16#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
17#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
18#endif
19
20namespace Eigen {
21namespace internal {
22
23// HVX utilities.
24
25template <int D>
26EIGEN_STRONG_INLINE HVX_Vector HVX_vmem(const void* m) {
27 HVX_Vector v;
28#if EIGEN_COMP_CLANG
29 // Use inlined assembly for aligned vmem load on unaligned memory.
30 // Using a type cast to HVX_Vector* may mess up the compiler data alignment.
31 __asm__("%0 = vmem(%1+#%2)" : "=v"(v) : "r"(m), "i"(D) : "memory");
32#else
33 void* aligned_mem =
34 reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(m) & ~(__HVX_LENGTH__ - 1)) + D * __HVX_LENGTH__);
35 memcpy(&v, aligned_mem, __HVX_LENGTH__);
36#endif
37 return v;
38}
39
40template <typename T>
41EIGEN_STRONG_INLINE HVX_Vector HVX_load(const T* mem) {
42 HVX_Vector v;
43 memcpy(&v, reinterpret_cast<const HVX_Vector*>(mem), __HVX_LENGTH__);
44 return v;
45}
46
47template <typename T>
48EIGEN_STRONG_INLINE HVX_Vector HVX_loadu(const T* mem) {
49 HVX_Vector v;
50 memcpy(&v, mem, __HVX_LENGTH__);
51 return v;
52}
53
54template <size_t Size, size_t Alignment, typename T>
55EIGEN_STRONG_INLINE HVX_Vector HVX_load_partial(const T* mem) {
56#if defined(EIGEN_HVX_FAST_PARTIAL_VECTOR_LOAD)
57 // Fast partial vector load through aligned vmem load.
58 // The load may past end of array but is aligned to prevent memory fault.
59 HVX_Vector v0 = HVX_vmem<0>(mem);
60 HVX_Vector v1 = v0;
61 uintptr_t mem_addr = reinterpret_cast<uintptr_t>(mem);
62 EIGEN_IF_CONSTEXPR (Size * sizeof(T) <= Alignment) {
63 // Data size less than alignment will never cross multiple aligned vectors.
64 v1 = v0;
65 } else {
66 uintptr_t left_off = mem_addr & (__HVX_LENGTH__ - 1);
67 if (left_off + Size * sizeof(T) > __HVX_LENGTH__) {
68 v1 = HVX_vmem<1>(mem);
69 } else {
70 v1 = v0;
71 }
72 }
73 return Q6_V_valign_VVR(v1, v0, mem_addr);
74#else
75 HVX_Vector v;
76 memcpy(&v, mem, Size * sizeof(T));
77 return v;
78#endif
79}
80
81template <typename T>
82EIGEN_STRONG_INLINE void HVX_store(T* mem, HVX_Vector v) {
83 memcpy(reinterpret_cast<HVX_Vector*>(mem), &v, __HVX_LENGTH__);
84}
85
86template <typename T>
87EIGEN_STRONG_INLINE void HVX_storeu(T* mem, HVX_Vector v) {
88 memcpy(mem, &v, __HVX_LENGTH__);
89}
90
91template <size_t Size, size_t Alignment, typename T>
92EIGEN_STRONG_INLINE void HVX_store_partial(T* mem, HVX_Vector v) {
93 uintptr_t mem_addr = reinterpret_cast<uintptr_t>(mem);
94 HVX_Vector value = Q6_V_vlalign_VVR(v, v, mem_addr);
95 uintptr_t left_off = mem_addr & (__HVX_LENGTH__ - 1);
96 uintptr_t right_off = left_off + Size * sizeof(T);
97
98 HVX_VectorPred ql_not = Q6_Q_vsetq_R(mem_addr);
99 HVX_VectorPred qr = Q6_Q_vsetq2_R(right_off);
100
101 EIGEN_IF_CONSTEXPR (Size * sizeof(T) > Alignment) {
102 if (right_off > __HVX_LENGTH__) {
103 Q6_vmem_QRIV(qr, mem + __HVX_LENGTH__ / sizeof(T), value);
104 qr = Q6_Q_vcmp_eq_VbVb(value, value);
105 }
106 }
107
108 ql_not = Q6_Q_or_QQn(ql_not, qr);
109 Q6_vmem_QnRIV(ql_not, mem, value);
110}
111
112// Packet definitions.
113enum class HVXPacketSize {
114 Full,
115 Half,
116 Quarter,
117};
118
119// Hexagon compiler uses same HVX_Vector to represent all HVX vector types.
120// Wrap different vector type (float32, int32, etc) to different class with
121// explicit constructor and casting back-and-force to HVX_Vector.
122template <HVXPacketSize T>
123class HVXPacket {
124 public:
125 HVXPacket() = default;
126 static HVXPacket Create(HVX_Vector v) { return HVXPacket(v); }
127 HVX_Vector Get() const { return m_val; }
128
129 private:
130 explicit HVXPacket(HVX_Vector v) : m_val(v) {}
131 HVX_Vector m_val = Q6_V_vzero();
132};
133
134typedef HVXPacket<HVXPacketSize::Full> Packet32f;
135typedef HVXPacket<HVXPacketSize::Half> Packet16f;
136typedef HVXPacket<HVXPacketSize::Quarter> Packet8f;
137
138// Packet traits.
139template <>
140struct packet_traits<float> : default_packet_traits {
141 typedef Packet32f type;
142 typedef Packet16f half;
143 enum {
144 Vectorizable = 1,
145 AlignedOnScalar = 1,
146 size = 32,
147
148 HasCmp = 1,
149 HasAdd = 1,
150 HasSub = 1,
151 HasShift = 0,
152 HasMul = 1,
153 HasNegate = 1,
154 HasAbs = 1,
155 HasArg = 0,
156 HasAbsDiff = 0,
157 HasMin = 1,
158 HasMax = 1,
159 HasConj = 0,
160 HasSetLinear = 0,
161 HasDiv = 0,
162 HasSin = 0,
163 HasCos = 0,
164 HasACos = 0,
165 HasASin = 0,
166 HasATan = 0,
167 HasATanh = 0,
168 HasLog = 0,
169 HasExp = 0,
170 HasSqrt = 0,
171 HasRsqrt = 0,
172 HasTanh = 0,
173 HasErf = 0,
174 HasBessel = 0,
175 HasNdtri = 0
176 };
177};
178
179template <>
180struct unpacket_traits<Packet32f> {
181 typedef float type;
182 typedef Packet16f half;
183 enum {
184 size = 32,
185 alignment = Aligned128,
186 vectorizable = true,
187 masked_load_available = false,
188 masked_store_available = false
189 };
190};
191
192template <>
193struct unpacket_traits<Packet16f> {
194 typedef float type;
195 typedef Packet8f half;
196 enum {
197 size = 16,
198 // Much code assumes alignment on packet size instead of following the trait
199 // So we do not use Aligned128 to optimize aligned load/store,
200 alignment = Aligned64,
201 vectorizable = true,
202 masked_load_available = false,
203 masked_store_available = false
204 };
205};
206
207template <>
208struct unpacket_traits<Packet8f> {
209 typedef float type;
210 typedef Packet8f half;
211 enum {
212 size = 8,
213 // Much code assumes alignment on packet size instead of following the trait
214 // So we do not use Aligned128 to optimize aligned load/store,
215 alignment = Aligned32,
216 vectorizable = true,
217 masked_load_available = false,
218 masked_store_available = false
219 };
220};
221
222// float32 operations.
223template <HVXPacketSize T>
224EIGEN_STRONG_INLINE HVXPacket<T> pzero_hvx(const HVXPacket<T>&) {
225 return HVXPacket<T>::Create(Q6_V_vzero());
226}
227template <>
228EIGEN_STRONG_INLINE Packet32f pzero<Packet32f>(const Packet32f&) {
229 return pzero_hvx(Packet32f());
230}
231template <>
232EIGEN_STRONG_INLINE Packet16f pzero<Packet16f>(const Packet16f&) {
233 return pzero_hvx(Packet16f());
234}
235template <>
236EIGEN_STRONG_INLINE Packet8f pzero<Packet8f>(const Packet8f&) {
237 return pzero_hvx(Packet8f());
238}
239
240template <HVXPacketSize T>
241EIGEN_STRONG_INLINE typename unpacket_traits<HVXPacket<T>>::half predux_half_hvx(const HVXPacket<T>& a) {
242 const Index packet_size = unpacket_traits<HVXPacket<T>>::size;
243 return unpacket_traits<HVXPacket<T>>::half::Create(
244 Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_vror_VR(a.Get(), sizeof(float) * packet_size / 2), a.Get())));
245}
246template <>
247EIGEN_STRONG_INLINE Packet16f predux_half(const Packet32f& a) {
248 return predux_half_hvx(a);
249}
250template <>
251EIGEN_STRONG_INLINE Packet8f predux_half(const Packet16f& a) {
252 return predux_half_hvx(a);
253}
254
255template <HVXPacketSize T>
256EIGEN_STRONG_INLINE HVXPacket<T> pset1_hvx(const float& from) {
257 union {
258 float f;
259 int32_t i;
260 } u;
261 u.f = from;
262 return HVXPacket<T>::Create(Q6_V_vsplat_R(u.i));
263}
264template <>
265EIGEN_STRONG_INLINE Packet32f pset1<Packet32f>(const float& from) {
266 return pset1_hvx<HVXPacketSize::Full>(from);
267}
268template <>
269EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
270 return pset1_hvx<HVXPacketSize::Half>(from);
271}
272template <>
273EIGEN_STRONG_INLINE Packet8f pset1<Packet8f>(const float& from) {
274 return pset1_hvx<HVXPacketSize::Quarter>(from);
275}
276
277template <>
278EIGEN_STRONG_INLINE Packet32f pload<Packet32f>(const float* from) {
279 return Packet32f::Create(HVX_load(from));
280}
281template <>
282EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
283 return Packet16f::Create(
284 HVX_load_partial<unpacket_traits<Packet16f>::size, unpacket_traits<Packet16f>::alignment>(from));
285}
286template <>
287EIGEN_STRONG_INLINE Packet8f pload<Packet8f>(const float* from) {
288 return Packet8f::Create(
289 HVX_load_partial<unpacket_traits<Packet8f>::size, unpacket_traits<Packet8f>::alignment>(from));
290}
291
292template <>
293EIGEN_STRONG_INLINE Packet32f ploadu<Packet32f>(const float* from) {
294 return Packet32f::Create(HVX_loadu(from));
295}
296template <>
297EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
298 return Packet16f::Create(HVX_load_partial<unpacket_traits<Packet16f>::size, 0>(from));
299}
300template <>
301EIGEN_STRONG_INLINE Packet8f ploadu<Packet8f>(const float* from) {
302 return Packet8f::Create(HVX_load_partial<unpacket_traits<Packet8f>::size, 0>(from));
303}
304
305template <>
306EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet32f& from) {
307 HVX_store(to, from.Get());
308}
309template <>
310EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
311 HVX_store_partial<unpacket_traits<Packet16f>::size, unpacket_traits<Packet16f>::alignment>(to, from.Get());
312}
313template <>
314EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet8f& from) {
315 HVX_store_partial<unpacket_traits<Packet8f>::size, unpacket_traits<Packet8f>::alignment>(to, from.Get());
316}
317
318template <>
319EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet32f& from) {
320 HVX_storeu(to, from.Get());
321}
322template <>
323EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
324 HVX_store_partial<unpacket_traits<Packet16f>::size, 0>(to, from.Get());
325}
326template <>
327EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet8f& from) {
328 HVX_store_partial<unpacket_traits<Packet8f>::size, 0>(to, from.Get());
329}
330
331template <HVXPacketSize T>
332EIGEN_STRONG_INLINE HVXPacket<T> pmul_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
333 return HVXPacket<T>::Create(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a.Get(), b.Get())));
334}
335template <>
336EIGEN_STRONG_INLINE Packet32f pmul<Packet32f>(const Packet32f& a, const Packet32f& b) {
337 return pmul_hvx(a, b);
338}
339template <>
340EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a, const Packet16f& b) {
341 return pmul_hvx(a, b);
342}
343template <>
344EIGEN_STRONG_INLINE Packet8f pmul<Packet8f>(const Packet8f& a, const Packet8f& b) {
345 return pmul_hvx(a, b);
346}
347
348template <HVXPacketSize T>
349EIGEN_STRONG_INLINE HVXPacket<T> padd_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
350 return HVXPacket<T>::Create(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a.Get(), b.Get())));
351}
352template <>
353EIGEN_STRONG_INLINE Packet32f padd<Packet32f>(const Packet32f& a, const Packet32f& b) {
354 return padd_hvx(a, b);
355}
356template <>
357EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a, const Packet16f& b) {
358 return padd_hvx(a, b);
359}
360template <>
361EIGEN_STRONG_INLINE Packet8f padd<Packet8f>(const Packet8f& a, const Packet8f& b) {
362 return padd_hvx(a, b);
363}
364
365template <HVXPacketSize T>
366EIGEN_STRONG_INLINE HVXPacket<T> psub_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
367 return HVXPacket<T>::Create(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a.Get(), b.Get())));
368}
369template <>
370EIGEN_STRONG_INLINE Packet32f psub<Packet32f>(const Packet32f& a, const Packet32f& b) {
371 return psub_hvx(a, b);
372}
373template <>
374EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a, const Packet16f& b) {
375 return psub_hvx(a, b);
376}
377template <>
378EIGEN_STRONG_INLINE Packet8f psub<Packet8f>(const Packet8f& a, const Packet8f& b) {
379 return psub_hvx(a, b);
380}
381
382template <HVXPacketSize T>
383EIGEN_STRONG_INLINE HVXPacket<T> pnegate_hvx(const HVXPacket<T>& a) {
384 return HVXPacket<T>::Create(a.Get() ^ Q6_V_vsplat_R(0x80000000));
385}
386template <>
387EIGEN_STRONG_INLINE Packet32f pnegate(const Packet32f& a) {
388 return pnegate_hvx(a);
389}
390template <>
391EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
392 return pnegate_hvx(a);
393}
394template <>
395EIGEN_STRONG_INLINE Packet8f pnegate(const Packet8f& a) {
396 return pnegate_hvx(a);
397}
398
399template <HVXPacketSize T>
400EIGEN_STRONG_INLINE HVXPacket<T> ptrue_hvx(const HVXPacket<T>& a) {
401 return HVXPacket<T>::Create(Q6_V_vsplat_R(0x3f800000));
402}
403template <>
404EIGEN_STRONG_INLINE Packet32f ptrue(const Packet32f& a) {
405 return ptrue_hvx(a);
406}
407template <>
408EIGEN_STRONG_INLINE Packet16f ptrue(const Packet16f& a) {
409 return ptrue_hvx(a);
410}
411template <>
412EIGEN_STRONG_INLINE Packet8f ptrue(const Packet8f& a) {
413 return ptrue_hvx(a);
414}
415
416template <HVXPacketSize T>
417EIGEN_STRONG_INLINE HVXPacket<T> pcmp_le_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
418 HVX_Vector v_true = ptrue(a).Get();
419 HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(a.Get(), b.Get());
420 return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, Q6_V_vzero(), v_true));
421}
422template <>
423EIGEN_STRONG_INLINE Packet32f pcmp_le(const Packet32f& a, const Packet32f& b) {
424 return pcmp_le_hvx(a, b);
425}
426template <>
427EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) {
428 return pcmp_le_hvx(a, b);
429}
430template <>
431EIGEN_STRONG_INLINE Packet8f pcmp_le(const Packet8f& a, const Packet8f& b) {
432 return pcmp_le_hvx(a, b);
433}
434
435template <HVXPacketSize T>
436EIGEN_STRONG_INLINE HVXPacket<T> pcmp_eq_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
437 HVX_Vector v_true = ptrue(a).Get();
438 HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(a.Get(), b.Get());
439 return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero()));
440}
441template <>
442EIGEN_STRONG_INLINE Packet32f pcmp_eq(const Packet32f& a, const Packet32f& b) {
443 return pcmp_eq_hvx(a, b);
444}
445template <>
446EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) {
447 return pcmp_eq_hvx(a, b);
448}
449template <>
450EIGEN_STRONG_INLINE Packet8f pcmp_eq(const Packet8f& a, const Packet8f& b) {
451 return pcmp_eq_hvx(a, b);
452}
453
454template <HVXPacketSize T>
455EIGEN_STRONG_INLINE HVXPacket<T> pcmp_lt_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
456 HVX_Vector v_true = ptrue(a).Get();
457 HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get());
458 return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero()));
459}
460template <>
461EIGEN_STRONG_INLINE Packet32f pcmp_lt(const Packet32f& a, const Packet32f& b) {
462 return pcmp_lt_hvx(a, b);
463}
464template <>
465EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) {
466 return pcmp_lt_hvx(a, b);
467}
468template <>
469EIGEN_STRONG_INLINE Packet8f pcmp_lt(const Packet8f& a, const Packet8f& b) {
470 return pcmp_lt_hvx(a, b);
471}
472
473template <HVXPacketSize T>
474EIGEN_STRONG_INLINE HVXPacket<T> pcmp_lt_or_nan_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
475 HVX_Vector v_true = ptrue(a).Get();
476 HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get());
477 return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero()));
478}
479template <>
480EIGEN_STRONG_INLINE Packet32f pcmp_lt_or_nan(const Packet32f& a, const Packet32f& b) {
481 return pcmp_lt_or_nan_hvx(a, b);
482}
483template <>
484EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) {
485 return pcmp_lt_or_nan_hvx(a, b);
486}
487template <>
488EIGEN_STRONG_INLINE Packet8f pcmp_lt_or_nan(const Packet8f& a, const Packet8f& b) {
489 return pcmp_lt_or_nan_hvx(a, b);
490}
491
492template <HVXPacketSize T>
493EIGEN_STRONG_INLINE HVXPacket<T> pabs_hvx(const HVXPacket<T>& a) {
494 return HVXPacket<T>::Create(a.Get() & Q6_V_vsplat_R(0x7FFFFFFF));
495}
496template <>
497EIGEN_STRONG_INLINE Packet32f pabs(const Packet32f& a) {
498 return pabs_hvx(a);
499}
500template <>
501EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) {
502 return pabs_hvx(a);
503}
504template <>
505EIGEN_STRONG_INLINE Packet8f pabs(const Packet8f& a) {
506 return pabs_hvx(a);
507}
508
509template <HVXPacketSize T>
510EIGEN_STRONG_INLINE float pfirst_hvx(const HVXPacket<T>& a) {
511 union {
512 float array[1];
513 HVX_Vector vector;
514 } HVX_and_array;
515 HVX_and_array.vector = a.Get();
516 return HVX_and_array.array[0];
517}
518template <>
519EIGEN_STRONG_INLINE float pfirst(const Packet32f& a) {
520 return pfirst_hvx(a);
521}
522template <>
523EIGEN_STRONG_INLINE float pfirst(const Packet16f& a) {
524 return pfirst_hvx(a);
525}
526template <>
527EIGEN_STRONG_INLINE float pfirst(const Packet8f& a) {
528 return pfirst_hvx(a);
529}
530
531EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet32f, 4>& kernel) {
532 // Shuffle the 32-bit lanes.
533 HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4);
534 HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4);
535
536 // Shuffle the 64-bit lanes.
537 HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8);
538 HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_3_2), HEXAGON_HVX_GET_V1(v_0_1_0), -8);
539 kernel.packet[0] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_1_1_0));
540 kernel.packet[1] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_1_1_0));
541 kernel.packet[2] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_1_3_2));
542 kernel.packet[3] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_1_3_2));
543}
544EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
545 // Shuffle the 32-bit lanes.
546 HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4);
547 HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4);
548
549 // Shuffle the 64-bit lanes.
550 HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8);
551
552 kernel.packet[0] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_1_0));
553 kernel.packet[1] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 64));
554 kernel.packet[2] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_1_0));
555 kernel.packet[3] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_1_0), HEXAGON_HVX_GET_V1(v_1_1_0), 64));
556}
557EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8f, 4>& kernel) {
558 // Shuffle the 32-bit lanes.
559 HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4);
560 HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4);
561
562 // Shuffle the 64-bit lanes.
563 HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8);
564
565 kernel.packet[0] = Packet8f::Create(HEXAGON_HVX_GET_V0(v_1_1_0));
566 kernel.packet[1] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 32));
567 kernel.packet[2] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 64));
568 kernel.packet[3] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 96));
569}
570
571EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8f, 8>& kernel) {
572 // Shuffle the 32-bit lanes.
573 HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4);
574 HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4);
575 HVX_VectorPair v_0_5_4 = Q6_W_vshuff_VVR(kernel.packet[5].Get(), kernel.packet[4].Get(), -4);
576 HVX_VectorPair v_0_7_6 = Q6_W_vshuff_VVR(kernel.packet[7].Get(), kernel.packet[6].Get(), -4);
577
578 // Shuffle the 64-bit lanes.
579 HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8);
580 HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_7_6), HEXAGON_HVX_GET_V0(v_0_5_4), -8);
581
582 // Shuffle the 128-bit lanes.
583 v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_3_2), HEXAGON_HVX_GET_V0(v_1_1_0), -16);
584
585 kernel.packet[0] = Packet8f::Create(HEXAGON_HVX_GET_V0(v_0_1_0));
586 kernel.packet[1] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_0_1_0), HEXAGON_HVX_GET_V0(v_0_1_0), 32));
587 kernel.packet[2] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_0_1_0), HEXAGON_HVX_GET_V0(v_0_1_0), 64));
588 kernel.packet[3] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_0_1_0), HEXAGON_HVX_GET_V0(v_0_1_0), 96));
589 kernel.packet[4] = Packet8f::Create(HEXAGON_HVX_GET_V1(v_0_1_0));
590 kernel.packet[5] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_0_1_0), HEXAGON_HVX_GET_V1(v_0_1_0), 32));
591 kernel.packet[6] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_0_1_0), HEXAGON_HVX_GET_V1(v_0_1_0), 64));
592 kernel.packet[7] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_0_1_0), HEXAGON_HVX_GET_V1(v_0_1_0), 96));
593}
594EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
595 // Shuffle the 32-bit lanes.
596 HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4);
597 HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4);
598 HVX_VectorPair v_0_5_4 = Q6_W_vshuff_VVR(kernel.packet[5].Get(), kernel.packet[4].Get(), -4);
599 HVX_VectorPair v_0_7_6 = Q6_W_vshuff_VVR(kernel.packet[7].Get(), kernel.packet[6].Get(), -4);
600 HVX_VectorPair v_0_9_8 = Q6_W_vshuff_VVR(kernel.packet[9].Get(), kernel.packet[8].Get(), -4);
601 HVX_VectorPair v_0_11_10 = Q6_W_vshuff_VVR(kernel.packet[11].Get(), kernel.packet[10].Get(), -4);
602 HVX_VectorPair v_0_13_12 = Q6_W_vshuff_VVR(kernel.packet[13].Get(), kernel.packet[12].Get(), -4);
603 HVX_VectorPair v_0_15_14 = Q6_W_vshuff_VVR(kernel.packet[15].Get(), kernel.packet[14].Get(), -4);
604
605 // Shuffle the 64-bit lanes.
606 HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8);
607 HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_7_6), HEXAGON_HVX_GET_V0(v_0_5_4), -8);
608 HVX_VectorPair v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_9_8), -8);
609 HVX_VectorPair v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), HEXAGON_HVX_GET_V0(v_0_13_12), -8);
610
611 // Shuffle the 128-bit lanes.
612 v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_3_2), HEXAGON_HVX_GET_V0(v_1_1_0), -16);
613 v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_3_2), HEXAGON_HVX_GET_V1(v_1_1_0), -16);
614 v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_7_6), HEXAGON_HVX_GET_V0(v_1_5_4), -16);
615 v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_7_6), HEXAGON_HVX_GET_V1(v_1_5_4), -16);
616
617 // Shuffle the 256-bit lanes.
618 v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_9_8), HEXAGON_HVX_GET_V0(v_0_1_0), -32);
619 v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_9_8), HEXAGON_HVX_GET_V1(v_0_1_0), -32);
620 v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_3_2), -32);
621 v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), HEXAGON_HVX_GET_V1(v_0_3_2), -32);
622
623 kernel.packet[0] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_1_0));
624 kernel.packet[1] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 64));
625 kernel.packet[2] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_1_0));
626 kernel.packet[3] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_1_0), HEXAGON_HVX_GET_V1(v_1_1_0), 64));
627 kernel.packet[4] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_3_2));
628 kernel.packet[5] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_3_2), HEXAGON_HVX_GET_V0(v_1_3_2), 64));
629 kernel.packet[6] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_3_2));
630 kernel.packet[7] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_3_2), HEXAGON_HVX_GET_V1(v_1_3_2), 64));
631 kernel.packet[8] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_5_4));
632 kernel.packet[9] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_5_4), HEXAGON_HVX_GET_V0(v_1_5_4), 64));
633 kernel.packet[10] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_5_4));
634 kernel.packet[11] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_5_4), HEXAGON_HVX_GET_V1(v_1_5_4), 64));
635 kernel.packet[12] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_7_6));
636 kernel.packet[13] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_7_6), HEXAGON_HVX_GET_V0(v_1_7_6), 64));
637 kernel.packet[14] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_7_6));
638 kernel.packet[15] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_7_6), HEXAGON_HVX_GET_V1(v_1_7_6), 64));
639}
640EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet32f, 32>& kernel) {
641 // Shuffle the 32-bit lanes.
642 HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4);
643 HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4);
644 HVX_VectorPair v_0_5_4 = Q6_W_vshuff_VVR(kernel.packet[5].Get(), kernel.packet[4].Get(), -4);
645 HVX_VectorPair v_0_7_6 = Q6_W_vshuff_VVR(kernel.packet[7].Get(), kernel.packet[6].Get(), -4);
646 HVX_VectorPair v_0_9_8 = Q6_W_vshuff_VVR(kernel.packet[9].Get(), kernel.packet[8].Get(), -4);
647 HVX_VectorPair v_0_11_10 = Q6_W_vshuff_VVR(kernel.packet[11].Get(), kernel.packet[10].Get(), -4);
648 HVX_VectorPair v_0_13_12 = Q6_W_vshuff_VVR(kernel.packet[13].Get(), kernel.packet[12].Get(), -4);
649 HVX_VectorPair v_0_15_14 = Q6_W_vshuff_VVR(kernel.packet[15].Get(), kernel.packet[14].Get(), -4);
650 HVX_VectorPair v_0_17_16 = Q6_W_vshuff_VVR(kernel.packet[17].Get(), kernel.packet[16].Get(), -4);
651 HVX_VectorPair v_0_19_18 = Q6_W_vshuff_VVR(kernel.packet[19].Get(), kernel.packet[18].Get(), -4);
652 HVX_VectorPair v_0_21_20 = Q6_W_vshuff_VVR(kernel.packet[21].Get(), kernel.packet[20].Get(), -4);
653 HVX_VectorPair v_0_23_22 = Q6_W_vshuff_VVR(kernel.packet[23].Get(), kernel.packet[22].Get(), -4);
654 HVX_VectorPair v_0_25_24 = Q6_W_vshuff_VVR(kernel.packet[25].Get(), kernel.packet[24].Get(), -4);
655 HVX_VectorPair v_0_27_26 = Q6_W_vshuff_VVR(kernel.packet[27].Get(), kernel.packet[26].Get(), -4);
656 HVX_VectorPair v_0_29_28 = Q6_W_vshuff_VVR(kernel.packet[29].Get(), kernel.packet[28].Get(), -4);
657 HVX_VectorPair v_0_31_30 = Q6_W_vshuff_VVR(kernel.packet[31].Get(), kernel.packet[30].Get(), -4);
658
659 // Shuffle the 64-bit lanes.
660 HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8);
661 HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_3_2), HEXAGON_HVX_GET_V1(v_0_1_0), -8);
662 HVX_VectorPair v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_7_6), HEXAGON_HVX_GET_V0(v_0_5_4), -8);
663 HVX_VectorPair v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_7_6), HEXAGON_HVX_GET_V1(v_0_5_4), -8);
664 HVX_VectorPair v_1_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_9_8), -8);
665 HVX_VectorPair v_1_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), HEXAGON_HVX_GET_V1(v_0_9_8), -8);
666 HVX_VectorPair v_1_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), HEXAGON_HVX_GET_V0(v_0_13_12), -8);
667 HVX_VectorPair v_1_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_15_14), HEXAGON_HVX_GET_V1(v_0_13_12), -8);
668 HVX_VectorPair v_1_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_19_18), HEXAGON_HVX_GET_V0(v_0_17_16), -8);
669 HVX_VectorPair v_1_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_19_18), HEXAGON_HVX_GET_V1(v_0_17_16), -8);
670 HVX_VectorPair v_1_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_23_22), HEXAGON_HVX_GET_V0(v_0_21_20), -8);
671 HVX_VectorPair v_1_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_23_22), HEXAGON_HVX_GET_V1(v_0_21_20), -8);
672 HVX_VectorPair v_1_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_27_26), HEXAGON_HVX_GET_V0(v_0_25_24), -8);
673 HVX_VectorPair v_1_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_27_26), HEXAGON_HVX_GET_V1(v_0_25_24), -8);
674 HVX_VectorPair v_1_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_31_30), HEXAGON_HVX_GET_V0(v_0_29_28), -8);
675 HVX_VectorPair v_1_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_31_30), HEXAGON_HVX_GET_V1(v_0_29_28), -8);
676
677 // Shuffle the 128-bit lanes.
678 v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_5_4), HEXAGON_HVX_GET_V0(v_1_1_0), -16);
679 v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_5_4), HEXAGON_HVX_GET_V1(v_1_1_0), -16);
680 v_0_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_7_6), HEXAGON_HVX_GET_V0(v_1_3_2), -16);
681 v_0_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_7_6), HEXAGON_HVX_GET_V1(v_1_3_2), -16);
682 v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_13_12), HEXAGON_HVX_GET_V0(v_1_9_8), -16);
683 v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_13_12), HEXAGON_HVX_GET_V1(v_1_9_8), -16);
684 v_0_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_15_14), HEXAGON_HVX_GET_V0(v_1_11_10), -16);
685 v_0_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_15_14), HEXAGON_HVX_GET_V1(v_1_11_10), -16);
686 v_0_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_21_20), HEXAGON_HVX_GET_V0(v_1_17_16), -16);
687 v_0_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_21_20), HEXAGON_HVX_GET_V1(v_1_17_16), -16);
688 v_0_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_23_22), HEXAGON_HVX_GET_V0(v_1_19_18), -16);
689 v_0_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_23_22), HEXAGON_HVX_GET_V1(v_1_19_18), -16);
690 v_0_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_29_28), HEXAGON_HVX_GET_V0(v_1_25_24), -16);
691 v_0_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_29_28), HEXAGON_HVX_GET_V1(v_1_25_24), -16);
692 v_0_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_31_30), HEXAGON_HVX_GET_V0(v_1_27_26), -16);
693 v_0_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_31_30), HEXAGON_HVX_GET_V1(v_1_27_26), -16);
694
695 // Shuffle the 256-bit lanes.
696 v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_9_8), HEXAGON_HVX_GET_V0(v_0_1_0), -32);
697 v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_9_8), HEXAGON_HVX_GET_V1(v_0_1_0), -32);
698 v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_3_2), -32);
699 v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), HEXAGON_HVX_GET_V1(v_0_3_2), -32);
700 v_1_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_13_12), HEXAGON_HVX_GET_V0(v_0_5_4), -32);
701 v_1_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_13_12), HEXAGON_HVX_GET_V1(v_0_5_4), -32);
702 v_1_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), HEXAGON_HVX_GET_V0(v_0_7_6), -32);
703 v_1_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_15_14), HEXAGON_HVX_GET_V1(v_0_7_6), -32);
704 v_1_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_25_24), HEXAGON_HVX_GET_V0(v_0_17_16), -32);
705 v_1_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_25_24), HEXAGON_HVX_GET_V1(v_0_17_16), -32);
706 v_1_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_27_26), HEXAGON_HVX_GET_V0(v_0_19_18), -32);
707 v_1_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_27_26), HEXAGON_HVX_GET_V1(v_0_19_18), -32);
708 v_1_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_29_28), HEXAGON_HVX_GET_V0(v_0_21_20), -32);
709 v_1_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_29_28), HEXAGON_HVX_GET_V1(v_0_21_20), -32);
710 v_1_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_31_30), HEXAGON_HVX_GET_V0(v_0_23_22), -32);
711 v_1_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_31_30), HEXAGON_HVX_GET_V1(v_0_23_22), -32);
712
713 // Shuffle the 512-bit lanes.
714 v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_17_16), HEXAGON_HVX_GET_V0(v_1_1_0), -64);
715 v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_17_16), HEXAGON_HVX_GET_V1(v_1_1_0), -64);
716 v_0_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_19_18), HEXAGON_HVX_GET_V0(v_1_3_2), -64);
717 v_0_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_19_18), HEXAGON_HVX_GET_V1(v_1_3_2), -64);
718 v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_21_20), HEXAGON_HVX_GET_V0(v_1_5_4), -64);
719 v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_21_20), HEXAGON_HVX_GET_V1(v_1_5_4), -64);
720 v_0_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_23_22), HEXAGON_HVX_GET_V0(v_1_7_6), -64);
721 v_0_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_23_22), HEXAGON_HVX_GET_V1(v_1_7_6), -64);
722 v_0_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_25_24), HEXAGON_HVX_GET_V0(v_1_9_8), -64);
723 v_0_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_25_24), HEXAGON_HVX_GET_V1(v_1_9_8), -64);
724 v_0_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_27_26), HEXAGON_HVX_GET_V0(v_1_11_10), -64);
725 v_0_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_27_26), HEXAGON_HVX_GET_V1(v_1_11_10), -64);
726 v_0_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_29_28), HEXAGON_HVX_GET_V0(v_1_13_12), -64);
727 v_0_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_29_28), HEXAGON_HVX_GET_V1(v_1_13_12), -64);
728 v_0_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_31_30), HEXAGON_HVX_GET_V0(v_1_15_14), -64);
729 v_0_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_31_30), HEXAGON_HVX_GET_V1(v_1_15_14), -64);
730
731 kernel.packet[0] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_1_0));
732 kernel.packet[1] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_1_0));
733 kernel.packet[2] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_3_2));
734 kernel.packet[3] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_3_2));
735 kernel.packet[4] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_5_4));
736 kernel.packet[5] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_5_4));
737 kernel.packet[6] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_7_6));
738 kernel.packet[7] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_7_6));
739 kernel.packet[8] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_9_8));
740 kernel.packet[9] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_9_8));
741 kernel.packet[10] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_11_10));
742 kernel.packet[11] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_11_10));
743 kernel.packet[12] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_13_12));
744 kernel.packet[13] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_13_12));
745 kernel.packet[14] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_15_14));
746 kernel.packet[15] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_15_14));
747 kernel.packet[16] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_17_16));
748 kernel.packet[17] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_17_16));
749 kernel.packet[18] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_19_18));
750 kernel.packet[19] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_19_18));
751 kernel.packet[20] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_21_20));
752 kernel.packet[21] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_21_20));
753 kernel.packet[22] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_23_22));
754 kernel.packet[23] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_23_22));
755 kernel.packet[24] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_25_24));
756 kernel.packet[25] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_25_24));
757 kernel.packet[26] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_27_26));
758 kernel.packet[27] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_27_26));
759 kernel.packet[28] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_29_28));
760 kernel.packet[29] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_29_28));
761 kernel.packet[30] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_31_30));
762 kernel.packet[31] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_31_30));
763}
764
765template <HVXPacketSize T>
766EIGEN_STRONG_INLINE float predux_hvx(const HVXPacket<T>& a) {
767 const Index packet_size = unpacket_traits<HVXPacket<T>>::size;
768#if __HVX_ARCH__ >= 79
769 HVX_Vector vsum = Q6_Vsf_vadd_VsfVsf(a.Get(), Q6_V_vror_VR(a.Get(), sizeof(float)));
770 for (int i = 2; i < packet_size; i <<= 1) {
771 vsum = Q6_Vsf_vadd_VsfVsf(vsum, Q6_V_vror_VR(vsum, i * sizeof(float)));
772 }
773 return pfirst(HVXPacket<T>::Create(vsum));
774#else
775 HVX_Vector vsum = Q6_Vqf32_vadd_VsfVsf(a.Get(), Q6_V_vror_VR(a.Get(), sizeof(float)));
776 for (int i = 2; i < packet_size; i <<= 1) {
777 vsum = Q6_Vqf32_vadd_Vqf32Vqf32(vsum, Q6_V_vror_VR(vsum, i * sizeof(float)));
778 }
779 return pfirst(HVXPacket<T>::Create(Q6_Vsf_equals_Vqf32(vsum)));
780#endif
781}
782template <>
783EIGEN_STRONG_INLINE float predux<Packet32f>(const Packet32f& a) {
784 return predux_hvx(a);
785}
786template <>
787EIGEN_STRONG_INLINE float predux<Packet16f>(const Packet16f& a) {
788 return predux_hvx(a);
789}
790template <>
791EIGEN_STRONG_INLINE float predux<Packet8f>(const Packet8f& a) {
792 return predux_hvx(a);
793}
794
795template <HVXPacketSize T>
796EIGEN_STRONG_INLINE HVXPacket<T> ploaddup_hvx(const float* from) {
797 constexpr Index size = unpacket_traits<HVXPacket<T>>::size / 2;
798 HVX_Vector load = HVX_load_partial<size, 0>(from);
799 HVX_VectorPair dup = Q6_W_vshuff_VVR(load, load, -4);
800 return HVXPacket<T>::Create(HEXAGON_HVX_GET_V0(dup));
801}
802template <>
803EIGEN_STRONG_INLINE Packet32f ploaddup(const float* from) {
804 return ploaddup_hvx<HVXPacketSize::Full>(from);
805}
806template <>
807EIGEN_STRONG_INLINE Packet16f ploaddup(const float* from) {
808 return ploaddup_hvx<HVXPacketSize::Half>(from);
809}
810template <>
811EIGEN_STRONG_INLINE Packet8f ploaddup(const float* from) {
812 return ploaddup_hvx<HVXPacketSize::Quarter>(from);
813}
814
815template <HVXPacketSize T>
816EIGEN_STRONG_INLINE HVXPacket<T> ploadquad_hvx(const float* from) {
817 constexpr Index size = unpacket_traits<HVXPacket<T>>::size / 4;
818 HVX_Vector load = HVX_load_partial<size, 0>(from);
819 HVX_VectorPair dup = Q6_W_vshuff_VVR(load, load, -4);
820 HVX_VectorPair quad = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(dup), HEXAGON_HVX_GET_V0(dup), -8);
821 return HVXPacket<T>::Create(HEXAGON_HVX_GET_V0(quad));
822}
823template <>
824EIGEN_STRONG_INLINE Packet32f ploadquad(const float* from) {
825 return ploadquad_hvx<HVXPacketSize::Full>(from);
826}
827template <>
828EIGEN_STRONG_INLINE Packet16f ploadquad(const float* from) {
829 return ploadquad_hvx<HVXPacketSize::Half>(from);
830}
831template <>
832EIGEN_STRONG_INLINE Packet8f ploadquad(const float* from) {
833 return ploadquad_hvx<HVXPacketSize::Quarter>(from);
834}
835
836template <>
837EIGEN_STRONG_INLINE Packet32f preverse(const Packet32f& a) {
838 HVX_Vector delta = Q6_Vb_vsplat_R(0x7c);
839 return Packet32f::Create(Q6_V_vdelta_VV(a.Get(), delta));
840}
841
842template <>
843EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) {
844 HVX_Vector delta = Q6_Vb_vsplat_R(0x3c);
845 return Packet16f::Create(Q6_V_vdelta_VV(a.Get(), delta));
846}
847
848template <>
849EIGEN_STRONG_INLINE Packet8f preverse(const Packet8f& a) {
850 HVX_Vector delta = Q6_Vb_vsplat_R(0x1c);
851 return Packet8f::Create(Q6_V_vdelta_VV(a.Get(), delta));
852}
853
854template <HVXPacketSize T>
855EIGEN_STRONG_INLINE HVXPacket<T> pmin_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
856 return HVXPacket<T>::Create(Q6_Vsf_vmin_VsfVsf(a.Get(), b.Get()));
857}
858template <>
859EIGEN_STRONG_INLINE Packet32f pmin(const Packet32f& a, const Packet32f& b) {
860 return pmin_hvx(a, b);
861}
862template <>
863EIGEN_STRONG_INLINE Packet16f pmin(const Packet16f& a, const Packet16f& b) {
864 return pmin_hvx(a, b);
865}
866template <>
867EIGEN_STRONG_INLINE Packet8f pmin(const Packet8f& a, const Packet8f& b) {
868 return pmin_hvx(a, b);
869}
870
871template <HVXPacketSize T>
872EIGEN_STRONG_INLINE HVXPacket<T> pmax_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
873 return HVXPacket<T>::Create(Q6_Vsf_vmax_VsfVsf(a.Get(), b.Get()));
874}
875template <>
876EIGEN_STRONG_INLINE Packet32f pmax(const Packet32f& a, const Packet32f& b) {
877 return pmax_hvx(a, b);
878}
879template <>
880EIGEN_STRONG_INLINE Packet16f pmax(const Packet16f& a, const Packet16f& b) {
881 return pmax_hvx(a, b);
882}
883template <>
884EIGEN_STRONG_INLINE Packet8f pmax(const Packet8f& a, const Packet8f& b) {
885 return pmax_hvx(a, b);
886}
887
888template <HVXPacketSize T>
889EIGEN_STRONG_INLINE HVXPacket<T> pand_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
890 return HVXPacket<T>::Create(a.Get() & b.Get());
891}
892template <>
893EIGEN_STRONG_INLINE Packet32f pand(const Packet32f& a, const Packet32f& b) {
894 return pand_hvx(a, b);
895}
896template <>
897EIGEN_STRONG_INLINE Packet16f pand(const Packet16f& a, const Packet16f& b) {
898 return pand_hvx(a, b);
899}
900template <>
901EIGEN_STRONG_INLINE Packet8f pand(const Packet8f& a, const Packet8f& b) {
902 return pand_hvx(a, b);
903}
904
905template <HVXPacketSize T>
906EIGEN_STRONG_INLINE HVXPacket<T> por_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
907 return HVXPacket<T>::Create(a.Get() | b.Get());
908}
909template <>
910EIGEN_STRONG_INLINE Packet32f por(const Packet32f& a, const Packet32f& b) {
911 return por_hvx(a, b);
912}
913template <>
914EIGEN_STRONG_INLINE Packet16f por(const Packet16f& a, const Packet16f& b) {
915 return por_hvx(a, b);
916}
917template <>
918EIGEN_STRONG_INLINE Packet8f por(const Packet8f& a, const Packet8f& b) {
919 return por_hvx(a, b);
920}
921
922template <HVXPacketSize T>
923EIGEN_STRONG_INLINE HVXPacket<T> pxor_hvx(const HVXPacket<T>& a, const HVXPacket<T>& b) {
924 return HVXPacket<T>::Create(a.Get() ^ b.Get());
925}
926template <>
927EIGEN_STRONG_INLINE Packet32f pxor(const Packet32f& a, const Packet32f& b) {
928 return pxor_hvx(a, b);
929}
930template <>
931EIGEN_STRONG_INLINE Packet16f pxor(const Packet16f& a, const Packet16f& b) {
932 return pxor_hvx(a, b);
933}
934template <>
935EIGEN_STRONG_INLINE Packet8f pxor(const Packet8f& a, const Packet8f& b) {
936 return pxor_hvx(a, b);
937}
938
939template <HVXPacketSize T>
940EIGEN_STRONG_INLINE HVXPacket<T> pnot_hvx(const HVXPacket<T>& a) {
941 return HVXPacket<T>::Create(~a.Get());
942}
943template <>
944EIGEN_STRONG_INLINE Packet32f pnot(const Packet32f& a) {
945 return pnot_hvx(a);
946}
947template <>
948EIGEN_STRONG_INLINE Packet16f pnot(const Packet16f& a) {
949 return pnot_hvx(a);
950}
951template <>
952EIGEN_STRONG_INLINE Packet8f pnot(const Packet8f& a) {
953 return pnot_hvx(a);
954}
955
956template <HVXPacketSize T>
957EIGEN_STRONG_INLINE HVXPacket<T> pselect_hvx(const HVXPacket<T>& mask, const HVXPacket<T>& a, const HVXPacket<T>& b) {
958 HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(mask.Get(), Q6_V_vzero());
959 return HVXPacket<T>::Create(Q6_V_vmux_QVV(pred, b.Get(), a.Get()));
960}
961template <>
962EIGEN_STRONG_INLINE Packet32f pselect(const Packet32f& mask, const Packet32f& a, const Packet32f& b) {
963 return pselect_hvx(mask, a, b);
964}
965template <>
966EIGEN_STRONG_INLINE Packet16f pselect(const Packet16f& mask, const Packet16f& a, const Packet16f& b) {
967 return pselect_hvx(mask, a, b);
968}
969template <>
970EIGEN_STRONG_INLINE Packet8f pselect(const Packet8f& mask, const Packet8f& a, const Packet8f& b) {
971 return pselect_hvx(mask, a, b);
972}
973
974template <HVXPacketSize T, typename Op>
975EIGEN_STRONG_INLINE float predux_generic(const HVXPacket<T>& a, Op op) {
976 const Index packet_size = unpacket_traits<HVXPacket<T>>::size;
977 HVXPacket<T> vredux = a;
978 for (int i = 1; i < packet_size; i <<= 1) {
979 vredux = op(vredux, HVXPacket<T>::Create(Q6_V_vror_VR(vredux.Get(), i * sizeof(float))));
980 }
981 return pfirst(vredux);
982}
983
984template <>
985EIGEN_STRONG_INLINE float predux_max(const Packet32f& a) {
986 return predux_generic(a, pmax<Packet32f>);
987}
988template <>
989EIGEN_STRONG_INLINE float predux_max(const Packet16f& a) {
990 return predux_generic(a, pmax<Packet16f>);
991}
992template <>
993EIGEN_STRONG_INLINE float predux_max(const Packet8f& a) {
994 return predux_generic(a, pmax<Packet8f>);
995}
996
997template <>
998EIGEN_STRONG_INLINE float predux_min(const Packet32f& a) {
999 return predux_generic(a, pmin<Packet32f>);
1000}
1001template <>
1002EIGEN_STRONG_INLINE float predux_min(const Packet16f& a) {
1003 return predux_generic(a, pmin<Packet16f>);
1004}
1005template <>
1006EIGEN_STRONG_INLINE float predux_min(const Packet8f& a) {
1007 return predux_generic(a, pmin<Packet8f>);
1008}
1009
1010template <HVXPacketSize T>
1011EIGEN_STRONG_INLINE bool predux_any_hvx(const HVXPacket<T>& a) {
1012 const Index packet_size = unpacket_traits<HVXPacket<T>>::size;
1013 HVX_Vector reduced = a.Get();
1014 for (int i = 1; i < packet_size; i <<= 1) {
1015 reduced |= Q6_V_vror_VR(reduced, i * sizeof(float));
1016 }
1017 return Q6_R_vextract_VR(reduced, 0) != 0;
1018}
1019
1020template <>
1021EIGEN_STRONG_INLINE bool predux_any(const Packet32f& a) {
1022 return predux_any_hvx(a);
1023}
1024template <>
1025EIGEN_STRONG_INLINE bool predux_any(const Packet16f& a) {
1026 return predux_any_hvx(a);
1027}
1028template <>
1029EIGEN_STRONG_INLINE bool predux_any(const Packet8f& a) {
1030 return predux_any_hvx(a);
1031}
1032
1033static const float index_vsf[32]
1034 __attribute__((aligned(__HVX_LENGTH__))) = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
1035 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
1036
1037template <HVXPacketSize T>
1038EIGEN_STRONG_INLINE HVXPacket<T> plset_hvx(const float& a) {
1039 return padd(pload<HVXPacket<T>>(index_vsf), pset1<HVXPacket<T>>(a));
1040}
1041template <>
1042EIGEN_STRONG_INLINE Packet32f plset(const float& a) {
1043 return plset_hvx<HVXPacketSize::Full>(a);
1044}
1045template <>
1046EIGEN_STRONG_INLINE Packet16f plset(const float& a) {
1047 return plset_hvx<HVXPacketSize::Half>(a);
1048}
1049template <>
1050EIGEN_STRONG_INLINE Packet8f plset(const float& a) {
1051 return plset_hvx<HVXPacketSize::Quarter>(a);
1052}
1053
1054template <HVXPacketSize T>
1055EIGEN_STRONG_INLINE void pscatter_hvx(float* to, const HVXPacket<T>& from, Index stride) {
1056 const Index packet_size = unpacket_traits<HVXPacket<T>>::size;
1057 float elements[packet_size] __attribute__((aligned(__HVX_LENGTH__)));
1058 pstore<float>(elements, from);
1059 for (Index i = 0; i < packet_size; ++i) {
1060 to[i * stride] = elements[i];
1061 }
1062}
1063template <>
1064EIGEN_STRONG_INLINE void pscatter<float, Packet32f>(float* to, const Packet32f& from, Index stride) {
1065 pscatter_hvx(to, from, stride);
1066}
1067template <>
1068EIGEN_STRONG_INLINE void pscatter<float, Packet16f>(float* to, const Packet16f& from, Index stride) {
1069 pscatter_hvx(to, from, stride);
1070}
1071template <>
1072EIGEN_STRONG_INLINE void pscatter<float, Packet8f>(float* to, const Packet8f& from, Index stride) {
1073 pscatter_hvx(to, from, stride);
1074}
1075
1076template <HVXPacketSize T>
1077EIGEN_STRONG_INLINE HVXPacket<T> pgather_hvx(const float* from, Index stride) {
1078 const Index packet_size = unpacket_traits<HVXPacket<T>>::size;
1079 float elements[packet_size] __attribute__((aligned(__HVX_LENGTH__)));
1080 for (Index i = 0; i < packet_size; i++) {
1081 elements[i] = from[i * stride];
1082 }
1083 return pload<HVXPacket<T>>(elements);
1084}
1085template <>
1086EIGEN_STRONG_INLINE Packet32f pgather<float, Packet32f>(const float* from, Index stride) {
1087 return pgather_hvx<HVXPacketSize::Full>(from, stride);
1088}
1089template <>
1090EIGEN_STRONG_INLINE Packet16f pgather<float, Packet16f>(const float* from, Index stride) {
1091 return pgather_hvx<HVXPacketSize::Half>(from, stride);
1092}
1093template <>
1094EIGEN_STRONG_INLINE Packet8f pgather<float, Packet8f>(const float* from, Index stride) {
1095 return pgather_hvx<HVXPacketSize::Quarter>(from, stride);
1096}
1097
1098} // end namespace internal
1099} // end namespace Eigen
1100
1101#endif // __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68
1102
1103#endif // EIGEN_HVX_PACKET_MATH_H
Eigen::Aligned64
@ Aligned64
Definition Constants.h:240
Eigen::Aligned128
@ Aligned128
Definition Constants.h:241
Eigen::Aligned32
@ Aligned32
Definition Constants.h:239