12#ifndef EIGEN_PACKET_MATH_LSX_H
13#define EIGEN_PACKET_MATH_LSX_H
16#include "../../InternalHeaderCheck.h"
22#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
23#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
26#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
27#if EIGEN_ARCH_LOONGARCH64
28#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
32#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
33#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
36using Packet4f = __m128;
37using Packet2d = __m128d;
39using Packet16c = eigen_packet_wrapper<__m128i, 0>;
40using Packet8s = eigen_packet_wrapper<__m128i, 1>;
41using Packet4i = eigen_packet_wrapper<__m128i, 2>;
42using Packet2l = eigen_packet_wrapper<__m128i, 3>;
43using Packet16uc = eigen_packet_wrapper<__m128i, 4>;
44using Packet8us = eigen_packet_wrapper<__m128i, 5>;
45using Packet4ui = eigen_packet_wrapper<__m128i, 6>;
46using Packet2ul = eigen_packet_wrapper<__m128i, 7>;
49struct is_arithmetic<__m128> : std::true_type {};
51struct is_arithmetic<__m128i> : std::true_type {};
53struct is_arithmetic<__m128d> : std::true_type {};
55struct is_arithmetic<Packet16c> : std::true_type {};
57struct is_arithmetic<Packet8s> : std::true_type {};
59struct is_arithmetic<Packet4i> : std::true_type {};
61struct is_arithmetic<Packet2l> : std::true_type {};
63struct is_arithmetic<Packet16uc> : std::false_type {};
65struct is_arithmetic<Packet8us> : std::false_type {};
67struct is_arithmetic<Packet4ui> : std::false_type {};
69struct is_arithmetic<Packet2ul> : std::false_type {};
71EIGEN_ALWAYS_INLINE Packet4f make_packet4f(
float a,
float b,
float c,
float d) {
72 float from[4] = {a, b, c, d};
73 return (Packet4f)__lsx_vld(from, 0);
76EIGEN_STRONG_INLINE Packet4f shuffle1(
const Packet4f& m,
int mask) {
77 const float* a =
reinterpret_cast<const float*
>(&m);
79 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3)));
83template <
bool interleave>
84EIGEN_STRONG_INLINE Packet4f shuffle2(
const Packet4f& m,
const Packet4f& n,
int mask) {
85 const float* a =
reinterpret_cast<const float*
>(&m);
86 const float* b =
reinterpret_cast<const float*
>(&n);
88 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
93EIGEN_STRONG_INLINE Packet4f shuffle2<true>(
const Packet4f& m,
const Packet4f& n,
int mask) {
94 const float* a =
reinterpret_cast<const float*
>(&m);
95 const float* b =
reinterpret_cast<const float*
>(&n);
97 make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
101EIGEN_STRONG_INLINE
static int eigen_lsx_shuffle_mask(
int p,
int q,
int r,
int s) {
102 return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
105EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(
const Packet4f& a,
int p,
int q,
int r,
int s) {
106 return shuffle1(a, eigen_lsx_shuffle_mask(p, q, r, s));
108EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(
const Packet4f& a,
const Packet4f& b,
int p,
int q,
int r,
int s) {
109 return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(p, q, r, s));
111EIGEN_STRONG_INLINE Packet4f vec4f_movelh(
const Packet4f& a,
const Packet4f& b) {
112 return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(0, 1, 0, 1));
114EIGEN_STRONG_INLINE Packet4f vec4f_movehl(
const Packet4f& a,
const Packet4f& b) {
115 return shuffle2<false>(b, a, eigen_lsx_shuffle_mask(2, 3, 2, 3));
117EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(
const Packet4f& a,
const Packet4f& b) {
118 return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(0, 0, 1, 1));
120EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(
const Packet4f& a,
const Packet4f& b) {
121 return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(2, 2, 3, 3));
124EIGEN_ALWAYS_INLINE Packet2d make_packet2d(
double a,
double b) {
125 double from[2] = {a, b};
126 return (Packet2d)__lsx_vld(from, 0);
129EIGEN_STRONG_INLINE Packet2d shuffle(
const Packet2d& m,
const Packet2d& n,
int mask) {
130 const double* a =
reinterpret_cast<const double*
>(&m);
131 const double* b =
reinterpret_cast<const double*
>(&n);
132 Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1)));
136EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(
const Packet2d& a,
const Packet2d& b,
int mask) {
137 return shuffle(a, b, mask);
139EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(
const Packet2d& a,
const Packet2d& b) {
return shuffle(a, b, 0); }
140EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(
const Packet2d& a,
const Packet2d& b) {
return shuffle(a, b, 3); }
143struct packet_traits<int8_t> : default_packet_traits {
144 using type = Packet16c;
145 using half = Packet16c;
157struct packet_traits<int16_t> : default_packet_traits {
158 using type = Packet8s;
159 using half = Packet8s;
172struct packet_traits<int32_t> : default_packet_traits {
173 using type = Packet4i;
174 using half = Packet4i;
187struct packet_traits<int64_t> : default_packet_traits {
188 using type = Packet2l;
189 using half = Packet2l;
202struct packet_traits<uint8_t> : default_packet_traits {
203 using type = Packet16uc;
204 using half = Packet16uc;
217struct packet_traits<uint16_t> : default_packet_traits {
218 using type = Packet8us;
219 using half = Packet8us;
233struct packet_traits<uint32_t> : default_packet_traits {
234 using type = Packet4ui;
235 using half = Packet4ui;
249struct packet_traits<uint64_t> : default_packet_traits {
250 using type = Packet2ul;
251 using half = Packet2ul;
265struct packet_traits<float> : default_packet_traits {
266 using type = Packet4f;
267 using half = Packet4f;
286struct packet_traits<double> : default_packet_traits {
287 using type = Packet2d;
288 using half = Packet2d;
306struct unpacket_traits<Packet16c> {
308 using half = Packet16c;
313 masked_load_available =
false,
314 masked_store_available =
false
318struct unpacket_traits<Packet8s> {
319 using type = int16_t;
320 using half = Packet8s;
325 masked_load_available =
false,
326 masked_store_available =
false
330struct unpacket_traits<Packet4i> {
331 using type = int32_t;
332 using half = Packet4i;
337 masked_load_available =
false,
338 masked_store_available =
false
342struct unpacket_traits<Packet2l> {
343 using type = int64_t;
344 using half = Packet2l;
349 masked_load_available =
false,
350 masked_store_available =
false
354struct unpacket_traits<Packet16uc> {
355 using type = uint8_t;
356 using half = Packet16uc;
361 masked_load_available =
false,
362 masked_store_available =
false
366struct unpacket_traits<Packet8us> {
367 using type = uint16_t;
368 using half = Packet8us;
373 masked_load_available =
false,
374 masked_store_available =
false
378struct unpacket_traits<Packet4ui> {
379 using type = uint32_t;
380 using half = Packet4ui;
385 masked_load_available =
false,
386 masked_store_available =
false
390struct unpacket_traits<Packet2ul> {
391 using type = uint64_t;
392 using half = Packet2ul;
397 masked_load_available =
false,
398 masked_store_available =
false
402struct unpacket_traits<Packet4f> {
404 using half = Packet4f;
405 using integer_packet = Packet4i;
410 masked_load_available =
false,
411 masked_store_available =
false
415struct unpacket_traits<Packet2d> {
417 using half = Packet2d;
418 using integer_packet = Packet2l;
423 masked_load_available =
false,
424 masked_store_available =
false
429EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(
const int8_t& from) {
430 return __lsx_vreplgr2vr_b(from);
433EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(
const int16_t& from) {
434 return __lsx_vreplgr2vr_h(from);
437EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int32_t& from) {
438 return __lsx_vreplgr2vr_w(from);
441EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(
const int64_t& from) {
442 return __lsx_vreplgr2vr_d(from);
445EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(
const uint8_t& from) {
446 return __lsx_vreplgr2vr_b(from);
449EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(
const uint16_t& from) {
450 return __lsx_vreplgr2vr_h(from);
453EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(
const uint32_t& from) {
454 return __lsx_vreplgr2vr_w(from);
457EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(
const uint64_t& from) {
458 return __lsx_vreplgr2vr_d(from);
461EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
462 Packet4f v = {from, from, from, from};
466EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
467 Packet2d v = {from, from};
472EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
473 return reinterpret_cast<__m128
>((__m128i)pset1<Packet4ui>(from));
476EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
477 return reinterpret_cast<__m128d
>((__m128i)pset1<Packet2ul>(from));
481EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(
const int8_t& a) {
482 const int8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
483 return __lsx_vadd_b(pset1<Packet16c>(a), __lsx_vld(countdown, 0));
486EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(
const int16_t& a) {
487 const int16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
488 return __lsx_vadd_h(pset1<Packet8s>(a), __lsx_vld(countdown, 0));
491EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int32_t& a) {
492 const int32_t countdown[] = {0, 1, 2, 3};
493 return __lsx_vadd_w(pset1<Packet4i>(a), __lsx_vld(countdown, 0));
496EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(
const int64_t& a) {
497 const int64_t countdown[] = {0, 1};
498 return __lsx_vadd_d(pset1<Packet2l>(a), __lsx_vld(countdown, 0));
501EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(
const uint8_t& a) {
502 const uint8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
503 return __lsx_vadd_b(pset1<Packet16uc>(a), __lsx_vld(countdown, 0));
506EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(
const uint16_t& a) {
507 const uint16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
508 return __lsx_vadd_h(pset1<Packet8us>(a), __lsx_vld(countdown, 0));
511EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(
const uint32_t& a) {
512 const uint32_t countdown[] = {0, 1, 2, 3};
513 return __lsx_vadd_w(pset1<Packet4ui>(a), __lsx_vld(countdown, 0));
516EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(
const uint64_t& a) {
517 const uint64_t countdown[] = {0, 1};
518 return __lsx_vadd_d(pset1<Packet2ul>(a), __lsx_vld(countdown, 0));
521EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a) {
522 static const Packet4f countdown = {0.0f, 1.0f, 2.0f, 3.0f};
523 return __lsx_vfadd_s(pset1<Packet4f>(a), countdown);
526EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a) {
527 static const Packet2d countdown = {0.0f, 1.0f};
528 return __lsx_vfadd_d(pset1<Packet2d>(a), countdown);
532EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
533 return __lsx_vadd_b(a, b);
536EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
537 return __lsx_vadd_h(a, b);
540EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
541 return __lsx_vadd_w(a, b);
544EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
545 return __lsx_vadd_d(a, b);
548EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
549 return __lsx_vadd_b(a, b);
552EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
553 return __lsx_vadd_h(a, b);
556EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
557 return __lsx_vadd_w(a, b);
560EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
561 return __lsx_vadd_d(a, b);
564EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
565 return __lsx_vfadd_s(a, b);
568EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
569 return __lsx_vfadd_d(a, b);
573EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
574 return __lsx_vsub_b(a, b);
577EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
578 return __lsx_vsub_h(a, b);
581EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
582 return __lsx_vsub_w(a, b);
585EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
586 return __lsx_vsub_d(a, b);
589EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
590 return __lsx_vsub_b(a, b);
593EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
594 return __lsx_vsub_h(a, b);
597EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
598 return __lsx_vsub_w(a, b);
601EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
602 return __lsx_vsub_d(a, b);
605EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
606 return __lsx_vfsub_s(a, b);
609EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
610 return __lsx_vfsub_d(a, b);
614EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b);
616EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
617 const Packet4f mask =
618 make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
619 return padd(a, pxor(mask, b));
622EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b);
624EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
625 const Packet2d mask = make_packet2d(numext::bit_cast<double>(0x8000000000000000ull), 0.0);
626 return padd(a, pxor(mask, b));
630EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
631 Packet4f mask = make_packet4f(numext::bit_cast<float>(0x80000000), numext::bit_cast<float>(0x80000000),
632 numext::bit_cast<float>(0x80000000), numext::bit_cast<float>(0x80000000));
633 return (Packet4f)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a));
636EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
638 make_packet2d(numext::bit_cast<double>(0x8000000000000000), numext::bit_cast<double>(0x8000000000000000));
639 return (Packet2d)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a));
642EIGEN_STRONG_INLINE Packet16c pnegate(
const Packet16c& a) {
643 return __lsx_vneg_b(a);
646EIGEN_STRONG_INLINE Packet8s pnegate(
const Packet8s& a) {
647 return __lsx_vneg_h(a);
650EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
651 return __lsx_vneg_w(a);
654EIGEN_STRONG_INLINE Packet2l pnegate(
const Packet2l& a) {
655 return __lsx_vneg_d(a);
659EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
660 return __lsx_vfmul_s(a, b);
663EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
664 return __lsx_vfmul_d(a, b);
667EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
668 return __lsx_vmul_b(a, b);
671EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
672 return __lsx_vmul_h(a, b);
675EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
676 return __lsx_vmul_w(a, b);
679EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
680 return __lsx_vmul_d(a, b);
683EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
684 return __lsx_vmul_b(a, b);
687EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
688 return __lsx_vmul_h(a, b);
691EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
692 return __lsx_vmul_w(a, b);
695EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
696 return __lsx_vmul_d(a, b);
700EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
701 return __lsx_vfdiv_s(a, b);
704EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
705 return __lsx_vfdiv_d(a, b);
708EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
709 return __lsx_vdiv_h(a, b);
712EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
713 return __lsx_vdiv_w(a, b);
716EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
717 return __lsx_vdiv_d(a, b);
720EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
721 return __lsx_vdiv_hu(a, b);
724EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
725 return __lsx_vdiv_wu(a, b);
728EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
729 return __lsx_vdiv_du(a, b);
733struct is_fused_madd<Packet4f> : true_type {};
735EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
736 return __lsx_vfmadd_s(a, b, c);
739struct is_fused_madd<Packet2d> : true_type {};
741EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
742 return __lsx_vfmadd_d(a, b, c);
745EIGEN_STRONG_INLINE Packet4f pmsub(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
746 return __lsx_vfmsub_s(a, b, c);
749EIGEN_STRONG_INLINE Packet2d pmsub(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
750 return __lsx_vfmsub_d(a, b, c);
753EIGEN_STRONG_INLINE Packet4f pnmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
754 return __lsx_vfnmsub_s(a, b, c);
757EIGEN_STRONG_INLINE Packet2d pnmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
758 return __lsx_vfnmsub_d(a, b, c);
761EIGEN_STRONG_INLINE Packet4f pnmsub(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
762 return __lsx_vfnmadd_s(a, b, c);
765EIGEN_STRONG_INLINE Packet2d pnmsub(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
766 return __lsx_vfnmadd_d(a, b, c);
769EIGEN_STRONG_INLINE Packet16c pmadd(
const Packet16c& a,
const Packet16c& b,
const Packet16c& c) {
770 return __lsx_vmadd_b(c, a, b);
773EIGEN_STRONG_INLINE Packet8s pmadd(
const Packet8s& a,
const Packet8s& b,
const Packet8s& c) {
774 return __lsx_vmadd_h(c, a, b);
777EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
778 return __lsx_vmadd_w(c, a, b);
781EIGEN_STRONG_INLINE Packet2l pmadd(
const Packet2l& a,
const Packet2l& b,
const Packet2l& c) {
782 return __lsx_vmadd_d(c, a, b);
785EIGEN_STRONG_INLINE Packet16uc pmadd(
const Packet16uc& a,
const Packet16uc& b,
const Packet16uc& c) {
786 return __lsx_vmadd_b(c, a, b);
789EIGEN_STRONG_INLINE Packet8us pmadd(
const Packet8us& a,
const Packet8us& b,
const Packet8us& c) {
790 return __lsx_vmadd_h(c, a, b);
793EIGEN_STRONG_INLINE Packet4ui pmadd(
const Packet4ui& a,
const Packet4ui& b,
const Packet4ui& c) {
794 return __lsx_vmadd_w(c, a, b);
797EIGEN_STRONG_INLINE Packet2ul pmadd(
const Packet2ul& a,
const Packet2ul& b,
const Packet2ul& c) {
798 return __lsx_vmadd_d(c, a, b);
802EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
803 return (Packet4f)__lsx_vand_v((__m128i)a, (__m128i)b);
806EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
807 return (Packet2d)__lsx_vand_v((__m128i)a, (__m128i)b);
810EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
811 return __lsx_vand_v(a, b);
814EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
815 return __lsx_vand_v(a, b);
818EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
819 return __lsx_vand_v(a, b);
822EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
823 return __lsx_vand_v(a, b);
826EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
827 return __lsx_vand_v(a, b);
830EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
831 return __lsx_vand_v(a, b);
834EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
835 return __lsx_vand_v(a, b);
838EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
839 return __lsx_vand_v(a, b);
843EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
844 return (Packet4f)__lsx_vor_v((__m128i)a, (__m128i)b);
847EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
848 return (Packet2d)__lsx_vor_v((__m128i)a, (__m128i)b);
851EIGEN_STRONG_INLINE Packet16c por<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
852 return __lsx_vor_v(a, b);
855EIGEN_STRONG_INLINE Packet8s por<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
856 return __lsx_vor_v(a, b);
859EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
860 return __lsx_vor_v(a, b);
863EIGEN_STRONG_INLINE Packet2l por<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
864 return __lsx_vor_v(a, b);
867EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
868 return __lsx_vor_v(a, b);
871EIGEN_STRONG_INLINE Packet8us por<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
872 return __lsx_vor_v(a, b);
875EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
876 return __lsx_vor_v(a, b);
879EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
880 return __lsx_vor_v(a, b);
884EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
885 return (Packet4f)__lsx_vxor_v((__m128i)a, (__m128i)b);
888EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
889 return (Packet2d)__lsx_vxor_v((__m128i)a, (__m128i)b);
892EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
893 return __lsx_vxor_v(a, b);
896EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
897 return __lsx_vxor_v(a, b);
900EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
901 return __lsx_vxor_v(a, b);
904EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
905 return __lsx_vxor_v(a, b);
908EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
909 return __lsx_vxor_v(a, b);
912EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
913 return __lsx_vxor_v(a, b);
916EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
917 return __lsx_vxor_v(a, b);
920EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
921 return __lsx_vxor_v(a, b);
925EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
926 return (Packet4f)__lsx_vandn_v((__m128i)b, (__m128i)a);
929EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
930 return (Packet2d)__lsx_vandn_v((__m128i)b, (__m128i)a);
933EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
934 return __lsx_vandn_v(b, a);
937EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
938 return __lsx_vandn_v(b, a);
941EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
942 return __lsx_vandn_v(b, a);
945EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
946 return __lsx_vandn_v(b, a);
949EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
950 return __lsx_vandn_v(b, a);
953EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
954 return __lsx_vandn_v(b, a);
957EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
958 return __lsx_vandn_v(b, a);
961EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
962 return __lsx_vandn_v(b, a);
966EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
967 return (Packet4f)__lsx_vfcmp_cle_s(a, b);
970EIGEN_STRONG_INLINE Packet2d pcmp_le<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
971 return (Packet2d)__lsx_vfcmp_cle_d(a, b);
974EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
975 return __lsx_vsle_b(a, b);
978EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
979 return __lsx_vsle_h(a, b);
982EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
983 return __lsx_vsle_w(a, b);
986EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
987 return __lsx_vsle_d(a, b);
990EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
991 return __lsx_vsle_bu(a, b);
994EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
995 return __lsx_vsle_hu(a, b);
998EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
999 return __lsx_vsle_wu(a, b);
1002EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1003 return __lsx_vsle_du(a, b);
1007EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1008 return (Packet4f)__lsx_vfcmp_clt_s(a, b);
1011EIGEN_STRONG_INLINE Packet2d pcmp_lt<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1012 return (Packet2d)__lsx_vfcmp_clt_d(a, b);
1015EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1016 return __lsx_vslt_b(a, b);
1019EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1020 return __lsx_vslt_h(a, b);
1023EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1024 return __lsx_vslt_w(a, b);
1027EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1028 return __lsx_vslt_d(a, b);
1031EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1032 return __lsx_vslt_bu(a, b);
1035EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1036 return __lsx_vslt_hu(a, b);
1039EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1040 return __lsx_vslt_wu(a, b);
1043EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1044 return __lsx_vslt_du(a, b);
1048EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1049 return (Packet4f)__lsx_vfcmp_sult_s(a, b);
1052EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1053 return (Packet2d)__lsx_vfcmp_sult_d(a, b);
1057EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1058 return (Packet4f)__lsx_vfcmp_seq_s(a, b);
1061EIGEN_STRONG_INLINE Packet2d pcmp_eq<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1062 return (Packet2d)__lsx_vfcmp_seq_d(a, b);
1065EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1066 return __lsx_vseq_b(a, b);
1069EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1070 return __lsx_vseq_h(a, b);
1073EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1074 return __lsx_vseq_w(a, b);
1077EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1078 return __lsx_vseq_d(a, b);
1081EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1082 return __lsx_vseq_b(a, b);
1085EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1086 return __lsx_vseq_h(a, b);
1089EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1090 return __lsx_vseq_w(a, b);
1093EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1094 return __lsx_vseq_d(a, b);
1098EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1099 return __lsx_vmin_b(a, b);
1102EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1103 return __lsx_vmin_h(a, b);
1106EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1107 return __lsx_vmin_w(a, b);
1110EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1111 return __lsx_vmin_d(a, b);
1114EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1115 return __lsx_vmin_bu(a, b);
1118EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1119 return __lsx_vmin_hu(a, b);
1122EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1123 return __lsx_vmin_wu(a, b);
1126EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1127 return __lsx_vmin_du(a, b);
1131EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(
const Packet16c& a,
const Packet16c& b) {
1132 return __lsx_vmax_b(a, b);
1135EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(
const Packet8s& a,
const Packet8s& b) {
1136 return __lsx_vmax_h(a, b);
1139EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
1140 return __lsx_vmax_w(a, b);
1143EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
1144 return __lsx_vmax_d(a, b);
1147EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
1148 return __lsx_vmax_bu(a, b);
1151EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(
const Packet8us& a,
const Packet8us& b) {
1152 return __lsx_vmax_hu(a, b);
1155EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(
const Packet4ui& a,
const Packet4ui& b) {
1156 return __lsx_vmax_wu(a, b);
1159EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(
const Packet2ul& a,
const Packet2ul& b) {
1160 return __lsx_vmax_du(a, b);
1164EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1165 Packet4i aNaN = __lsx_vfcmp_cun_s(a, a);
1166 Packet4i aMinOrNaN = por<Packet4i>(__lsx_vfcmp_clt_s(a, b), aNaN);
1167 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN);
1170EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1171 Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
1172 Packet2l aMinOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(a, b), aNaN);
1173 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN);
1176EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
1177 Packet4i aNaN = __lsx_vfcmp_cun_s(a, a);
1178 Packet4i aMaxOrNaN = por<Packet4i>(__lsx_vfcmp_clt_s(b, a), aNaN);
1179 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN);
1182EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1183 Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
1184 Packet2l aMaxOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(b, a), aNaN);
1185 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN);
1189EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(
const Packet16c& a) {
1190 return __lsx_vsrai_b((__m128i)a, N);
1193EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(
const Packet8s& a) {
1194 return __lsx_vsrai_h((__m128i)a, N);
1197EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(
const Packet4i& a) {
1198 return __lsx_vsrai_w((__m128i)a, N);
1201EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(
const Packet2l& a) {
1202 return __lsx_vsrai_d((__m128i)a, N);
1205EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(
const Packet16uc& a) {
1206 return __lsx_vsrli_b((__m128i)a, N);
1209EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(
const Packet8us& a) {
1210 return __lsx_vsrli_h((__m128i)a, N);
1213EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(
const Packet4ui& a) {
1214 return __lsx_vsrli_w((__m128i)a, N);
1217EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(
const Packet2ul& a) {
1218 return __lsx_vsrli_d((__m128i)a, N);
1222EIGEN_STRONG_INLINE Packet16c plogical_shift_right(
const Packet16c& a) {
1223 return __lsx_vsrli_b((__m128i)a, N);
1226EIGEN_STRONG_INLINE Packet8s plogical_shift_right(
const Packet8s& a) {
1227 return __lsx_vsrli_h((__m128i)a, N);
1230EIGEN_STRONG_INLINE Packet4i plogical_shift_right(
const Packet4i& a) {
1231 return __lsx_vsrli_w((__m128i)a, N);
1234EIGEN_STRONG_INLINE Packet2l plogical_shift_right(
const Packet2l& a) {
1235 return __lsx_vsrli_d((__m128i)a, N);
1238EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(
const Packet16uc& a) {
1239 return __lsx_vsrli_b((__m128i)a, N);
1242EIGEN_STRONG_INLINE Packet8us plogical_shift_right(
const Packet8us& a) {
1243 return __lsx_vsrli_h((__m128i)a, N);
1246EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(
const Packet4ui& a) {
1247 return __lsx_vsrli_w((__m128i)a, N);
1250EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(
const Packet2ul& a) {
1251 return __lsx_vsrli_d((__m128i)a, N);
1255EIGEN_STRONG_INLINE Packet16c plogical_shift_left(
const Packet16c& a) {
1256 return __lsx_vslli_b((__m128i)a, N);
1259EIGEN_STRONG_INLINE Packet8s plogical_shift_left(
const Packet8s& a) {
1260 return __lsx_vslli_h((__m128i)a, N);
1263EIGEN_STRONG_INLINE Packet4i plogical_shift_left(
const Packet4i& a) {
1264 return __lsx_vslli_w((__m128i)a, N);
1267EIGEN_STRONG_INLINE Packet2l plogical_shift_left(
const Packet2l& a) {
1268 return __lsx_vslli_d((__m128i)a, N);
1271EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(
const Packet16uc& a) {
1272 return __lsx_vslli_b((__m128i)a, N);
1275EIGEN_STRONG_INLINE Packet8us plogical_shift_left(
const Packet8us& a) {
1276 return __lsx_vslli_h((__m128i)a, N);
1279EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(
const Packet4ui& a) {
1280 return __lsx_vslli_w((__m128i)a, N);
1283EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(
const Packet2ul& a) {
1284 return __lsx_vslli_d((__m128i)a, N);
1288EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
1289 return (Packet4f)__lsx_vbitclri_w((__m128i)a, 31);
1292EIGEN_STRONG_INLINE Packet2d pabs(
const Packet2d& a) {
1293 return (Packet2d)__lsx_vbitclri_d((__m128i)a, 63);
1296EIGEN_STRONG_INLINE Packet16c pabs(
const Packet16c& a) {
1297 return __lsx_vabsd_b(a, pzero(a));
1300EIGEN_STRONG_INLINE Packet8s pabs(
const Packet8s& a) {
1301 return __lsx_vabsd_h(a, pzero(a));
1304EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
1305 return __lsx_vabsd_w(a, pzero(a));
1308EIGEN_STRONG_INLINE Packet2l pabs(
const Packet2l& a) {
1309 return __lsx_vabsd_d(a, pzero(a));
1312EIGEN_STRONG_INLINE Packet16c pabsdiff(
const Packet16c& a,
const Packet16c& b) {
1313 return __lsx_vabsd_b(a, b);
1316EIGEN_STRONG_INLINE Packet8s pabsdiff(
const Packet8s& a,
const Packet8s& b) {
1317 return __lsx_vabsd_h(a, b);
1320EIGEN_STRONG_INLINE Packet4i pabsdiff(
const Packet4i& a,
const Packet4i& b) {
1321 return __lsx_vabsd_w(a, b);
1324EIGEN_STRONG_INLINE Packet2l pabsdiff(
const Packet2l& a,
const Packet2l& b) {
1325 return __lsx_vabsd_d(a, b);
1328EIGEN_STRONG_INLINE Packet16uc pabsdiff(
const Packet16uc& a,
const Packet16uc& b) {
1329 return __lsx_vabsd_bu(a, b);
1332EIGEN_STRONG_INLINE Packet8us pabsdiff(
const Packet8us& a,
const Packet8us& b) {
1333 return __lsx_vabsd_hu(a, b);
1336EIGEN_STRONG_INLINE Packet4ui pabsdiff(
const Packet4ui& a,
const Packet4ui& b) {
1337 return __lsx_vabsd_wu(a, b);
1340EIGEN_STRONG_INLINE Packet2ul pabsdiff(
const Packet2ul& a,
const Packet2ul& b) {
1341 return __lsx_vabsd_du(a, b);
1344EIGEN_STRONG_INLINE Packet4f pabsdiff(
const Packet4f& a,
const Packet4f& b) {
1345 return pabs(psub(a, b));
1348EIGEN_STRONG_INLINE Packet2d pabsdiff(
const Packet2d& a,
const Packet2d& b) {
1349 return pabs(psub(a, b));
1353EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from) {
1354 EIGEN_DEBUG_ALIGNED_LOAD
return (Packet4f)__lsx_vld(from, 0);
1357EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from) {
1358 EIGEN_DEBUG_ALIGNED_LOAD
return (Packet2d)__lsx_vld(from, 0);
1361EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(
const int8_t* from) {
1362 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1365EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(
const int16_t* from) {
1366 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1369EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int32_t* from) {
1370 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1373EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(
const int64_t* from) {
1374 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1377EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(
const uint8_t* from) {
1378 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1381EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(
const uint16_t* from) {
1382 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1385EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(
const uint32_t* from) {
1386 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1389EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(
const uint64_t* from) {
1390 EIGEN_DEBUG_ALIGNED_LOAD
return __lsx_vld(from, 0);
1394EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from) {
1395 EIGEN_DEBUG_UNALIGNED_LOAD
return (Packet4f)__lsx_vld(from, 0);
1398EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from) {
1399 EIGEN_DEBUG_UNALIGNED_LOAD
return (Packet2d)__lsx_vld(from, 0);
1402EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(
const int8_t* from) {
1403 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1406EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(
const int16_t* from) {
1407 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1410EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int32_t* from) {
1411 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1414EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(
const int64_t* from) {
1415 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1418EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(
const uint8_t* from) {
1419 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1422EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(
const uint16_t* from) {
1423 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1426EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(
const uint32_t* from) {
1427 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1430EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(
const uint64_t* from) {
1431 EIGEN_DEBUG_UNALIGNED_LOAD
return __lsx_vld(from, 0);
1435EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from) {
1436 float f0 = from[0], f1 = from[1];
1437 return make_packet4f(f0, f0, f1, f1);
1440EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(
const int8_t* from) {
1442 memcpy(&tmp, from,
sizeof(tmp) / 2);
1443 return __lsx_vilvl_b(tmp, tmp);
1446EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(
const int16_t* from) {
1448 memcpy(&tmp, from,
sizeof(tmp) / 2);
1449 return __lsx_vilvl_h(tmp, tmp);
1452EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int32_t* from) {
1454 memcpy(&tmp, from,
sizeof(tmp) / 2);
1455 return __lsx_vilvl_w(tmp, tmp);
1458EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(
const uint8_t* from) {
1460 memcpy(&tmp, from,
sizeof(tmp) / 2);
1461 return __lsx_vilvl_b(tmp, tmp);
1464EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(
const uint16_t* from) {
1466 memcpy(&tmp, from,
sizeof(tmp) / 2);
1467 return __lsx_vilvl_h(tmp, tmp);
1470EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(
const uint32_t* from) {
1472 memcpy(&tmp, from,
sizeof(tmp) / 2);
1473 return __lsx_vilvl_w(tmp, tmp);
1476EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from) {
1477 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0);
1480EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from) {
1481 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0);
1484EIGEN_STRONG_INLINE
void pstore<int8_t>(int8_t* to,
const Packet16c& from) {
1485 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1488EIGEN_STRONG_INLINE
void pstore<int16_t>(int16_t* to,
const Packet8s& from) {
1489 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1492EIGEN_STRONG_INLINE
void pstore<int32_t>(int32_t* to,
const Packet4i& from) {
1493 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1496EIGEN_STRONG_INLINE
void pstore<int64_t>(int64_t* to,
const Packet2l& from) {
1497 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1500EIGEN_STRONG_INLINE
void pstore<uint8_t>(uint8_t* to,
const Packet16uc& from) {
1501 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1504EIGEN_STRONG_INLINE
void pstore<uint16_t>(uint16_t* to,
const Packet8us& from) {
1505 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1508EIGEN_STRONG_INLINE
void pstore<uint32_t>(uint32_t* to,
const Packet4ui& from) {
1509 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1512EIGEN_STRONG_INLINE
void pstore<uint64_t>(uint64_t* to,
const Packet2ul& from) {
1513 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1517EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from) {
1518 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0);
1521EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from) {
1522 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0);
1526EIGEN_STRONG_INLINE
void pstoreu<int8_t>(int8_t* to,
const Packet16c& from) {
1527 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1530EIGEN_STRONG_INLINE
void pstoreu<int16_t>(int16_t* to,
const Packet8s& from) {
1531 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1534EIGEN_STRONG_INLINE
void pstoreu<int32_t>(int32_t* to,
const Packet4i& from) {
1535 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1538EIGEN_STRONG_INLINE
void pstoreu<int64_t>(int64_t* to,
const Packet2l& from) {
1539 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1542EIGEN_STRONG_INLINE
void pstoreu<uint8_t>(uint8_t* to,
const Packet16uc& from) {
1543 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1546EIGEN_STRONG_INLINE
void pstoreu<uint16_t>(uint16_t* to,
const Packet8us& from) {
1547 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1550EIGEN_STRONG_INLINE
void pstoreu<uint32_t>(uint32_t* to,
const Packet4ui& from) {
1551 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1554EIGEN_STRONG_INLINE
void pstoreu<uint64_t>(uint64_t* to,
const Packet2ul& from) {
1555 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1559EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(
const float* from, Index stride) {
1560 Packet4f v = {from[0], from[stride], from[2 * stride], from[3 * stride]};
1564EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(
const double* from, Index stride) {
1565 Packet2d v = {from[0], from[stride]};
1569EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(
const int8_t* from, Index stride) {
1570 int8_t v[16] __attribute__((aligned(16)));
1572 v[1] = from[stride];
1573 v[2] = from[2 * stride];
1574 v[3] = from[3 * stride];
1575 v[4] = from[4 * stride];
1576 v[5] = from[5 * stride];
1577 v[6] = from[6 * stride];
1578 v[7] = from[7 * stride];
1579 v[8] = from[8 * stride];
1580 v[9] = from[9 * stride];
1581 v[10] = from[10 * stride];
1582 v[11] = from[11 * stride];
1583 v[12] = from[12 * stride];
1584 v[13] = from[13 * stride];
1585 v[14] = from[14 * stride];
1586 v[15] = from[15 * stride];
1587 return __lsx_vld(v, 0);
1590EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(
const int16_t* from, Index stride) {
1591 int16_t v[8] __attribute__((aligned(16)));
1593 v[1] = from[stride];
1594 v[2] = from[2 * stride];
1595 v[3] = from[3 * stride];
1596 v[4] = from[4 * stride];
1597 v[5] = from[5 * stride];
1598 v[6] = from[6 * stride];
1599 v[7] = from[7 * stride];
1600 return __lsx_vld(v, 0);
1603EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(
const int32_t* from, Index stride) {
1604 int32_t v[4] __attribute__((aligned(16)));
1606 v[1] = from[stride];
1607 v[2] = from[2 * stride];
1608 v[3] = from[3 * stride];
1609 return __lsx_vld(v, 0);
1612EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(
const int64_t* from, Index stride) {
1613 int64_t v[2] __attribute__((aligned(16)));
1615 v[1] = from[stride];
1616 return __lsx_vld(v, 0);
1619EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(
const uint8_t* from, Index stride) {
1620 uint8_t v[16] __attribute__((aligned(16)));
1622 v[1] = from[stride];
1623 v[2] = from[2 * stride];
1624 v[3] = from[3 * stride];
1625 v[4] = from[4 * stride];
1626 v[5] = from[5 * stride];
1627 v[6] = from[6 * stride];
1628 v[7] = from[7 * stride];
1629 v[8] = from[8 * stride];
1630 v[9] = from[9 * stride];
1631 v[10] = from[10 * stride];
1632 v[11] = from[11 * stride];
1633 v[12] = from[12 * stride];
1634 v[13] = from[13 * stride];
1635 v[14] = from[14 * stride];
1636 v[15] = from[15 * stride];
1637 return __lsx_vld(v, 0);
1640EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(
const uint16_t* from, Index stride) {
1641 uint16_t v[8] __attribute__((aligned(16)));
1643 v[1] = from[stride];
1644 v[2] = from[2 * stride];
1645 v[3] = from[3 * stride];
1646 v[4] = from[4 * stride];
1647 v[5] = from[5 * stride];
1648 v[6] = from[6 * stride];
1649 v[7] = from[7 * stride];
1650 return __lsx_vld(v, 0);
1653EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(
const uint32_t* from, Index stride) {
1654 uint32_t v[4] __attribute__((aligned(16)));
1656 v[1] = from[stride];
1657 v[2] = from[2 * stride];
1658 v[3] = from[3 * stride];
1659 return __lsx_vld(v, 0);
1662EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(
const uint64_t* from, Index stride) {
1663 uint64_t v[2] __attribute__((aligned(16)));
1665 v[1] = from[stride];
1666 return __lsx_vld(v, 0);
1670EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<float, Packet4f>(
float* to,
const Packet4f& from, Index stride) {
1671 __lsx_vstelm_w(from, to, 0, 0);
1672 __lsx_vstelm_w(from, to + stride * 1, 0, 1);
1673 __lsx_vstelm_w(from, to + stride * 2, 0, 2);
1674 __lsx_vstelm_w(from, to + stride * 3, 0, 3);
1677EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<double, Packet2d>(
double* to,
const Packet2d& from, Index stride) {
1678 __lsx_vstelm_d(from, to, 0, 0);
1679 __lsx_vstelm_d(from, to + stride, 0, 1);
1682EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int8_t, Packet16c>(int8_t* to,
const Packet16c& from,
1684 __lsx_vstelm_b((__m128i)from, to, 0, 0);
1685 __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
1686 __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
1687 __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
1688 __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
1689 __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
1690 __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
1691 __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
1692 __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
1693 __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
1694 __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
1695 __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
1696 __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
1697 __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
1698 __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
1699 __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
1702EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int16_t, Packet8s>(int16_t* to,
const Packet8s& from,
1704 __lsx_vstelm_h((__m128i)from, to, 0, 0);
1705 __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
1706 __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
1707 __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
1708 __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
1709 __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
1710 __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
1711 __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
1714EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int32_t, Packet4i>(int32_t* to,
const Packet4i& from,
1716 __lsx_vstelm_w((__m128i)from, to, 0, 0);
1717 __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
1718 __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
1719 __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
1722EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<int64_t, Packet2l>(int64_t* to,
const Packet2l& from,
1724 __lsx_vstelm_d((__m128i)from, to, 0, 0);
1725 __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
1728EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint8_t, Packet16uc>(uint8_t* to,
const Packet16uc& from,
1730 __lsx_vstelm_b((__m128i)from, to, 0, 0);
1731 __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
1732 __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
1733 __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
1734 __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
1735 __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
1736 __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
1737 __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
1738 __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
1739 __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
1740 __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
1741 __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
1742 __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
1743 __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
1744 __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
1745 __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
1748EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint16_t, Packet8us>(uint16_t* to,
const Packet8us& from,
1750 __lsx_vstelm_h((__m128i)from, to, 0, 0);
1751 __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
1752 __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
1753 __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
1754 __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
1755 __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
1756 __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
1757 __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
1760EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint32_t, Packet4ui>(uint32_t* to,
const Packet4ui& from,
1762 __lsx_vstelm_w((__m128i)from, to, 0, 0);
1763 __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
1764 __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
1765 __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
1768EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void pscatter<uint64_t, Packet2ul>(uint64_t* to,
const Packet2ul& from,
1770 __lsx_vstelm_d((__m128i)from, to, 0, 0);
1771 __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
1775EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) {
1776 __builtin_prefetch(addr);
1779EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) {
1780 __builtin_prefetch(addr);
1783EIGEN_STRONG_INLINE
void prefetch<int8_t>(
const int8_t* addr) {
1784 __builtin_prefetch(addr);
1787EIGEN_STRONG_INLINE
void prefetch<int16_t>(
const int16_t* addr) {
1788 __builtin_prefetch(addr);
1791EIGEN_STRONG_INLINE
void prefetch<int32_t>(
const int32_t* addr) {
1792 __builtin_prefetch(addr);
1795EIGEN_STRONG_INLINE
void prefetch<int64_t>(
const int64_t* addr) {
1796 __builtin_prefetch(addr);
1799EIGEN_STRONG_INLINE
void prefetch<uint8_t>(
const uint8_t* addr) {
1800 __builtin_prefetch(addr);
1803EIGEN_STRONG_INLINE
void prefetch<uint16_t>(
const uint16_t* addr) {
1804 __builtin_prefetch(addr);
1807EIGEN_STRONG_INLINE
void prefetch<uint32_t>(
const uint32_t* addr) {
1808 __builtin_prefetch(addr);
1811EIGEN_STRONG_INLINE
void prefetch<uint64_t>(
const uint64_t* addr) {
1812 __builtin_prefetch(addr);
1816EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
1818 __lsx_vstelm_w(a, &v, 0, 0);
1822EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
1824 __lsx_vstelm_d(a, &v, 0, 0);
1829EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(
const Packet16c& a) {
1830 return (int8_t)__lsx_vpickve2gr_b((__m128i)a, 0);
1833EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(
const Packet8s& a) {
1834 return (int16_t)__lsx_vpickve2gr_h((__m128i)a, 0);
1837EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(
const Packet4i& a) {
1838 return __lsx_vpickve2gr_w((__m128i)a, 0);
1841EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(
const Packet2l& a) {
1842 return __lsx_vpickve2gr_d((__m128i)a, 0);
1845EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(
const Packet16uc& a) {
1846 return (uint8_t)__lsx_vpickve2gr_bu((__m128i)a, 0);
1849EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(
const Packet8us& a) {
1850 return (uint16_t)__lsx_vpickve2gr_hu((__m128i)a, 0);
1853EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(
const Packet4ui& a) {
1854 return __lsx_vpickve2gr_wu((__m128i)a, 0);
1857EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(
const Packet2ul& a) {
1858 return __lsx_vpickve2gr_du((__m128i)a, 0);
1862EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a) {
1863 return (Packet4f)__lsx_vshuf4i_w(a, 0x1B);
1866EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a) {
1867 return (Packet2d)__lsx_vshuf4i_d(a, a, 0x1);
1870EIGEN_STRONG_INLINE Packet16c preverse(
const Packet16c& a) {
1871 return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B);
1874EIGEN_STRONG_INLINE Packet8s preverse(
const Packet8s& a) {
1875 return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B);
1878EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a) {
1879 return __lsx_vshuf4i_w((__m128i)a, 0x1B);
1882EIGEN_STRONG_INLINE Packet2l preverse(
const Packet2l& a) {
1883 return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
1886EIGEN_STRONG_INLINE Packet16uc preverse(
const Packet16uc& a) {
1887 return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B);
1890EIGEN_STRONG_INLINE Packet8us preverse(
const Packet8us& a) {
1891 return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B);
1894EIGEN_STRONG_INLINE Packet4ui preverse(
const Packet4ui& a) {
1895 return __lsx_vshuf4i_w((__m128i)a, 0x1B);
1898EIGEN_STRONG_INLINE Packet2ul preverse(
const Packet2ul& a) {
1899 return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
1903EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a) {
1904 Packet4f tmp = __lsx_vfadd_s(a, vec4f_swizzle1(a, 2, 3, 2, 3));
1905 return pfirst<Packet4f>(__lsx_vfadd_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1)));
1908EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a) {
1909 return pfirst<Packet2d>(__lsx_vfadd_d(a, preverse(a)));
1912template <
typename Packet>
1913EIGEN_STRONG_INLINE
bool predux_any_lsx(
const Packet& a) {
1914 return __lsx_bnz_v((__m128i)a);
1918EIGEN_STRONG_INLINE
bool predux_any(
const Packet4f& a) {
1919 return predux_any_lsx(a);
1923EIGEN_STRONG_INLINE
bool predux_any(
const Packet2d& a) {
1924 return predux_any_lsx(a);
1928EIGEN_STRONG_INLINE
bool predux_any(
const Packet16c& a) {
1929 return predux_any_lsx(a);
1933EIGEN_STRONG_INLINE
bool predux_any(
const Packet8s& a) {
1934 return predux_any_lsx(a);
1938EIGEN_STRONG_INLINE
bool predux_any(
const Packet4i& a) {
1939 return predux_any_lsx(a);
1943EIGEN_STRONG_INLINE
bool predux_any(
const Packet2l& a) {
1944 return predux_any_lsx(a);
1948EIGEN_STRONG_INLINE
bool predux_any(
const Packet16uc& a) {
1949 return predux_any_lsx(a);
1953EIGEN_STRONG_INLINE
bool predux_any(
const Packet8us& a) {
1954 return predux_any_lsx(a);
1958EIGEN_STRONG_INLINE
bool predux_any(
const Packet4ui& a) {
1959 return predux_any_lsx(a);
1963EIGEN_STRONG_INLINE
bool predux_any(
const Packet2ul& a) {
1964 return predux_any_lsx(a);
1968EIGEN_STRONG_INLINE int8_t predux<Packet16c>(
const Packet16c& a) {
1969 Packet8s tmp1 = __lsx_vhaddw_h_b(a, a);
1970 Packet4i tmp2 = __lsx_vhaddw_w_h(tmp1, tmp1);
1971 Packet2l tmp3 = __lsx_vhaddw_d_w(tmp2, tmp2);
1972 return (int8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp3, tmp3), 0);
1975EIGEN_STRONG_INLINE int16_t predux<Packet8s>(
const Packet8s& a) {
1976 Packet4i tmp1 = __lsx_vhaddw_w_h(a, a);
1977 Packet2l tmp2 = __lsx_vhaddw_d_w(tmp1, tmp1);
1978 return (int16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp2, tmp2), 0);
1981EIGEN_STRONG_INLINE int32_t predux<Packet4i>(
const Packet4i& a) {
1982 Packet2l tmp = __lsx_vhaddw_d_w(a, a);
1983 return (int32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp, tmp), 0);
1986EIGEN_STRONG_INLINE int64_t predux<Packet2l>(
const Packet2l& a) {
1987 return (int64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(a, a), 0);
1990EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(
const Packet16uc& a) {
1991 Packet8us tmp1 = __lsx_vhaddw_hu_bu(a, a);
1992 Packet4ui tmp2 = __lsx_vhaddw_wu_hu(tmp1, tmp1);
1993 Packet2ul tmp3 = __lsx_vhaddw_du_wu(tmp2, tmp2);
1994 return (uint8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp3, tmp3), 0);
1997EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(
const Packet8us& a) {
1998 Packet4ui tmp1 = __lsx_vhaddw_wu_hu(a, a);
1999 Packet2ul tmp2 = __lsx_vhaddw_du_wu(tmp1, tmp1);
2000 return (uint16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp2, tmp2), 0);
2003EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(
const Packet4ui& a) {
2004 Packet2ul tmp = __lsx_vhaddw_du_wu(a, a);
2005 return (uint32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp, tmp), 0);
2008EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(
const Packet2ul& a) {
2009 return (uint64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(a, a), 0);
2013EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a) {
2014 Packet4f tmp = __lsx_vfmul_s(a, vec4f_swizzle1(a, 2, 3, 2, 3));
2015 return pfirst<Packet4f>(__lsx_vfmul_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1)));
2018EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a) {
2019 return pfirst<Packet2d>(__lsx_vfmul_d(a, preverse(a)));
2022EIGEN_STRONG_INLINE int8_t predux_mul<Packet16c>(
const Packet16c& a) {
2023 Packet8s tmp1 = __lsx_vmulwev_h_b(a, preverse(a));
2024 Packet4i tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1));
2025 Packet2l tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2));
2026 return (int8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0);
2029EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(
const Packet8s& a) {
2030 Packet4i tmp1 = __lsx_vmulwev_w_h(a, preverse(a));
2031 Packet2l tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1));
2032 return (int16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0);
2035EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(
const Packet4i& a) {
2036 Packet2l tmp = __lsx_vmulwev_d_w(a, preverse(a));
2037 return (int32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0);
2040EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(
const Packet2l& a) {
2041 return (int64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(a, preverse(a)), 0);
2044EIGEN_STRONG_INLINE uint8_t predux_mul<Packet16uc>(
const Packet16uc& a) {
2045 Packet8us tmp1 = __lsx_vmulwev_h_bu(a, preverse(a));
2046 Packet4ui tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1));
2047 Packet2ul tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2));
2048 return (uint8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0);
2051EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(
const Packet8us& a) {
2052 Packet4ui tmp1 = __lsx_vmulwev_w_hu(a, preverse(a));
2053 Packet2ul tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1));
2054 return (uint16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0);
2057EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(
const Packet4ui& a) {
2058 Packet2ul tmp = __lsx_vmulwev_d_wu(a, preverse(a));
2059 return (uint32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0);
2062EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(
const Packet2ul& a) {
2063 return (uint64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_du(a, preverse(a)), 0);
2067EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a) {
2068 Packet4f tmp = __lsx_vfmin_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E));
2069 return pfirst(__lsx_vfmin_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1)));
2072EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a) {
2073 return pfirst(__lsx_vfmin_d(a, preverse(a)));
2076EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(
const Packet16c& a) {
2077 Packet16c tmp1 = __lsx_vmin_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2078 Packet16c tmp2 = __lsx_vmin_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2079 Packet16c tmp3 = __lsx_vmin_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2080 return pfirst((Packet16c)__lsx_vmin_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2083EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(
const Packet8s& a) {
2084 Packet8s tmp1 = __lsx_vmin_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2085 Packet8s tmp2 = __lsx_vmin_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2086 return pfirst((Packet8s)__lsx_vmin_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2089EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(
const Packet4i& a) {
2090 Packet4i tmp = __lsx_vmin_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2091 return pfirst((Packet4i)__lsx_vmin_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2094EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(
const Packet2l& a) {
2095 return pfirst((Packet2l)__lsx_vmin_d(a, preverse(a)));
2098EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(
const Packet16uc& a) {
2099 Packet16uc tmp1 = __lsx_vmin_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2100 Packet16uc tmp2 = __lsx_vmin_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2101 Packet16uc tmp3 = __lsx_vmin_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2102 return pfirst((Packet16uc)__lsx_vmin_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2105EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(
const Packet8us& a) {
2106 Packet8us tmp1 = __lsx_vmin_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2107 Packet8us tmp2 = __lsx_vmin_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2108 return pfirst((Packet8us)__lsx_vmin_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2111EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(
const Packet4ui& a) {
2112 Packet4ui tmp = __lsx_vmin_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2113 return pfirst((Packet4ui)__lsx_vmin_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2116EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(
const Packet2ul& a) {
2117 return pfirst((Packet2ul)__lsx_vmin_du(a, preverse(a)));
2121EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a) {
2122 Packet4f tmp = __lsx_vfmax_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E));
2123 return pfirst(__lsx_vfmax_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1)));
2126EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a) {
2127 return pfirst(__lsx_vfmax_d(a, preverse(a)));
2130EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(
const Packet16c& a) {
2131 Packet16c tmp1 = __lsx_vmax_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2132 Packet16c tmp2 = __lsx_vmax_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2133 Packet16c tmp3 = __lsx_vmax_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2134 return pfirst((Packet16c)__lsx_vmax_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2137EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(
const Packet8s& a) {
2138 Packet8s tmp1 = __lsx_vmax_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2139 Packet8s tmp2 = __lsx_vmax_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2140 return pfirst((Packet8s)__lsx_vmax_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2143EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(
const Packet4i& a) {
2144 Packet4i tmp = __lsx_vmax_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2145 return pfirst((Packet4i)__lsx_vmax_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2148EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(
const Packet2l& a) {
2149 return pfirst((Packet2l)__lsx_vmax_d(a, preverse(a)));
2152EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(
const Packet16uc& a) {
2153 Packet16uc tmp1 = __lsx_vmax_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2154 Packet16uc tmp2 = __lsx_vmax_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2155 Packet16uc tmp3 = __lsx_vmax_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2156 return pfirst((Packet16uc)__lsx_vmax_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2159EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(
const Packet8us& a) {
2160 Packet8us tmp1 = __lsx_vmax_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2161 Packet8us tmp2 = __lsx_vmax_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2162 return pfirst((Packet8us)__lsx_vmax_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2165EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(
const Packet4ui& a) {
2166 Packet4ui tmp = __lsx_vmax_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2167 return pfirst((Packet4ui)__lsx_vmax_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2170EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(
const Packet2ul& a) {
2171 return pfirst((Packet2ul)__lsx_vmax_du(a, preverse(a)));
2175EIGEN_STRONG_INLINE Packet4f psqrt(
const Packet4f& a) {
2176 return __lsx_vfsqrt_s(a);
2179EIGEN_STRONG_INLINE Packet2d psqrt(
const Packet2d& a) {
2180 return __lsx_vfsqrt_d(a);
2183EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
2184 Packet4f T0 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2185 Packet4f T1 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2186 Packet4f T2 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]);
2187 Packet4f T3 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]);
2189 kernel.packet[0] = (Packet4f)__lsx_vilvl_d((__m128i)T2, (__m128i)T0);
2190 kernel.packet[1] = (Packet4f)__lsx_vilvh_d((__m128i)T2, (__m128i)T0);
2191 kernel.packet[2] = (Packet4f)__lsx_vilvl_d((__m128i)T3, (__m128i)T1);
2192 kernel.packet[3] = (Packet4f)__lsx_vilvh_d((__m128i)T3, (__m128i)T1);
2194EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
2195 Packet2d tmp = (Packet2d)__lsx_vilvh_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2196 kernel.packet[0] = (Packet2d)__lsx_vilvl_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2197 kernel.packet[1] = tmp;
2199EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
2200 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2201 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2202 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2203 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2204 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2205 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2206 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2207 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2208 __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]);
2209 __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]);
2210 __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]);
2211 __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]);
2212 __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]);
2213 __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]);
2214 __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]);
2215 __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]);
2217 __m128i s0 = __lsx_vilvl_h(t2, t0);
2218 __m128i s1 = __lsx_vilvh_h(t2, t0);
2219 __m128i s2 = __lsx_vilvl_h(t3, t1);
2220 __m128i s3 = __lsx_vilvh_h(t3, t1);
2221 __m128i s4 = __lsx_vilvl_h(t6, t4);
2222 __m128i s5 = __lsx_vilvh_h(t6, t4);
2223 __m128i s6 = __lsx_vilvl_h(t7, t5);
2224 __m128i s7 = __lsx_vilvh_h(t7, t5);
2225 __m128i s8 = __lsx_vilvl_h(ta, t8);
2226 __m128i s9 = __lsx_vilvh_h(ta, t8);
2227 __m128i sa = __lsx_vilvl_h(tb, t9);
2228 __m128i sb = __lsx_vilvh_h(tb, t9);
2229 __m128i sc = __lsx_vilvl_h(te, tc);
2230 __m128i sd = __lsx_vilvh_h(te, tc);
2231 __m128i se = __lsx_vilvl_h(tf, td);
2232 __m128i sf = __lsx_vilvh_h(tf, td);
2234 __m128i u0 = __lsx_vilvl_w(s4, s0);
2235 __m128i u1 = __lsx_vilvh_w(s4, s0);
2236 __m128i u2 = __lsx_vilvl_w(s5, s1);
2237 __m128i u3 = __lsx_vilvh_w(s5, s1);
2238 __m128i u4 = __lsx_vilvl_w(s6, s2);
2239 __m128i u5 = __lsx_vilvh_w(s6, s2);
2240 __m128i u6 = __lsx_vilvl_w(s7, s3);
2241 __m128i u7 = __lsx_vilvh_w(s7, s3);
2242 __m128i u8 = __lsx_vilvl_w(sc, s8);
2243 __m128i u9 = __lsx_vilvh_w(sc, s8);
2244 __m128i ua = __lsx_vilvl_w(sd, s9);
2245 __m128i ub = __lsx_vilvh_w(sd, s9);
2246 __m128i uc = __lsx_vilvl_w(se, sa);
2247 __m128i ud = __lsx_vilvh_w(se, sa);
2248 __m128i ue = __lsx_vilvl_w(sf, sb);
2249 __m128i uf = __lsx_vilvh_w(sf, sb);
2251 kernel.packet[0] = __lsx_vilvl_d(u8, u0);
2252 kernel.packet[1] = __lsx_vilvh_d(u8, u0);
2253 kernel.packet[2] = __lsx_vilvl_d(u9, u1);
2254 kernel.packet[3] = __lsx_vilvh_d(u9, u1);
2255 kernel.packet[4] = __lsx_vilvl_d(ua, u2);
2256 kernel.packet[5] = __lsx_vilvh_d(ua, u2);
2257 kernel.packet[6] = __lsx_vilvl_d(ub, u3);
2258 kernel.packet[7] = __lsx_vilvh_d(ub, u3);
2259 kernel.packet[8] = __lsx_vilvl_d(uc, u4);
2260 kernel.packet[9] = __lsx_vilvh_d(uc, u4);
2261 kernel.packet[10] = __lsx_vilvl_d(ud, u5);
2262 kernel.packet[11] = __lsx_vilvh_d(ud, u5);
2263 kernel.packet[12] = __lsx_vilvl_d(ue, u6);
2264 kernel.packet[13] = __lsx_vilvh_d(ue, u6);
2265 kernel.packet[14] = __lsx_vilvl_d(uf, u7);
2266 kernel.packet[15] = __lsx_vilvh_d(uf, u7);
2268EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 8>& kernel) {
2269 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2270 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2271 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2272 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2273 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2274 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2275 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2276 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2278 __m128i s0 = __lsx_vilvl_h(t2, t0);
2279 __m128i s1 = __lsx_vilvh_h(t2, t0);
2280 __m128i s2 = __lsx_vilvl_h(t3, t1);
2281 __m128i s3 = __lsx_vilvh_h(t3, t1);
2282 __m128i s4 = __lsx_vilvl_h(t6, t4);
2283 __m128i s5 = __lsx_vilvh_h(t6, t4);
2284 __m128i s6 = __lsx_vilvl_h(t7, t5);
2285 __m128i s7 = __lsx_vilvh_h(t7, t5);
2287 kernel.packet[0] = __lsx_vilvl_w(s4, s0);
2288 kernel.packet[1] = __lsx_vilvh_w(s4, s0);
2289 kernel.packet[2] = __lsx_vilvl_w(s5, s1);
2290 kernel.packet[3] = __lsx_vilvh_w(s5, s1);
2291 kernel.packet[4] = __lsx_vilvl_w(s6, s2);
2292 kernel.packet[5] = __lsx_vilvh_w(s6, s2);
2293 kernel.packet[6] = __lsx_vilvl_w(s7, s3);
2294 kernel.packet[7] = __lsx_vilvh_w(s7, s3);
2296EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
2297 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2298 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2299 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2300 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2302 kernel.packet[0] = __lsx_vilvl_h(t2, t0);
2303 kernel.packet[1] = __lsx_vilvh_h(t2, t0);
2304 kernel.packet[2] = __lsx_vilvl_h(t3, t1);
2305 kernel.packet[3] = __lsx_vilvh_h(t3, t1);
2307EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
2308 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2309 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2310 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2311 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2312 __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]);
2313 __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]);
2314 __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]);
2315 __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]);
2317 __m128i s0 = __lsx_vilvl_w(t2, t0);
2318 __m128i s1 = __lsx_vilvh_w(t2, t0);
2319 __m128i s2 = __lsx_vilvl_w(t3, t1);
2320 __m128i s3 = __lsx_vilvh_w(t3, t1);
2321 __m128i s4 = __lsx_vilvl_w(t6, t4);
2322 __m128i s5 = __lsx_vilvh_w(t6, t4);
2323 __m128i s6 = __lsx_vilvl_w(t7, t5);
2324 __m128i s7 = __lsx_vilvh_w(t7, t5);
2326 kernel.packet[0] = __lsx_vilvl_d(s4, s0);
2327 kernel.packet[1] = __lsx_vilvh_d(s4, s0);
2328 kernel.packet[2] = __lsx_vilvl_d(s5, s1);
2329 kernel.packet[3] = __lsx_vilvh_d(s5, s1);
2330 kernel.packet[4] = __lsx_vilvl_d(s6, s2);
2331 kernel.packet[5] = __lsx_vilvh_d(s6, s2);
2332 kernel.packet[6] = __lsx_vilvl_d(s7, s3);
2333 kernel.packet[7] = __lsx_vilvh_d(s7, s3);
2335EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
2336 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2337 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2338 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2339 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2341 kernel.packet[0] = __lsx_vilvl_w(t2, t0);
2342 kernel.packet[1] = __lsx_vilvh_w(t2, t0);
2343 kernel.packet[2] = __lsx_vilvl_w(t3, t1);
2344 kernel.packet[3] = __lsx_vilvh_w(t3, t1);
2346EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
2347 __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]);
2348 __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]);
2349 __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]);
2350 __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]);
2352 kernel.packet[0] = __lsx_vilvl_d(T2, T0);
2353 kernel.packet[1] = __lsx_vilvh_d(T2, T0);
2354 kernel.packet[2] = __lsx_vilvl_d(T3, T1);
2355 kernel.packet[3] = __lsx_vilvh_d(T3, T1);
2357EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2l, 2>& kernel) {
2358 __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
2359 kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
2360 kernel.packet[1] = tmp;
2362EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
2363 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2364 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2365 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2366 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2367 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2368 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2369 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2370 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2371 __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]);
2372 __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]);
2373 __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]);
2374 __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]);
2375 __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]);
2376 __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]);
2377 __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]);
2378 __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]);
2380 __m128i s0 = __lsx_vilvl_h(t2, t0);
2381 __m128i s1 = __lsx_vilvh_h(t2, t0);
2382 __m128i s2 = __lsx_vilvl_h(t3, t1);
2383 __m128i s3 = __lsx_vilvh_h(t3, t1);
2384 __m128i s4 = __lsx_vilvl_h(t6, t4);
2385 __m128i s5 = __lsx_vilvh_h(t6, t4);
2386 __m128i s6 = __lsx_vilvl_h(t7, t5);
2387 __m128i s7 = __lsx_vilvh_h(t7, t5);
2388 __m128i s8 = __lsx_vilvl_h(ta, t8);
2389 __m128i s9 = __lsx_vilvh_h(ta, t8);
2390 __m128i sa = __lsx_vilvl_h(tb, t9);
2391 __m128i sb = __lsx_vilvh_h(tb, t9);
2392 __m128i sc = __lsx_vilvl_h(te, tc);
2393 __m128i sd = __lsx_vilvh_h(te, tc);
2394 __m128i se = __lsx_vilvl_h(tf, td);
2395 __m128i sf = __lsx_vilvh_h(tf, td);
2397 __m128i u0 = __lsx_vilvl_w(s4, s0);
2398 __m128i u1 = __lsx_vilvh_w(s4, s0);
2399 __m128i u2 = __lsx_vilvl_w(s5, s1);
2400 __m128i u3 = __lsx_vilvh_w(s5, s1);
2401 __m128i u4 = __lsx_vilvl_w(s6, s2);
2402 __m128i u5 = __lsx_vilvh_w(s6, s2);
2403 __m128i u6 = __lsx_vilvl_w(s7, s3);
2404 __m128i u7 = __lsx_vilvh_w(s7, s3);
2405 __m128i u8 = __lsx_vilvl_w(sc, s8);
2406 __m128i u9 = __lsx_vilvh_w(sc, s8);
2407 __m128i ua = __lsx_vilvl_w(sd, s9);
2408 __m128i ub = __lsx_vilvh_w(sd, s9);
2409 __m128i uc = __lsx_vilvl_w(se, sa);
2410 __m128i ud = __lsx_vilvh_w(se, sa);
2411 __m128i ue = __lsx_vilvl_w(sf, sb);
2412 __m128i uf = __lsx_vilvh_w(sf, sb);
2414 kernel.packet[0] = __lsx_vilvl_d(u8, u0);
2415 kernel.packet[1] = __lsx_vilvh_d(u8, u0);
2416 kernel.packet[2] = __lsx_vilvl_d(u9, u1);
2417 kernel.packet[3] = __lsx_vilvh_d(u9, u1);
2418 kernel.packet[4] = __lsx_vilvl_d(ua, u2);
2419 kernel.packet[5] = __lsx_vilvh_d(ua, u2);
2420 kernel.packet[6] = __lsx_vilvl_d(ub, u3);
2421 kernel.packet[7] = __lsx_vilvh_d(ub, u3);
2422 kernel.packet[8] = __lsx_vilvl_d(uc, u4);
2423 kernel.packet[9] = __lsx_vilvh_d(uc, u4);
2424 kernel.packet[10] = __lsx_vilvl_d(ud, u5);
2425 kernel.packet[11] = __lsx_vilvh_d(ud, u5);
2426 kernel.packet[12] = __lsx_vilvl_d(ue, u6);
2427 kernel.packet[13] = __lsx_vilvh_d(ue, u6);
2428 kernel.packet[14] = __lsx_vilvl_d(uf, u7);
2429 kernel.packet[15] = __lsx_vilvh_d(uf, u7);
2431EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 8>& kernel) {
2432 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2433 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2434 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2435 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2436 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2437 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2438 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2439 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2441 __m128i s0 = __lsx_vilvl_h(t2, t0);
2442 __m128i s1 = __lsx_vilvh_h(t2, t0);
2443 __m128i s2 = __lsx_vilvl_h(t3, t1);
2444 __m128i s3 = __lsx_vilvh_h(t3, t1);
2445 __m128i s4 = __lsx_vilvl_h(t6, t4);
2446 __m128i s5 = __lsx_vilvh_h(t6, t4);
2447 __m128i s6 = __lsx_vilvl_h(t7, t5);
2448 __m128i s7 = __lsx_vilvh_h(t7, t5);
2450 kernel.packet[0] = __lsx_vilvl_w(s4, s0);
2451 kernel.packet[1] = __lsx_vilvh_w(s4, s0);
2452 kernel.packet[2] = __lsx_vilvl_w(s5, s1);
2453 kernel.packet[3] = __lsx_vilvh_w(s5, s1);
2454 kernel.packet[4] = __lsx_vilvl_w(s6, s2);
2455 kernel.packet[5] = __lsx_vilvh_w(s6, s2);
2456 kernel.packet[6] = __lsx_vilvl_w(s7, s3);
2457 kernel.packet[7] = __lsx_vilvh_w(s7, s3);
2459EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
2460 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2461 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2462 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2463 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2465 kernel.packet[0] = __lsx_vilvl_h(t2, t0);
2466 kernel.packet[1] = __lsx_vilvh_h(t2, t0);
2467 kernel.packet[2] = __lsx_vilvl_h(t3, t1);
2468 kernel.packet[3] = __lsx_vilvh_h(t3, t1);
2470EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
2471 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2472 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2473 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2474 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2475 __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]);
2476 __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]);
2477 __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]);
2478 __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]);
2480 __m128i s0 = __lsx_vilvl_w(t2, t0);
2481 __m128i s1 = __lsx_vilvh_w(t2, t0);
2482 __m128i s2 = __lsx_vilvl_w(t3, t1);
2483 __m128i s3 = __lsx_vilvh_w(t3, t1);
2484 __m128i s4 = __lsx_vilvl_w(t6, t4);
2485 __m128i s5 = __lsx_vilvh_w(t6, t4);
2486 __m128i s6 = __lsx_vilvl_w(t7, t5);
2487 __m128i s7 = __lsx_vilvh_w(t7, t5);
2489 kernel.packet[0] = __lsx_vilvl_d(s4, s0);
2490 kernel.packet[1] = __lsx_vilvh_d(s4, s0);
2491 kernel.packet[2] = __lsx_vilvl_d(s5, s1);
2492 kernel.packet[3] = __lsx_vilvh_d(s5, s1);
2493 kernel.packet[4] = __lsx_vilvl_d(s6, s2);
2494 kernel.packet[5] = __lsx_vilvh_d(s6, s2);
2495 kernel.packet[6] = __lsx_vilvl_d(s7, s3);
2496 kernel.packet[7] = __lsx_vilvh_d(s7, s3);
2498EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
2499 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2500 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2501 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2502 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2504 kernel.packet[0] = __lsx_vilvl_w(t2, t0);
2505 kernel.packet[1] = __lsx_vilvh_w(t2, t0);
2506 kernel.packet[2] = __lsx_vilvl_w(t3, t1);
2507 kernel.packet[3] = __lsx_vilvh_w(t3, t1);
2509EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
2510 __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]);
2511 __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]);
2512 __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]);
2513 __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]);
2515 kernel.packet[0] = __lsx_vilvl_d(T2, T0);
2516 kernel.packet[1] = __lsx_vilvh_d(T2, T0);
2517 kernel.packet[2] = __lsx_vilvl_d(T3, T1);
2518 kernel.packet[3] = __lsx_vilvh_d(T3, T1);
2520EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void ptranspose(PacketBlock<Packet2ul, 2>& kernel) {
2521 __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
2522 kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
2523 kernel.packet[1] = tmp;
2527EIGEN_STRONG_INLINE Packet4f prsqrt(
const Packet4f& a) {
2528 return __lsx_vfrsqrt_s(a);
2531EIGEN_STRONG_INLINE Packet2d prsqrt(
const Packet2d& a) {
2532 return __lsx_vfrsqrt_d(a);
2536EIGEN_STRONG_INLINE Packet4f pfloor(
const Packet4f& a) {
2537 return __lsx_vfrintrm_s(a);
2540EIGEN_STRONG_INLINE Packet2d pfloor(
const Packet2d& a) {
2541 return __lsx_vfrintrm_d(a);
2545EIGEN_STRONG_INLINE Packet4f pceil(
const Packet4f& a) {
2546 return __lsx_vfrintrp_s(a);
2549EIGEN_STRONG_INLINE Packet2d pceil(
const Packet2d& a) {
2550 return __lsx_vfrintrp_d(a);
2554EIGEN_STRONG_INLINE Packet4f pround(
const Packet4f& a) {
2555 const Packet4f mask = pset1frombits<Packet4f>(
static_cast<numext::uint32_t
>(0x80000000u));
2556 const Packet4f prev0dot5 = pset1frombits<Packet4f>(
static_cast<numext::uint32_t
>(0x3EFFFFFFu));
2557 return __lsx_vfrintrz_s(padd(pxor(pand(a, mask), prev0dot5), a));
2560EIGEN_STRONG_INLINE Packet2d pround(
const Packet2d& a) {
2561 const Packet2d mask = pset1frombits<Packet2d>(
static_cast<numext::uint64_t
>(0x8000000000000000ull));
2562 const Packet2d prev0dot5 = pset1frombits<Packet2d>(
static_cast<numext::uint64_t
>(0x3FDFFFFFFFFFFFFFull));
2563 return __lsx_vfrintrz_d(padd(por(pand(a, mask), prev0dot5), a));
2567EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(
const Packet4f& mask,
const Packet4f& a,
const Packet4f& b) {
2568 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2571EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(
const Packet16c& mask,
const Packet16c& a,
const Packet16c& b) {
2572 return (Packet16c)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2576EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(
const int8_t* from) {
2577 int8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1),
2578 *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
2579 *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
2580 return __lsx_vld(tmp, 0);
2583EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(
const uint8_t* from) {
2584 uint8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1),
2585 *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
2586 *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
2587 return __lsx_vld(tmp, 0);
2590EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(
const int16_t* from) {
2591 int16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
2592 return __lsx_vld(tmp, 0);
2595EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(
const uint16_t* from) {
2596 uint16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
2597 return __lsx_vld(tmp, 0);
2600EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(
const int32_t* from) {
2601 int32_t tmp[4] = {*from, *from, *from, *from};
2602 return __lsx_vld(tmp, 0);
2605EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(
const uint32_t* from) {
2606 uint32_t tmp[4] = {*from, *from, *from, *from};
2607 return __lsx_vld(tmp, 0);
2611EIGEN_STRONG_INLINE Packet16c pnmsub(
const Packet16c& a,
const Packet16c& b,
const Packet16c& c) {
2612 return __lsx_vmsub_b(pnegate(c), a, b);
2615EIGEN_STRONG_INLINE Packet8s pnmsub(
const Packet8s& a,
const Packet8s& b,
const Packet8s& c) {
2616 return __lsx_vmsub_h(pnegate(c), a, b);
2619EIGEN_STRONG_INLINE Packet4i pnmsub(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
2620 return __lsx_vmsub_w(pnegate(c), a, b);
2623EIGEN_STRONG_INLINE Packet2l pnmsub(
const Packet2l& a,
const Packet2l& b,
const Packet2l& c) {
2624 return __lsx_vmsub_d(pnegate(c), a, b);
2628EIGEN_STRONG_INLINE Packet16c pmsub(
const Packet16c& a,
const Packet16c& b,
const Packet16c& c) {
2629 return __lsx_vmadd_b(pnegate(c), a, b);
2632EIGEN_STRONG_INLINE Packet8s pmsub(
const Packet8s& a,
const Packet8s& b,
const Packet8s& c) {
2633 return __lsx_vmadd_h(pnegate(c), a, b);
2636EIGEN_STRONG_INLINE Packet4i pmsub(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
2637 return __lsx_vmadd_w(pnegate(c), a, b);
2640EIGEN_STRONG_INLINE Packet2l pmsub(
const Packet2l& a,
const Packet2l& b,
const Packet2l& c) {
2641 return __lsx_vmadd_d(pnegate(c), a, b);
2645EIGEN_STRONG_INLINE Packet16c pnmadd(
const Packet16c& a,
const Packet16c& b,
const Packet16c& c) {
2646 return __lsx_vmsub_b(c, a, b);
2649EIGEN_STRONG_INLINE Packet8s pnmadd(
const Packet8s& a,
const Packet8s& b,
const Packet8s& c) {
2650 return __lsx_vmsub_h(c, a, b);
2653EIGEN_STRONG_INLINE Packet4i pnmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
2654 return __lsx_vmsub_w(c, a, b);
2657EIGEN_STRONG_INLINE Packet2l pnmadd(
const Packet2l& a,
const Packet2l& b,
const Packet2l& c) {
2658 return __lsx_vmsub_d(c, a, b);
2662EIGEN_STRONG_INLINE Packet4f pexp(
const Packet4f& _x) {
2663 return pexp_float(_x);
2666EIGEN_STRONG_INLINE Packet2d pexp(
const Packet2d& _x) {
2667 return pexp_double(_x);
2671EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(
const Packet4f& a,
const Packet4f& exponent) {
2672 return pldexp_generic(a, exponent);
2676EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(
const Packet2d& a, Packet2d& exponent) {
2677 return pfrexp_generic(a, exponent);
2680EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(
const Packet4f& a, Packet4f& exponent) {
2681 return pfrexp_generic(a, exponent);
2684EIGEN_STRONG_INLINE Packet4f pzero(
const Packet4f& ) {
2685 Packet4f v = {0.0f, 0.0f, 0.0f, 0.0f};
2689struct pminmax_propagates_nan<Packet4f> : bool_constant<true> {};
2691EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(
const float* from) {
2692 return (__m128)__lsx_vldrepl_w(from, 0);
2695EIGEN_STRONG_INLINE Packet4f psignbit(
const Packet4f& a) {
2696 return (__m128)__lsx_vsrai_w((__m128i)a, 31);
2699EIGEN_STRONG_INLINE Packet4f print<Packet4f>(
const Packet4f& a) {
2700 return __lsx_vfrintrne_s(a);
2703EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(
const Packet4f& a) {
2704 return __lsx_vfrintrz_s(a);
2707EIGEN_STRONG_INLINE Packet4f preciprocal<Packet4f>(
const Packet4f& a) {
2708 return __lsx_vfrecip_s(a);
2712EIGEN_STRONG_INLINE Packet2d pzero(
const Packet2d& ) {
2713 Packet2d v = {0.0, 0.0};
2717struct pminmax_propagates_nan<Packet2d> : bool_constant<true> {};
2719EIGEN_STRONG_INLINE Packet2d psignbit(
const Packet2d& a) {
2720 return (__m128d)(__lsx_vsrai_d((__m128i)a, 63));
2723EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect(
const Packet2d& mask,
const Packet2d& a,
const Packet2d& b) {
2724 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2727EIGEN_STRONG_INLINE Packet2d print<Packet2d>(
const Packet2d& a) {
2728 return __lsx_vfrintrne_d(a);
2731EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(
const Packet2d& a) {
2732 return __lsx_vfrintrz_d(a);
2735EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(
const Packet2d& a,
const Packet2d& exponent) {
2736 return pldexp_generic(a, exponent);
2739EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(
const Packet8s& mask,
const Packet8s& a,
const Packet8s& b) {
2740 return __lsx_vbitsel_v(b, a, mask);
2743EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(
const Packet4i& mask,
const Packet4i& a,
const Packet4i& b) {
2744 return __lsx_vbitsel_v(b, a, mask);
2747EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(
const Packet2l& mask,
const Packet2l& a,
const Packet2l& b) {
2748 return __lsx_vbitsel_v(b, a, mask);
2751EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(
const Packet16uc& a,
const Packet16uc& b) {
2752 return __lsx_vdiv_bu(a, b);
2755EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(
const Packet16uc& mask,
const Packet16uc& a,
2756 const Packet16uc& b) {
2757 return __lsx_vbitsel_v(b, a, mask);
2760EIGEN_STRONG_INLINE Packet16uc psqrt(
const Packet16uc& a) {
2761 __m128i res = {0, 0};
2762 __m128i add = {0x0808080808080808, 0x0808080808080808};
2763 for (
int i = 0; i < 4; i++) {
2764 const __m128i temp = __lsx_vor_v(res, add);
2765 const __m128i tmul = __lsx_vpackev_b(__lsx_vmulwod_h_bu(temp, temp), __lsx_vmulwev_h_bu(temp, temp));
2766 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_bu(tmul, a));
2767 add = __lsx_vsrli_b(add, 1);
2772EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(
const Packet8us& mask,
const Packet8us& a,
const Packet8us& b) {
2773 return __lsx_vbitsel_v(b, a, mask);
2776EIGEN_STRONG_INLINE Packet8us psqrt(
const Packet8us& a) {
2777 __m128i res = {0, 0};
2778 __m128i add = {0x0080008000800080, 0x0080008000800080};
2779 for (
int i = 0; i < 4; i++) {
2780 const __m128i temp = __lsx_vor_v(res, add);
2781 const __m128i tmul = __lsx_vpackev_h(__lsx_vmulwod_w_hu(temp, temp), __lsx_vmulwev_w_hu(temp, temp));
2782 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_hu(tmul, a));
2783 add = __lsx_vsrli_h(add, 1);
2789EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(
const Packet4ui& mask,
const Packet4ui& a,
const Packet4ui& b) {
2790 return __lsx_vbitsel_v(b, a, mask);
2793EIGEN_STRONG_INLINE Packet4ui psqrt(
const Packet4ui& a) {
2794 __m128i res = {0, 0};
2795 __m128i add = {0x0000800000008000, 0x0000800000008000};
2796 for (
int i = 0; i < 4; i++) {
2797 const __m128i temp = __lsx_vor_v(res, add);
2798 const __m128i tmul = __lsx_vpackev_w(__lsx_vmulwod_d_wu(temp, temp), __lsx_vmulwev_d_wu(temp, temp));
2799 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_wu(tmul, a));
2800 add = __lsx_vsrli_w(add, 1);
2806EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(
const Packet2ul& mask,
const Packet2ul& a,
const Packet2ul& b) {
2807 return __lsx_vbitsel_v(b, a, mask);
@ Aligned16
Definition Constants.h:238