Eigen  5.0.1
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2023 Zang Ruochen <zangruochen@loongson.cn>
5// Copyright (C) 2024 XiWei Gu <guxiwei-hf@loongson.cn>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_PACKET_MATH_LSX_H
13#define EIGEN_PACKET_MATH_LSX_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
23#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
24#endif
25
26#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
27#if EIGEN_ARCH_LOONGARCH64
28#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
29#endif
30#endif
31
32#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
33#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
34#endif
35
36typedef __m128 Packet4f;
37typedef __m128d Packet2d;
38
39typedef eigen_packet_wrapper<__m128i, 0> Packet16c;
40typedef eigen_packet_wrapper<__m128i, 1> Packet8s;
41typedef eigen_packet_wrapper<__m128i, 2> Packet4i;
42typedef eigen_packet_wrapper<__m128i, 3> Packet2l;
43typedef eigen_packet_wrapper<__m128i, 4> Packet16uc;
44typedef eigen_packet_wrapper<__m128i, 5> Packet8us;
45typedef eigen_packet_wrapper<__m128i, 6> Packet4ui;
46typedef eigen_packet_wrapper<__m128i, 7> Packet2ul;
47
48template <>
49struct is_arithmetic<__m128> : std::true_type {};
50template <>
51struct is_arithmetic<__m128i> : std::true_type {};
52template <>
53struct is_arithmetic<__m128d> : std::true_type {};
54template <>
55struct is_arithmetic<Packet16c> : std::true_type {};
56template <>
57struct is_arithmetic<Packet8s> : std::true_type {};
58template <>
59struct is_arithmetic<Packet4i> : std::true_type {};
60template <>
61struct is_arithmetic<Packet2l> : std::true_type {};
62template <>
63struct is_arithmetic<Packet16uc> : std::false_type {};
64template <>
65struct is_arithmetic<Packet8us> : std::false_type {};
66template <>
67struct is_arithmetic<Packet4ui> : std::false_type {};
68template <>
69struct is_arithmetic<Packet2ul> : std::false_type {};
70
71EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) {
72 float from[4] = {a, b, c, d};
73 return (Packet4f)__lsx_vld(from, 0);
74}
75
76EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask) {
77 const float* a = reinterpret_cast<const float*>(&m);
78 Packet4f res =
79 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3)));
80 return res;
81}
82
83template <bool interleave>
84EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f& m, const Packet4f& n, int mask) {
85 const float* a = reinterpret_cast<const float*>(&m);
86 const float* b = reinterpret_cast<const float*>(&n);
87 Packet4f res =
88 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
89 return res;
90}
91
92template <>
93EIGEN_STRONG_INLINE Packet4f shuffle2<true>(const Packet4f& m, const Packet4f& n, int mask) {
94 const float* a = reinterpret_cast<const float*>(&m);
95 const float* b = reinterpret_cast<const float*>(&n);
96 Packet4f res =
97 make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
98 return res;
99}
100
101EIGEN_STRONG_INLINE static int eigen_lsx_shuffle_mask(int p, int q, int r, int s) {
102 return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
103}
104
105EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s) {
106 return shuffle1(a, eigen_lsx_shuffle_mask(p, q, r, s));
107}
108EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s) {
109 return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(p, q, r, s));
110}
111EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) {
112 return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(0, 1, 0, 1));
113}
114EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) {
115 return shuffle2<false>(b, a, eigen_lsx_shuffle_mask(2, 3, 2, 3));
116}
117EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) {
118 return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(0, 0, 1, 1));
119}
120EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) {
121 return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(2, 2, 3, 3));
122}
123
124EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) {
125 double from[2] = {a, b};
126 return (Packet2d)__lsx_vld(from, 0);
127}
128
129EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask) {
130 const double* a = reinterpret_cast<const double*>(&m);
131 const double* b = reinterpret_cast<const double*>(&n);
132 Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1)));
133 return res;
134}
135
136EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(const Packet2d& a, const Packet2d& b, int mask) {
137 return shuffle(a, b, mask);
138}
139EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 0); }
140EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 3); }
141
142template <>
143struct packet_traits<int8_t> : default_packet_traits {
144 typedef Packet16c type;
145 typedef Packet16c half;
146 enum {
147 Vectorizable = 1,
148 AlignedOnScalar = 1,
149 size = 16,
150
151 HasSetLinear = 0,
152 HasCmp = 1,
153 };
154};
155
156template <>
157struct packet_traits<int16_t> : default_packet_traits {
158 typedef Packet8s type;
159 typedef Packet8s half;
160 enum {
161 Vectorizable = 1,
162 AlignedOnScalar = 1,
163 size = 8,
164
165 HasSetLinear = 0,
166 HasCmp = 1,
167 HasDiv = 1,
168 };
169};
170
171template <>
172struct packet_traits<int32_t> : default_packet_traits {
173 typedef Packet4i type;
174 typedef Packet4i half;
175 enum {
176 Vectorizable = 1,
177 AlignedOnScalar = 1,
178 size = 4,
179
180 HasSetLinear = 0,
181 HasCmp = 1,
182 HasDiv = 1,
183 };
184};
185
186template <>
187struct packet_traits<int64_t> : default_packet_traits {
188 typedef Packet2l type;
189 typedef Packet2l half;
190 enum {
191 Vectorizable = 1,
192 AlignedOnScalar = 1,
193 size = 2,
194
195 HasSetLinear = 0,
196 HasCmp = 1,
197 HasDiv = 1,
198 };
199};
200
201template <>
202struct packet_traits<uint8_t> : default_packet_traits {
203 typedef Packet16uc type;
204 typedef Packet16uc half;
205 enum {
206 Vectorizable = 1,
207 AlignedOnScalar = 1,
208 size = 16,
209
210 HasSetLinear = 0,
211 HasNegate = 0,
212 HasCmp = 1,
213 };
214};
215
216template <>
217struct packet_traits<uint16_t> : default_packet_traits {
218 typedef Packet8us type;
219 typedef Packet8us half;
220 enum {
221 Vectorizable = 1,
222 AlignedOnScalar = 1,
223 size = 8,
224
225 HasSetLinear = 0,
226 HasNegate = 0,
227 HasCmp = 1,
228 HasDiv = 1,
229 };
230};
231
232template <>
233struct packet_traits<uint32_t> : default_packet_traits {
234 typedef Packet4ui type;
235 typedef Packet4ui half;
236 enum {
237 Vectorizable = 1,
238 AlignedOnScalar = 1,
239 size = 4,
240
241 HasSetLinear = 0,
242 HasNegate = 0,
243 HasCmp = 1,
244 HasDiv = 1,
245 };
246};
247
248template <>
249struct packet_traits<uint64_t> : default_packet_traits {
250 typedef Packet2ul type;
251 typedef Packet2ul half;
252 enum {
253 Vectorizable = 1,
254 AlignedOnScalar = 1,
255 size = 2,
256
257 HasSetLinear = 0,
258 HasNegate = 0,
259 HasCmp = 1,
260 HasDiv = 1,
261 };
262};
263
264template <>
265struct packet_traits<float> : default_packet_traits {
266 typedef Packet4f type;
267 typedef Packet4f half;
268 enum {
269 Vectorizable = 1,
270 AlignedOnScalar = 1,
271 size = 4,
272
273 HasCmp = 1,
274 HasSetLinear = 0,
275 HasSign = 0,
276 HasDiv = 1,
277 HasExp = 1,
278 HasSqrt = 1,
279 HasLog = 1,
280 HasRsqrt = 1
281 };
282};
283
284template <>
285struct packet_traits<double> : default_packet_traits {
286 typedef Packet2d type;
287 typedef Packet2d half;
288 enum {
289 Vectorizable = 1,
290 AlignedOnScalar = 1,
291 size = 2,
292
293 HasCmp = 1,
294 HasSetLinear = 0,
295 HasSign = 0,
296 HasDiv = 1,
297 HasSqrt = 1,
298 HasLog = 1,
299 HasRsqrt = 1
300 };
301};
302
303template <>
304struct unpacket_traits<Packet16c> {
305 typedef int8_t type;
306 typedef Packet16c half;
307 enum {
308 size = 16,
309 alignment = Aligned16,
310 vectorizable = true,
311 masked_load_available = false,
312 masked_store_available = false
313 };
314};
315template <>
316struct unpacket_traits<Packet8s> {
317 typedef int16_t type;
318 typedef Packet8s half;
319 enum {
320 size = 8,
321 alignment = Aligned16,
322 vectorizable = true,
323 masked_load_available = false,
324 masked_store_available = false
325 };
326};
327template <>
328struct unpacket_traits<Packet4i> {
329 typedef int32_t type;
330 typedef Packet4i half;
331 enum {
332 size = 4,
333 alignment = Aligned16,
334 vectorizable = true,
335 masked_load_available = false,
336 masked_store_available = false
337 };
338};
339template <>
340struct unpacket_traits<Packet2l> {
341 typedef int64_t type;
342 typedef Packet2l half;
343 enum {
344 size = 2,
345 alignment = Aligned16,
346 vectorizable = true,
347 masked_load_available = false,
348 masked_store_available = false
349 };
350};
351template <>
352struct unpacket_traits<Packet16uc> {
353 typedef uint8_t type;
354 typedef Packet16uc half;
355 enum {
356 size = 16,
357 alignment = Aligned16,
358 vectorizable = true,
359 masked_load_available = false,
360 masked_store_available = false
361 };
362};
363template <>
364struct unpacket_traits<Packet8us> {
365 typedef uint16_t type;
366 typedef Packet8us half;
367 enum {
368 size = 8,
369 alignment = Aligned16,
370 vectorizable = true,
371 masked_load_available = false,
372 masked_store_available = false
373 };
374};
375template <>
376struct unpacket_traits<Packet4ui> {
377 typedef uint32_t type;
378 typedef Packet4ui half;
379 enum {
380 size = 4,
381 alignment = Aligned16,
382 vectorizable = true,
383 masked_load_available = false,
384 masked_store_available = false
385 };
386};
387template <>
388struct unpacket_traits<Packet2ul> {
389 typedef uint64_t type;
390 typedef Packet2ul half;
391 enum {
392 size = 2,
393 alignment = Aligned16,
394 vectorizable = true,
395 masked_load_available = false,
396 masked_store_available = false
397 };
398};
399template <>
400struct unpacket_traits<Packet4f> {
401 typedef float type;
402 typedef Packet4f half;
403 typedef Packet4i integer_packet;
404 enum {
405 size = 4,
406 alignment = Aligned16,
407 vectorizable = true,
408 masked_load_available = false,
409 masked_store_available = false
410 };
411};
412template <>
413struct unpacket_traits<Packet2d> {
414 typedef double type;
415 typedef Packet2d half;
416 typedef Packet2l integer_packet;
417 enum {
418 size = 2,
419 alignment = Aligned16,
420 vectorizable = true,
421 masked_load_available = false,
422 masked_store_available = false
423 };
424};
425
426template <>
427EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(const int8_t& from) {
428 return __lsx_vreplgr2vr_b(from);
429}
430template <>
431EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const int16_t& from) {
432 return __lsx_vreplgr2vr_h(from);
433}
434template <>
435EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) {
436 return __lsx_vreplgr2vr_w(from);
437}
438template <>
439EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) {
440 return __lsx_vreplgr2vr_d(from);
441}
442template <>
443EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(const uint8_t& from) {
444 return __lsx_vreplgr2vr_b(from);
445}
446template <>
447EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(const uint16_t& from) {
448 return __lsx_vreplgr2vr_h(from);
449}
450template <>
451EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(const uint32_t& from) {
452 return __lsx_vreplgr2vr_w(from);
453}
454template <>
455EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(const uint64_t& from) {
456 return __lsx_vreplgr2vr_d(from);
457}
458template <>
459EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
460 Packet4f v = {from, from, from, from};
461 return v;
462}
463template <>
464EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
465 Packet2d v = {from, from};
466 return v;
467}
468
469template <>
470EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
471 return reinterpret_cast<__m128>((__m128i)pset1<Packet4ui>(from));
472}
473template <>
474EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
475 return reinterpret_cast<__m128d>((__m128i)pset1<Packet2ul>(from));
476}
477
478template <>
479EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(const int8_t& a) {
480 const int8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
481 return __lsx_vadd_b(pset1<Packet16c>(a), __lsx_vld(countdown, 0));
482}
483template <>
484EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(const int16_t& a) {
485 const int16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
486 return __lsx_vadd_h(pset1<Packet8s>(a), __lsx_vld(countdown, 0));
487}
488template <>
489EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) {
490 const int32_t countdown[] = {0, 1, 2, 3};
491 return __lsx_vadd_w(pset1<Packet4i>(a), __lsx_vld(countdown, 0));
492}
493template <>
494EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(const int64_t& a) {
495 const int64_t countdown[] = {0, 1};
496 return __lsx_vadd_d(pset1<Packet2l>(a), __lsx_vld(countdown, 0));
497}
498template <>
499EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(const uint8_t& a) {
500 const uint8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
501 return __lsx_vadd_b(pset1<Packet16uc>(a), __lsx_vld(countdown, 0));
502}
503template <>
504EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(const uint16_t& a) {
505 const uint16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
506 return __lsx_vadd_h(pset1<Packet8us>(a), __lsx_vld(countdown, 0));
507}
508template <>
509EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(const uint32_t& a) {
510 const uint32_t countdown[] = {0, 1, 2, 3};
511 return __lsx_vadd_w(pset1<Packet4ui>(a), __lsx_vld(countdown, 0));
512}
513template <>
514EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(const uint64_t& a) {
515 const uint64_t countdown[] = {0, 1};
516 return __lsx_vadd_d(pset1<Packet2ul>(a), __lsx_vld(countdown, 0));
517}
518template <>
519EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
520 static const Packet4f countdown = {0.0f, 1.0f, 2.0f, 3.0f};
521 return __lsx_vfadd_s(pset1<Packet4f>(a), countdown);
522}
523template <>
524EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
525 static const Packet2d countdown = {0.0f, 1.0f};
526 return __lsx_vfadd_d(pset1<Packet2d>(a), countdown);
527}
528
529template <>
530EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(const Packet16c& a, const Packet16c& b) {
531 return __lsx_vadd_b(a, b);
532}
533template <>
534EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(const Packet8s& a, const Packet8s& b) {
535 return __lsx_vadd_h(a, b);
536}
537template <>
538EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
539 return __lsx_vadd_w(a, b);
540}
541template <>
542EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) {
543 return __lsx_vadd_d(a, b);
544}
545template <>
546EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
547 return __lsx_vadd_b(a, b);
548}
549template <>
550EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(const Packet8us& a, const Packet8us& b) {
551 return __lsx_vadd_h(a, b);
552}
553template <>
554EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
555 return __lsx_vadd_w(a, b);
556}
557template <>
558EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
559 return __lsx_vadd_d(a, b);
560}
561template <>
562EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
563 return __lsx_vfadd_s(a, b);
564}
565template <>
566EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
567 return __lsx_vfadd_d(a, b);
568}
569
570template <>
571EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(const Packet16c& a, const Packet16c& b) {
572 return __lsx_vsub_b(a, b);
573}
574template <>
575EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(const Packet8s& a, const Packet8s& b) {
576 return __lsx_vsub_h(a, b);
577}
578template <>
579EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
580 return __lsx_vsub_w(a, b);
581}
582template <>
583EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) {
584 return __lsx_vsub_d(a, b);
585}
586template <>
587EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
588 return __lsx_vsub_b(a, b);
589}
590template <>
591EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(const Packet8us& a, const Packet8us& b) {
592 return __lsx_vsub_h(a, b);
593}
594template <>
595EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
596 return __lsx_vsub_w(a, b);
597}
598template <>
599EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
600 return __lsx_vsub_d(a, b);
601}
602template <>
603EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
604 return __lsx_vfsub_s(a, b);
605}
606template <>
607EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
608 return __lsx_vfsub_d(a, b);
609}
610
611template <>
612EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b);
613template <>
614EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(const Packet4f& a, const Packet4f& b) {
615 const Packet4f mask =
616 make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
617 return padd(a, pxor(mask, b));
618}
619template <>
620EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b);
621template <>
622EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(const Packet2d& a, const Packet2d& b) {
623 const Packet2d mask = make_packet2d(numext::bit_cast<double>(0x8000000000000000ull), 0.0);
624 return padd(a, pxor(mask, b));
625}
626
627template <>
628EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
629 Packet4f mask = make_packet4f(numext::bit_cast<float>(0x80000000), numext::bit_cast<float>(0x80000000),
630 numext::bit_cast<float>(0x80000000), numext::bit_cast<float>(0x80000000));
631 return (Packet4f)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a));
632}
633template <>
634EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
635 Packet2d mask =
636 make_packet2d(numext::bit_cast<double>(0x8000000000000000), numext::bit_cast<double>(0x8000000000000000));
637 return (Packet2d)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a));
638}
639template <>
640EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) {
641 return __lsx_vneg_b(a);
642}
643template <>
644EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
645 return __lsx_vneg_h(a);
646}
647template <>
648EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
649 return __lsx_vneg_w(a);
650}
651template <>
652EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) {
653 return __lsx_vneg_d(a);
654}
655
656template <>
657EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
658 return __lsx_vfmul_s(a, b);
659}
660template <>
661EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
662 return __lsx_vfmul_d(a, b);
663}
664template <>
665EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(const Packet16c& a, const Packet16c& b) {
666 return __lsx_vmul_b(a, b);
667}
668template <>
669EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(const Packet8s& a, const Packet8s& b) {
670 return __lsx_vmul_h(a, b);
671}
672template <>
673EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
674 return __lsx_vmul_w(a, b);
675}
676template <>
677EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(const Packet2l& a, const Packet2l& b) {
678 return __lsx_vmul_d(a, b);
679}
680template <>
681EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
682 return __lsx_vmul_b(a, b);
683}
684template <>
685EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(const Packet8us& a, const Packet8us& b) {
686 return __lsx_vmul_h(a, b);
687}
688template <>
689EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
690 return __lsx_vmul_w(a, b);
691}
692template <>
693EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
694 return __lsx_vmul_d(a, b);
695}
696
697template <>
698EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
699 return __lsx_vfdiv_s(a, b);
700}
701template <>
702EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
703 return __lsx_vfdiv_d(a, b);
704}
705template <>
706EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(const Packet8s& a, const Packet8s& b) {
707 return __lsx_vdiv_h(a, b);
708}
709template <>
710EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) {
711 return __lsx_vdiv_w(a, b);
712}
713template <>
714EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(const Packet2l& a, const Packet2l& b) {
715 return __lsx_vdiv_d(a, b);
716}
717template <>
718EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(const Packet8us& a, const Packet8us& b) {
719 return __lsx_vdiv_hu(a, b);
720}
721template <>
722EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
723 return __lsx_vdiv_wu(a, b);
724}
725template <>
726EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
727 return __lsx_vdiv_du(a, b);
728}
729
730template <>
731EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
732 return __lsx_vfmadd_s(a, b, c);
733}
734template <>
735EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
736 return __lsx_vfmadd_d(a, b, c);
737}
738template <>
739EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
740 return __lsx_vfmsub_s(a, b, c);
741}
742template <>
743EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
744 return __lsx_vfmsub_d(a, b, c);
745}
746template <>
747EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
748 return __lsx_vfnmsub_s(a, b, c);
749}
750template <>
751EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
752 return __lsx_vfnmsub_d(a, b, c);
753}
754template <>
755EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
756 return __lsx_vfnmadd_s(a, b, c);
757}
758template <>
759EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
760 return __lsx_vfnmadd_d(a, b, c);
761}
762template <>
763EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
764 return __lsx_vmadd_b(c, a, b);
765}
766template <>
767EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
768 return __lsx_vmadd_h(c, a, b);
769}
770template <>
771EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
772 return __lsx_vmadd_w(c, a, b);
773}
774template <>
775EIGEN_STRONG_INLINE Packet2l pmadd(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
776 return __lsx_vmadd_d(c, a, b);
777}
778template <>
779EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c) {
780 return __lsx_vmadd_b(c, a, b);
781}
782template <>
783EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) {
784 return __lsx_vmadd_h(c, a, b);
785}
786template <>
787EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c) {
788 return __lsx_vmadd_w(c, a, b);
789}
790template <>
791EIGEN_STRONG_INLINE Packet2ul pmadd(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c) {
792 return __lsx_vmadd_d(c, a, b);
793}
794
795template <>
796EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
797 return (Packet4f)__lsx_vand_v((__m128i)a, (__m128i)b);
798}
799template <>
800EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
801 return (Packet2d)__lsx_vand_v((__m128i)a, (__m128i)b);
802}
803template <>
804EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(const Packet16c& a, const Packet16c& b) {
805 return __lsx_vand_v(a, b);
806}
807template <>
808EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(const Packet8s& a, const Packet8s& b) {
809 return __lsx_vand_v(a, b);
810}
811template <>
812EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
813 return __lsx_vand_v(a, b);
814}
815template <>
816EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) {
817 return __lsx_vand_v(a, b);
818}
819template <>
820EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
821 return __lsx_vand_v(a, b);
822}
823template <>
824EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(const Packet8us& a, const Packet8us& b) {
825 return __lsx_vand_v(a, b);
826}
827template <>
828EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
829 return __lsx_vand_v(a, b);
830}
831template <>
832EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
833 return __lsx_vand_v(a, b);
834}
835
836template <>
837EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
838 return (Packet4f)__lsx_vor_v((__m128i)a, (__m128i)b);
839}
840template <>
841EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
842 return (Packet2d)__lsx_vor_v((__m128i)a, (__m128i)b);
843}
844template <>
845EIGEN_STRONG_INLINE Packet16c por<Packet16c>(const Packet16c& a, const Packet16c& b) {
846 return __lsx_vor_v(a, b);
847}
848template <>
849EIGEN_STRONG_INLINE Packet8s por<Packet8s>(const Packet8s& a, const Packet8s& b) {
850 return __lsx_vor_v(a, b);
851}
852template <>
853EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
854 return __lsx_vor_v(a, b);
855}
856template <>
857EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b) {
858 return __lsx_vor_v(a, b);
859}
860template <>
861EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
862 return __lsx_vor_v(a, b);
863}
864template <>
865EIGEN_STRONG_INLINE Packet8us por<Packet8us>(const Packet8us& a, const Packet8us& b) {
866 return __lsx_vor_v(a, b);
867}
868template <>
869EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
870 return __lsx_vor_v(a, b);
871}
872template <>
873EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
874 return __lsx_vor_v(a, b);
875}
876
877template <>
878EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
879 return (Packet4f)__lsx_vxor_v((__m128i)a, (__m128i)b);
880}
881template <>
882EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
883 return (Packet2d)__lsx_vxor_v((__m128i)a, (__m128i)b);
884}
885template <>
886EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(const Packet16c& a, const Packet16c& b) {
887 return __lsx_vxor_v(a, b);
888}
889template <>
890EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(const Packet8s& a, const Packet8s& b) {
891 return __lsx_vxor_v(a, b);
892}
893template <>
894EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
895 return __lsx_vxor_v(a, b);
896}
897template <>
898EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b) {
899 return __lsx_vxor_v(a, b);
900}
901template <>
902EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
903 return __lsx_vxor_v(a, b);
904}
905template <>
906EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(const Packet8us& a, const Packet8us& b) {
907 return __lsx_vxor_v(a, b);
908}
909template <>
910EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
911 return __lsx_vxor_v(a, b);
912}
913template <>
914EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
915 return __lsx_vxor_v(a, b);
916}
917
918template <>
919EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
920 return (Packet4f)__lsx_vandn_v((__m128i)b, (__m128i)a);
921}
922template <>
923EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
924 return (Packet2d)__lsx_vandn_v((__m128i)b, (__m128i)a);
925}
926template <>
927EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(const Packet16c& a, const Packet16c& b) {
928 return __lsx_vandn_v(b, a);
929}
930template <>
931EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(const Packet8s& a, const Packet8s& b) {
932 return __lsx_vandn_v(b, a);
933}
934template <>
935EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
936 return __lsx_vandn_v(b, a);
937}
938template <>
939EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b) {
940 return __lsx_vandn_v(b, a);
941}
942template <>
943EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
944 return __lsx_vandn_v(b, a);
945}
946template <>
947EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(const Packet8us& a, const Packet8us& b) {
948 return __lsx_vandn_v(b, a);
949}
950template <>
951EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
952 return __lsx_vandn_v(b, a);
953}
954template <>
955EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
956 return __lsx_vandn_v(b, a);
957}
958
959template <>
960EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b) {
961 return (Packet4f)__lsx_vfcmp_cle_s(a, b);
962}
963template <>
964EIGEN_STRONG_INLINE Packet2d pcmp_le<Packet2d>(const Packet2d& a, const Packet2d& b) {
965 return (Packet2d)__lsx_vfcmp_cle_d(a, b);
966}
967template <>
968EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(const Packet16c& a, const Packet16c& b) {
969 return __lsx_vsle_b(a, b);
970}
971template <>
972EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(const Packet8s& a, const Packet8s& b) {
973 return __lsx_vsle_h(a, b);
974}
975template <>
976EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(const Packet4i& a, const Packet4i& b) {
977 return __lsx_vsle_w(a, b);
978}
979template <>
980EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(const Packet2l& a, const Packet2l& b) {
981 return __lsx_vsle_d(a, b);
982}
983template <>
984EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
985 return __lsx_vsle_bu(a, b);
986}
987template <>
988EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(const Packet8us& a, const Packet8us& b) {
989 return __lsx_vsle_hu(a, b);
990}
991template <>
992EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
993 return __lsx_vsle_wu(a, b);
994}
995template <>
996EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
997 return __lsx_vsle_du(a, b);
998}
999
1000template <>
1001EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b) {
1002 return (Packet4f)__lsx_vfcmp_clt_s(a, b);
1003}
1004template <>
1005EIGEN_STRONG_INLINE Packet2d pcmp_lt<Packet2d>(const Packet2d& a, const Packet2d& b) {
1006 return (Packet2d)__lsx_vfcmp_clt_d(a, b);
1007}
1008template <>
1009EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(const Packet16c& a, const Packet16c& b) {
1010 return __lsx_vslt_b(a, b);
1011}
1012template <>
1013EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(const Packet8s& a, const Packet8s& b) {
1014 return __lsx_vslt_h(a, b);
1015}
1016template <>
1017EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(const Packet4i& a, const Packet4i& b) {
1018 return __lsx_vslt_w(a, b);
1019}
1020template <>
1021EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(const Packet2l& a, const Packet2l& b) {
1022 return __lsx_vslt_d(a, b);
1023}
1024template <>
1025EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1026 return __lsx_vslt_bu(a, b);
1027}
1028template <>
1029EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(const Packet8us& a, const Packet8us& b) {
1030 return __lsx_vslt_hu(a, b);
1031}
1032template <>
1033EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1034 return __lsx_vslt_wu(a, b);
1035}
1036template <>
1037EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1038 return __lsx_vslt_du(a, b);
1039}
1040
1041template <>
1042EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b) {
1043 return (Packet4f)__lsx_vfcmp_sult_s(a, b);
1044}
1045template <>
1046EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan<Packet2d>(const Packet2d& a, const Packet2d& b) {
1047 return (Packet2d)__lsx_vfcmp_sult_d(a, b);
1048}
1049
1050template <>
1051EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(const Packet4f& a, const Packet4f& b) {
1052 return (Packet4f)__lsx_vfcmp_seq_s(a, b);
1053}
1054template <>
1055EIGEN_STRONG_INLINE Packet2d pcmp_eq<Packet2d>(const Packet2d& a, const Packet2d& b) {
1056 return (Packet2d)__lsx_vfcmp_seq_d(a, b);
1057}
1058template <>
1059EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(const Packet16c& a, const Packet16c& b) {
1060 return __lsx_vseq_b(a, b);
1061}
1062template <>
1063EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(const Packet8s& a, const Packet8s& b) {
1064 return __lsx_vseq_h(a, b);
1065}
1066template <>
1067EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(const Packet4i& a, const Packet4i& b) {
1068 return __lsx_vseq_w(a, b);
1069}
1070template <>
1071EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(const Packet2l& a, const Packet2l& b) {
1072 return __lsx_vseq_d(a, b);
1073}
1074template <>
1075EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1076 return __lsx_vseq_b(a, b);
1077}
1078template <>
1079EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(const Packet8us& a, const Packet8us& b) {
1080 return __lsx_vseq_h(a, b);
1081}
1082template <>
1083EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1084 return __lsx_vseq_w(a, b);
1085}
1086template <>
1087EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1088 return __lsx_vseq_d(a, b);
1089}
1090
1091template <>
1092EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(const Packet16c& a, const Packet16c& b) {
1093 return __lsx_vmin_b(a, b);
1094}
1095template <>
1096EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(const Packet8s& a, const Packet8s& b) {
1097 return __lsx_vmin_h(a, b);
1098}
1099template <>
1100EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
1101 return __lsx_vmin_w(a, b);
1102}
1103template <>
1104EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(const Packet2l& a, const Packet2l& b) {
1105 return __lsx_vmin_d(a, b);
1106}
1107template <>
1108EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1109 return __lsx_vmin_bu(a, b);
1110}
1111template <>
1112EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(const Packet8us& a, const Packet8us& b) {
1113 return __lsx_vmin_hu(a, b);
1114}
1115template <>
1116EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1117 return __lsx_vmin_wu(a, b);
1118}
1119template <>
1120EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1121 return __lsx_vmin_du(a, b);
1122}
1123
1124template <>
1125EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(const Packet16c& a, const Packet16c& b) {
1126 return __lsx_vmax_b(a, b);
1127}
1128template <>
1129EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(const Packet8s& a, const Packet8s& b) {
1130 return __lsx_vmax_h(a, b);
1131}
1132template <>
1133EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
1134 return __lsx_vmax_w(a, b);
1135}
1136template <>
1137EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(const Packet2l& a, const Packet2l& b) {
1138 return __lsx_vmax_d(a, b);
1139}
1140template <>
1141EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1142 return __lsx_vmax_bu(a, b);
1143}
1144template <>
1145EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(const Packet8us& a, const Packet8us& b) {
1146 return __lsx_vmax_hu(a, b);
1147}
1148template <>
1149EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1150 return __lsx_vmax_wu(a, b);
1151}
1152template <>
1153EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1154 return __lsx_vmax_du(a, b);
1155}
1156
1157template <>
1158EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
1159 Packet4i aNaN = __lsx_vfcmp_cun_s(a, a);
1160 Packet4i aMinOrNaN = por<Packet4i>(__lsx_vfcmp_clt_s(a, b), aNaN);
1161 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN);
1162}
1163template <>
1164EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
1165 Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
1166 Packet2l aMinOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(a, b), aNaN);
1167 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN);
1168}
1169template <>
1170EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
1171 Packet4i aNaN = __lsx_vfcmp_cun_s(a, a);
1172 Packet4i aMaxOrNaN = por<Packet4i>(__lsx_vfcmp_clt_s(b, a), aNaN);
1173 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN);
1174}
1175template <>
1176EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
1177 Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
1178 Packet2l aMaxOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(b, a), aNaN);
1179 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN);
1180}
1181
1182template <int N>
1183EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(const Packet16c& a) {
1184 return __lsx_vsrai_b((__m128i)a, N);
1185}
1186template <int N>
1187EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(const Packet8s& a) {
1188 return __lsx_vsrai_h((__m128i)a, N);
1189}
1190template <int N>
1191EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) {
1192 return __lsx_vsrai_w((__m128i)a, N);
1193}
1194template <int N>
1195EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(const Packet2l& a) {
1196 return __lsx_vsrai_d((__m128i)a, N);
1197}
1198template <int N>
1199EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(const Packet16uc& a) {
1200 return __lsx_vsrli_b((__m128i)a, N);
1201}
1202template <int N>
1203EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(const Packet8us& a) {
1204 return __lsx_vsrli_h((__m128i)a, N);
1205}
1206template <int N>
1207EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(const Packet4ui& a) {
1208 return __lsx_vsrli_w((__m128i)a, N);
1209}
1210template <int N>
1211EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(const Packet2ul& a) {
1212 return __lsx_vsrli_d((__m128i)a, N);
1213}
1214
1215template <int N>
1216EIGEN_STRONG_INLINE Packet16c plogical_shift_right(const Packet16c& a) {
1217 return __lsx_vsrli_b((__m128i)a, N);
1218}
1219template <int N>
1220EIGEN_STRONG_INLINE Packet8s plogical_shift_right(const Packet8s& a) {
1221 return __lsx_vsrli_h((__m128i)a, N);
1222}
1223template <int N>
1224EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) {
1225 return __lsx_vsrli_w((__m128i)a, N);
1226}
1227template <int N>
1228EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
1229 return __lsx_vsrli_d((__m128i)a, N);
1230}
1231template <int N>
1232EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(const Packet16uc& a) {
1233 return __lsx_vsrli_b((__m128i)a, N);
1234}
1235template <int N>
1236EIGEN_STRONG_INLINE Packet8us plogical_shift_right(const Packet8us& a) {
1237 return __lsx_vsrli_h((__m128i)a, N);
1238}
1239template <int N>
1240EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) {
1241 return __lsx_vsrli_w((__m128i)a, N);
1242}
1243template <int N>
1244EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(const Packet2ul& a) {
1245 return __lsx_vsrli_d((__m128i)a, N);
1246}
1247
1248template <int N>
1249EIGEN_STRONG_INLINE Packet16c plogical_shift_left(const Packet16c& a) {
1250 return __lsx_vslli_b((__m128i)a, N);
1251}
1252template <int N>
1253EIGEN_STRONG_INLINE Packet8s plogical_shift_left(const Packet8s& a) {
1254 return __lsx_vslli_h((__m128i)a, N);
1255}
1256template <int N>
1257EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) {
1258 return __lsx_vslli_w((__m128i)a, N);
1259}
1260template <int N>
1261EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
1262 return __lsx_vslli_d((__m128i)a, N);
1263}
1264template <int N>
1265EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(const Packet16uc& a) {
1266 return __lsx_vslli_b((__m128i)a, N);
1267}
1268template <int N>
1269EIGEN_STRONG_INLINE Packet8us plogical_shift_left(const Packet8us& a) {
1270 return __lsx_vslli_h((__m128i)a, N);
1271}
1272template <int N>
1273EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) {
1274 return __lsx_vslli_w((__m128i)a, N);
1275}
1276template <int N>
1277EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(const Packet2ul& a) {
1278 return __lsx_vslli_d((__m128i)a, N);
1279}
1280
1281template <>
1282EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) {
1283 return (Packet4f)__lsx_vbitclri_w((__m128i)a, 31);
1284}
1285template <>
1286EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) {
1287 return (Packet2d)__lsx_vbitclri_d((__m128i)a, 63);
1288}
1289template <>
1290EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) {
1291 return __lsx_vabsd_b(a, pzero(a));
1292}
1293template <>
1294EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) {
1295 return __lsx_vabsd_h(a, pzero(a));
1296}
1297template <>
1298EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) {
1299 return __lsx_vabsd_w(a, pzero(a));
1300}
1301template <>
1302EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) {
1303 return __lsx_vabsd_d(a, pzero(a));
1304}
1305template <>
1306EIGEN_STRONG_INLINE Packet16c pabsdiff(const Packet16c& a, const Packet16c& b) {
1307 return __lsx_vabsd_b(a, b);
1308}
1309template <>
1310EIGEN_STRONG_INLINE Packet8s pabsdiff(const Packet8s& a, const Packet8s& b) {
1311 return __lsx_vabsd_h(a, b);
1312}
1313template <>
1314EIGEN_STRONG_INLINE Packet4i pabsdiff(const Packet4i& a, const Packet4i& b) {
1315 return __lsx_vabsd_w(a, b);
1316}
1317template <>
1318EIGEN_STRONG_INLINE Packet2l pabsdiff(const Packet2l& a, const Packet2l& b) {
1319 return __lsx_vabsd_d(a, b);
1320}
1321template <>
1322EIGEN_STRONG_INLINE Packet16uc pabsdiff(const Packet16uc& a, const Packet16uc& b) {
1323 return __lsx_vabsd_bu(a, b);
1324}
1325template <>
1326EIGEN_STRONG_INLINE Packet8us pabsdiff(const Packet8us& a, const Packet8us& b) {
1327 return __lsx_vabsd_hu(a, b);
1328}
1329template <>
1330EIGEN_STRONG_INLINE Packet4ui pabsdiff(const Packet4ui& a, const Packet4ui& b) {
1331 return __lsx_vabsd_wu(a, b);
1332}
1333template <>
1334EIGEN_STRONG_INLINE Packet2ul pabsdiff(const Packet2ul& a, const Packet2ul& b) {
1335 return __lsx_vabsd_du(a, b);
1336}
1337template <>
1338EIGEN_STRONG_INLINE Packet4f pabsdiff(const Packet4f& a, const Packet4f& b) {
1339 return pabs(psub(a, b));
1340}
1341template <>
1342EIGEN_STRONG_INLINE Packet2d pabsdiff(const Packet2d& a, const Packet2d& b) {
1343 return pabs(psub(a, b));
1344}
1345
1346template <>
1347EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
1348 EIGEN_DEBUG_ALIGNED_LOAD return (Packet4f)__lsx_vld(from, 0);
1349}
1350template <>
1351EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
1352 EIGEN_DEBUG_ALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0);
1353}
1354template <>
1355EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(const int8_t* from) {
1356 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1357}
1358template <>
1359EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(const int16_t* from) {
1360 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1361}
1362template <>
1363EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) {
1364 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1365}
1366template <>
1367EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(const int64_t* from) {
1368 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1369}
1370template <>
1371EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(const uint8_t* from) {
1372 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1373}
1374template <>
1375EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(const uint16_t* from) {
1376 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1377}
1378template <>
1379EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(const uint32_t* from) {
1380 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1381}
1382template <>
1383EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(const uint64_t* from) {
1384 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1385}
1386
1387template <>
1388EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
1389 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4f)__lsx_vld(from, 0);
1390}
1391template <>
1392EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
1393 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0);
1394}
1395template <>
1396EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(const int8_t* from) {
1397 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1398}
1399template <>
1400EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(const int16_t* from) {
1401 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1402}
1403template <>
1404EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) {
1405 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1406}
1407template <>
1408EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(const int64_t* from) {
1409 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1410}
1411template <>
1412EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(const uint8_t* from) {
1413 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1414}
1415template <>
1416EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(const uint16_t* from) {
1417 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1418}
1419template <>
1420EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(const uint32_t* from) {
1421 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1422}
1423template <>
1424EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(const uint64_t* from) {
1425 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1426}
1427
1428template <>
1429EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
1430 float f0 = from[0], f1 = from[1];
1431 return make_packet4f(f0, f0, f1, f1);
1432}
1433template <>
1434EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(const int8_t* from) {
1435 Packet16c tmp = pload<Packet16c>(from);
1436 return __lsx_vilvl_b(tmp, tmp);
1437}
1438template <>
1439EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(const int16_t* from) {
1440 Packet8s tmp = pload<Packet8s>(from);
1441 return __lsx_vilvl_h(tmp, tmp);
1442}
1443template <>
1444EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from) {
1445 Packet4i tmp = pload<Packet4i>(from);
1446 return __lsx_vilvl_w(tmp, tmp);
1447}
1448template <>
1449EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(const uint8_t* from) {
1450 Packet16uc tmp = pload<Packet16uc>(from);
1451 return __lsx_vilvl_b(tmp, tmp);
1452}
1453template <>
1454EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(const uint16_t* from) {
1455 Packet8us tmp = pload<Packet8us>(from);
1456 return __lsx_vilvl_h(tmp, tmp);
1457}
1458template <>
1459EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(const uint32_t* from) {
1460 Packet4ui tmp = pload<Packet4ui>(from);
1461 return __lsx_vilvl_w(tmp, tmp);
1462}
1463template <>
1464EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
1465 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0);
1466}
1467template <>
1468EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
1469 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0);
1470}
1471template <>
1472EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet16c& from) {
1473 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1474}
1475template <>
1476EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet8s& from) {
1477 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1478}
1479template <>
1480EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) {
1481 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1482}
1483template <>
1484EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet2l& from) {
1485 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1486}
1487template <>
1488EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet16uc& from) {
1489 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1490}
1491template <>
1492EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet8us& from) {
1493 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1494}
1495template <>
1496EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet4ui& from) {
1497 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1498}
1499template <>
1500EIGEN_STRONG_INLINE void pstore<uint64_t>(uint64_t* to, const Packet2ul& from) {
1501 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1502}
1503
1504template <>
1505EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
1506 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0);
1507}
1508template <>
1509EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
1510 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0);
1511}
1512
1513template <>
1514EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet16c& from) {
1515 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1516}
1517template <>
1518EIGEN_STRONG_INLINE void pstoreu<int16_t>(int16_t* to, const Packet8s& from) {
1519 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1520}
1521template <>
1522EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from) {
1523 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1524}
1525template <>
1526EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet2l& from) {
1527 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1528}
1529template <>
1530EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet16uc& from) {
1531 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1532}
1533template <>
1534EIGEN_STRONG_INLINE void pstoreu<uint16_t>(uint16_t* to, const Packet8us& from) {
1535 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1536}
1537template <>
1538EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet4ui& from) {
1539 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1540}
1541template <>
1542EIGEN_STRONG_INLINE void pstoreu<uint64_t>(uint64_t* to, const Packet2ul& from) {
1543 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1544}
1545
1546template <>
1547EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
1548 Packet4f v = {from[0], from[stride], from[2 * stride], from[3 * stride]};
1549 return v;
1550}
1551template <>
1552EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
1553 Packet2d v = {from[0], from[stride]};
1554 return v;
1555}
1556template <>
1557EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(const int8_t* from, Index stride) {
1558 int8_t v[16] __attribute__((aligned(16)));
1559 v[0] = from[0];
1560 v[1] = from[stride];
1561 v[2] = from[2 * stride];
1562 v[3] = from[3 * stride];
1563 v[4] = from[4 * stride];
1564 v[5] = from[5 * stride];
1565 v[6] = from[6 * stride];
1566 v[7] = from[7 * stride];
1567 v[8] = from[8 * stride];
1568 v[9] = from[9 * stride];
1569 v[10] = from[10 * stride];
1570 v[11] = from[11 * stride];
1571 v[12] = from[12 * stride];
1572 v[13] = from[13 * stride];
1573 v[14] = from[14 * stride];
1574 v[15] = from[15 * stride];
1575 return __lsx_vld(v, 0);
1576}
1577template <>
1578EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(const int16_t* from, Index stride) {
1579 int16_t v[8] __attribute__((aligned(16)));
1580 v[0] = from[0];
1581 v[1] = from[stride];
1582 v[2] = from[2 * stride];
1583 v[3] = from[3 * stride];
1584 v[4] = from[4 * stride];
1585 v[5] = from[5 * stride];
1586 v[6] = from[6 * stride];
1587 v[7] = from[7 * stride];
1588 return __lsx_vld(v, 0);
1589}
1590template <>
1591EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride) {
1592 int32_t v[4] __attribute__((aligned(16)));
1593 v[0] = from[0];
1594 v[1] = from[stride];
1595 v[2] = from[2 * stride];
1596 v[3] = from[3 * stride];
1597 return __lsx_vld(v, 0);
1598}
1599template <>
1600EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(const int64_t* from, Index stride) {
1601 int64_t v[2] __attribute__((aligned(16)));
1602 v[0] = from[0];
1603 v[1] = from[stride];
1604 return __lsx_vld(v, 0);
1605}
1606template <>
1607EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(const uint8_t* from, Index stride) {
1608 uint8_t v[16] __attribute__((aligned(16)));
1609 v[0] = from[0];
1610 v[1] = from[stride];
1611 v[2] = from[2 * stride];
1612 v[3] = from[3 * stride];
1613 v[4] = from[4 * stride];
1614 v[5] = from[5 * stride];
1615 v[6] = from[6 * stride];
1616 v[7] = from[7 * stride];
1617 v[8] = from[8 * stride];
1618 v[9] = from[9 * stride];
1619 v[10] = from[10 * stride];
1620 v[11] = from[11 * stride];
1621 v[12] = from[12 * stride];
1622 v[13] = from[13 * stride];
1623 v[14] = from[14 * stride];
1624 v[15] = from[15 * stride];
1625 return __lsx_vld(v, 0);
1626}
1627template <>
1628EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(const uint16_t* from, Index stride) {
1629 uint16_t v[8] __attribute__((aligned(16)));
1630 v[0] = from[0];
1631 v[1] = from[stride];
1632 v[2] = from[2 * stride];
1633 v[3] = from[3 * stride];
1634 v[4] = from[4 * stride];
1635 v[5] = from[5 * stride];
1636 v[6] = from[6 * stride];
1637 v[7] = from[7 * stride];
1638 return __lsx_vld(v, 0);
1639}
1640template <>
1641EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(const uint32_t* from, Index stride) {
1642 uint32_t v[4] __attribute__((aligned(16)));
1643 v[0] = from[0];
1644 v[1] = from[stride];
1645 v[2] = from[2 * stride];
1646 v[3] = from[3 * stride];
1647 return __lsx_vld(v, 0);
1648}
1649template <>
1650EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(const uint64_t* from, Index stride) {
1651 uint64_t v[2] __attribute__((aligned(16)));
1652 v[0] = from[0];
1653 v[1] = from[stride];
1654 return __lsx_vld(v, 0);
1655}
1656
1657template <>
1658EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
1659 __lsx_vstelm_w(from, to, 0, 0);
1660 __lsx_vstelm_w(from, to + stride * 1, 0, 1);
1661 __lsx_vstelm_w(from, to + stride * 2, 0, 2);
1662 __lsx_vstelm_w(from, to + stride * 3, 0, 3);
1663}
1664template <>
1665EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
1666 __lsx_vstelm_d(from, to, 0, 0);
1667 __lsx_vstelm_d(from, to + stride, 0, 1);
1668}
1669template <>
1670EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet16c>(int8_t* to, const Packet16c& from,
1671 Index stride) {
1672 __lsx_vstelm_b((__m128i)from, to, 0, 0);
1673 __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
1674 __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
1675 __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
1676 __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
1677 __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
1678 __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
1679 __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
1680 __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
1681 __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
1682 __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
1683 __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
1684 __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
1685 __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
1686 __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
1687 __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
1688}
1689template <>
1690EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet8s>(int16_t* to, const Packet8s& from,
1691 Index stride) {
1692 __lsx_vstelm_h((__m128i)from, to, 0, 0);
1693 __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
1694 __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
1695 __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
1696 __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
1697 __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
1698 __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
1699 __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
1700}
1701template <>
1702EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from,
1703 Index stride) {
1704 __lsx_vstelm_w((__m128i)from, to, 0, 0);
1705 __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
1706 __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
1707 __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
1708}
1709template <>
1710EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int64_t, Packet2l>(int64_t* to, const Packet2l& from,
1711 Index stride) {
1712 __lsx_vstelm_d((__m128i)from, to, 0, 0);
1713 __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
1714}
1715template <>
1716EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet16uc>(uint8_t* to, const Packet16uc& from,
1717 Index stride) {
1718 __lsx_vstelm_b((__m128i)from, to, 0, 0);
1719 __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
1720 __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
1721 __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
1722 __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
1723 __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
1724 __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
1725 __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
1726 __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
1727 __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
1728 __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
1729 __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
1730 __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
1731 __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
1732 __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
1733 __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
1734}
1735template <>
1736EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet8us>(uint16_t* to, const Packet8us& from,
1737 Index stride) {
1738 __lsx_vstelm_h((__m128i)from, to, 0, 0);
1739 __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
1740 __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
1741 __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
1742 __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
1743 __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
1744 __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
1745 __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
1746}
1747template <>
1748EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from,
1749 Index stride) {
1750 __lsx_vstelm_w((__m128i)from, to, 0, 0);
1751 __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
1752 __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
1753 __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
1754}
1755template <>
1756EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint64_t, Packet2ul>(uint64_t* to, const Packet2ul& from,
1757 Index stride) {
1758 __lsx_vstelm_d((__m128i)from, to, 0, 0);
1759 __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
1760}
1761
1762template <>
1763EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1764 __builtin_prefetch(addr);
1765}
1766template <>
1767EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1768 __builtin_prefetch(addr);
1769}
1770template <>
1771EIGEN_STRONG_INLINE void prefetch<int8_t>(const int8_t* addr) {
1772 __builtin_prefetch(addr);
1773}
1774template <>
1775EIGEN_STRONG_INLINE void prefetch<int16_t>(const int16_t* addr) {
1776 __builtin_prefetch(addr);
1777}
1778template <>
1779EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) {
1780 __builtin_prefetch(addr);
1781}
1782template <>
1783EIGEN_STRONG_INLINE void prefetch<int64_t>(const int64_t* addr) {
1784 __builtin_prefetch(addr);
1785}
1786template <>
1787EIGEN_STRONG_INLINE void prefetch<uint8_t>(const uint8_t* addr) {
1788 __builtin_prefetch(addr);
1789}
1790template <>
1791EIGEN_STRONG_INLINE void prefetch<uint16_t>(const uint16_t* addr) {
1792 __builtin_prefetch(addr);
1793}
1794template <>
1795EIGEN_STRONG_INLINE void prefetch<uint32_t>(const uint32_t* addr) {
1796 __builtin_prefetch(addr);
1797}
1798template <>
1799EIGEN_STRONG_INLINE void prefetch<uint64_t>(const uint64_t* addr) {
1800 __builtin_prefetch(addr);
1801}
1802
1803template <>
1804EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
1805 float v;
1806 __lsx_vstelm_w(a, &v, 0, 0);
1807 return v;
1808}
1809template <>
1810EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
1811 double v;
1812 __lsx_vstelm_d(a, &v, 0, 0);
1813 return v;
1814}
1815
1816template <>
1817EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(const Packet16c& a) {
1818 return (int8_t)__lsx_vpickve2gr_b((__m128i)a, 0);
1819}
1820template <>
1821EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(const Packet8s& a) {
1822 return (int16_t)__lsx_vpickve2gr_h((__m128i)a, 0);
1823}
1824template <>
1825EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) {
1826 return __lsx_vpickve2gr_w((__m128i)a, 0);
1827}
1828template <>
1829EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(const Packet2l& a) {
1830 return __lsx_vpickve2gr_d((__m128i)a, 0);
1831}
1832template <>
1833EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(const Packet16uc& a) {
1834 return (uint8_t)__lsx_vpickve2gr_bu((__m128i)a, 0);
1835}
1836template <>
1837EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(const Packet8us& a) {
1838 return (uint16_t)__lsx_vpickve2gr_hu((__m128i)a, 0);
1839}
1840template <>
1841EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(const Packet4ui& a) {
1842 return __lsx_vpickve2gr_wu((__m128i)a, 0);
1843}
1844template <>
1845EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(const Packet2ul& a) {
1846 return __lsx_vpickve2gr_du((__m128i)a, 0);
1847}
1848
1849template <>
1850EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
1851 return (Packet4f)__lsx_vshuf4i_w(a, 0x1B);
1852}
1853template <>
1854EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
1855 return (Packet2d)__lsx_vshuf4i_d(a, a, 0x1);
1856}
1857template <>
1858EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) {
1859 return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B);
1860}
1861template <>
1862EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a) {
1863 return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B);
1864}
1865template <>
1866EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
1867 return __lsx_vshuf4i_w((__m128i)a, 0x1B);
1868}
1869template <>
1870EIGEN_STRONG_INLINE Packet2l preverse(const Packet2l& a) {
1871 return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
1872}
1873template <>
1874EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) {
1875 return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B);
1876}
1877template <>
1878EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) {
1879 return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B);
1880}
1881template <>
1882EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a) {
1883 return __lsx_vshuf4i_w((__m128i)a, 0x1B);
1884}
1885template <>
1886EIGEN_STRONG_INLINE Packet2ul preverse(const Packet2ul& a) {
1887 return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
1888}
1889
1890template <>
1891EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
1892 Packet4f tmp = __lsx_vfadd_s(a, vec4f_swizzle1(a, 2, 3, 2, 3));
1893 return pfirst<Packet4f>(__lsx_vfadd_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1)));
1894}
1895template <>
1896EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
1897 return pfirst<Packet2d>(__lsx_vfadd_d(a, preverse(a)));
1898}
1899
1900template <typename Packet>
1901EIGEN_STRONG_INLINE bool predux_any_lsx(const Packet& a) {
1902 return __lsx_bnz_v((__m128i)a);
1903}
1904
1905template <>
1906EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) {
1907 return predux_any_lsx(a);
1908}
1909
1910template <>
1911EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) {
1912 return predux_any_lsx(a);
1913}
1914
1915template <>
1916EIGEN_STRONG_INLINE bool predux_any(const Packet16c& a) {
1917 return predux_any_lsx(a);
1918}
1919
1920template <>
1921EIGEN_STRONG_INLINE bool predux_any(const Packet8s& a) {
1922 return predux_any_lsx(a);
1923}
1924
1925template <>
1926EIGEN_STRONG_INLINE bool predux_any(const Packet4i& a) {
1927 return predux_any_lsx(a);
1928}
1929
1930template <>
1931EIGEN_STRONG_INLINE bool predux_any(const Packet2l& a) {
1932 return predux_any_lsx(a);
1933}
1934
1935template <>
1936EIGEN_STRONG_INLINE bool predux_any(const Packet16uc& a) {
1937 return predux_any_lsx(a);
1938}
1939
1940template <>
1941EIGEN_STRONG_INLINE bool predux_any(const Packet8us& a) {
1942 return predux_any_lsx(a);
1943}
1944
1945template <>
1946EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& a) {
1947 return predux_any_lsx(a);
1948}
1949
1950template <>
1951EIGEN_STRONG_INLINE bool predux_any(const Packet2ul& a) {
1952 return predux_any_lsx(a);
1953}
1954
1955template <>
1956EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a) {
1957 Packet8s tmp1 = __lsx_vhaddw_h_b(a, a);
1958 Packet4i tmp2 = __lsx_vhaddw_w_h(tmp1, tmp1);
1959 Packet2l tmp3 = __lsx_vhaddw_d_w(tmp2, tmp2);
1960 return (int8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp3, tmp3), 0);
1961}
1962template <>
1963EIGEN_STRONG_INLINE int16_t predux<Packet8s>(const Packet8s& a) {
1964 Packet4i tmp1 = __lsx_vhaddw_w_h(a, a);
1965 Packet2l tmp2 = __lsx_vhaddw_d_w(tmp1, tmp1);
1966 return (int16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp2, tmp2), 0);
1967}
1968template <>
1969EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) {
1970 Packet2l tmp = __lsx_vhaddw_d_w(a, a);
1971 return (int32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp, tmp), 0);
1972}
1973template <>
1974EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a) {
1975 return (int64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(a, a), 0);
1976}
1977template <>
1978EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(const Packet16uc& a) {
1979 Packet8us tmp1 = __lsx_vhaddw_hu_bu(a, a);
1980 Packet4ui tmp2 = __lsx_vhaddw_wu_hu(tmp1, tmp1);
1981 Packet2ul tmp3 = __lsx_vhaddw_du_wu(tmp2, tmp2);
1982 return (uint8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp3, tmp3), 0);
1983}
1984template <>
1985EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(const Packet8us& a) {
1986 Packet4ui tmp1 = __lsx_vhaddw_wu_hu(a, a);
1987 Packet2ul tmp2 = __lsx_vhaddw_du_wu(tmp1, tmp1);
1988 return (uint16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp2, tmp2), 0);
1989}
1990template <>
1991EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a) {
1992 Packet2ul tmp = __lsx_vhaddw_du_wu(a, a);
1993 return (uint32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp, tmp), 0);
1994}
1995template <>
1996EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(const Packet2ul& a) {
1997 return (uint64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(a, a), 0);
1998}
1999
2000template <>
2001EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
2002 Packet4f tmp = __lsx_vfmul_s(a, vec4f_swizzle1(a, 2, 3, 2, 3));
2003 return pfirst<Packet4f>(__lsx_vfmul_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1)));
2004}
2005template <>
2006EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
2007 return pfirst<Packet2d>(__lsx_vfmul_d(a, preverse(a)));
2008}
2009template <>
2010EIGEN_STRONG_INLINE int8_t predux_mul<Packet16c>(const Packet16c& a) {
2011 Packet8s tmp1 = __lsx_vmulwev_h_b(a, preverse(a));
2012 Packet4i tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1));
2013 Packet2l tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2));
2014 return (int8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0);
2015}
2016template <>
2017EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(const Packet8s& a) {
2018 Packet4i tmp1 = __lsx_vmulwev_w_h(a, preverse(a));
2019 Packet2l tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1));
2020 return (int16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0);
2021}
2022template <>
2023EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a) {
2024 Packet2l tmp = __lsx_vmulwev_d_w(a, preverse(a));
2025 return (int32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0);
2026}
2027template <>
2028EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(const Packet2l& a) {
2029 return (int64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(a, preverse(a)), 0);
2030}
2031template <>
2032EIGEN_STRONG_INLINE uint8_t predux_mul<Packet16uc>(const Packet16uc& a) {
2033 Packet8us tmp1 = __lsx_vmulwev_h_bu(a, preverse(a));
2034 Packet4ui tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1));
2035 Packet2ul tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2));
2036 return (uint8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0);
2037}
2038template <>
2039EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(const Packet8us& a) {
2040 Packet4ui tmp1 = __lsx_vmulwev_w_hu(a, preverse(a));
2041 Packet2ul tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1));
2042 return (uint16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0);
2043}
2044template <>
2045EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(const Packet4ui& a) {
2046 Packet2ul tmp = __lsx_vmulwev_d_wu(a, preverse(a));
2047 return (uint32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0);
2048}
2049template <>
2050EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(const Packet2ul& a) {
2051 return (uint64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_du(a, preverse(a)), 0);
2052}
2053
2054template <>
2055EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
2056 Packet4f tmp = __lsx_vfmin_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E));
2057 return pfirst(__lsx_vfmin_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1)));
2058}
2059template <>
2060EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
2061 return pfirst(__lsx_vfmin_d(a, preverse(a)));
2062}
2063template <>
2064EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(const Packet16c& a) {
2065 Packet16c tmp1 = __lsx_vmin_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2066 Packet16c tmp2 = __lsx_vmin_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2067 Packet16c tmp3 = __lsx_vmin_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2068 return pfirst((Packet16c)__lsx_vmin_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2069}
2070template <>
2071EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(const Packet8s& a) {
2072 Packet8s tmp1 = __lsx_vmin_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2073 Packet8s tmp2 = __lsx_vmin_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2074 return pfirst((Packet8s)__lsx_vmin_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2075}
2076template <>
2077EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) {
2078 Packet4i tmp = __lsx_vmin_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2079 return pfirst((Packet4i)__lsx_vmin_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2080}
2081template <>
2082EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(const Packet2l& a) {
2083 return pfirst((Packet2l)__lsx_vmin_d(a, preverse(a)));
2084}
2085template <>
2086EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(const Packet16uc& a) {
2087 Packet16uc tmp1 = __lsx_vmin_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2088 Packet16uc tmp2 = __lsx_vmin_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2089 Packet16uc tmp3 = __lsx_vmin_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2090 return pfirst((Packet16uc)__lsx_vmin_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2091}
2092template <>
2093EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(const Packet8us& a) {
2094 Packet8us tmp1 = __lsx_vmin_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2095 Packet8us tmp2 = __lsx_vmin_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2096 return pfirst((Packet8us)__lsx_vmin_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2097}
2098template <>
2099EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a) {
2100 Packet4ui tmp = __lsx_vmin_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2101 return pfirst((Packet4ui)__lsx_vmin_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2102}
2103template <>
2104EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(const Packet2ul& a) {
2105 return pfirst((Packet2ul)__lsx_vmin_du(a, preverse(a)));
2106}
2107
2108template <>
2109EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
2110 Packet4f tmp = __lsx_vfmax_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E));
2111 return pfirst(__lsx_vfmax_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1)));
2112}
2113template <>
2114EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
2115 return pfirst(__lsx_vfmax_d(a, preverse(a)));
2116}
2117template <>
2118EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(const Packet16c& a) {
2119 Packet16c tmp1 = __lsx_vmax_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2120 Packet16c tmp2 = __lsx_vmax_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2121 Packet16c tmp3 = __lsx_vmax_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2122 return pfirst((Packet16c)__lsx_vmax_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2123}
2124template <>
2125EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(const Packet8s& a) {
2126 Packet8s tmp1 = __lsx_vmax_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2127 Packet8s tmp2 = __lsx_vmax_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2128 return pfirst((Packet8s)__lsx_vmax_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2129}
2130template <>
2131EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) {
2132 Packet4i tmp = __lsx_vmax_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2133 return pfirst((Packet4i)__lsx_vmax_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2134}
2135template <>
2136EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(const Packet2l& a) {
2137 return pfirst((Packet2l)__lsx_vmax_d(a, preverse(a)));
2138}
2139template <>
2140EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(const Packet16uc& a) {
2141 Packet16uc tmp1 = __lsx_vmax_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2142 Packet16uc tmp2 = __lsx_vmax_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2143 Packet16uc tmp3 = __lsx_vmax_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2144 return pfirst((Packet16uc)__lsx_vmax_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2145}
2146template <>
2147EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(const Packet8us& a) {
2148 Packet8us tmp1 = __lsx_vmax_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2149 Packet8us tmp2 = __lsx_vmax_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2150 return pfirst((Packet8us)__lsx_vmax_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2151}
2152template <>
2153EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a) {
2154 Packet4ui tmp = __lsx_vmax_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2155 return pfirst((Packet4ui)__lsx_vmax_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2156}
2157template <>
2158EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(const Packet2ul& a) {
2159 return pfirst((Packet2ul)__lsx_vmax_du(a, preverse(a)));
2160}
2161
2162template <>
2163EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) {
2164 return __lsx_vfsqrt_s(a);
2165}
2166template <>
2167EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& a) {
2168 return __lsx_vfsqrt_d(a);
2169}
2170
2171EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
2172 Packet4f T0 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2173 Packet4f T1 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2174 Packet4f T2 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]);
2175 Packet4f T3 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]);
2176
2177 kernel.packet[0] = (Packet4f)__lsx_vilvl_d((__m128i)T2, (__m128i)T0);
2178 kernel.packet[1] = (Packet4f)__lsx_vilvh_d((__m128i)T2, (__m128i)T0);
2179 kernel.packet[2] = (Packet4f)__lsx_vilvl_d((__m128i)T3, (__m128i)T1);
2180 kernel.packet[3] = (Packet4f)__lsx_vilvh_d((__m128i)T3, (__m128i)T1);
2181}
2182EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
2183 Packet2d tmp = (Packet2d)__lsx_vilvh_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2184 kernel.packet[0] = (Packet2d)__lsx_vilvl_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2185 kernel.packet[1] = tmp;
2186}
2187EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
2188 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2189 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2190 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2191 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2192 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2193 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2194 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2195 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2196 __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]);
2197 __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]);
2198 __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]);
2199 __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]);
2200 __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]);
2201 __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]);
2202 __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]);
2203 __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]);
2204
2205 __m128i s0 = __lsx_vilvl_h(t2, t0);
2206 __m128i s1 = __lsx_vilvh_h(t2, t0);
2207 __m128i s2 = __lsx_vilvl_h(t3, t1);
2208 __m128i s3 = __lsx_vilvh_h(t3, t1);
2209 __m128i s4 = __lsx_vilvl_h(t6, t4);
2210 __m128i s5 = __lsx_vilvh_h(t6, t4);
2211 __m128i s6 = __lsx_vilvl_h(t7, t5);
2212 __m128i s7 = __lsx_vilvh_h(t7, t5);
2213 __m128i s8 = __lsx_vilvl_h(ta, t8);
2214 __m128i s9 = __lsx_vilvh_h(ta, t8);
2215 __m128i sa = __lsx_vilvl_h(tb, t9);
2216 __m128i sb = __lsx_vilvh_h(tb, t9);
2217 __m128i sc = __lsx_vilvl_h(te, tc);
2218 __m128i sd = __lsx_vilvh_h(te, tc);
2219 __m128i se = __lsx_vilvl_h(tf, td);
2220 __m128i sf = __lsx_vilvh_h(tf, td);
2221
2222 __m128i u0 = __lsx_vilvl_w(s4, s0);
2223 __m128i u1 = __lsx_vilvh_w(s4, s0);
2224 __m128i u2 = __lsx_vilvl_w(s5, s1);
2225 __m128i u3 = __lsx_vilvh_w(s5, s1);
2226 __m128i u4 = __lsx_vilvl_w(s6, s2);
2227 __m128i u5 = __lsx_vilvh_w(s6, s2);
2228 __m128i u6 = __lsx_vilvl_w(s7, s3);
2229 __m128i u7 = __lsx_vilvh_w(s7, s3);
2230 __m128i u8 = __lsx_vilvl_w(sc, s8);
2231 __m128i u9 = __lsx_vilvh_w(sc, s8);
2232 __m128i ua = __lsx_vilvl_w(sd, s9);
2233 __m128i ub = __lsx_vilvh_w(sd, s9);
2234 __m128i uc = __lsx_vilvl_w(se, sa);
2235 __m128i ud = __lsx_vilvh_w(se, sa);
2236 __m128i ue = __lsx_vilvl_w(sf, sb);
2237 __m128i uf = __lsx_vilvh_w(sf, sb);
2238
2239 kernel.packet[0] = __lsx_vilvl_d(u8, u0);
2240 kernel.packet[1] = __lsx_vilvh_d(u8, u0);
2241 kernel.packet[2] = __lsx_vilvl_d(u9, u1);
2242 kernel.packet[3] = __lsx_vilvh_d(u9, u1);
2243 kernel.packet[4] = __lsx_vilvl_d(ua, u2);
2244 kernel.packet[5] = __lsx_vilvh_d(ua, u2);
2245 kernel.packet[6] = __lsx_vilvl_d(ub, u3);
2246 kernel.packet[7] = __lsx_vilvh_d(ub, u3);
2247 kernel.packet[8] = __lsx_vilvl_d(uc, u4);
2248 kernel.packet[9] = __lsx_vilvh_d(uc, u4);
2249 kernel.packet[10] = __lsx_vilvl_d(ud, u5);
2250 kernel.packet[11] = __lsx_vilvh_d(ud, u5);
2251 kernel.packet[12] = __lsx_vilvl_d(ue, u6);
2252 kernel.packet[13] = __lsx_vilvh_d(ue, u6);
2253 kernel.packet[14] = __lsx_vilvl_d(uf, u7);
2254 kernel.packet[15] = __lsx_vilvh_d(uf, u7);
2255}
2256EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 8>& kernel) {
2257 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2258 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2259 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2260 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2261 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2262 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2263 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2264 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2265
2266 __m128i s0 = __lsx_vilvl_h(t2, t0);
2267 __m128i s1 = __lsx_vilvh_h(t2, t0);
2268 __m128i s2 = __lsx_vilvl_h(t3, t1);
2269 __m128i s3 = __lsx_vilvh_h(t3, t1);
2270 __m128i s4 = __lsx_vilvl_h(t6, t4);
2271 __m128i s5 = __lsx_vilvh_h(t6, t4);
2272 __m128i s6 = __lsx_vilvl_h(t7, t5);
2273 __m128i s7 = __lsx_vilvh_h(t7, t5);
2274
2275 kernel.packet[0] = __lsx_vilvl_w(s4, s0);
2276 kernel.packet[1] = __lsx_vilvh_w(s4, s0);
2277 kernel.packet[2] = __lsx_vilvl_w(s5, s1);
2278 kernel.packet[3] = __lsx_vilvh_w(s5, s1);
2279 kernel.packet[4] = __lsx_vilvl_w(s6, s2);
2280 kernel.packet[5] = __lsx_vilvh_w(s6, s2);
2281 kernel.packet[6] = __lsx_vilvl_w(s7, s3);
2282 kernel.packet[7] = __lsx_vilvh_w(s7, s3);
2283}
2284EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
2285 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2286 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2287 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2288 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2289
2290 kernel.packet[0] = __lsx_vilvl_h(t2, t0);
2291 kernel.packet[1] = __lsx_vilvh_h(t2, t0);
2292 kernel.packet[2] = __lsx_vilvl_h(t3, t1);
2293 kernel.packet[3] = __lsx_vilvh_h(t3, t1);
2294}
2295EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
2296 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2297 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2298 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2299 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2300 __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]);
2301 __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]);
2302 __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]);
2303 __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]);
2304
2305 __m128i s0 = __lsx_vilvl_w(t2, t0);
2306 __m128i s1 = __lsx_vilvh_w(t2, t0);
2307 __m128i s2 = __lsx_vilvl_w(t3, t1);
2308 __m128i s3 = __lsx_vilvh_w(t3, t1);
2309 __m128i s4 = __lsx_vilvl_w(t6, t4);
2310 __m128i s5 = __lsx_vilvh_w(t6, t4);
2311 __m128i s6 = __lsx_vilvl_w(t7, t5);
2312 __m128i s7 = __lsx_vilvh_w(t7, t5);
2313
2314 kernel.packet[0] = __lsx_vilvl_d(s4, s0);
2315 kernel.packet[1] = __lsx_vilvh_d(s4, s0);
2316 kernel.packet[2] = __lsx_vilvl_d(s5, s1);
2317 kernel.packet[3] = __lsx_vilvh_d(s5, s1);
2318 kernel.packet[4] = __lsx_vilvl_d(s6, s2);
2319 kernel.packet[5] = __lsx_vilvh_d(s6, s2);
2320 kernel.packet[6] = __lsx_vilvl_d(s7, s3);
2321 kernel.packet[7] = __lsx_vilvh_d(s7, s3);
2322}
2323EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
2324 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2325 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2326 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2327 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2328
2329 kernel.packet[0] = __lsx_vilvl_w(t2, t0);
2330 kernel.packet[1] = __lsx_vilvh_w(t2, t0);
2331 kernel.packet[2] = __lsx_vilvl_w(t3, t1);
2332 kernel.packet[3] = __lsx_vilvh_w(t3, t1);
2333}
2334EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
2335 __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]);
2336 __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]);
2337 __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]);
2338 __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]);
2339
2340 kernel.packet[0] = __lsx_vilvl_d(T2, T0);
2341 kernel.packet[1] = __lsx_vilvh_d(T2, T0);
2342 kernel.packet[2] = __lsx_vilvl_d(T3, T1);
2343 kernel.packet[3] = __lsx_vilvh_d(T3, T1);
2344}
2345EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2l, 2>& kernel) {
2346 __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
2347 kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
2348 kernel.packet[1] = tmp;
2349}
2350EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
2351 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2352 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2353 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2354 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2355 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2356 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2357 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2358 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2359 __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]);
2360 __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]);
2361 __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]);
2362 __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]);
2363 __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]);
2364 __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]);
2365 __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]);
2366 __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]);
2367
2368 __m128i s0 = __lsx_vilvl_h(t2, t0);
2369 __m128i s1 = __lsx_vilvh_h(t2, t0);
2370 __m128i s2 = __lsx_vilvl_h(t3, t1);
2371 __m128i s3 = __lsx_vilvh_h(t3, t1);
2372 __m128i s4 = __lsx_vilvl_h(t6, t4);
2373 __m128i s5 = __lsx_vilvh_h(t6, t4);
2374 __m128i s6 = __lsx_vilvl_h(t7, t5);
2375 __m128i s7 = __lsx_vilvh_h(t7, t5);
2376 __m128i s8 = __lsx_vilvl_h(ta, t8);
2377 __m128i s9 = __lsx_vilvh_h(ta, t8);
2378 __m128i sa = __lsx_vilvl_h(tb, t9);
2379 __m128i sb = __lsx_vilvh_h(tb, t9);
2380 __m128i sc = __lsx_vilvl_h(te, tc);
2381 __m128i sd = __lsx_vilvh_h(te, tc);
2382 __m128i se = __lsx_vilvl_h(tf, td);
2383 __m128i sf = __lsx_vilvh_h(tf, td);
2384
2385 __m128i u0 = __lsx_vilvl_w(s4, s0);
2386 __m128i u1 = __lsx_vilvh_w(s4, s0);
2387 __m128i u2 = __lsx_vilvl_w(s5, s1);
2388 __m128i u3 = __lsx_vilvh_w(s5, s1);
2389 __m128i u4 = __lsx_vilvl_w(s6, s2);
2390 __m128i u5 = __lsx_vilvh_w(s6, s2);
2391 __m128i u6 = __lsx_vilvl_w(s7, s3);
2392 __m128i u7 = __lsx_vilvh_w(s7, s3);
2393 __m128i u8 = __lsx_vilvl_w(sc, s8);
2394 __m128i u9 = __lsx_vilvh_w(sc, s8);
2395 __m128i ua = __lsx_vilvl_w(sd, s9);
2396 __m128i ub = __lsx_vilvh_w(sd, s9);
2397 __m128i uc = __lsx_vilvl_w(se, sa);
2398 __m128i ud = __lsx_vilvh_w(se, sa);
2399 __m128i ue = __lsx_vilvl_w(sf, sb);
2400 __m128i uf = __lsx_vilvh_w(sf, sb);
2401
2402 kernel.packet[0] = __lsx_vilvl_d(u8, u0);
2403 kernel.packet[1] = __lsx_vilvh_d(u8, u0);
2404 kernel.packet[2] = __lsx_vilvl_d(u9, u1);
2405 kernel.packet[3] = __lsx_vilvh_d(u9, u1);
2406 kernel.packet[4] = __lsx_vilvl_d(ua, u2);
2407 kernel.packet[5] = __lsx_vilvh_d(ua, u2);
2408 kernel.packet[6] = __lsx_vilvl_d(ub, u3);
2409 kernel.packet[7] = __lsx_vilvh_d(ub, u3);
2410 kernel.packet[8] = __lsx_vilvl_d(uc, u4);
2411 kernel.packet[9] = __lsx_vilvh_d(uc, u4);
2412 kernel.packet[10] = __lsx_vilvl_d(ud, u5);
2413 kernel.packet[11] = __lsx_vilvh_d(ud, u5);
2414 kernel.packet[12] = __lsx_vilvl_d(ue, u6);
2415 kernel.packet[13] = __lsx_vilvh_d(ue, u6);
2416 kernel.packet[14] = __lsx_vilvl_d(uf, u7);
2417 kernel.packet[15] = __lsx_vilvh_d(uf, u7);
2418}
2419EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 8>& kernel) {
2420 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2421 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2422 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2423 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2424 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2425 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2426 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2427 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2428
2429 __m128i s0 = __lsx_vilvl_h(t2, t0);
2430 __m128i s1 = __lsx_vilvh_h(t2, t0);
2431 __m128i s2 = __lsx_vilvl_h(t3, t1);
2432 __m128i s3 = __lsx_vilvh_h(t3, t1);
2433 __m128i s4 = __lsx_vilvl_h(t6, t4);
2434 __m128i s5 = __lsx_vilvh_h(t6, t4);
2435 __m128i s6 = __lsx_vilvl_h(t7, t5);
2436 __m128i s7 = __lsx_vilvh_h(t7, t5);
2437
2438 kernel.packet[0] = __lsx_vilvl_w(s4, s0);
2439 kernel.packet[1] = __lsx_vilvh_w(s4, s0);
2440 kernel.packet[2] = __lsx_vilvl_w(s5, s1);
2441 kernel.packet[3] = __lsx_vilvh_w(s5, s1);
2442 kernel.packet[4] = __lsx_vilvl_w(s6, s2);
2443 kernel.packet[5] = __lsx_vilvh_w(s6, s2);
2444 kernel.packet[6] = __lsx_vilvl_w(s7, s3);
2445 kernel.packet[7] = __lsx_vilvh_w(s7, s3);
2446}
2447EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
2448 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2449 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2450 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2451 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2452
2453 kernel.packet[0] = __lsx_vilvl_h(t2, t0);
2454 kernel.packet[1] = __lsx_vilvh_h(t2, t0);
2455 kernel.packet[2] = __lsx_vilvl_h(t3, t1);
2456 kernel.packet[3] = __lsx_vilvh_h(t3, t1);
2457}
2458EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
2459 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2460 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2461 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2462 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2463 __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]);
2464 __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]);
2465 __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]);
2466 __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]);
2467
2468 __m128i s0 = __lsx_vilvl_w(t2, t0);
2469 __m128i s1 = __lsx_vilvh_w(t2, t0);
2470 __m128i s2 = __lsx_vilvl_w(t3, t1);
2471 __m128i s3 = __lsx_vilvh_w(t3, t1);
2472 __m128i s4 = __lsx_vilvl_w(t6, t4);
2473 __m128i s5 = __lsx_vilvh_w(t6, t4);
2474 __m128i s6 = __lsx_vilvl_w(t7, t5);
2475 __m128i s7 = __lsx_vilvh_w(t7, t5);
2476
2477 kernel.packet[0] = __lsx_vilvl_d(s4, s0);
2478 kernel.packet[1] = __lsx_vilvh_d(s4, s0);
2479 kernel.packet[2] = __lsx_vilvl_d(s5, s1);
2480 kernel.packet[3] = __lsx_vilvh_d(s5, s1);
2481 kernel.packet[4] = __lsx_vilvl_d(s6, s2);
2482 kernel.packet[5] = __lsx_vilvh_d(s6, s2);
2483 kernel.packet[6] = __lsx_vilvl_d(s7, s3);
2484 kernel.packet[7] = __lsx_vilvh_d(s7, s3);
2485}
2486EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
2487 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2488 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2489 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2490 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2491
2492 kernel.packet[0] = __lsx_vilvl_w(t2, t0);
2493 kernel.packet[1] = __lsx_vilvh_w(t2, t0);
2494 kernel.packet[2] = __lsx_vilvl_w(t3, t1);
2495 kernel.packet[3] = __lsx_vilvh_w(t3, t1);
2496}
2497EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
2498 __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]);
2499 __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]);
2500 __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]);
2501 __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]);
2502
2503 kernel.packet[0] = __lsx_vilvl_d(T2, T0);
2504 kernel.packet[1] = __lsx_vilvh_d(T2, T0);
2505 kernel.packet[2] = __lsx_vilvl_d(T3, T1);
2506 kernel.packet[3] = __lsx_vilvh_d(T3, T1);
2507}
2508EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2ul, 2>& kernel) {
2509 __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
2510 kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
2511 kernel.packet[1] = tmp;
2512}
2513
2514template <>
2515EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) {
2516 return __lsx_vfrsqrt_s(a);
2517}
2518template <>
2519EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) {
2520 return __lsx_vfrsqrt_d(a);
2521}
2522
2523template <>
2524EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) {
2525 return __lsx_vfrintrm_s(a);
2526}
2527template <>
2528EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) {
2529 return __lsx_vfrintrm_d(a);
2530}
2531
2532template <>
2533EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) {
2534 return __lsx_vfrintrp_s(a);
2535}
2536template <>
2537EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) {
2538 return __lsx_vfrintrp_d(a);
2539}
2540
2541template <>
2542EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) {
2543 const Packet4f mask = pset1frombits<Packet4f>(static_cast<numext::uint32_t>(0x80000000u));
2544 const Packet4f prev0dot5 = pset1frombits<Packet4f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
2545 return __lsx_vfrintrz_s(padd(pxor(pand(a, mask), prev0dot5), a));
2546}
2547template <>
2548EIGEN_STRONG_INLINE Packet2d pround(const Packet2d& a) {
2549 const Packet2d mask = pset1frombits<Packet2d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
2550 const Packet2d prev0dot5 = pset1frombits<Packet2d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
2551 return __lsx_vfrintrz_d(padd(por(pand(a, mask), prev0dot5), a));
2552}
2553
2554template <>
2555EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) {
2556 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2557}
2558template <>
2559EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b) {
2560 return (Packet16c)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2561}
2562
2563template <>
2564EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(const int8_t* from) {
2565 int8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1),
2566 *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
2567 *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
2568 return __lsx_vld(tmp, 0);
2569}
2570template <>
2571EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(const uint8_t* from) {
2572 uint8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1),
2573 *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
2574 *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
2575 return __lsx_vld(tmp, 0);
2576}
2577template <>
2578EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(const int16_t* from) {
2579 int16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
2580 return __lsx_vld(tmp, 0);
2581}
2582template <>
2583EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(const uint16_t* from) {
2584 uint16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
2585 return __lsx_vld(tmp, 0);
2586}
2587template <>
2588EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(const int32_t* from) {
2589 int32_t tmp[4] = {*from, *from, *from, *from};
2590 return __lsx_vld(tmp, 0);
2591}
2592template <>
2593EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(const uint32_t* from) {
2594 uint32_t tmp[4] = {*from, *from, *from, *from};
2595 return __lsx_vld(tmp, 0);
2596}
2597
2598template <>
2599EIGEN_STRONG_INLINE Packet16c pnmsub(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
2600 return __lsx_vmsub_b(pnegate(c), a, b);
2601}
2602template <>
2603EIGEN_STRONG_INLINE Packet8s pnmsub(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
2604 return __lsx_vmsub_h(pnegate(c), a, b);
2605}
2606template <>
2607EIGEN_STRONG_INLINE Packet4i pnmsub(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
2608 return __lsx_vmsub_w(pnegate(c), a, b);
2609}
2610template <>
2611EIGEN_STRONG_INLINE Packet2l pnmsub(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
2612 return __lsx_vmsub_d(pnegate(c), a, b);
2613}
2614
2615template <>
2616EIGEN_STRONG_INLINE Packet16c pmsub(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
2617 return __lsx_vmadd_b(pnegate(c), a, b);
2618}
2619template <>
2620EIGEN_STRONG_INLINE Packet8s pmsub(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
2621 return __lsx_vmadd_h(pnegate(c), a, b);
2622}
2623template <>
2624EIGEN_STRONG_INLINE Packet4i pmsub(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
2625 return __lsx_vmadd_w(pnegate(c), a, b);
2626}
2627template <>
2628EIGEN_STRONG_INLINE Packet2l pmsub(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
2629 return __lsx_vmadd_d(pnegate(c), a, b);
2630}
2631
2632template <>
2633EIGEN_STRONG_INLINE Packet16c pnmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
2634 return __lsx_vmsub_b(c, a, b);
2635}
2636template <>
2637EIGEN_STRONG_INLINE Packet8s pnmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
2638 return __lsx_vmsub_h(c, a, b);
2639}
2640template <>
2641EIGEN_STRONG_INLINE Packet4i pnmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
2642 return __lsx_vmsub_w(c, a, b);
2643}
2644template <>
2645EIGEN_STRONG_INLINE Packet2l pnmadd(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
2646 return __lsx_vmsub_d(c, a, b);
2647}
2648
2649template <>
2650EIGEN_STRONG_INLINE Packet4f pexp(const Packet4f& _x) {
2651 return pexp_float(_x);
2652}
2653template <>
2654EIGEN_STRONG_INLINE Packet2d pexp(const Packet2d& _x) {
2655 return pexp_double(_x);
2656}
2657
2658template <>
2659EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
2660 return pldexp_generic(a, exponent);
2661}
2662
2663template <>
2664EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent) {
2665 return pfrexp_generic(a, exponent);
2666}
2667template <>
2668EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) {
2669 return pfrexp_generic(a, exponent);
2670}
2671template <>
2672EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /* a */) {
2673 Packet4f v = {0.0f, 0.0f, 0.0f, 0.0f};
2674 return v;
2675}
2676template <>
2677struct pminmax_propagates_nan<Packet4f> : bool_constant<true> {};
2678template <>
2679EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(const float* from) {
2680 return (__m128)__lsx_vldrepl_w(from, 0);
2681}
2682template <>
2683EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) {
2684 return (__m128)__lsx_vsrai_w((__m128i)a, 31);
2685}
2686template <>
2687EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
2688 return __lsx_vfrintrne_s(a);
2689}
2690template <>
2691EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
2692 return __lsx_vfrintrz_s(a);
2693}
2694template <>
2695EIGEN_STRONG_INLINE Packet4f preciprocal<Packet4f>(const Packet4f& a) {
2696 return __lsx_vfrecip_s(a);
2697}
2698
2699template <>
2700EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /* a */) {
2701 Packet2d v = {0.0, 0.0};
2702 return v;
2703}
2704template <>
2705struct pminmax_propagates_nan<Packet2d> : bool_constant<true> {};
2706template <>
2707EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) {
2708 return (__m128d)(__lsx_vsrai_d((__m128i)a, 63));
2709}
2710template <>
2711EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) {
2712 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2713}
2714template <>
2715EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
2716 return __lsx_vfrintrne_d(a);
2717}
2718template <>
2719EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
2720 return __lsx_vfrintrz_d(a);
2721}
2722template <>
2723EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
2724 return pldexp_generic(a, exponent);
2725}
2726template <>
2727EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) {
2728 return __lsx_vbitsel_v(b, a, mask);
2729}
2730template <>
2731EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) {
2732 return __lsx_vbitsel_v(b, a, mask);
2733}
2734template <>
2735EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) {
2736 return __lsx_vbitsel_v(b, a, mask);
2737}
2738template <>
2739EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
2740 return __lsx_vdiv_bu(a, b);
2741}
2742template <>
2743EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a,
2744 const Packet16uc& b) {
2745 return __lsx_vbitsel_v(b, a, mask);
2746}
2747template <>
2748EIGEN_STRONG_INLINE Packet16uc psqrt(const Packet16uc& a) {
2749 __m128i res = {0, 0};
2750 __m128i add = {0x0808080808080808, 0x0808080808080808};
2751 for (int i = 0; i < 4; i++) {
2752 const __m128i temp = __lsx_vor_v(res, add);
2753 const __m128i tmul = __lsx_vpackev_b(__lsx_vmulwod_h_bu(temp, temp), __lsx_vmulwev_h_bu(temp, temp));
2754 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_bu(tmul, a));
2755 add = __lsx_vsrli_b(add, 1);
2756 }
2757 return res;
2758}
2759template <>
2760EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) {
2761 return __lsx_vbitsel_v(b, a, mask);
2762}
2763template <>
2764EIGEN_STRONG_INLINE Packet8us psqrt(const Packet8us& a) {
2765 __m128i res = {0, 0};
2766 __m128i add = {0x0080008000800080, 0x0080008000800080};
2767 for (int i = 0; i < 4; i++) {
2768 const __m128i temp = __lsx_vor_v(res, add);
2769 const __m128i tmul = __lsx_vpackev_h(__lsx_vmulwod_w_hu(temp, temp), __lsx_vmulwev_w_hu(temp, temp));
2770 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_hu(tmul, a));
2771 add = __lsx_vsrli_h(add, 1);
2772 }
2773 return res;
2774}
2775
2776template <>
2777EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) {
2778 return __lsx_vbitsel_v(b, a, mask);
2779}
2780template <>
2781EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) {
2782 __m128i res = {0, 0};
2783 __m128i add = {0x0000800000008000, 0x0000800000008000};
2784 for (int i = 0; i < 4; i++) {
2785 const __m128i temp = __lsx_vor_v(res, add);
2786 const __m128i tmul = __lsx_vpackev_w(__lsx_vmulwod_d_wu(temp, temp), __lsx_vmulwev_d_wu(temp, temp));
2787 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_wu(tmul, a));
2788 add = __lsx_vsrli_w(add, 1);
2789 }
2790 return res;
2791}
2792
2793template <>
2794EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b) {
2795 return __lsx_vbitsel_v(b, a, mask);
2796}
2797
2798} // namespace internal
2799} // namespace Eigen
2800#endif
Eigen::Aligned16
@ Aligned16
Definition Constants.h:238