Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2023 Zang Ruochen <zangruochen@loongson.cn>
5// Copyright (C) 2024 XiWei Gu <guxiwei-hf@loongson.cn>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_PACKET_MATH_LSX_H
13#define EIGEN_PACKET_MATH_LSX_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
23#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
24#endif
25
26#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
27#if EIGEN_ARCH_LOONGARCH64
28#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
29#endif
30#endif
31
32#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
33#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
34#endif
35
36using Packet4f = __m128;
37using Packet2d = __m128d;
38
39using Packet16c = eigen_packet_wrapper<__m128i, 0>;
40using Packet8s = eigen_packet_wrapper<__m128i, 1>;
41using Packet4i = eigen_packet_wrapper<__m128i, 2>;
42using Packet2l = eigen_packet_wrapper<__m128i, 3>;
43using Packet16uc = eigen_packet_wrapper<__m128i, 4>;
44using Packet8us = eigen_packet_wrapper<__m128i, 5>;
45using Packet4ui = eigen_packet_wrapper<__m128i, 6>;
46using Packet2ul = eigen_packet_wrapper<__m128i, 7>;
47
48template <>
49struct is_arithmetic<__m128> : std::true_type {};
50template <>
51struct is_arithmetic<__m128i> : std::true_type {};
52template <>
53struct is_arithmetic<__m128d> : std::true_type {};
54template <>
55struct is_arithmetic<Packet16c> : std::true_type {};
56template <>
57struct is_arithmetic<Packet8s> : std::true_type {};
58template <>
59struct is_arithmetic<Packet4i> : std::true_type {};
60template <>
61struct is_arithmetic<Packet2l> : std::true_type {};
62template <>
63struct is_arithmetic<Packet16uc> : std::false_type {};
64template <>
65struct is_arithmetic<Packet8us> : std::false_type {};
66template <>
67struct is_arithmetic<Packet4ui> : std::false_type {};
68template <>
69struct is_arithmetic<Packet2ul> : std::false_type {};
70
71EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) {
72 float from[4] = {a, b, c, d};
73 return (Packet4f)__lsx_vld(from, 0);
74}
75
76EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask) {
77 const float* a = reinterpret_cast<const float*>(&m);
78 Packet4f res =
79 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3)));
80 return res;
81}
82
83template <bool interleave>
84EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f& m, const Packet4f& n, int mask) {
85 const float* a = reinterpret_cast<const float*>(&m);
86 const float* b = reinterpret_cast<const float*>(&n);
87 Packet4f res =
88 make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
89 return res;
90}
91
92template <>
93EIGEN_STRONG_INLINE Packet4f shuffle2<true>(const Packet4f& m, const Packet4f& n, int mask) {
94 const float* a = reinterpret_cast<const float*>(&m);
95 const float* b = reinterpret_cast<const float*>(&n);
96 Packet4f res =
97 make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
98 return res;
99}
100
101EIGEN_STRONG_INLINE static int eigen_lsx_shuffle_mask(int p, int q, int r, int s) {
102 return ((s) << 6 | (r) << 4 | (q) << 2 | (p));
103}
104
105EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s) {
106 return shuffle1(a, eigen_lsx_shuffle_mask(p, q, r, s));
107}
108EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s) {
109 return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(p, q, r, s));
110}
111EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) {
112 return shuffle2<false>(a, b, eigen_lsx_shuffle_mask(0, 1, 0, 1));
113}
114EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) {
115 return shuffle2<false>(b, a, eigen_lsx_shuffle_mask(2, 3, 2, 3));
116}
117EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) {
118 return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(0, 0, 1, 1));
119}
120EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) {
121 return shuffle2<true>(a, b, eigen_lsx_shuffle_mask(2, 2, 3, 3));
122}
123
124EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) {
125 double from[2] = {a, b};
126 return (Packet2d)__lsx_vld(from, 0);
127}
128
129EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask) {
130 const double* a = reinterpret_cast<const double*>(&m);
131 const double* b = reinterpret_cast<const double*>(&n);
132 Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1)));
133 return res;
134}
135
136EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(const Packet2d& a, const Packet2d& b, int mask) {
137 return shuffle(a, b, mask);
138}
139EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 0); }
140EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 3); }
141
142template <>
143struct packet_traits<int8_t> : default_packet_traits {
144 using type = Packet16c;
145 using half = Packet16c;
146 enum {
147 Vectorizable = 1,
148 AlignedOnScalar = 1,
149 size = 16,
150
151 HasSetLinear = 0,
152 HasCmp = 1,
153 };
154};
155
156template <>
157struct packet_traits<int16_t> : default_packet_traits {
158 using type = Packet8s;
159 using half = Packet8s;
160 enum {
161 Vectorizable = 1,
162 AlignedOnScalar = 1,
163 size = 8,
164
165 HasSetLinear = 0,
166 HasCmp = 1,
167 HasDiv = 1,
168 };
169};
170
171template <>
172struct packet_traits<int32_t> : default_packet_traits {
173 using type = Packet4i;
174 using half = Packet4i;
175 enum {
176 Vectorizable = 1,
177 AlignedOnScalar = 1,
178 size = 4,
179
180 HasSetLinear = 0,
181 HasCmp = 1,
182 HasDiv = 1,
183 };
184};
185
186template <>
187struct packet_traits<int64_t> : default_packet_traits {
188 using type = Packet2l;
189 using half = Packet2l;
190 enum {
191 Vectorizable = 1,
192 AlignedOnScalar = 1,
193 size = 2,
194
195 HasSetLinear = 0,
196 HasCmp = 1,
197 HasDiv = 1,
198 };
199};
200
201template <>
202struct packet_traits<uint8_t> : default_packet_traits {
203 using type = Packet16uc;
204 using half = Packet16uc;
205 enum {
206 Vectorizable = 1,
207 AlignedOnScalar = 1,
208 size = 16,
209
210 HasSetLinear = 0,
211 HasNegate = 0,
212 HasCmp = 1,
213 };
214};
215
216template <>
217struct packet_traits<uint16_t> : default_packet_traits {
218 using type = Packet8us;
219 using half = Packet8us;
220 enum {
221 Vectorizable = 1,
222 AlignedOnScalar = 1,
223 size = 8,
224
225 HasSetLinear = 0,
226 HasNegate = 0,
227 HasCmp = 1,
228 HasDiv = 1,
229 };
230};
231
232template <>
233struct packet_traits<uint32_t> : default_packet_traits {
234 using type = Packet4ui;
235 using half = Packet4ui;
236 enum {
237 Vectorizable = 1,
238 AlignedOnScalar = 1,
239 size = 4,
240
241 HasSetLinear = 0,
242 HasNegate = 0,
243 HasCmp = 1,
244 HasDiv = 1,
245 };
246};
247
248template <>
249struct packet_traits<uint64_t> : default_packet_traits {
250 using type = Packet2ul;
251 using half = Packet2ul;
252 enum {
253 Vectorizable = 1,
254 AlignedOnScalar = 1,
255 size = 2,
256
257 HasSetLinear = 0,
258 HasNegate = 0,
259 HasCmp = 1,
260 HasDiv = 1,
261 };
262};
263
264template <>
265struct packet_traits<float> : default_packet_traits {
266 using type = Packet4f;
267 using half = Packet4f;
268 enum {
269 Vectorizable = 1,
270 AlignedOnScalar = 1,
271 size = 4,
272
273 HasCmp = 1,
274 HasSetLinear = 0,
275 HasSign = 0,
276 HasDiv = 1,
277 HasExp = 1,
278 HasExpm1 = 1,
279 HasSqrt = 1,
280 HasLog = 1,
281 HasRsqrt = 1
282 };
283};
284
285template <>
286struct packet_traits<double> : default_packet_traits {
287 using type = Packet2d;
288 using half = Packet2d;
289 enum {
290 Vectorizable = 1,
291 AlignedOnScalar = 1,
292 size = 2,
293
294 HasCmp = 1,
295 HasSetLinear = 0,
296 HasSign = 0,
297 HasDiv = 1,
298 HasExpm1 = 1,
299 HasSqrt = 1,
300 HasLog = 1,
301 HasRsqrt = 1
302 };
303};
304
305template <>
306struct unpacket_traits<Packet16c> {
307 using type = int8_t;
308 using half = Packet16c;
309 enum {
310 size = 16,
311 alignment = Aligned16,
312 vectorizable = true,
313 masked_load_available = false,
314 masked_store_available = false
315 };
316};
317template <>
318struct unpacket_traits<Packet8s> {
319 using type = int16_t;
320 using half = Packet8s;
321 enum {
322 size = 8,
323 alignment = Aligned16,
324 vectorizable = true,
325 masked_load_available = false,
326 masked_store_available = false
327 };
328};
329template <>
330struct unpacket_traits<Packet4i> {
331 using type = int32_t;
332 using half = Packet4i;
333 enum {
334 size = 4,
335 alignment = Aligned16,
336 vectorizable = true,
337 masked_load_available = false,
338 masked_store_available = false
339 };
340};
341template <>
342struct unpacket_traits<Packet2l> {
343 using type = int64_t;
344 using half = Packet2l;
345 enum {
346 size = 2,
347 alignment = Aligned16,
348 vectorizable = true,
349 masked_load_available = false,
350 masked_store_available = false
351 };
352};
353template <>
354struct unpacket_traits<Packet16uc> {
355 using type = uint8_t;
356 using half = Packet16uc;
357 enum {
358 size = 16,
359 alignment = Aligned16,
360 vectorizable = true,
361 masked_load_available = false,
362 masked_store_available = false
363 };
364};
365template <>
366struct unpacket_traits<Packet8us> {
367 using type = uint16_t;
368 using half = Packet8us;
369 enum {
370 size = 8,
371 alignment = Aligned16,
372 vectorizable = true,
373 masked_load_available = false,
374 masked_store_available = false
375 };
376};
377template <>
378struct unpacket_traits<Packet4ui> {
379 using type = uint32_t;
380 using half = Packet4ui;
381 enum {
382 size = 4,
383 alignment = Aligned16,
384 vectorizable = true,
385 masked_load_available = false,
386 masked_store_available = false
387 };
388};
389template <>
390struct unpacket_traits<Packet2ul> {
391 using type = uint64_t;
392 using half = Packet2ul;
393 enum {
394 size = 2,
395 alignment = Aligned16,
396 vectorizable = true,
397 masked_load_available = false,
398 masked_store_available = false
399 };
400};
401template <>
402struct unpacket_traits<Packet4f> {
403 using type = float;
404 using half = Packet4f;
405 using integer_packet = Packet4i;
406 enum {
407 size = 4,
408 alignment = Aligned16,
409 vectorizable = true,
410 masked_load_available = false,
411 masked_store_available = false
412 };
413};
414template <>
415struct unpacket_traits<Packet2d> {
416 using type = double;
417 using half = Packet2d;
418 using integer_packet = Packet2l;
419 enum {
420 size = 2,
421 alignment = Aligned16,
422 vectorizable = true,
423 masked_load_available = false,
424 masked_store_available = false
425 };
426};
427
428template <>
429EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(const int8_t& from) {
430 return __lsx_vreplgr2vr_b(from);
431}
432template <>
433EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const int16_t& from) {
434 return __lsx_vreplgr2vr_h(from);
435}
436template <>
437EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) {
438 return __lsx_vreplgr2vr_w(from);
439}
440template <>
441EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) {
442 return __lsx_vreplgr2vr_d(from);
443}
444template <>
445EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(const uint8_t& from) {
446 return __lsx_vreplgr2vr_b(from);
447}
448template <>
449EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(const uint16_t& from) {
450 return __lsx_vreplgr2vr_h(from);
451}
452template <>
453EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(const uint32_t& from) {
454 return __lsx_vreplgr2vr_w(from);
455}
456template <>
457EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(const uint64_t& from) {
458 return __lsx_vreplgr2vr_d(from);
459}
460template <>
461EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) {
462 Packet4f v = {from, from, from, from};
463 return v;
464}
465template <>
466EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) {
467 Packet2d v = {from, from};
468 return v;
469}
470
471template <>
472EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from) {
473 return reinterpret_cast<__m128>((__m128i)pset1<Packet4ui>(from));
474}
475template <>
476EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from) {
477 return reinterpret_cast<__m128d>((__m128i)pset1<Packet2ul>(from));
478}
479
480template <>
481EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(const int8_t& a) {
482 const int8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
483 return __lsx_vadd_b(pset1<Packet16c>(a), __lsx_vld(countdown, 0));
484}
485template <>
486EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(const int16_t& a) {
487 const int16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
488 return __lsx_vadd_h(pset1<Packet8s>(a), __lsx_vld(countdown, 0));
489}
490template <>
491EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a) {
492 const int32_t countdown[] = {0, 1, 2, 3};
493 return __lsx_vadd_w(pset1<Packet4i>(a), __lsx_vld(countdown, 0));
494}
495template <>
496EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(const int64_t& a) {
497 const int64_t countdown[] = {0, 1};
498 return __lsx_vadd_d(pset1<Packet2l>(a), __lsx_vld(countdown, 0));
499}
500template <>
501EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(const uint8_t& a) {
502 const uint8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
503 return __lsx_vadd_b(pset1<Packet16uc>(a), __lsx_vld(countdown, 0));
504}
505template <>
506EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(const uint16_t& a) {
507 const uint16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7};
508 return __lsx_vadd_h(pset1<Packet8us>(a), __lsx_vld(countdown, 0));
509}
510template <>
511EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(const uint32_t& a) {
512 const uint32_t countdown[] = {0, 1, 2, 3};
513 return __lsx_vadd_w(pset1<Packet4ui>(a), __lsx_vld(countdown, 0));
514}
515template <>
516EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(const uint64_t& a) {
517 const uint64_t countdown[] = {0, 1};
518 return __lsx_vadd_d(pset1<Packet2ul>(a), __lsx_vld(countdown, 0));
519}
520template <>
521EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a) {
522 static const Packet4f countdown = {0.0f, 1.0f, 2.0f, 3.0f};
523 return __lsx_vfadd_s(pset1<Packet4f>(a), countdown);
524}
525template <>
526EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a) {
527 static const Packet2d countdown = {0.0f, 1.0f};
528 return __lsx_vfadd_d(pset1<Packet2d>(a), countdown);
529}
530
531template <>
532EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(const Packet16c& a, const Packet16c& b) {
533 return __lsx_vadd_b(a, b);
534}
535template <>
536EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(const Packet8s& a, const Packet8s& b) {
537 return __lsx_vadd_h(a, b);
538}
539template <>
540EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) {
541 return __lsx_vadd_w(a, b);
542}
543template <>
544EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) {
545 return __lsx_vadd_d(a, b);
546}
547template <>
548EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
549 return __lsx_vadd_b(a, b);
550}
551template <>
552EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(const Packet8us& a, const Packet8us& b) {
553 return __lsx_vadd_h(a, b);
554}
555template <>
556EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
557 return __lsx_vadd_w(a, b);
558}
559template <>
560EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
561 return __lsx_vadd_d(a, b);
562}
563template <>
564EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) {
565 return __lsx_vfadd_s(a, b);
566}
567template <>
568EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) {
569 return __lsx_vfadd_d(a, b);
570}
571
572template <>
573EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(const Packet16c& a, const Packet16c& b) {
574 return __lsx_vsub_b(a, b);
575}
576template <>
577EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(const Packet8s& a, const Packet8s& b) {
578 return __lsx_vsub_h(a, b);
579}
580template <>
581EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) {
582 return __lsx_vsub_w(a, b);
583}
584template <>
585EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) {
586 return __lsx_vsub_d(a, b);
587}
588template <>
589EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
590 return __lsx_vsub_b(a, b);
591}
592template <>
593EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(const Packet8us& a, const Packet8us& b) {
594 return __lsx_vsub_h(a, b);
595}
596template <>
597EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
598 return __lsx_vsub_w(a, b);
599}
600template <>
601EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
602 return __lsx_vsub_d(a, b);
603}
604template <>
605EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) {
606 return __lsx_vfsub_s(a, b);
607}
608template <>
609EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) {
610 return __lsx_vfsub_d(a, b);
611}
612
613template <>
614EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b);
615template <>
616EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(const Packet4f& a, const Packet4f& b) {
617 const Packet4f mask =
618 make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
619 return padd(a, pxor(mask, b));
620}
621template <>
622EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b);
623template <>
624EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(const Packet2d& a, const Packet2d& b) {
625 const Packet2d mask = make_packet2d(numext::bit_cast<double>(0x8000000000000000ull), 0.0);
626 return padd(a, pxor(mask, b));
627}
628
629template <>
630EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) {
631 Packet4f mask = make_packet4f(numext::bit_cast<float>(0x80000000), numext::bit_cast<float>(0x80000000),
632 numext::bit_cast<float>(0x80000000), numext::bit_cast<float>(0x80000000));
633 return (Packet4f)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a));
634}
635template <>
636EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) {
637 Packet2d mask =
638 make_packet2d(numext::bit_cast<double>(0x8000000000000000), numext::bit_cast<double>(0x8000000000000000));
639 return (Packet2d)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a));
640}
641template <>
642EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) {
643 return __lsx_vneg_b(a);
644}
645template <>
646EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) {
647 return __lsx_vneg_h(a);
648}
649template <>
650EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) {
651 return __lsx_vneg_w(a);
652}
653template <>
654EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) {
655 return __lsx_vneg_d(a);
656}
657
658template <>
659EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) {
660 return __lsx_vfmul_s(a, b);
661}
662template <>
663EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) {
664 return __lsx_vfmul_d(a, b);
665}
666template <>
667EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(const Packet16c& a, const Packet16c& b) {
668 return __lsx_vmul_b(a, b);
669}
670template <>
671EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(const Packet8s& a, const Packet8s& b) {
672 return __lsx_vmul_h(a, b);
673}
674template <>
675EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) {
676 return __lsx_vmul_w(a, b);
677}
678template <>
679EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(const Packet2l& a, const Packet2l& b) {
680 return __lsx_vmul_d(a, b);
681}
682template <>
683EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
684 return __lsx_vmul_b(a, b);
685}
686template <>
687EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(const Packet8us& a, const Packet8us& b) {
688 return __lsx_vmul_h(a, b);
689}
690template <>
691EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
692 return __lsx_vmul_w(a, b);
693}
694template <>
695EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
696 return __lsx_vmul_d(a, b);
697}
698
699template <>
700EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b) {
701 return __lsx_vfdiv_s(a, b);
702}
703template <>
704EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) {
705 return __lsx_vfdiv_d(a, b);
706}
707template <>
708EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(const Packet8s& a, const Packet8s& b) {
709 return __lsx_vdiv_h(a, b);
710}
711template <>
712EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& a, const Packet4i& b) {
713 return __lsx_vdiv_w(a, b);
714}
715template <>
716EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(const Packet2l& a, const Packet2l& b) {
717 return __lsx_vdiv_d(a, b);
718}
719template <>
720EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(const Packet8us& a, const Packet8us& b) {
721 return __lsx_vdiv_hu(a, b);
722}
723template <>
724EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
725 return __lsx_vdiv_wu(a, b);
726}
727template <>
728EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
729 return __lsx_vdiv_du(a, b);
730}
731
732template <>
733struct is_fused_madd<Packet4f> : true_type {};
734template <>
735EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
736 return __lsx_vfmadd_s(a, b, c);
737}
738template <>
739struct is_fused_madd<Packet2d> : true_type {};
740template <>
741EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
742 return __lsx_vfmadd_d(a, b, c);
743}
744template <>
745EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
746 return __lsx_vfmsub_s(a, b, c);
747}
748template <>
749EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
750 return __lsx_vfmsub_d(a, b, c);
751}
752template <>
753EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
754 return __lsx_vfnmsub_s(a, b, c);
755}
756template <>
757EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
758 return __lsx_vfnmsub_d(a, b, c);
759}
760template <>
761EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) {
762 return __lsx_vfnmadd_s(a, b, c);
763}
764template <>
765EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) {
766 return __lsx_vfnmadd_d(a, b, c);
767}
768template <>
769EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
770 return __lsx_vmadd_b(c, a, b);
771}
772template <>
773EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
774 return __lsx_vmadd_h(c, a, b);
775}
776template <>
777EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
778 return __lsx_vmadd_w(c, a, b);
779}
780template <>
781EIGEN_STRONG_INLINE Packet2l pmadd(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
782 return __lsx_vmadd_d(c, a, b);
783}
784template <>
785EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c) {
786 return __lsx_vmadd_b(c, a, b);
787}
788template <>
789EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) {
790 return __lsx_vmadd_h(c, a, b);
791}
792template <>
793EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c) {
794 return __lsx_vmadd_w(c, a, b);
795}
796template <>
797EIGEN_STRONG_INLINE Packet2ul pmadd(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c) {
798 return __lsx_vmadd_d(c, a, b);
799}
800
801template <>
802EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b) {
803 return (Packet4f)__lsx_vand_v((__m128i)a, (__m128i)b);
804}
805template <>
806EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b) {
807 return (Packet2d)__lsx_vand_v((__m128i)a, (__m128i)b);
808}
809template <>
810EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(const Packet16c& a, const Packet16c& b) {
811 return __lsx_vand_v(a, b);
812}
813template <>
814EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(const Packet8s& a, const Packet8s& b) {
815 return __lsx_vand_v(a, b);
816}
817template <>
818EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) {
819 return __lsx_vand_v(a, b);
820}
821template <>
822EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) {
823 return __lsx_vand_v(a, b);
824}
825template <>
826EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
827 return __lsx_vand_v(a, b);
828}
829template <>
830EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(const Packet8us& a, const Packet8us& b) {
831 return __lsx_vand_v(a, b);
832}
833template <>
834EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
835 return __lsx_vand_v(a, b);
836}
837template <>
838EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
839 return __lsx_vand_v(a, b);
840}
841
842template <>
843EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b) {
844 return (Packet4f)__lsx_vor_v((__m128i)a, (__m128i)b);
845}
846template <>
847EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b) {
848 return (Packet2d)__lsx_vor_v((__m128i)a, (__m128i)b);
849}
850template <>
851EIGEN_STRONG_INLINE Packet16c por<Packet16c>(const Packet16c& a, const Packet16c& b) {
852 return __lsx_vor_v(a, b);
853}
854template <>
855EIGEN_STRONG_INLINE Packet8s por<Packet8s>(const Packet8s& a, const Packet8s& b) {
856 return __lsx_vor_v(a, b);
857}
858template <>
859EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) {
860 return __lsx_vor_v(a, b);
861}
862template <>
863EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b) {
864 return __lsx_vor_v(a, b);
865}
866template <>
867EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
868 return __lsx_vor_v(a, b);
869}
870template <>
871EIGEN_STRONG_INLINE Packet8us por<Packet8us>(const Packet8us& a, const Packet8us& b) {
872 return __lsx_vor_v(a, b);
873}
874template <>
875EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
876 return __lsx_vor_v(a, b);
877}
878template <>
879EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
880 return __lsx_vor_v(a, b);
881}
882
883template <>
884EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b) {
885 return (Packet4f)__lsx_vxor_v((__m128i)a, (__m128i)b);
886}
887template <>
888EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b) {
889 return (Packet2d)__lsx_vxor_v((__m128i)a, (__m128i)b);
890}
891template <>
892EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(const Packet16c& a, const Packet16c& b) {
893 return __lsx_vxor_v(a, b);
894}
895template <>
896EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(const Packet8s& a, const Packet8s& b) {
897 return __lsx_vxor_v(a, b);
898}
899template <>
900EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) {
901 return __lsx_vxor_v(a, b);
902}
903template <>
904EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b) {
905 return __lsx_vxor_v(a, b);
906}
907template <>
908EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
909 return __lsx_vxor_v(a, b);
910}
911template <>
912EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(const Packet8us& a, const Packet8us& b) {
913 return __lsx_vxor_v(a, b);
914}
915template <>
916EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
917 return __lsx_vxor_v(a, b);
918}
919template <>
920EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
921 return __lsx_vxor_v(a, b);
922}
923
924template <>
925EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b) {
926 return (Packet4f)__lsx_vandn_v((__m128i)b, (__m128i)a);
927}
928template <>
929EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b) {
930 return (Packet2d)__lsx_vandn_v((__m128i)b, (__m128i)a);
931}
932template <>
933EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(const Packet16c& a, const Packet16c& b) {
934 return __lsx_vandn_v(b, a);
935}
936template <>
937EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(const Packet8s& a, const Packet8s& b) {
938 return __lsx_vandn_v(b, a);
939}
940template <>
941EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b) {
942 return __lsx_vandn_v(b, a);
943}
944template <>
945EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b) {
946 return __lsx_vandn_v(b, a);
947}
948template <>
949EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
950 return __lsx_vandn_v(b, a);
951}
952template <>
953EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(const Packet8us& a, const Packet8us& b) {
954 return __lsx_vandn_v(b, a);
955}
956template <>
957EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
958 return __lsx_vandn_v(b, a);
959}
960template <>
961EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
962 return __lsx_vandn_v(b, a);
963}
964
965template <>
966EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b) {
967 return (Packet4f)__lsx_vfcmp_cle_s(a, b);
968}
969template <>
970EIGEN_STRONG_INLINE Packet2d pcmp_le<Packet2d>(const Packet2d& a, const Packet2d& b) {
971 return (Packet2d)__lsx_vfcmp_cle_d(a, b);
972}
973template <>
974EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(const Packet16c& a, const Packet16c& b) {
975 return __lsx_vsle_b(a, b);
976}
977template <>
978EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(const Packet8s& a, const Packet8s& b) {
979 return __lsx_vsle_h(a, b);
980}
981template <>
982EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(const Packet4i& a, const Packet4i& b) {
983 return __lsx_vsle_w(a, b);
984}
985template <>
986EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(const Packet2l& a, const Packet2l& b) {
987 return __lsx_vsle_d(a, b);
988}
989template <>
990EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
991 return __lsx_vsle_bu(a, b);
992}
993template <>
994EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(const Packet8us& a, const Packet8us& b) {
995 return __lsx_vsle_hu(a, b);
996}
997template <>
998EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
999 return __lsx_vsle_wu(a, b);
1000}
1001template <>
1002EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1003 return __lsx_vsle_du(a, b);
1004}
1005
1006template <>
1007EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b) {
1008 return (Packet4f)__lsx_vfcmp_clt_s(a, b);
1009}
1010template <>
1011EIGEN_STRONG_INLINE Packet2d pcmp_lt<Packet2d>(const Packet2d& a, const Packet2d& b) {
1012 return (Packet2d)__lsx_vfcmp_clt_d(a, b);
1013}
1014template <>
1015EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(const Packet16c& a, const Packet16c& b) {
1016 return __lsx_vslt_b(a, b);
1017}
1018template <>
1019EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(const Packet8s& a, const Packet8s& b) {
1020 return __lsx_vslt_h(a, b);
1021}
1022template <>
1023EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(const Packet4i& a, const Packet4i& b) {
1024 return __lsx_vslt_w(a, b);
1025}
1026template <>
1027EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(const Packet2l& a, const Packet2l& b) {
1028 return __lsx_vslt_d(a, b);
1029}
1030template <>
1031EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1032 return __lsx_vslt_bu(a, b);
1033}
1034template <>
1035EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(const Packet8us& a, const Packet8us& b) {
1036 return __lsx_vslt_hu(a, b);
1037}
1038template <>
1039EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1040 return __lsx_vslt_wu(a, b);
1041}
1042template <>
1043EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1044 return __lsx_vslt_du(a, b);
1045}
1046
1047template <>
1048EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b) {
1049 return (Packet4f)__lsx_vfcmp_sult_s(a, b);
1050}
1051template <>
1052EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan<Packet2d>(const Packet2d& a, const Packet2d& b) {
1053 return (Packet2d)__lsx_vfcmp_sult_d(a, b);
1054}
1055
1056template <>
1057EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(const Packet4f& a, const Packet4f& b) {
1058 return (Packet4f)__lsx_vfcmp_seq_s(a, b);
1059}
1060template <>
1061EIGEN_STRONG_INLINE Packet2d pcmp_eq<Packet2d>(const Packet2d& a, const Packet2d& b) {
1062 return (Packet2d)__lsx_vfcmp_seq_d(a, b);
1063}
1064template <>
1065EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(const Packet16c& a, const Packet16c& b) {
1066 return __lsx_vseq_b(a, b);
1067}
1068template <>
1069EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(const Packet8s& a, const Packet8s& b) {
1070 return __lsx_vseq_h(a, b);
1071}
1072template <>
1073EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(const Packet4i& a, const Packet4i& b) {
1074 return __lsx_vseq_w(a, b);
1075}
1076template <>
1077EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(const Packet2l& a, const Packet2l& b) {
1078 return __lsx_vseq_d(a, b);
1079}
1080template <>
1081EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1082 return __lsx_vseq_b(a, b);
1083}
1084template <>
1085EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(const Packet8us& a, const Packet8us& b) {
1086 return __lsx_vseq_h(a, b);
1087}
1088template <>
1089EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1090 return __lsx_vseq_w(a, b);
1091}
1092template <>
1093EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1094 return __lsx_vseq_d(a, b);
1095}
1096
1097template <>
1098EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(const Packet16c& a, const Packet16c& b) {
1099 return __lsx_vmin_b(a, b);
1100}
1101template <>
1102EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(const Packet8s& a, const Packet8s& b) {
1103 return __lsx_vmin_h(a, b);
1104}
1105template <>
1106EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) {
1107 return __lsx_vmin_w(a, b);
1108}
1109template <>
1110EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(const Packet2l& a, const Packet2l& b) {
1111 return __lsx_vmin_d(a, b);
1112}
1113template <>
1114EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1115 return __lsx_vmin_bu(a, b);
1116}
1117template <>
1118EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(const Packet8us& a, const Packet8us& b) {
1119 return __lsx_vmin_hu(a, b);
1120}
1121template <>
1122EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1123 return __lsx_vmin_wu(a, b);
1124}
1125template <>
1126EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1127 return __lsx_vmin_du(a, b);
1128}
1129
1130template <>
1131EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(const Packet16c& a, const Packet16c& b) {
1132 return __lsx_vmax_b(a, b);
1133}
1134template <>
1135EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(const Packet8s& a, const Packet8s& b) {
1136 return __lsx_vmax_h(a, b);
1137}
1138template <>
1139EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) {
1140 return __lsx_vmax_w(a, b);
1141}
1142template <>
1143EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(const Packet2l& a, const Packet2l& b) {
1144 return __lsx_vmax_d(a, b);
1145}
1146template <>
1147EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
1148 return __lsx_vmax_bu(a, b);
1149}
1150template <>
1151EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(const Packet8us& a, const Packet8us& b) {
1152 return __lsx_vmax_hu(a, b);
1153}
1154template <>
1155EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(const Packet4ui& a, const Packet4ui& b) {
1156 return __lsx_vmax_wu(a, b);
1157}
1158template <>
1159EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1160 return __lsx_vmax_du(a, b);
1161}
1162
1163template <>
1164EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) {
1165 Packet4i aNaN = __lsx_vfcmp_cun_s(a, a);
1166 Packet4i aMinOrNaN = por<Packet4i>(__lsx_vfcmp_clt_s(a, b), aNaN);
1167 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN);
1168}
1169template <>
1170EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) {
1171 Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
1172 Packet2l aMinOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(a, b), aNaN);
1173 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN);
1174}
1175template <>
1176EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) {
1177 Packet4i aNaN = __lsx_vfcmp_cun_s(a, a);
1178 Packet4i aMaxOrNaN = por<Packet4i>(__lsx_vfcmp_clt_s(b, a), aNaN);
1179 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN);
1180}
1181template <>
1182EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) {
1183 Packet2l aNaN = __lsx_vfcmp_cun_d(a, a);
1184 Packet2l aMaxOrNaN = por<Packet2l>(__lsx_vfcmp_clt_d(b, a), aNaN);
1185 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN);
1186}
1187
1188template <int N>
1189EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(const Packet16c& a) {
1190 return __lsx_vsrai_b((__m128i)a, N);
1191}
1192template <int N>
1193EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(const Packet8s& a) {
1194 return __lsx_vsrai_h((__m128i)a, N);
1195}
1196template <int N>
1197EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) {
1198 return __lsx_vsrai_w((__m128i)a, N);
1199}
1200template <int N>
1201EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(const Packet2l& a) {
1202 return __lsx_vsrai_d((__m128i)a, N);
1203}
1204template <int N>
1205EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(const Packet16uc& a) {
1206 return __lsx_vsrli_b((__m128i)a, N);
1207}
1208template <int N>
1209EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(const Packet8us& a) {
1210 return __lsx_vsrli_h((__m128i)a, N);
1211}
1212template <int N>
1213EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(const Packet4ui& a) {
1214 return __lsx_vsrli_w((__m128i)a, N);
1215}
1216template <int N>
1217EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(const Packet2ul& a) {
1218 return __lsx_vsrli_d((__m128i)a, N);
1219}
1220
1221template <int N>
1222EIGEN_STRONG_INLINE Packet16c plogical_shift_right(const Packet16c& a) {
1223 return __lsx_vsrli_b((__m128i)a, N);
1224}
1225template <int N>
1226EIGEN_STRONG_INLINE Packet8s plogical_shift_right(const Packet8s& a) {
1227 return __lsx_vsrli_h((__m128i)a, N);
1228}
1229template <int N>
1230EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) {
1231 return __lsx_vsrli_w((__m128i)a, N);
1232}
1233template <int N>
1234EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) {
1235 return __lsx_vsrli_d((__m128i)a, N);
1236}
1237template <int N>
1238EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(const Packet16uc& a) {
1239 return __lsx_vsrli_b((__m128i)a, N);
1240}
1241template <int N>
1242EIGEN_STRONG_INLINE Packet8us plogical_shift_right(const Packet8us& a) {
1243 return __lsx_vsrli_h((__m128i)a, N);
1244}
1245template <int N>
1246EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) {
1247 return __lsx_vsrli_w((__m128i)a, N);
1248}
1249template <int N>
1250EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(const Packet2ul& a) {
1251 return __lsx_vsrli_d((__m128i)a, N);
1252}
1253
1254template <int N>
1255EIGEN_STRONG_INLINE Packet16c plogical_shift_left(const Packet16c& a) {
1256 return __lsx_vslli_b((__m128i)a, N);
1257}
1258template <int N>
1259EIGEN_STRONG_INLINE Packet8s plogical_shift_left(const Packet8s& a) {
1260 return __lsx_vslli_h((__m128i)a, N);
1261}
1262template <int N>
1263EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) {
1264 return __lsx_vslli_w((__m128i)a, N);
1265}
1266template <int N>
1267EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) {
1268 return __lsx_vslli_d((__m128i)a, N);
1269}
1270template <int N>
1271EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(const Packet16uc& a) {
1272 return __lsx_vslli_b((__m128i)a, N);
1273}
1274template <int N>
1275EIGEN_STRONG_INLINE Packet8us plogical_shift_left(const Packet8us& a) {
1276 return __lsx_vslli_h((__m128i)a, N);
1277}
1278template <int N>
1279EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) {
1280 return __lsx_vslli_w((__m128i)a, N);
1281}
1282template <int N>
1283EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(const Packet2ul& a) {
1284 return __lsx_vslli_d((__m128i)a, N);
1285}
1286
1287template <>
1288EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) {
1289 return (Packet4f)__lsx_vbitclri_w((__m128i)a, 31);
1290}
1291template <>
1292EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) {
1293 return (Packet2d)__lsx_vbitclri_d((__m128i)a, 63);
1294}
1295template <>
1296EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) {
1297 return __lsx_vabsd_b(a, pzero(a));
1298}
1299template <>
1300EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) {
1301 return __lsx_vabsd_h(a, pzero(a));
1302}
1303template <>
1304EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) {
1305 return __lsx_vabsd_w(a, pzero(a));
1306}
1307template <>
1308EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) {
1309 return __lsx_vabsd_d(a, pzero(a));
1310}
1311template <>
1312EIGEN_STRONG_INLINE Packet16c pabsdiff(const Packet16c& a, const Packet16c& b) {
1313 return __lsx_vabsd_b(a, b);
1314}
1315template <>
1316EIGEN_STRONG_INLINE Packet8s pabsdiff(const Packet8s& a, const Packet8s& b) {
1317 return __lsx_vabsd_h(a, b);
1318}
1319template <>
1320EIGEN_STRONG_INLINE Packet4i pabsdiff(const Packet4i& a, const Packet4i& b) {
1321 return __lsx_vabsd_w(a, b);
1322}
1323template <>
1324EIGEN_STRONG_INLINE Packet2l pabsdiff(const Packet2l& a, const Packet2l& b) {
1325 return __lsx_vabsd_d(a, b);
1326}
1327template <>
1328EIGEN_STRONG_INLINE Packet16uc pabsdiff(const Packet16uc& a, const Packet16uc& b) {
1329 return __lsx_vabsd_bu(a, b);
1330}
1331template <>
1332EIGEN_STRONG_INLINE Packet8us pabsdiff(const Packet8us& a, const Packet8us& b) {
1333 return __lsx_vabsd_hu(a, b);
1334}
1335template <>
1336EIGEN_STRONG_INLINE Packet4ui pabsdiff(const Packet4ui& a, const Packet4ui& b) {
1337 return __lsx_vabsd_wu(a, b);
1338}
1339template <>
1340EIGEN_STRONG_INLINE Packet2ul pabsdiff(const Packet2ul& a, const Packet2ul& b) {
1341 return __lsx_vabsd_du(a, b);
1342}
1343template <>
1344EIGEN_STRONG_INLINE Packet4f pabsdiff(const Packet4f& a, const Packet4f& b) {
1345 return pabs(psub(a, b));
1346}
1347template <>
1348EIGEN_STRONG_INLINE Packet2d pabsdiff(const Packet2d& a, const Packet2d& b) {
1349 return pabs(psub(a, b));
1350}
1351
1352template <>
1353EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from) {
1354 EIGEN_DEBUG_ALIGNED_LOAD return (Packet4f)__lsx_vld(from, 0);
1355}
1356template <>
1357EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from) {
1358 EIGEN_DEBUG_ALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0);
1359}
1360template <>
1361EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(const int8_t* from) {
1362 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1363}
1364template <>
1365EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(const int16_t* from) {
1366 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1367}
1368template <>
1369EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from) {
1370 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1371}
1372template <>
1373EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(const int64_t* from) {
1374 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1375}
1376template <>
1377EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(const uint8_t* from) {
1378 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1379}
1380template <>
1381EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(const uint16_t* from) {
1382 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1383}
1384template <>
1385EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(const uint32_t* from) {
1386 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1387}
1388template <>
1389EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(const uint64_t* from) {
1390 EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0);
1391}
1392
1393template <>
1394EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from) {
1395 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4f)__lsx_vld(from, 0);
1396}
1397template <>
1398EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from) {
1399 EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0);
1400}
1401template <>
1402EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(const int8_t* from) {
1403 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1404}
1405template <>
1406EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(const int16_t* from) {
1407 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1408}
1409template <>
1410EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from) {
1411 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1412}
1413template <>
1414EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(const int64_t* from) {
1415 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1416}
1417template <>
1418EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(const uint8_t* from) {
1419 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1420}
1421template <>
1422EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(const uint16_t* from) {
1423 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1424}
1425template <>
1426EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(const uint32_t* from) {
1427 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1428}
1429template <>
1430EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(const uint64_t* from) {
1431 EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0);
1432}
1433
1434template <>
1435EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from) {
1436 float f0 = from[0], f1 = from[1];
1437 return make_packet4f(f0, f0, f1, f1);
1438}
1439template <>
1440EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(const int8_t* from) {
1441 __m128i tmp = {};
1442 memcpy(&tmp, from, sizeof(tmp) / 2);
1443 return __lsx_vilvl_b(tmp, tmp);
1444}
1445template <>
1446EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(const int16_t* from) {
1447 __m128i tmp = {};
1448 memcpy(&tmp, from, sizeof(tmp) / 2);
1449 return __lsx_vilvl_h(tmp, tmp);
1450}
1451template <>
1452EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from) {
1453 __m128i tmp = {};
1454 memcpy(&tmp, from, sizeof(tmp) / 2);
1455 return __lsx_vilvl_w(tmp, tmp);
1456}
1457template <>
1458EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(const uint8_t* from) {
1459 __m128i tmp = {};
1460 memcpy(&tmp, from, sizeof(tmp) / 2);
1461 return __lsx_vilvl_b(tmp, tmp);
1462}
1463template <>
1464EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(const uint16_t* from) {
1465 __m128i tmp = {};
1466 memcpy(&tmp, from, sizeof(tmp) / 2);
1467 return __lsx_vilvl_h(tmp, tmp);
1468}
1469template <>
1470EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(const uint32_t* from) {
1471 __m128i tmp = {};
1472 memcpy(&tmp, from, sizeof(tmp) / 2);
1473 return __lsx_vilvl_w(tmp, tmp);
1474}
1475template <>
1476EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from) {
1477 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0);
1478}
1479template <>
1480EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from) {
1481 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0);
1482}
1483template <>
1484EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet16c& from) {
1485 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1486}
1487template <>
1488EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet8s& from) {
1489 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1490}
1491template <>
1492EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from) {
1493 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1494}
1495template <>
1496EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet2l& from) {
1497 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1498}
1499template <>
1500EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet16uc& from) {
1501 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1502}
1503template <>
1504EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet8us& from) {
1505 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1506}
1507template <>
1508EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet4ui& from) {
1509 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1510}
1511template <>
1512EIGEN_STRONG_INLINE void pstore<uint64_t>(uint64_t* to, const Packet2ul& from) {
1513 EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1514}
1515
1516template <>
1517EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from) {
1518 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0);
1519}
1520template <>
1521EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from) {
1522 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0);
1523}
1524
1525template <>
1526EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet16c& from) {
1527 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1528}
1529template <>
1530EIGEN_STRONG_INLINE void pstoreu<int16_t>(int16_t* to, const Packet8s& from) {
1531 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1532}
1533template <>
1534EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from) {
1535 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1536}
1537template <>
1538EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet2l& from) {
1539 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1540}
1541template <>
1542EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet16uc& from) {
1543 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1544}
1545template <>
1546EIGEN_STRONG_INLINE void pstoreu<uint16_t>(uint16_t* to, const Packet8us& from) {
1547 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1548}
1549template <>
1550EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet4ui& from) {
1551 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1552}
1553template <>
1554EIGEN_STRONG_INLINE void pstoreu<uint64_t>(uint64_t* to, const Packet2ul& from) {
1555 EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0);
1556}
1557
1558template <>
1559EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride) {
1560 Packet4f v = {from[0], from[stride], from[2 * stride], from[3 * stride]};
1561 return v;
1562}
1563template <>
1564EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(const double* from, Index stride) {
1565 Packet2d v = {from[0], from[stride]};
1566 return v;
1567}
1568template <>
1569EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(const int8_t* from, Index stride) {
1570 int8_t v[16] __attribute__((aligned(16)));
1571 v[0] = from[0];
1572 v[1] = from[stride];
1573 v[2] = from[2 * stride];
1574 v[3] = from[3 * stride];
1575 v[4] = from[4 * stride];
1576 v[5] = from[5 * stride];
1577 v[6] = from[6 * stride];
1578 v[7] = from[7 * stride];
1579 v[8] = from[8 * stride];
1580 v[9] = from[9 * stride];
1581 v[10] = from[10 * stride];
1582 v[11] = from[11 * stride];
1583 v[12] = from[12 * stride];
1584 v[13] = from[13 * stride];
1585 v[14] = from[14 * stride];
1586 v[15] = from[15 * stride];
1587 return __lsx_vld(v, 0);
1588}
1589template <>
1590EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(const int16_t* from, Index stride) {
1591 int16_t v[8] __attribute__((aligned(16)));
1592 v[0] = from[0];
1593 v[1] = from[stride];
1594 v[2] = from[2 * stride];
1595 v[3] = from[3 * stride];
1596 v[4] = from[4 * stride];
1597 v[5] = from[5 * stride];
1598 v[6] = from[6 * stride];
1599 v[7] = from[7 * stride];
1600 return __lsx_vld(v, 0);
1601}
1602template <>
1603EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride) {
1604 int32_t v[4] __attribute__((aligned(16)));
1605 v[0] = from[0];
1606 v[1] = from[stride];
1607 v[2] = from[2 * stride];
1608 v[3] = from[3 * stride];
1609 return __lsx_vld(v, 0);
1610}
1611template <>
1612EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(const int64_t* from, Index stride) {
1613 int64_t v[2] __attribute__((aligned(16)));
1614 v[0] = from[0];
1615 v[1] = from[stride];
1616 return __lsx_vld(v, 0);
1617}
1618template <>
1619EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(const uint8_t* from, Index stride) {
1620 uint8_t v[16] __attribute__((aligned(16)));
1621 v[0] = from[0];
1622 v[1] = from[stride];
1623 v[2] = from[2 * stride];
1624 v[3] = from[3 * stride];
1625 v[4] = from[4 * stride];
1626 v[5] = from[5 * stride];
1627 v[6] = from[6 * stride];
1628 v[7] = from[7 * stride];
1629 v[8] = from[8 * stride];
1630 v[9] = from[9 * stride];
1631 v[10] = from[10 * stride];
1632 v[11] = from[11 * stride];
1633 v[12] = from[12 * stride];
1634 v[13] = from[13 * stride];
1635 v[14] = from[14 * stride];
1636 v[15] = from[15 * stride];
1637 return __lsx_vld(v, 0);
1638}
1639template <>
1640EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(const uint16_t* from, Index stride) {
1641 uint16_t v[8] __attribute__((aligned(16)));
1642 v[0] = from[0];
1643 v[1] = from[stride];
1644 v[2] = from[2 * stride];
1645 v[3] = from[3 * stride];
1646 v[4] = from[4 * stride];
1647 v[5] = from[5 * stride];
1648 v[6] = from[6 * stride];
1649 v[7] = from[7 * stride];
1650 return __lsx_vld(v, 0);
1651}
1652template <>
1653EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(const uint32_t* from, Index stride) {
1654 uint32_t v[4] __attribute__((aligned(16)));
1655 v[0] = from[0];
1656 v[1] = from[stride];
1657 v[2] = from[2 * stride];
1658 v[3] = from[3 * stride];
1659 return __lsx_vld(v, 0);
1660}
1661template <>
1662EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(const uint64_t* from, Index stride) {
1663 uint64_t v[2] __attribute__((aligned(16)));
1664 v[0] = from[0];
1665 v[1] = from[stride];
1666 return __lsx_vld(v, 0);
1667}
1668
1669template <>
1670EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride) {
1671 __lsx_vstelm_w(from, to, 0, 0);
1672 __lsx_vstelm_w(from, to + stride * 1, 0, 1);
1673 __lsx_vstelm_w(from, to + stride * 2, 0, 2);
1674 __lsx_vstelm_w(from, to + stride * 3, 0, 3);
1675}
1676template <>
1677EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride) {
1678 __lsx_vstelm_d(from, to, 0, 0);
1679 __lsx_vstelm_d(from, to + stride, 0, 1);
1680}
1681template <>
1682EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet16c>(int8_t* to, const Packet16c& from,
1683 Index stride) {
1684 __lsx_vstelm_b((__m128i)from, to, 0, 0);
1685 __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
1686 __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
1687 __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
1688 __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
1689 __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
1690 __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
1691 __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
1692 __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
1693 __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
1694 __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
1695 __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
1696 __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
1697 __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
1698 __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
1699 __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
1700}
1701template <>
1702EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet8s>(int16_t* to, const Packet8s& from,
1703 Index stride) {
1704 __lsx_vstelm_h((__m128i)from, to, 0, 0);
1705 __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
1706 __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
1707 __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
1708 __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
1709 __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
1710 __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
1711 __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
1712}
1713template <>
1714EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from,
1715 Index stride) {
1716 __lsx_vstelm_w((__m128i)from, to, 0, 0);
1717 __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
1718 __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
1719 __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
1720}
1721template <>
1722EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int64_t, Packet2l>(int64_t* to, const Packet2l& from,
1723 Index stride) {
1724 __lsx_vstelm_d((__m128i)from, to, 0, 0);
1725 __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
1726}
1727template <>
1728EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet16uc>(uint8_t* to, const Packet16uc& from,
1729 Index stride) {
1730 __lsx_vstelm_b((__m128i)from, to, 0, 0);
1731 __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1);
1732 __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2);
1733 __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3);
1734 __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4);
1735 __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5);
1736 __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6);
1737 __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7);
1738 __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8);
1739 __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9);
1740 __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10);
1741 __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11);
1742 __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12);
1743 __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13);
1744 __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14);
1745 __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15);
1746}
1747template <>
1748EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet8us>(uint16_t* to, const Packet8us& from,
1749 Index stride) {
1750 __lsx_vstelm_h((__m128i)from, to, 0, 0);
1751 __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1);
1752 __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2);
1753 __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3);
1754 __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4);
1755 __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5);
1756 __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6);
1757 __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7);
1758}
1759template <>
1760EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from,
1761 Index stride) {
1762 __lsx_vstelm_w((__m128i)from, to, 0, 0);
1763 __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1);
1764 __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2);
1765 __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3);
1766}
1767template <>
1768EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint64_t, Packet2ul>(uint64_t* to, const Packet2ul& from,
1769 Index stride) {
1770 __lsx_vstelm_d((__m128i)from, to, 0, 0);
1771 __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1);
1772}
1773
1774template <>
1775EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) {
1776 __builtin_prefetch(addr);
1777}
1778template <>
1779EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) {
1780 __builtin_prefetch(addr);
1781}
1782template <>
1783EIGEN_STRONG_INLINE void prefetch<int8_t>(const int8_t* addr) {
1784 __builtin_prefetch(addr);
1785}
1786template <>
1787EIGEN_STRONG_INLINE void prefetch<int16_t>(const int16_t* addr) {
1788 __builtin_prefetch(addr);
1789}
1790template <>
1791EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) {
1792 __builtin_prefetch(addr);
1793}
1794template <>
1795EIGEN_STRONG_INLINE void prefetch<int64_t>(const int64_t* addr) {
1796 __builtin_prefetch(addr);
1797}
1798template <>
1799EIGEN_STRONG_INLINE void prefetch<uint8_t>(const uint8_t* addr) {
1800 __builtin_prefetch(addr);
1801}
1802template <>
1803EIGEN_STRONG_INLINE void prefetch<uint16_t>(const uint16_t* addr) {
1804 __builtin_prefetch(addr);
1805}
1806template <>
1807EIGEN_STRONG_INLINE void prefetch<uint32_t>(const uint32_t* addr) {
1808 __builtin_prefetch(addr);
1809}
1810template <>
1811EIGEN_STRONG_INLINE void prefetch<uint64_t>(const uint64_t* addr) {
1812 __builtin_prefetch(addr);
1813}
1814
1815template <>
1816EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) {
1817 float v;
1818 __lsx_vstelm_w(a, &v, 0, 0);
1819 return v;
1820}
1821template <>
1822EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) {
1823 double v;
1824 __lsx_vstelm_d(a, &v, 0, 0);
1825 return v;
1826}
1827
1828template <>
1829EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(const Packet16c& a) {
1830 return (int8_t)__lsx_vpickve2gr_b((__m128i)a, 0);
1831}
1832template <>
1833EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(const Packet8s& a) {
1834 return (int16_t)__lsx_vpickve2gr_h((__m128i)a, 0);
1835}
1836template <>
1837EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) {
1838 return __lsx_vpickve2gr_w((__m128i)a, 0);
1839}
1840template <>
1841EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(const Packet2l& a) {
1842 return __lsx_vpickve2gr_d((__m128i)a, 0);
1843}
1844template <>
1845EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(const Packet16uc& a) {
1846 return (uint8_t)__lsx_vpickve2gr_bu((__m128i)a, 0);
1847}
1848template <>
1849EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(const Packet8us& a) {
1850 return (uint16_t)__lsx_vpickve2gr_hu((__m128i)a, 0);
1851}
1852template <>
1853EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(const Packet4ui& a) {
1854 return __lsx_vpickve2gr_wu((__m128i)a, 0);
1855}
1856template <>
1857EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(const Packet2ul& a) {
1858 return __lsx_vpickve2gr_du((__m128i)a, 0);
1859}
1860
1861template <>
1862EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) {
1863 return (Packet4f)__lsx_vshuf4i_w(a, 0x1B);
1864}
1865template <>
1866EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) {
1867 return (Packet2d)__lsx_vshuf4i_d(a, a, 0x1);
1868}
1869template <>
1870EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) {
1871 return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B);
1872}
1873template <>
1874EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a) {
1875 return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B);
1876}
1877template <>
1878EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) {
1879 return __lsx_vshuf4i_w((__m128i)a, 0x1B);
1880}
1881template <>
1882EIGEN_STRONG_INLINE Packet2l preverse(const Packet2l& a) {
1883 return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
1884}
1885template <>
1886EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) {
1887 return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B);
1888}
1889template <>
1890EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) {
1891 return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B);
1892}
1893template <>
1894EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a) {
1895 return __lsx_vshuf4i_w((__m128i)a, 0x1B);
1896}
1897template <>
1898EIGEN_STRONG_INLINE Packet2ul preverse(const Packet2ul& a) {
1899 return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1);
1900}
1901
1902template <>
1903EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) {
1904 Packet4f tmp = __lsx_vfadd_s(a, vec4f_swizzle1(a, 2, 3, 2, 3));
1905 return pfirst<Packet4f>(__lsx_vfadd_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1)));
1906}
1907template <>
1908EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a) {
1909 return pfirst<Packet2d>(__lsx_vfadd_d(a, preverse(a)));
1910}
1911
1912template <typename Packet>
1913EIGEN_STRONG_INLINE bool predux_any_lsx(const Packet& a) {
1914 return __lsx_bnz_v((__m128i)a);
1915}
1916
1917template <>
1918EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) {
1919 return predux_any_lsx(a);
1920}
1921
1922template <>
1923EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) {
1924 return predux_any_lsx(a);
1925}
1926
1927template <>
1928EIGEN_STRONG_INLINE bool predux_any(const Packet16c& a) {
1929 return predux_any_lsx(a);
1930}
1931
1932template <>
1933EIGEN_STRONG_INLINE bool predux_any(const Packet8s& a) {
1934 return predux_any_lsx(a);
1935}
1936
1937template <>
1938EIGEN_STRONG_INLINE bool predux_any(const Packet4i& a) {
1939 return predux_any_lsx(a);
1940}
1941
1942template <>
1943EIGEN_STRONG_INLINE bool predux_any(const Packet2l& a) {
1944 return predux_any_lsx(a);
1945}
1946
1947template <>
1948EIGEN_STRONG_INLINE bool predux_any(const Packet16uc& a) {
1949 return predux_any_lsx(a);
1950}
1951
1952template <>
1953EIGEN_STRONG_INLINE bool predux_any(const Packet8us& a) {
1954 return predux_any_lsx(a);
1955}
1956
1957template <>
1958EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& a) {
1959 return predux_any_lsx(a);
1960}
1961
1962template <>
1963EIGEN_STRONG_INLINE bool predux_any(const Packet2ul& a) {
1964 return predux_any_lsx(a);
1965}
1966
1967template <>
1968EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a) {
1969 Packet8s tmp1 = __lsx_vhaddw_h_b(a, a);
1970 Packet4i tmp2 = __lsx_vhaddw_w_h(tmp1, tmp1);
1971 Packet2l tmp3 = __lsx_vhaddw_d_w(tmp2, tmp2);
1972 return (int8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp3, tmp3), 0);
1973}
1974template <>
1975EIGEN_STRONG_INLINE int16_t predux<Packet8s>(const Packet8s& a) {
1976 Packet4i tmp1 = __lsx_vhaddw_w_h(a, a);
1977 Packet2l tmp2 = __lsx_vhaddw_d_w(tmp1, tmp1);
1978 return (int16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp2, tmp2), 0);
1979}
1980template <>
1981EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) {
1982 Packet2l tmp = __lsx_vhaddw_d_w(a, a);
1983 return (int32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp, tmp), 0);
1984}
1985template <>
1986EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a) {
1987 return (int64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(a, a), 0);
1988}
1989template <>
1990EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(const Packet16uc& a) {
1991 Packet8us tmp1 = __lsx_vhaddw_hu_bu(a, a);
1992 Packet4ui tmp2 = __lsx_vhaddw_wu_hu(tmp1, tmp1);
1993 Packet2ul tmp3 = __lsx_vhaddw_du_wu(tmp2, tmp2);
1994 return (uint8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp3, tmp3), 0);
1995}
1996template <>
1997EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(const Packet8us& a) {
1998 Packet4ui tmp1 = __lsx_vhaddw_wu_hu(a, a);
1999 Packet2ul tmp2 = __lsx_vhaddw_du_wu(tmp1, tmp1);
2000 return (uint16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp2, tmp2), 0);
2001}
2002template <>
2003EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a) {
2004 Packet2ul tmp = __lsx_vhaddw_du_wu(a, a);
2005 return (uint32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp, tmp), 0);
2006}
2007template <>
2008EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(const Packet2ul& a) {
2009 return (uint64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(a, a), 0);
2010}
2011
2012template <>
2013EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a) {
2014 Packet4f tmp = __lsx_vfmul_s(a, vec4f_swizzle1(a, 2, 3, 2, 3));
2015 return pfirst<Packet4f>(__lsx_vfmul_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1)));
2016}
2017template <>
2018EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a) {
2019 return pfirst<Packet2d>(__lsx_vfmul_d(a, preverse(a)));
2020}
2021template <>
2022EIGEN_STRONG_INLINE int8_t predux_mul<Packet16c>(const Packet16c& a) {
2023 Packet8s tmp1 = __lsx_vmulwev_h_b(a, preverse(a));
2024 Packet4i tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1));
2025 Packet2l tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2));
2026 return (int8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0);
2027}
2028template <>
2029EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(const Packet8s& a) {
2030 Packet4i tmp1 = __lsx_vmulwev_w_h(a, preverse(a));
2031 Packet2l tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1));
2032 return (int16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0);
2033}
2034template <>
2035EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a) {
2036 Packet2l tmp = __lsx_vmulwev_d_w(a, preverse(a));
2037 return (int32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0);
2038}
2039template <>
2040EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(const Packet2l& a) {
2041 return (int64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(a, preverse(a)), 0);
2042}
2043template <>
2044EIGEN_STRONG_INLINE uint8_t predux_mul<Packet16uc>(const Packet16uc& a) {
2045 Packet8us tmp1 = __lsx_vmulwev_h_bu(a, preverse(a));
2046 Packet4ui tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1));
2047 Packet2ul tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2));
2048 return (uint8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0);
2049}
2050template <>
2051EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(const Packet8us& a) {
2052 Packet4ui tmp1 = __lsx_vmulwev_w_hu(a, preverse(a));
2053 Packet2ul tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1));
2054 return (uint16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0);
2055}
2056template <>
2057EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(const Packet4ui& a) {
2058 Packet2ul tmp = __lsx_vmulwev_d_wu(a, preverse(a));
2059 return (uint32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0);
2060}
2061template <>
2062EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(const Packet2ul& a) {
2063 return (uint64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_du(a, preverse(a)), 0);
2064}
2065
2066template <>
2067EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) {
2068 Packet4f tmp = __lsx_vfmin_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E));
2069 return pfirst(__lsx_vfmin_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1)));
2070}
2071template <>
2072EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a) {
2073 return pfirst(__lsx_vfmin_d(a, preverse(a)));
2074}
2075template <>
2076EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(const Packet16c& a) {
2077 Packet16c tmp1 = __lsx_vmin_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2078 Packet16c tmp2 = __lsx_vmin_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2079 Packet16c tmp3 = __lsx_vmin_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2080 return pfirst((Packet16c)__lsx_vmin_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2081}
2082template <>
2083EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(const Packet8s& a) {
2084 Packet8s tmp1 = __lsx_vmin_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2085 Packet8s tmp2 = __lsx_vmin_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2086 return pfirst((Packet8s)__lsx_vmin_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2087}
2088template <>
2089EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) {
2090 Packet4i tmp = __lsx_vmin_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2091 return pfirst((Packet4i)__lsx_vmin_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2092}
2093template <>
2094EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(const Packet2l& a) {
2095 return pfirst((Packet2l)__lsx_vmin_d(a, preverse(a)));
2096}
2097template <>
2098EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(const Packet16uc& a) {
2099 Packet16uc tmp1 = __lsx_vmin_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2100 Packet16uc tmp2 = __lsx_vmin_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2101 Packet16uc tmp3 = __lsx_vmin_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2102 return pfirst((Packet16uc)__lsx_vmin_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2103}
2104template <>
2105EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(const Packet8us& a) {
2106 Packet8us tmp1 = __lsx_vmin_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2107 Packet8us tmp2 = __lsx_vmin_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2108 return pfirst((Packet8us)__lsx_vmin_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2109}
2110template <>
2111EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a) {
2112 Packet4ui tmp = __lsx_vmin_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2113 return pfirst((Packet4ui)__lsx_vmin_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2114}
2115template <>
2116EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(const Packet2ul& a) {
2117 return pfirst((Packet2ul)__lsx_vmin_du(a, preverse(a)));
2118}
2119
2120template <>
2121EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) {
2122 Packet4f tmp = __lsx_vfmax_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E));
2123 return pfirst(__lsx_vfmax_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1)));
2124}
2125template <>
2126EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a) {
2127 return pfirst(__lsx_vfmax_d(a, preverse(a)));
2128}
2129template <>
2130EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(const Packet16c& a) {
2131 Packet16c tmp1 = __lsx_vmax_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2132 Packet16c tmp2 = __lsx_vmax_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2133 Packet16c tmp3 = __lsx_vmax_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2134 return pfirst((Packet16c)__lsx_vmax_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2135}
2136template <>
2137EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(const Packet8s& a) {
2138 Packet8s tmp1 = __lsx_vmax_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2139 Packet8s tmp2 = __lsx_vmax_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2140 return pfirst((Packet8s)__lsx_vmax_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2141}
2142template <>
2143EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) {
2144 Packet4i tmp = __lsx_vmax_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2145 return pfirst((Packet4i)__lsx_vmax_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2146}
2147template <>
2148EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(const Packet2l& a) {
2149 return pfirst((Packet2l)__lsx_vmax_d(a, preverse(a)));
2150}
2151template <>
2152EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(const Packet16uc& a) {
2153 Packet16uc tmp1 = __lsx_vmax_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2154 Packet16uc tmp2 = __lsx_vmax_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2155 Packet16uc tmp3 = __lsx_vmax_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E));
2156 return pfirst((Packet16uc)__lsx_vmax_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1)));
2157}
2158template <>
2159EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(const Packet8us& a) {
2160 Packet8us tmp1 = __lsx_vmax_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2161 Packet8us tmp2 = __lsx_vmax_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E));
2162 return pfirst((Packet8us)__lsx_vmax_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1)));
2163}
2164template <>
2165EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a) {
2166 Packet4ui tmp = __lsx_vmax_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E));
2167 return pfirst((Packet4ui)__lsx_vmax_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1)));
2168}
2169template <>
2170EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(const Packet2ul& a) {
2171 return pfirst((Packet2ul)__lsx_vmax_du(a, preverse(a)));
2172}
2173
2174template <>
2175EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) {
2176 return __lsx_vfsqrt_s(a);
2177}
2178template <>
2179EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& a) {
2180 return __lsx_vfsqrt_d(a);
2181}
2182
2183EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
2184 Packet4f T0 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2185 Packet4f T1 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2186 Packet4f T2 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]);
2187 Packet4f T3 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]);
2188
2189 kernel.packet[0] = (Packet4f)__lsx_vilvl_d((__m128i)T2, (__m128i)T0);
2190 kernel.packet[1] = (Packet4f)__lsx_vilvh_d((__m128i)T2, (__m128i)T0);
2191 kernel.packet[2] = (Packet4f)__lsx_vilvl_d((__m128i)T3, (__m128i)T1);
2192 kernel.packet[3] = (Packet4f)__lsx_vilvh_d((__m128i)T3, (__m128i)T1);
2193}
2194EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
2195 Packet2d tmp = (Packet2d)__lsx_vilvh_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2196 kernel.packet[0] = (Packet2d)__lsx_vilvl_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]);
2197 kernel.packet[1] = tmp;
2198}
2199EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
2200 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2201 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2202 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2203 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2204 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2205 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2206 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2207 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2208 __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]);
2209 __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]);
2210 __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]);
2211 __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]);
2212 __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]);
2213 __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]);
2214 __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]);
2215 __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]);
2216
2217 __m128i s0 = __lsx_vilvl_h(t2, t0);
2218 __m128i s1 = __lsx_vilvh_h(t2, t0);
2219 __m128i s2 = __lsx_vilvl_h(t3, t1);
2220 __m128i s3 = __lsx_vilvh_h(t3, t1);
2221 __m128i s4 = __lsx_vilvl_h(t6, t4);
2222 __m128i s5 = __lsx_vilvh_h(t6, t4);
2223 __m128i s6 = __lsx_vilvl_h(t7, t5);
2224 __m128i s7 = __lsx_vilvh_h(t7, t5);
2225 __m128i s8 = __lsx_vilvl_h(ta, t8);
2226 __m128i s9 = __lsx_vilvh_h(ta, t8);
2227 __m128i sa = __lsx_vilvl_h(tb, t9);
2228 __m128i sb = __lsx_vilvh_h(tb, t9);
2229 __m128i sc = __lsx_vilvl_h(te, tc);
2230 __m128i sd = __lsx_vilvh_h(te, tc);
2231 __m128i se = __lsx_vilvl_h(tf, td);
2232 __m128i sf = __lsx_vilvh_h(tf, td);
2233
2234 __m128i u0 = __lsx_vilvl_w(s4, s0);
2235 __m128i u1 = __lsx_vilvh_w(s4, s0);
2236 __m128i u2 = __lsx_vilvl_w(s5, s1);
2237 __m128i u3 = __lsx_vilvh_w(s5, s1);
2238 __m128i u4 = __lsx_vilvl_w(s6, s2);
2239 __m128i u5 = __lsx_vilvh_w(s6, s2);
2240 __m128i u6 = __lsx_vilvl_w(s7, s3);
2241 __m128i u7 = __lsx_vilvh_w(s7, s3);
2242 __m128i u8 = __lsx_vilvl_w(sc, s8);
2243 __m128i u9 = __lsx_vilvh_w(sc, s8);
2244 __m128i ua = __lsx_vilvl_w(sd, s9);
2245 __m128i ub = __lsx_vilvh_w(sd, s9);
2246 __m128i uc = __lsx_vilvl_w(se, sa);
2247 __m128i ud = __lsx_vilvh_w(se, sa);
2248 __m128i ue = __lsx_vilvl_w(sf, sb);
2249 __m128i uf = __lsx_vilvh_w(sf, sb);
2250
2251 kernel.packet[0] = __lsx_vilvl_d(u8, u0);
2252 kernel.packet[1] = __lsx_vilvh_d(u8, u0);
2253 kernel.packet[2] = __lsx_vilvl_d(u9, u1);
2254 kernel.packet[3] = __lsx_vilvh_d(u9, u1);
2255 kernel.packet[4] = __lsx_vilvl_d(ua, u2);
2256 kernel.packet[5] = __lsx_vilvh_d(ua, u2);
2257 kernel.packet[6] = __lsx_vilvl_d(ub, u3);
2258 kernel.packet[7] = __lsx_vilvh_d(ub, u3);
2259 kernel.packet[8] = __lsx_vilvl_d(uc, u4);
2260 kernel.packet[9] = __lsx_vilvh_d(uc, u4);
2261 kernel.packet[10] = __lsx_vilvl_d(ud, u5);
2262 kernel.packet[11] = __lsx_vilvh_d(ud, u5);
2263 kernel.packet[12] = __lsx_vilvl_d(ue, u6);
2264 kernel.packet[13] = __lsx_vilvh_d(ue, u6);
2265 kernel.packet[14] = __lsx_vilvl_d(uf, u7);
2266 kernel.packet[15] = __lsx_vilvh_d(uf, u7);
2267}
2268EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 8>& kernel) {
2269 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2270 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2271 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2272 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2273 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2274 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2275 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2276 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2277
2278 __m128i s0 = __lsx_vilvl_h(t2, t0);
2279 __m128i s1 = __lsx_vilvh_h(t2, t0);
2280 __m128i s2 = __lsx_vilvl_h(t3, t1);
2281 __m128i s3 = __lsx_vilvh_h(t3, t1);
2282 __m128i s4 = __lsx_vilvl_h(t6, t4);
2283 __m128i s5 = __lsx_vilvh_h(t6, t4);
2284 __m128i s6 = __lsx_vilvl_h(t7, t5);
2285 __m128i s7 = __lsx_vilvh_h(t7, t5);
2286
2287 kernel.packet[0] = __lsx_vilvl_w(s4, s0);
2288 kernel.packet[1] = __lsx_vilvh_w(s4, s0);
2289 kernel.packet[2] = __lsx_vilvl_w(s5, s1);
2290 kernel.packet[3] = __lsx_vilvh_w(s5, s1);
2291 kernel.packet[4] = __lsx_vilvl_w(s6, s2);
2292 kernel.packet[5] = __lsx_vilvh_w(s6, s2);
2293 kernel.packet[6] = __lsx_vilvl_w(s7, s3);
2294 kernel.packet[7] = __lsx_vilvh_w(s7, s3);
2295}
2296EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
2297 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2298 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2299 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2300 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2301
2302 kernel.packet[0] = __lsx_vilvl_h(t2, t0);
2303 kernel.packet[1] = __lsx_vilvh_h(t2, t0);
2304 kernel.packet[2] = __lsx_vilvl_h(t3, t1);
2305 kernel.packet[3] = __lsx_vilvh_h(t3, t1);
2306}
2307EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
2308 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2309 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2310 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2311 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2312 __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]);
2313 __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]);
2314 __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]);
2315 __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]);
2316
2317 __m128i s0 = __lsx_vilvl_w(t2, t0);
2318 __m128i s1 = __lsx_vilvh_w(t2, t0);
2319 __m128i s2 = __lsx_vilvl_w(t3, t1);
2320 __m128i s3 = __lsx_vilvh_w(t3, t1);
2321 __m128i s4 = __lsx_vilvl_w(t6, t4);
2322 __m128i s5 = __lsx_vilvh_w(t6, t4);
2323 __m128i s6 = __lsx_vilvl_w(t7, t5);
2324 __m128i s7 = __lsx_vilvh_w(t7, t5);
2325
2326 kernel.packet[0] = __lsx_vilvl_d(s4, s0);
2327 kernel.packet[1] = __lsx_vilvh_d(s4, s0);
2328 kernel.packet[2] = __lsx_vilvl_d(s5, s1);
2329 kernel.packet[3] = __lsx_vilvh_d(s5, s1);
2330 kernel.packet[4] = __lsx_vilvl_d(s6, s2);
2331 kernel.packet[5] = __lsx_vilvh_d(s6, s2);
2332 kernel.packet[6] = __lsx_vilvl_d(s7, s3);
2333 kernel.packet[7] = __lsx_vilvh_d(s7, s3);
2334}
2335EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
2336 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2337 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2338 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2339 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2340
2341 kernel.packet[0] = __lsx_vilvl_w(t2, t0);
2342 kernel.packet[1] = __lsx_vilvh_w(t2, t0);
2343 kernel.packet[2] = __lsx_vilvl_w(t3, t1);
2344 kernel.packet[3] = __lsx_vilvh_w(t3, t1);
2345}
2346EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
2347 __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]);
2348 __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]);
2349 __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]);
2350 __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]);
2351
2352 kernel.packet[0] = __lsx_vilvl_d(T2, T0);
2353 kernel.packet[1] = __lsx_vilvh_d(T2, T0);
2354 kernel.packet[2] = __lsx_vilvl_d(T3, T1);
2355 kernel.packet[3] = __lsx_vilvh_d(T3, T1);
2356}
2357EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2l, 2>& kernel) {
2358 __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
2359 kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
2360 kernel.packet[1] = tmp;
2361}
2362EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
2363 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2364 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2365 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2366 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2367 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2368 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2369 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2370 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2371 __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]);
2372 __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]);
2373 __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]);
2374 __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]);
2375 __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]);
2376 __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]);
2377 __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]);
2378 __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]);
2379
2380 __m128i s0 = __lsx_vilvl_h(t2, t0);
2381 __m128i s1 = __lsx_vilvh_h(t2, t0);
2382 __m128i s2 = __lsx_vilvl_h(t3, t1);
2383 __m128i s3 = __lsx_vilvh_h(t3, t1);
2384 __m128i s4 = __lsx_vilvl_h(t6, t4);
2385 __m128i s5 = __lsx_vilvh_h(t6, t4);
2386 __m128i s6 = __lsx_vilvl_h(t7, t5);
2387 __m128i s7 = __lsx_vilvh_h(t7, t5);
2388 __m128i s8 = __lsx_vilvl_h(ta, t8);
2389 __m128i s9 = __lsx_vilvh_h(ta, t8);
2390 __m128i sa = __lsx_vilvl_h(tb, t9);
2391 __m128i sb = __lsx_vilvh_h(tb, t9);
2392 __m128i sc = __lsx_vilvl_h(te, tc);
2393 __m128i sd = __lsx_vilvh_h(te, tc);
2394 __m128i se = __lsx_vilvl_h(tf, td);
2395 __m128i sf = __lsx_vilvh_h(tf, td);
2396
2397 __m128i u0 = __lsx_vilvl_w(s4, s0);
2398 __m128i u1 = __lsx_vilvh_w(s4, s0);
2399 __m128i u2 = __lsx_vilvl_w(s5, s1);
2400 __m128i u3 = __lsx_vilvh_w(s5, s1);
2401 __m128i u4 = __lsx_vilvl_w(s6, s2);
2402 __m128i u5 = __lsx_vilvh_w(s6, s2);
2403 __m128i u6 = __lsx_vilvl_w(s7, s3);
2404 __m128i u7 = __lsx_vilvh_w(s7, s3);
2405 __m128i u8 = __lsx_vilvl_w(sc, s8);
2406 __m128i u9 = __lsx_vilvh_w(sc, s8);
2407 __m128i ua = __lsx_vilvl_w(sd, s9);
2408 __m128i ub = __lsx_vilvh_w(sd, s9);
2409 __m128i uc = __lsx_vilvl_w(se, sa);
2410 __m128i ud = __lsx_vilvh_w(se, sa);
2411 __m128i ue = __lsx_vilvl_w(sf, sb);
2412 __m128i uf = __lsx_vilvh_w(sf, sb);
2413
2414 kernel.packet[0] = __lsx_vilvl_d(u8, u0);
2415 kernel.packet[1] = __lsx_vilvh_d(u8, u0);
2416 kernel.packet[2] = __lsx_vilvl_d(u9, u1);
2417 kernel.packet[3] = __lsx_vilvh_d(u9, u1);
2418 kernel.packet[4] = __lsx_vilvl_d(ua, u2);
2419 kernel.packet[5] = __lsx_vilvh_d(ua, u2);
2420 kernel.packet[6] = __lsx_vilvl_d(ub, u3);
2421 kernel.packet[7] = __lsx_vilvh_d(ub, u3);
2422 kernel.packet[8] = __lsx_vilvl_d(uc, u4);
2423 kernel.packet[9] = __lsx_vilvh_d(uc, u4);
2424 kernel.packet[10] = __lsx_vilvl_d(ud, u5);
2425 kernel.packet[11] = __lsx_vilvh_d(ud, u5);
2426 kernel.packet[12] = __lsx_vilvl_d(ue, u6);
2427 kernel.packet[13] = __lsx_vilvh_d(ue, u6);
2428 kernel.packet[14] = __lsx_vilvl_d(uf, u7);
2429 kernel.packet[15] = __lsx_vilvh_d(uf, u7);
2430}
2431EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 8>& kernel) {
2432 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2433 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2434 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2435 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2436 __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]);
2437 __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]);
2438 __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]);
2439 __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]);
2440
2441 __m128i s0 = __lsx_vilvl_h(t2, t0);
2442 __m128i s1 = __lsx_vilvh_h(t2, t0);
2443 __m128i s2 = __lsx_vilvl_h(t3, t1);
2444 __m128i s3 = __lsx_vilvh_h(t3, t1);
2445 __m128i s4 = __lsx_vilvl_h(t6, t4);
2446 __m128i s5 = __lsx_vilvh_h(t6, t4);
2447 __m128i s6 = __lsx_vilvl_h(t7, t5);
2448 __m128i s7 = __lsx_vilvh_h(t7, t5);
2449
2450 kernel.packet[0] = __lsx_vilvl_w(s4, s0);
2451 kernel.packet[1] = __lsx_vilvh_w(s4, s0);
2452 kernel.packet[2] = __lsx_vilvl_w(s5, s1);
2453 kernel.packet[3] = __lsx_vilvh_w(s5, s1);
2454 kernel.packet[4] = __lsx_vilvl_w(s6, s2);
2455 kernel.packet[5] = __lsx_vilvh_w(s6, s2);
2456 kernel.packet[6] = __lsx_vilvl_w(s7, s3);
2457 kernel.packet[7] = __lsx_vilvh_w(s7, s3);
2458}
2459EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
2460 __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]);
2461 __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]);
2462 __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]);
2463 __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]);
2464
2465 kernel.packet[0] = __lsx_vilvl_h(t2, t0);
2466 kernel.packet[1] = __lsx_vilvh_h(t2, t0);
2467 kernel.packet[2] = __lsx_vilvl_h(t3, t1);
2468 kernel.packet[3] = __lsx_vilvh_h(t3, t1);
2469}
2470EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
2471 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2472 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2473 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2474 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2475 __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]);
2476 __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]);
2477 __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]);
2478 __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]);
2479
2480 __m128i s0 = __lsx_vilvl_w(t2, t0);
2481 __m128i s1 = __lsx_vilvh_w(t2, t0);
2482 __m128i s2 = __lsx_vilvl_w(t3, t1);
2483 __m128i s3 = __lsx_vilvh_w(t3, t1);
2484 __m128i s4 = __lsx_vilvl_w(t6, t4);
2485 __m128i s5 = __lsx_vilvh_w(t6, t4);
2486 __m128i s6 = __lsx_vilvl_w(t7, t5);
2487 __m128i s7 = __lsx_vilvh_w(t7, t5);
2488
2489 kernel.packet[0] = __lsx_vilvl_d(s4, s0);
2490 kernel.packet[1] = __lsx_vilvh_d(s4, s0);
2491 kernel.packet[2] = __lsx_vilvl_d(s5, s1);
2492 kernel.packet[3] = __lsx_vilvh_d(s5, s1);
2493 kernel.packet[4] = __lsx_vilvl_d(s6, s2);
2494 kernel.packet[5] = __lsx_vilvh_d(s6, s2);
2495 kernel.packet[6] = __lsx_vilvl_d(s7, s3);
2496 kernel.packet[7] = __lsx_vilvh_d(s7, s3);
2497}
2498EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
2499 __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]);
2500 __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]);
2501 __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]);
2502 __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]);
2503
2504 kernel.packet[0] = __lsx_vilvl_w(t2, t0);
2505 kernel.packet[1] = __lsx_vilvh_w(t2, t0);
2506 kernel.packet[2] = __lsx_vilvl_w(t3, t1);
2507 kernel.packet[3] = __lsx_vilvh_w(t3, t1);
2508}
2509EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
2510 __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]);
2511 __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]);
2512 __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]);
2513 __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]);
2514
2515 kernel.packet[0] = __lsx_vilvl_d(T2, T0);
2516 kernel.packet[1] = __lsx_vilvh_d(T2, T0);
2517 kernel.packet[2] = __lsx_vilvl_d(T3, T1);
2518 kernel.packet[3] = __lsx_vilvh_d(T3, T1);
2519}
2520EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2ul, 2>& kernel) {
2521 __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]);
2522 kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]);
2523 kernel.packet[1] = tmp;
2524}
2525
2526template <>
2527EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) {
2528 return __lsx_vfrsqrt_s(a);
2529}
2530template <>
2531EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) {
2532 return __lsx_vfrsqrt_d(a);
2533}
2534
2535template <>
2536EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) {
2537 return __lsx_vfrintrm_s(a);
2538}
2539template <>
2540EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) {
2541 return __lsx_vfrintrm_d(a);
2542}
2543
2544template <>
2545EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) {
2546 return __lsx_vfrintrp_s(a);
2547}
2548template <>
2549EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) {
2550 return __lsx_vfrintrp_d(a);
2551}
2552
2553template <>
2554EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) {
2555 const Packet4f mask = pset1frombits<Packet4f>(static_cast<numext::uint32_t>(0x80000000u));
2556 const Packet4f prev0dot5 = pset1frombits<Packet4f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
2557 return __lsx_vfrintrz_s(padd(pxor(pand(a, mask), prev0dot5), a));
2558}
2559template <>
2560EIGEN_STRONG_INLINE Packet2d pround(const Packet2d& a) {
2561 const Packet2d mask = pset1frombits<Packet2d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
2562 const Packet2d prev0dot5 = pset1frombits<Packet2d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
2563 return __lsx_vfrintrz_d(padd(por(pand(a, mask), prev0dot5), a));
2564}
2565
2566template <>
2567EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) {
2568 return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2569}
2570template <>
2571EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b) {
2572 return (Packet16c)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2573}
2574
2575template <>
2576EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(const int8_t* from) {
2577 int8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1),
2578 *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
2579 *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
2580 return __lsx_vld(tmp, 0);
2581}
2582template <>
2583EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(const uint8_t* from) {
2584 uint8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1),
2585 *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2),
2586 *(from + 3), *(from + 3), *(from + 3), *(from + 3)};
2587 return __lsx_vld(tmp, 0);
2588}
2589template <>
2590EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(const int16_t* from) {
2591 int16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
2592 return __lsx_vld(tmp, 0);
2593}
2594template <>
2595EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(const uint16_t* from) {
2596 uint16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)};
2597 return __lsx_vld(tmp, 0);
2598}
2599template <>
2600EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(const int32_t* from) {
2601 int32_t tmp[4] = {*from, *from, *from, *from};
2602 return __lsx_vld(tmp, 0);
2603}
2604template <>
2605EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(const uint32_t* from) {
2606 uint32_t tmp[4] = {*from, *from, *from, *from};
2607 return __lsx_vld(tmp, 0);
2608}
2609
2610template <>
2611EIGEN_STRONG_INLINE Packet16c pnmsub(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
2612 return __lsx_vmsub_b(pnegate(c), a, b);
2613}
2614template <>
2615EIGEN_STRONG_INLINE Packet8s pnmsub(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
2616 return __lsx_vmsub_h(pnegate(c), a, b);
2617}
2618template <>
2619EIGEN_STRONG_INLINE Packet4i pnmsub(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
2620 return __lsx_vmsub_w(pnegate(c), a, b);
2621}
2622template <>
2623EIGEN_STRONG_INLINE Packet2l pnmsub(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
2624 return __lsx_vmsub_d(pnegate(c), a, b);
2625}
2626
2627template <>
2628EIGEN_STRONG_INLINE Packet16c pmsub(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
2629 return __lsx_vmadd_b(pnegate(c), a, b);
2630}
2631template <>
2632EIGEN_STRONG_INLINE Packet8s pmsub(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
2633 return __lsx_vmadd_h(pnegate(c), a, b);
2634}
2635template <>
2636EIGEN_STRONG_INLINE Packet4i pmsub(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
2637 return __lsx_vmadd_w(pnegate(c), a, b);
2638}
2639template <>
2640EIGEN_STRONG_INLINE Packet2l pmsub(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
2641 return __lsx_vmadd_d(pnegate(c), a, b);
2642}
2643
2644template <>
2645EIGEN_STRONG_INLINE Packet16c pnmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) {
2646 return __lsx_vmsub_b(c, a, b);
2647}
2648template <>
2649EIGEN_STRONG_INLINE Packet8s pnmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) {
2650 return __lsx_vmsub_h(c, a, b);
2651}
2652template <>
2653EIGEN_STRONG_INLINE Packet4i pnmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) {
2654 return __lsx_vmsub_w(c, a, b);
2655}
2656template <>
2657EIGEN_STRONG_INLINE Packet2l pnmadd(const Packet2l& a, const Packet2l& b, const Packet2l& c) {
2658 return __lsx_vmsub_d(c, a, b);
2659}
2660
2661template <>
2662EIGEN_STRONG_INLINE Packet4f pexp(const Packet4f& _x) {
2663 return pexp_float(_x);
2664}
2665template <>
2666EIGEN_STRONG_INLINE Packet2d pexp(const Packet2d& _x) {
2667 return pexp_double(_x);
2668}
2669
2670template <>
2671EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent) {
2672 return pldexp_generic(a, exponent);
2673}
2674
2675template <>
2676EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent) {
2677 return pfrexp_generic(a, exponent);
2678}
2679template <>
2680EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent) {
2681 return pfrexp_generic(a, exponent);
2682}
2683template <>
2684EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /* a */) {
2685 Packet4f v = {0.0f, 0.0f, 0.0f, 0.0f};
2686 return v;
2687}
2688template <>
2689struct pminmax_propagates_nan<Packet4f> : bool_constant<true> {};
2690template <>
2691EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(const float* from) {
2692 return (__m128)__lsx_vldrepl_w(from, 0);
2693}
2694template <>
2695EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) {
2696 return (__m128)__lsx_vsrai_w((__m128i)a, 31);
2697}
2698template <>
2699EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a) {
2700 return __lsx_vfrintrne_s(a);
2701}
2702template <>
2703EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(const Packet4f& a) {
2704 return __lsx_vfrintrz_s(a);
2705}
2706template <>
2707EIGEN_STRONG_INLINE Packet4f preciprocal<Packet4f>(const Packet4f& a) {
2708 return __lsx_vfrecip_s(a);
2709}
2710
2711template <>
2712EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /* a */) {
2713 Packet2d v = {0.0, 0.0};
2714 return v;
2715}
2716template <>
2717struct pminmax_propagates_nan<Packet2d> : bool_constant<true> {};
2718template <>
2719EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) {
2720 return (__m128d)(__lsx_vsrai_d((__m128i)a, 63));
2721}
2722template <>
2723EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) {
2724 return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask);
2725}
2726template <>
2727EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a) {
2728 return __lsx_vfrintrne_d(a);
2729}
2730template <>
2731EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(const Packet2d& a) {
2732 return __lsx_vfrintrz_d(a);
2733}
2734template <>
2735EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent) {
2736 return pldexp_generic(a, exponent);
2737}
2738template <>
2739EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) {
2740 return __lsx_vbitsel_v(b, a, mask);
2741}
2742template <>
2743EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) {
2744 return __lsx_vbitsel_v(b, a, mask);
2745}
2746template <>
2747EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) {
2748 return __lsx_vbitsel_v(b, a, mask);
2749}
2750template <>
2751EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(const Packet16uc& a, const Packet16uc& b) {
2752 return __lsx_vdiv_bu(a, b);
2753}
2754template <>
2755EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a,
2756 const Packet16uc& b) {
2757 return __lsx_vbitsel_v(b, a, mask);
2758}
2759template <>
2760EIGEN_STRONG_INLINE Packet16uc psqrt(const Packet16uc& a) {
2761 __m128i res = {0, 0};
2762 __m128i add = {0x0808080808080808, 0x0808080808080808};
2763 for (int i = 0; i < 4; i++) {
2764 const __m128i temp = __lsx_vor_v(res, add);
2765 const __m128i tmul = __lsx_vpackev_b(__lsx_vmulwod_h_bu(temp, temp), __lsx_vmulwev_h_bu(temp, temp));
2766 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_bu(tmul, a));
2767 add = __lsx_vsrli_b(add, 1);
2768 }
2769 return res;
2770}
2771template <>
2772EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) {
2773 return __lsx_vbitsel_v(b, a, mask);
2774}
2775template <>
2776EIGEN_STRONG_INLINE Packet8us psqrt(const Packet8us& a) {
2777 __m128i res = {0, 0};
2778 __m128i add = {0x0080008000800080, 0x0080008000800080};
2779 for (int i = 0; i < 4; i++) {
2780 const __m128i temp = __lsx_vor_v(res, add);
2781 const __m128i tmul = __lsx_vpackev_h(__lsx_vmulwod_w_hu(temp, temp), __lsx_vmulwev_w_hu(temp, temp));
2782 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_hu(tmul, a));
2783 add = __lsx_vsrli_h(add, 1);
2784 }
2785 return res;
2786}
2787
2788template <>
2789EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) {
2790 return __lsx_vbitsel_v(b, a, mask);
2791}
2792template <>
2793EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) {
2794 __m128i res = {0, 0};
2795 __m128i add = {0x0000800000008000, 0x0000800000008000};
2796 for (int i = 0; i < 4; i++) {
2797 const __m128i temp = __lsx_vor_v(res, add);
2798 const __m128i tmul = __lsx_vpackev_w(__lsx_vmulwod_d_wu(temp, temp), __lsx_vmulwev_d_wu(temp, temp));
2799 res = __lsx_vbitsel_v(res, temp, __lsx_vsle_wu(tmul, a));
2800 add = __lsx_vsrli_w(add, 1);
2801 }
2802 return res;
2803}
2804
2805template <>
2806EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b) {
2807 return __lsx_vbitsel_v(b, a, mask);
2808}
2809
2810} // namespace internal
2811} // namespace Eigen
2812#endif
@ Aligned16
Definition Constants.h:238