14#ifndef EIGEN_PACKET_MATH_MSA_H
15#define EIGEN_PACKET_MATH_MSA_H
21#include "../../InternalHeaderCheck.h"
27#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
28#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
31#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
32#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
35#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
36#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
39#define EIGEN_MSA_DEBUG
41#define EIGEN_MSA_SHF_I8(a, b, c, d) (((d) << 6) | ((c) << 4) | ((b) << 2) | (a))
43using Packet4f = v4f32;
44using Packet4i = v4i32;
45using Packet4ui = v4u32;
47#define EIGEN_DECLARE_CONST_Packet4f(NAME, X) const Packet4f p4f_##NAME = {X, X, X, X}
48#define EIGEN_DECLARE_CONST_Packet4i(NAME, X) const Packet4i p4i_##NAME = {X, X, X, X}
49#define EIGEN_DECLARE_CONST_Packet4ui(NAME, X) const Packet4ui p4ui_##NAME = {X, X, X, X}
51inline std::ostream& operator<<(std::ostream& os,
const Packet4f& value) {
52 os <<
"[ " << value[0] <<
", " << value[1] <<
", " << value[2] <<
", " << value[3] <<
" ]";
56inline std::ostream& operator<<(std::ostream& os,
const Packet4i& value) {
57 os <<
"[ " << value[0] <<
", " << value[1] <<
", " << value[2] <<
", " << value[3] <<
" ]";
61inline std::ostream& operator<<(std::ostream& os,
const Packet4ui& value) {
62 os <<
"[ " << value[0] <<
", " << value[1] <<
", " << value[2] <<
", " << value[3] <<
" ]";
67struct packet_traits<float> : default_packet_traits {
68 using type = Packet4f;
69 using half = Packet4f;
76 HasSin = EIGEN_FAST_MATH,
77 HasCos = EIGEN_FAST_MATH,
78 HasTanh = EIGEN_FAST_MATH,
79 HasErf = EIGEN_FAST_MATH,
88struct packet_traits<int32_t> : default_packet_traits {
89 using type = Packet4i;
90 using half = Packet4i;
101struct unpacket_traits<Packet4f> {
107 masked_load_available =
false,
108 masked_store_available =
false
110 using half = Packet4f;
111 using integer_packet = Packet4i;
115struct unpacket_traits<Packet4i> {
116 using type = int32_t;
121 masked_load_available =
false,
122 masked_store_available =
false
124 using half = Packet4i;
128EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
131 Packet4f v = {from, from, from, from};
136EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int32_t& from) {
139 return __builtin_msa_fill_w(from);
143EIGEN_STRONG_INLINE Packet4f pload1<Packet4f>(
const float* from) {
147 Packet4f v = {f, f, f, f};
152EIGEN_STRONG_INLINE Packet4i pload1<Packet4i>(
const int32_t* from) {
155 return __builtin_msa_fill_w(*from);
159EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
162 return __builtin_msa_fadd_w(a, b);
166EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
169 return __builtin_msa_addv_w(a, b);
173EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a) {
176 static const Packet4f countdown = {0.0f, 1.0f, 2.0f, 3.0f};
177 return padd(pset1<Packet4f>(a), countdown);
181EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int32_t& a) {
184 static const Packet4i countdown = {0, 1, 2, 3};
185 return padd(pset1<Packet4i>(a), countdown);
189EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
192 return __builtin_msa_fsub_w(a, b);
196EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
199 return __builtin_msa_subv_w(a, b);
203EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
206 return (Packet4f)__builtin_msa_bnegi_w((v4u32)a, 31);
210EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
213 return __builtin_msa_addvi_w((v4i32)__builtin_msa_nori_b((v16u8)a, 0), 1);
217EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
220 return __builtin_msa_fmul_w(a, b);
224EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
227 return __builtin_msa_mulv_w(a, b);
231EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
234 return __builtin_msa_fdiv_w(a, b);
238EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
241 return __builtin_msa_div_s_w(a, b);
245struct is_fused_madd<Packet4f> : true_type {};
247EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
250 return __builtin_msa_fmadd_w(c, a, b);
254EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
259 __asm__(
"maddv.w %w[value], %w[a], %w[b]\n"
261 : [value]
"+f"(value)
263 : [a]
"f"(a), [b]
"f"(b));
268EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
271 return (Packet4f)__builtin_msa_and_v((v16u8)a, (v16u8)b);
275EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
278 return (Packet4i)__builtin_msa_and_v((v16u8)a, (v16u8)b);
282EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
285 return (Packet4f)__builtin_msa_or_v((v16u8)a, (v16u8)b);
289EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
292 return (Packet4i)__builtin_msa_or_v((v16u8)a, (v16u8)b);
296EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
299 return (Packet4f)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
303EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
306 return (Packet4i)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
310EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
313 return pand(a, (Packet4f)__builtin_msa_xori_b((v16u8)b, 255));
317EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
320 return pand(a, (Packet4i)__builtin_msa_xori_b((v16u8)b, 255));
324EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
329 return __builtin_msa_fmin_w(a, b);
332 Packet4i aNaN = __builtin_msa_fcun_w(a, a);
333 Packet4i aMinOrNaN = por(__builtin_msa_fclt_w(a, b), aNaN);
334 return (Packet4f)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a);
339EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
342 return __builtin_msa_min_s_w(a, b);
346EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
351 return __builtin_msa_fmax_w(a, b);
354 Packet4i aNaN = __builtin_msa_fcun_w(a, a);
355 Packet4i aMaxOrNaN = por(__builtin_msa_fclt_w(b, a), aNaN);
356 return (Packet4f)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a);
361EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
364 return __builtin_msa_max_s_w(a, b);
368EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from) {
371 EIGEN_DEBUG_ALIGNED_LOAD
return (Packet4f)__builtin_msa_ld_w(
const_cast<float*
>(from), 0);
375EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int32_t* from) {
378 EIGEN_DEBUG_ALIGNED_LOAD
return __builtin_msa_ld_w(
const_cast<int32_t*
>(from), 0);
382EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from) {
385 EIGEN_DEBUG_UNALIGNED_LOAD
return (Packet4f)__builtin_msa_ld_w(
const_cast<float*
>(from), 0);
389EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int32_t* from) {
392 EIGEN_DEBUG_UNALIGNED_LOAD
return (Packet4i)__builtin_msa_ld_w(
const_cast<int32_t*
>(from), 0);
396EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from) {
399 float f0 = from[0], f1 = from[1];
400 Packet4f v0 = {f0, f0, f0, f0};
401 Packet4f v1 = {f1, f1, f1, f1};
402 return (Packet4f)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0);
406EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int32_t* from) {
409 int32_t i0 = from[0], i1 = from[1];
410 Packet4i v0 = {i0, i0, i0, i0};
411 Packet4i v1 = {i1, i1, i1, i1};
412 return (Packet4i)__builtin_msa_ilvr_d((v2i64)v1, (v2i64)v0);
416EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from) {
419 EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0);
423EIGEN_STRONG_INLINE
void pstore<int32_t>(int32_t* to,
const Packet4i& from) {
426 EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_w(from, to, 0);
430EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from) {
433 EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w((Packet4i)from, to, 0);
437EIGEN_STRONG_INLINE
void pstoreu<int32_t>(int32_t* to,
const Packet4i& from) {
440 EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_w(from, to, 0);
444EIGEN_DEVICE_FUNC
inline Packet4f pgather<float, Packet4f>(
const float* from, Index stride) {
448 Packet4f v = {f, f, f, f};
450 v[2] = from[2 * stride];
451 v[3] = from[3 * stride];
456EIGEN_DEVICE_FUNC
inline Packet4i pgather<int32_t, Packet4i>(
const int32_t* from, Index stride) {
460 Packet4i v = {i, i, i, i};
462 v[2] = from[2 * stride];
463 v[3] = from[3 * stride];
468EIGEN_DEVICE_FUNC
inline void pscatter<float, Packet4f>(
float* to,
const Packet4f& from, Index stride) {
481EIGEN_DEVICE_FUNC
inline void pscatter<int32_t, Packet4i>(int32_t* to,
const Packet4i& from, Index stride) {
494EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) {
497 __builtin_prefetch(addr);
501EIGEN_STRONG_INLINE
void prefetch<int32_t>(
const int32_t* addr) {
504 __builtin_prefetch(addr);
508EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
515EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(
const Packet4i& a) {
522EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a) {
525 return (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(3, 2, 1, 0));
529EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a) {
532 return __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(3, 2, 1, 0));
536EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
539 return (Packet4f)__builtin_msa_bclri_w((v4u32)a, 31);
543EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
546 Packet4i zero = __builtin_msa_ldi_w(0);
547 return __builtin_msa_add_a_w(zero, a);
551EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(
const Packet4i& a) {
552 return __builtin_msa_srai_w(a, N);
556EIGEN_STRONG_INLINE Packet4i plogical_shift_right(
const Packet4i& a) {
557 return __builtin_msa_srli_w(a, N);
561EIGEN_STRONG_INLINE Packet4i plogical_shift_left(
const Packet4i& a) {
562 return __builtin_msa_slli_w(a, N);
566EIGEN_STRONG_INLINE Packet4i pcast<Packet4f, Packet4i>(
const Packet4f& a) {
567 return __builtin_msa_ftrunc_s_w(a);
571EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(
const Packet4f& a,
const Packet4f& exponent) {
572 return pldexp_generic(a, exponent);
576EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a) {
579 Packet4f s = padd(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
580 s = padd(s, (Packet4f)__builtin_msa_shf_w((v4i32)s, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
585EIGEN_STRONG_INLINE
bool predux_any(
const Packet4f& a) {
586 return __builtin_msa_bnz_v((v16u8)a);
590EIGEN_STRONG_INLINE int32_t predux<Packet4i>(
const Packet4i& a) {
593 Packet4i s = padd(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
594 s = padd(s, __builtin_msa_shf_w(s, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
599EIGEN_STRONG_INLINE
bool predux_any(
const Packet4i& a) {
600 return __builtin_msa_bnz_v((v16u8)a);
606EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a) {
609 Packet4f p = pmul(a, (Packet4f)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
610 p = pmul(p, (Packet4f)__builtin_msa_shf_w((v4i32)p, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
615EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(
const Packet4i& a) {
618 Packet4i p = pmul(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
619 p = pmul(p, __builtin_msa_shf_w(p, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
625EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a) {
629 Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
633 v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped);
635 unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0);
638 Packet4f v = __builtin_msa_fmin_w(a, swapped);
639 v = __builtin_msa_fmin_w(v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
642 v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000);
643 v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v);
649EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(
const Packet4i& a) {
652 Packet4i m = pmin(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
653 m = pmin(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
659EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a) {
663 Packet4f swapped = (Packet4f)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
667 v16u8 unord = (v16u8)__builtin_msa_fcun_w(a, swapped);
669 unord = (v16u8)__builtin_msa_ceqi_d((v2i64)unord, 0);
672 Packet4f v = __builtin_msa_fmax_w(a, swapped);
673 v = __builtin_msa_fmax_w(v, (Packet4f)__builtin_msa_shf_w((Packet4i)v, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
676 v16u8 qnans = (v16u8)__builtin_msa_fill_w(0x7FC00000);
677 v = (Packet4f)__builtin_msa_bsel_v(unord, qnans, (v16u8)v);
683EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(
const Packet4i& a) {
686 Packet4i m = pmax(a, __builtin_msa_shf_w(a, EIGEN_MSA_SHF_I8(2, 3, 0, 1)));
687 m = pmax(m, __builtin_msa_shf_w(m, EIGEN_MSA_SHF_I8(1, 0, 3, 2)));
691inline std::ostream& operator<<(std::ostream& os,
const PacketBlock<Packet4f, 4>& value) {
692 os <<
"[ " << value.packet[0] <<
"," << std::endl
693 <<
" " << value.packet[1] <<
"," << std::endl
694 <<
" " << value.packet[2] <<
"," << std::endl
695 <<
" " << value.packet[3] <<
" ]";
699EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
702 v4i32 tmp1, tmp2, tmp3, tmp4;
704 tmp1 = __builtin_msa_ilvr_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]);
705 tmp2 = __builtin_msa_ilvr_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]);
706 tmp3 = __builtin_msa_ilvl_w((v4i32)kernel.packet[1], (v4i32)kernel.packet[0]);
707 tmp4 = __builtin_msa_ilvl_w((v4i32)kernel.packet[3], (v4i32)kernel.packet[2]);
709 kernel.packet[0] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1);
710 kernel.packet[1] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1);
711 kernel.packet[2] = (Packet4f)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3);
712 kernel.packet[3] = (Packet4f)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3);
715inline std::ostream& operator<<(std::ostream& os,
const PacketBlock<Packet4i, 4>& value) {
716 os <<
"[ " << value.packet[0] <<
"," << std::endl
717 <<
" " << value.packet[1] <<
"," << std::endl
718 <<
" " << value.packet[2] <<
"," << std::endl
719 <<
" " << value.packet[3] <<
" ]";
723EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
726 v4i32 tmp1, tmp2, tmp3, tmp4;
728 tmp1 = __builtin_msa_ilvr_w(kernel.packet[1], kernel.packet[0]);
729 tmp2 = __builtin_msa_ilvr_w(kernel.packet[3], kernel.packet[2]);
730 tmp3 = __builtin_msa_ilvl_w(kernel.packet[1], kernel.packet[0]);
731 tmp4 = __builtin_msa_ilvl_w(kernel.packet[3], kernel.packet[2]);
733 kernel.packet[0] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp2, (v2i64)tmp1);
734 kernel.packet[1] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp2, (v2i64)tmp1);
735 kernel.packet[2] = (Packet4i)__builtin_msa_ilvr_d((v2i64)tmp4, (v2i64)tmp3);
736 kernel.packet[3] = (Packet4i)__builtin_msa_ilvod_d((v2i64)tmp4, (v2i64)tmp3);
740EIGEN_STRONG_INLINE Packet4f psqrt(
const Packet4f& a) {
743 return __builtin_msa_fsqrt_w(a);
747EIGEN_STRONG_INLINE Packet4f prsqrt(
const Packet4f& a) {
751 return __builtin_msa_frsqrt_w(a);
753 Packet4f ones = __builtin_msa_ffint_s_w(__builtin_msa_ldi_w(1));
754 return pdiv(ones, psqrt(a));
759EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a) {
761 int32_t old_mode, new_mode;
763 "cfcmsa %[old_mode], $1\n"
764 "ori %[new_mode], %[old_mode], 3\n"
765 "ctcmsa $1, %[new_mode]\n"
766 "frint.w %w[v], %w[v]\n"
767 "ctcmsa $1, %[old_mode]\n"
769 [old_mode]
"=r"(old_mode), [new_mode]
"=r"(new_mode),
778EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a) {
780 int32_t old_mode, new_mode;
782 "cfcmsa %[old_mode], $1\n"
783 "ori %[new_mode], %[old_mode], 3\n"
784 "xori %[new_mode], %[new_mode], 1\n"
785 "ctcmsa $1, %[new_mode]\n"
786 "frint.w %w[v], %w[v]\n"
787 "ctcmsa $1, %[old_mode]\n"
789 [old_mode]
"=r"(old_mode), [new_mode]
"=r"(new_mode),
798EIGEN_STRONG_INLINE Packet4f print<Packet4f>(
const Packet4f& a) {
801 asm volatile(
"frint.w %w[v], %w[v]\n" : [v]
"+f"(v));
806EIGEN_STRONG_INLINE Packet4f ptrunc<Packet4f>(
const Packet4f& a) {
808 int32_t old_mode, new_mode;
810 "cfcmsa %[old_mode], $1\n"
811 "ori %[new_mode], %[old_mode], 3\n"
812 "xori %[new_mode], %[new_mode], 2\n"
813 "ctcmsa $1, %[new_mode]\n"
814 "frint.w %w[v], %w[v]\n"
815 "ctcmsa $1, %[old_mode]\n"
817 [old_mode]
"=r"(old_mode), [new_mode]
"=r"(new_mode),
826EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
827 return (Packet4f)__builtin_msa_fcult_w(a, b);
832using Packet2d = v2f64;
833using Packet2l = v2i64;
834using Packet2ul = v2u64;
836#define EIGEN_DECLARE_CONST_Packet2d(NAME, X) const Packet2d p2d_##NAME = {X, X}
837#define EIGEN_DECLARE_CONST_Packet2l(NAME, X) const Packet2l p2l_##NAME = {X, X}
838#define EIGEN_DECLARE_CONST_Packet2ul(NAME, X) const Packet2ul p2ul_##NAME = {X, X}
840inline std::ostream& operator<<(std::ostream& os,
const Packet2d& value) {
841 os <<
"[ " << value[0] <<
", " << value[1] <<
" ]";
845inline std::ostream& operator<<(std::ostream& os,
const Packet2l& value) {
846 os <<
"[ " << value[0] <<
", " << value[1] <<
" ]";
850inline std::ostream& operator<<(std::ostream& os,
const Packet2ul& value) {
851 os <<
"[ " << value[0] <<
", " << value[1] <<
" ]";
856struct packet_traits<double> : default_packet_traits {
857 using type = Packet2d;
858 using half = Packet2d;
872struct unpacket_traits<Packet2d> {
878 masked_load_available =
false,
879 masked_store_available =
false
881 using half = Packet2d;
882 using integer_packet = Packet2l;
886struct unpacket_traits<Packet2l> {
887 using type = int64_t;
892 masked_load_available =
false,
893 masked_store_available =
false
895 using half = Packet2l;
899EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(
const int64_t& from) {
900 return __builtin_msa_fill_d(from);
904EIGEN_STRONG_INLINE Packet2l plogical_shift_right(
const Packet2l& a) {
905 return __builtin_msa_srli_d(a, N);
909EIGEN_STRONG_INLINE Packet2l plogical_shift_left(
const Packet2l& a) {
910 return __builtin_msa_slli_d(a, N);
914EIGEN_STRONG_INLINE Packet2l pcast<Packet2d, Packet2l>(
const Packet2d& a) {
915 return __builtin_msa_ftrunc_s_d(a);
919EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
920 return (Packet2l)__builtin_msa_and_v((v16u8)a, (v16u8)b);
924EIGEN_STRONG_INLINE Packet2l por<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
925 return (Packet2l)__builtin_msa_or_v((v16u8)a, (v16u8)b);
929EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
930 return (Packet2l)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
934EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(
const Packet2l& a,
const Packet2l& b) {
935 return pand(a, (Packet2l)__builtin_msa_xori_b((v16u8)b, 255));
939EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
942 Packet2d value = {from, from};
947EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
950 return __builtin_msa_fadd_d(a, b);
954EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a) {
957 static const Packet2d countdown = {0.0, 1.0};
958 return padd(pset1<Packet2d>(a), countdown);
962EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
965 return __builtin_msa_fsub_d(a, b);
969EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
972 return (Packet2d)__builtin_msa_bnegi_d((v2u64)a, 63);
976EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
979 return __builtin_msa_fmul_d(a, b);
983EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
986 return __builtin_msa_fdiv_d(a, b);
990struct is_fused_madd<Packet2d> : true_type {};
992EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
995 return __builtin_msa_fmadd_d(c, a, b);
1001EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1004 return (Packet2d)__builtin_msa_and_v((v16u8)a, (v16u8)b);
1008EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1011 return (Packet2d)__builtin_msa_or_v((v16u8)a, (v16u8)b);
1015EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1018 return (Packet2d)__builtin_msa_xor_v((v16u8)a, (v16u8)b);
1022EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1025 return pand(a, (Packet2d)__builtin_msa_xori_b((v16u8)b, 255));
1029EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from) {
1032 EIGEN_DEBUG_UNALIGNED_LOAD
return (Packet2d)__builtin_msa_ld_d(
const_cast<double*
>(from), 0);
1036EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1041 return __builtin_msa_fmin_d(a, b);
1044 v2i64 aNaN = __builtin_msa_fcun_d(a, a);
1045 v2i64 aMinOrNaN = por(__builtin_msa_fclt_d(a, b), aNaN);
1046 return (Packet2d)__builtin_msa_bsel_v((v16u8)aMinOrNaN, (v16u8)b, (v16u8)a);
1051EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1056 return __builtin_msa_fmax_d(a, b);
1059 v2i64 aNaN = __builtin_msa_fcun_d(a, a);
1060 v2i64 aMaxOrNaN = por(__builtin_msa_fclt_d(b, a), aNaN);
1061 return (Packet2d)__builtin_msa_bsel_v((v16u8)aMaxOrNaN, (v16u8)b, (v16u8)a);
1066EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from) {
1069 EIGEN_DEBUG_UNALIGNED_LOAD
return (Packet2d)__builtin_msa_ld_d(
const_cast<double*
>(from), 0);
1073EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(
const double* from) {
1076 Packet2d value = {*from, *from};
1081EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from) {
1084 EIGEN_DEBUG_ALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0);
1088EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from) {
1091 EIGEN_DEBUG_UNALIGNED_STORE __builtin_msa_st_d((v2i64)from, to, 0);
1095EIGEN_DEVICE_FUNC
inline Packet2d pgather<double, Packet2d>(
const double* from, Index stride) {
1106EIGEN_DEVICE_FUNC
inline void pscatter<double, Packet2d>(
double* to,
const Packet2d& from, Index stride) {
1115EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) {
1118 __builtin_prefetch(addr);
1122EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
1129EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a) {
1132 return (Packet2d)__builtin_msa_shf_w((v4i32)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
1136EIGEN_STRONG_INLINE Packet2d pabs(
const Packet2d& a) {
1139 return (Packet2d)__builtin_msa_bclri_d((v2u64)a, 63);
1143EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a) {
1146 Packet2d s = padd(a, preverse(a));
1151EIGEN_STRONG_INLINE
bool predux_any(
const Packet2d& a) {
1152 return __builtin_msa_bnz_v((v16u8)a);
1158EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a) {
1161 Packet2d p = pmul(a, preverse(a));
1167EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a) {
1171 Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
1172 Packet2d v = __builtin_msa_fmin_d(a, swapped);
1175 double a0 = a[0], a1 = a[1];
1176 return ((numext::isnan)(a0) || a0 < a1) ? a0 : a1;
1182EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a) {
1186 Packet2d swapped = (Packet2d)__builtin_msa_shf_w((Packet4i)a, EIGEN_MSA_SHF_I8(2, 3, 0, 1));
1187 Packet2d v = __builtin_msa_fmax_d(a, swapped);
1190 double a0 = a[0], a1 = a[1];
1191 return ((numext::isnan)(a0) || a0 > a1) ? a0 : a1;
1196EIGEN_STRONG_INLINE Packet2d psqrt(
const Packet2d& a) {
1199 return __builtin_msa_fsqrt_d(a);
1203EIGEN_STRONG_INLINE Packet2d prsqrt(
const Packet2d& a) {
1207 return __builtin_msa_frsqrt_d(a);
1209 Packet2d ones = __builtin_msa_ffint_s_d(__builtin_msa_ldi_d(1));
1210 return pdiv(ones, psqrt(a));
1214inline std::ostream& operator<<(std::ostream& os,
const PacketBlock<Packet2d, 2>& value) {
1215 os <<
"[ " << value.packet[0] <<
"," << std::endl <<
" " << value.packet[1] <<
" ]";
1219EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet2d, 2>& kernel) {
1222 Packet2d trn1 = (Packet2d)__builtin_msa_ilvev_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]);
1223 Packet2d trn2 = (Packet2d)__builtin_msa_ilvod_d((v2i64)kernel.packet[1], (v2i64)kernel.packet[0]);
1224 kernel.packet[0] = trn1;
1225 kernel.packet[1] = trn2;
1229EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(
const Packet2d& a) {
1231 int32_t old_mode, new_mode;
1233 "cfcmsa %[old_mode], $1\n"
1234 "ori %[new_mode], %[old_mode], 3\n"
1235 "ctcmsa $1, %[new_mode]\n"
1236 "frint.d %w[v], %w[v]\n"
1237 "ctcmsa $1, %[old_mode]\n"
1239 [old_mode]
"=r"(old_mode), [new_mode]
"=r"(new_mode),
1248EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(
const Packet2d& a) {
1250 int32_t old_mode, new_mode;
1252 "cfcmsa %[old_mode], $1\n"
1253 "ori %[new_mode], %[old_mode], 3\n"
1254 "xori %[new_mode], %[new_mode], 1\n"
1255 "ctcmsa $1, %[new_mode]\n"
1256 "frint.d %w[v], %w[v]\n"
1257 "ctcmsa $1, %[old_mode]\n"
1259 [old_mode]
"=r"(old_mode), [new_mode]
"=r"(new_mode),
1268EIGEN_STRONG_INLINE Packet2d print<Packet2d>(
const Packet2d& a) {
1271 asm volatile(
"frint.d %w[v], %w[v]\n" : [v]
"+f"(v));
1276EIGEN_STRONG_INLINE Packet2d ptrunc<Packet2d>(
const Packet2d& a) {
1278 int32_t old_mode, new_mode;
1280 "cfcmsa %[old_mode], $1\n"
1281 "ori %[new_mode], %[old_mode], 3\n"
1282 "xori %[new_mode], %[new_mode], 2\n"
1283 "ctcmsa $1, %[new_mode]\n"
1284 "frint.d %w[v], %w[v]\n"
1285 "ctcmsa $1, %[old_mode]\n"
1287 [old_mode]
"=r"(old_mode), [new_mode]
"=r"(new_mode),
1296EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
1297 return (Packet2d)__builtin_msa_fcult_d(a, b);
1301EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(
const Packet2d& a,
const Packet2d& exponent) {
1302 return pldexp_generic(a, exponent);
@ Aligned16
Definition Constants.h:238