12#ifndef EIGEN_PACKET4_MATH_RVV10_H
13#define EIGEN_PACKET4_MATH_RVV10_H
16#include "../../InternalHeaderCheck.h"
23EIGEN_STRONG_INLINE Packet4Xi __riscv_vreinterpret_v_u64m4_i32m4(
const Packet4Xul& a) {
24 return __riscv_vreinterpret_v_i64m4_i32m4(__riscv_vreinterpret_v_u64m4_i64m4(a));
28EIGEN_STRONG_INLINE Packet4Xi pset1<Packet4Xi>(
const numext::int32_t& from) {
29 return __riscv_vmv_v_x_i32m4(from, unpacket_traits<Packet4Xi>::size);
33EIGEN_STRONG_INLINE Packet4Xi plset<Packet4Xi>(
const numext::int32_t& a) {
34 Packet4Xi idx = __riscv_vreinterpret_v_u32m4_i32m4(__riscv_vid_v_u32m4(unpacket_traits<Packet4Xi>::size));
35 return __riscv_vadd_vx_i32m4(idx, a, unpacket_traits<Packet4Xi>::size);
39EIGEN_STRONG_INLINE Packet4Xi pzero<Packet4Xi>(
const Packet4Xi& ) {
40 return __riscv_vmv_v_x_i32m4(0, unpacket_traits<Packet4Xi>::size);
44EIGEN_STRONG_INLINE Packet4Xi padd<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
45 return __riscv_vadd_vv_i32m4(a, b, unpacket_traits<Packet4Xi>::size);
49EIGEN_STRONG_INLINE Packet4Xi psub<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
50 return __riscv_vsub(a, b, unpacket_traits<Packet4Xi>::size);
54EIGEN_STRONG_INLINE Packet4Xi pnegate(
const Packet4Xi& a) {
55 return __riscv_vneg(a, unpacket_traits<Packet4Xi>::size);
59EIGEN_STRONG_INLINE Packet4Xi pmul<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
60 return __riscv_vmul(a, b, unpacket_traits<Packet4Xi>::size);
64EIGEN_STRONG_INLINE Packet4Xi pdiv<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
65 return __riscv_vdiv(a, b, unpacket_traits<Packet4Xi>::size);
69EIGEN_STRONG_INLINE Packet4Xi pmadd(
const Packet4Xi& a,
const Packet4Xi& b,
const Packet4Xi& c) {
70 return __riscv_vmadd(a, b, c, unpacket_traits<Packet4Xi>::size);
74EIGEN_STRONG_INLINE Packet4Xi pmsub(
const Packet4Xi& a,
const Packet4Xi& b,
const Packet4Xi& c) {
75 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet4Xi>::size);
79EIGEN_STRONG_INLINE Packet4Xi pnmadd(
const Packet4Xi& a,
const Packet4Xi& b,
const Packet4Xi& c) {
80 return __riscv_vnmsub_vv_i32m4(a, b, c, unpacket_traits<Packet4Xi>::size);
84EIGEN_STRONG_INLINE Packet4Xi pnmsub(
const Packet4Xi& a,
const Packet4Xi& b,
const Packet4Xi& c) {
85 return __riscv_vnmsub_vv_i32m4(a, b, pnegate(c), unpacket_traits<Packet4Xi>::size);
89EIGEN_STRONG_INLINE Packet4Xi pmin<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
90 return __riscv_vmin(a, b, unpacket_traits<Packet4Xi>::size);
94EIGEN_STRONG_INLINE Packet4Xi pmax<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
95 return __riscv_vmax(a, b, unpacket_traits<Packet4Xi>::size);
99EIGEN_STRONG_INLINE Packet4Xi pcmp_le<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
100 PacketMask8 mask = __riscv_vmsle_vv_i32m4_b8(a, b, unpacket_traits<Packet4Xi>::size);
101 return __riscv_vmerge_vxm_i32m4(pzero(a), 0xffffffff, mask, unpacket_traits<Packet4Xi>::size);
105EIGEN_STRONG_INLINE Packet4Xi pcmp_lt<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
106 PacketMask8 mask = __riscv_vmslt_vv_i32m4_b8(a, b, unpacket_traits<Packet4Xi>::size);
107 return __riscv_vmerge_vxm_i32m4(pzero(a), 0xffffffff, mask, unpacket_traits<Packet4Xi>::size);
111EIGEN_STRONG_INLINE Packet4Xi pcmp_eq<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
112 PacketMask8 mask = __riscv_vmseq_vv_i32m4_b8(a, b, unpacket_traits<Packet4Xi>::size);
113 return __riscv_vmerge_vxm_i32m4(pzero(a), 0xffffffff, mask, unpacket_traits<Packet4Xi>::size);
117EIGEN_STRONG_INLINE Packet4Xi ptrue<Packet4Xi>(
const Packet4Xi& ) {
118 return __riscv_vmv_v_x_i32m4(0xffffffffu, unpacket_traits<Packet4Xi>::size);
122EIGEN_STRONG_INLINE Packet4Xi pand<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
123 return __riscv_vand_vv_i32m4(a, b, unpacket_traits<Packet4Xi>::size);
127EIGEN_STRONG_INLINE Packet4Xi por<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
128 return __riscv_vor_vv_i32m4(a, b, unpacket_traits<Packet4Xi>::size);
132EIGEN_STRONG_INLINE Packet4Xi pxor<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
133 return __riscv_vxor_vv_i32m4(a, b, unpacket_traits<Packet4Xi>::size);
137EIGEN_STRONG_INLINE Packet4Xi pandnot<Packet4Xi>(
const Packet4Xi& a,
const Packet4Xi& b) {
139 return __riscv_vand_vv_i32m4(a, __riscv_vnot_v_i32m4(b, unpacket_traits<Packet4Xi>::size),
140 unpacket_traits<Packet4Xi>::size);
142 return __riscv_vreinterpret_v_u32m4_i32m4(__riscv_vandn_vv_u32m4(
143 __riscv_vreinterpret_v_i32m4_u32m4(a), __riscv_vreinterpret_v_i32m4_u32m4(b), unpacket_traits<Packet4Xi>::size));
148EIGEN_STRONG_INLINE Packet4Xi pnot<Packet4Xi>(
const Packet4Xi& a) {
149 return __riscv_vnot_v_i32m4(a, unpacket_traits<Packet4Xi>::size);
153EIGEN_STRONG_INLINE Packet4Xi parithmetic_shift_right(
const Packet4Xi& a) {
154 return __riscv_vsra_vx_i32m4(a, N, unpacket_traits<Packet4Xi>::size);
158EIGEN_STRONG_INLINE Packet4Xi plogical_shift_right(
const Packet4Xi& a) {
159 return __riscv_vreinterpret_i32m4(
160 __riscv_vsrl_vx_u32m4(__riscv_vreinterpret_u32m4(a), N, unpacket_traits<Packet4Xi>::size));
164EIGEN_STRONG_INLINE Packet4Xi plogical_shift_left(
const Packet4Xi& a) {
165 return __riscv_vsll_vx_i32m4(a, N, unpacket_traits<Packet4Xi>::size);
169EIGEN_STRONG_INLINE Packet4Xi pload<Packet4Xi>(
const numext::int32_t* from) {
170 EIGEN_DEBUG_ALIGNED_LOAD
return __riscv_vle32_v_i32m4(from, unpacket_traits<Packet4Xi>::size);
174EIGEN_STRONG_INLINE Packet4Xi ploadu<Packet4Xi>(
const numext::int32_t* from) {
175 EIGEN_DEBUG_UNALIGNED_LOAD
return __riscv_vle32_v_i32m4(from, unpacket_traits<Packet4Xi>::size);
179EIGEN_STRONG_INLINE Packet4Xi ploaddup<Packet4Xi>(
const numext::int32_t* from) {
180 Packet4Xul data = __riscv_vwcvtu_x_x_v_u64m4(__riscv_vreinterpret_v_i32m2_u32m2(pload<Packet2Xi>(from)),
181 unpacket_traits<Packet2Xi>::size);
182 return __riscv_vreinterpret_v_u64m4_i32m4(__riscv_vadd_vv_u64m4(
183 __riscv_vsll_vx_u64m4(data, 32, unpacket_traits<Packet4Xl>::size), data, unpacket_traits<Packet4Xl>::size));
187EIGEN_STRONG_INLINE Packet4Xi ploadquad<Packet4Xi>(
const numext::int32_t* from) {
189 __riscv_vsrl_vx_u32m4(__riscv_vid_v_u32m4(unpacket_traits<Packet4Xi>::size), 2, unpacket_traits<Packet4Xi>::size);
190 return __riscv_vrgather_vv_i32m4(__riscv_vlmul_ext_v_i32m1_i32m4(pload<Packet1Xi>(from)), idx,
191 unpacket_traits<Packet4Xi>::size);
195EIGEN_STRONG_INLINE
void pstore<numext::int32_t>(numext::int32_t* to,
const Packet4Xi& from) {
196 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse32_v_i32m4(to, from, unpacket_traits<Packet4Xi>::size);
200EIGEN_STRONG_INLINE
void pstoreu<numext::int32_t>(numext::int32_t* to,
const Packet4Xi& from) {
201 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse32_v_i32m4(to, from, unpacket_traits<Packet4Xi>::size);
205EIGEN_DEVICE_FUNC
inline Packet4Xi pgather<numext::int32_t, Packet4Xi>(
const numext::int32_t* from, Index stride) {
206 return __riscv_vlse32_v_i32m4(from, stride *
sizeof(numext::int32_t), unpacket_traits<Packet4Xi>::size);
210EIGEN_DEVICE_FUNC
inline void pscatter<numext::int32_t, Packet4Xi>(numext::int32_t* to,
const Packet4Xi& from,
212 __riscv_vsse32(to, stride *
sizeof(numext::int32_t), from, unpacket_traits<Packet4Xi>::size);
216EIGEN_STRONG_INLINE numext::int32_t pfirst<Packet4Xi>(
const Packet4Xi& a) {
217 return __riscv_vmv_x_s_i32m4_i32(a);
221EIGEN_STRONG_INLINE Packet4Xi preverse(
const Packet4Xi& a) {
222 Packet4Xu idx = __riscv_vrsub_vx_u32m4(__riscv_vid_v_u32m4(unpacket_traits<Packet4Xi>::size),
223 unpacket_traits<Packet4Xi>::size - 1, unpacket_traits<Packet4Xi>::size);
224 return __riscv_vrgather_vv_i32m4(a, idx, unpacket_traits<Packet4Xi>::size);
228EIGEN_STRONG_INLINE Packet4Xi pabs(
const Packet4Xi& a) {
229 Packet4Xi mask = __riscv_vsra_vx_i32m4(a, 31, unpacket_traits<Packet4Xi>::size);
230 return __riscv_vsub_vv_i32m4(__riscv_vxor_vv_i32m4(a, mask, unpacket_traits<Packet4Xi>::size), mask,
231 unpacket_traits<Packet4Xi>::size);
235EIGEN_STRONG_INLINE numext::int32_t predux<Packet4Xi>(
const Packet4Xi& a) {
236 return __riscv_vmv_x(__riscv_vredsum_vs_i32m4_i32m1(a, __riscv_vmv_v_x_i32m1(0, unpacket_traits<Packet4Xi>::size / 4),
237 unpacket_traits<Packet4Xi>::size));
241EIGEN_STRONG_INLINE
bool predux_all(
const Packet4Xi& a) {
242 const PacketMask8 mask = __riscv_vmseq_vx_i32m4_b8(a, 0, unpacket_traits<Packet4Xi>::size);
243 return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xi>::size) == 0;
247EIGEN_STRONG_INLINE numext::int32_t predux_mul<Packet4Xi>(
const Packet4Xi& a) {
248 Packet1Xi half1 = __riscv_vmul_vv_i32m1(__riscv_vget_v_i32m4_i32m1(a, 0), __riscv_vget_v_i32m4_i32m1(a, 1),
249 unpacket_traits<Packet1Xi>::size);
250 Packet1Xi half2 = __riscv_vmul_vv_i32m1(__riscv_vget_v_i32m4_i32m1(a, 2), __riscv_vget_v_i32m4_i32m1(a, 3),
251 unpacket_traits<Packet1Xi>::size);
252 return predux_mul<Packet1Xi>(__riscv_vmul_vv_i32m1(half1, half2, unpacket_traits<Packet1Xi>::size));
256EIGEN_STRONG_INLINE numext::int32_t predux_min<Packet4Xi>(
const Packet4Xi& a) {
257 return __riscv_vmv_x(__riscv_vredmin_vs_i32m4_i32m1(
258 a, __riscv_vmv_v_x_i32m1((std::numeric_limits<numext::int32_t>::max)(), unpacket_traits<Packet4Xi>::size / 4),
259 unpacket_traits<Packet4Xi>::size));
263EIGEN_STRONG_INLINE numext::int32_t predux_max<Packet4Xi>(
const Packet4Xi& a) {
264 return __riscv_vmv_x(__riscv_vredmax_vs_i32m4_i32m1(
265 a, __riscv_vmv_v_x_i32m1((std::numeric_limits<numext::int32_t>::min)(), unpacket_traits<Packet4Xi>::size / 4),
266 unpacket_traits<Packet4Xi>::size));
270EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4Xi, N>& kernel) {
271 numext::int32_t buffer[unpacket_traits<Packet4Xi>::size * N] = {0};
274 for (i = 0; i < N; i++) {
275 __riscv_vsse32(&buffer[i], N *
sizeof(numext::int32_t), kernel.packet[i], unpacket_traits<Packet4Xi>::size);
277 for (i = 0; i < N; i++) {
279 __riscv_vle32_v_i32m4(&buffer[i * unpacket_traits<Packet4Xi>::size], unpacket_traits<Packet4Xi>::size);
283template <
typename Packet = Packet4Xi>
285 std::enable_if_t<std::is_same<Packet, Packet4Xi>::value && (unpacket_traits<Packet4Xi>::size % 8) == 0, Packet2Xi>
286 predux_half(
const Packet4Xi& a) {
287 return __riscv_vadd_vv_i32m2(__riscv_vget_v_i32m4_i32m2(a, 0), __riscv_vget_v_i32m4_i32m2(a, 1),
288 unpacket_traits<Packet2Xi>::size);
294EIGEN_STRONG_INLINE Packet4Xf ptrue<Packet4Xf>(
const Packet4Xf& ) {
295 Packet4Xf r = __riscv_vreinterpret_f32m4(__riscv_vmv_v_x_u32m4(0xffffffffu, unpacket_traits<Packet4Xf>::size));
296 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
301EIGEN_STRONG_INLINE Packet4Xf pzero<Packet4Xf>(
const Packet4Xf& ) {
302 return __riscv_vfmv_v_f_f32m4(0.0f, unpacket_traits<Packet4Xf>::size);
306EIGEN_STRONG_INLINE Packet4Xf pabs(
const Packet4Xf& a) {
307 return __riscv_vfabs_v_f32m4(a, unpacket_traits<Packet4Xf>::size);
311EIGEN_STRONG_INLINE Packet4Xf pabsdiff(
const Packet4Xf& a,
const Packet4Xf& b) {
312 return __riscv_vfabs_v_f32m4(__riscv_vfsub_vv_f32m4(a, b, unpacket_traits<Packet4Xf>::size),
313 unpacket_traits<Packet4Xf>::size);
317EIGEN_STRONG_INLINE Packet4Xf pset1<Packet4Xf>(
const float& from) {
318 return __riscv_vfmv_v_f_f32m4(from, unpacket_traits<Packet4Xf>::size);
322EIGEN_STRONG_INLINE Packet4Xf pset1frombits<Packet4Xf>(numext::uint32_t from) {
323 return __riscv_vreinterpret_f32m4(__riscv_vmv_v_x_u32m4(from, unpacket_traits<Packet4Xf>::size));
327EIGEN_STRONG_INLINE Packet4Xf plset<Packet4Xf>(
const float& a) {
328 Packet4Xf idx = __riscv_vfcvt_f_x_v_f32m4(
329 __riscv_vreinterpret_v_u32m4_i32m4(__riscv_vid_v_u32m4(unpacket_traits<Packet4Xi>::size)),
330 unpacket_traits<Packet4Xf>::size);
331 return __riscv_vfadd_vf_f32m4(idx, a, unpacket_traits<Packet4Xf>::size);
335EIGEN_STRONG_INLINE
void pbroadcast4<Packet4Xf>(
const float* a, Packet4Xf& a0, Packet4Xf& a1, Packet4Xf& a2,
337 Packet4Xf aa = __riscv_vlmul_ext_f32m4(__riscv_vle32_v_f32m1(a, 4));
338 a0 = __riscv_vrgather_vx_f32m4(aa, 0, unpacket_traits<Packet4Xf>::size);
339 a1 = __riscv_vrgather_vx_f32m4(aa, 1, unpacket_traits<Packet4Xf>::size);
340 a2 = __riscv_vrgather_vx_f32m4(aa, 2, unpacket_traits<Packet4Xf>::size);
341 a3 = __riscv_vrgather_vx_f32m4(aa, 3, unpacket_traits<Packet4Xf>::size);
345EIGEN_STRONG_INLINE Packet4Xf padd<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
346 return __riscv_vfadd_vv_f32m4(a, b, unpacket_traits<Packet4Xf>::size);
350EIGEN_STRONG_INLINE Packet4Xf psub<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
351 return __riscv_vfsub_vv_f32m4(a, b, unpacket_traits<Packet4Xf>::size);
355EIGEN_STRONG_INLINE Packet4Xf pnegate(
const Packet4Xf& a) {
356 return __riscv_vfneg_v_f32m4(a, unpacket_traits<Packet4Xf>::size);
360EIGEN_STRONG_INLINE Packet4Xf psignbit(
const Packet4Xf& a) {
361 return __riscv_vreinterpret_v_i32m4_f32m4(
362 __riscv_vsra_vx_i32m4(__riscv_vreinterpret_v_f32m4_i32m4(a), 31, unpacket_traits<Packet4Xi>::size));
366EIGEN_STRONG_INLINE Packet4Xf pmul<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
367 return __riscv_vfmul_vv_f32m4(a, b, unpacket_traits<Packet4Xf>::size);
371EIGEN_STRONG_INLINE Packet4Xf pdiv<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
372 return __riscv_vfdiv_vv_f32m4(a, b, unpacket_traits<Packet4Xf>::size);
376struct is_fused_madd<Packet4Xf> : true_type {};
378EIGEN_STRONG_INLINE Packet4Xf pmadd(
const Packet4Xf& a,
const Packet4Xf& b,
const Packet4Xf& c) {
379 return __riscv_vfmadd_vv_f32m4(a, b, c, unpacket_traits<Packet4Xf>::size);
383EIGEN_STRONG_INLINE Packet4Xf pmsub(
const Packet4Xf& a,
const Packet4Xf& b,
const Packet4Xf& c) {
384 return __riscv_vfmsub_vv_f32m4(a, b, c, unpacket_traits<Packet4Xf>::size);
388EIGEN_STRONG_INLINE Packet4Xf pnmadd(
const Packet4Xf& a,
const Packet4Xf& b,
const Packet4Xf& c) {
389 return __riscv_vfnmsub_vv_f32m4(a, b, c, unpacket_traits<Packet4Xf>::size);
393EIGEN_STRONG_INLINE Packet4Xf pnmsub(
const Packet4Xf& a,
const Packet4Xf& b,
const Packet4Xf& c) {
394 return __riscv_vfnmadd_vv_f32m4(a, b, c, unpacket_traits<Packet4Xf>::size);
398struct pminmax_propagates_nan<Packet4Xf> : bool_constant<true> {};
401EIGEN_STRONG_INLINE Packet4Xf pmin<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
402 Packet4Xf nans = __riscv_vfmv_v_f_f32m4((std::numeric_limits<float>::quiet_NaN)(), unpacket_traits<Packet4Xf>::size);
403 PacketMask8 mask = __riscv_vmfeq_vv_f32m4_b8(a, a, unpacket_traits<Packet4Xf>::size);
404 PacketMask8 mask2 = __riscv_vmfeq_vv_f32m4_b8(b, b, unpacket_traits<Packet4Xf>::size);
405 mask = __riscv_vmand_mm_b8(mask, mask2, unpacket_traits<Packet4Xf>::size);
407 return __riscv_vfmin_vv_f32m4_tumu(mask, nans, a, b, unpacket_traits<Packet4Xf>::size);
411EIGEN_STRONG_INLINE Packet4Xf pmin<PropagateNumbers, Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
412 return __riscv_vfmin_vv_f32m4(a, b, unpacket_traits<Packet4Xf>::size);
416EIGEN_STRONG_INLINE Packet4Xf pmax<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
417 Packet4Xf nans = __riscv_vfmv_v_f_f32m4((std::numeric_limits<float>::quiet_NaN)(), unpacket_traits<Packet4Xf>::size);
418 PacketMask8 mask = __riscv_vmfeq_vv_f32m4_b8(a, a, unpacket_traits<Packet4Xf>::size);
419 PacketMask8 mask2 = __riscv_vmfeq_vv_f32m4_b8(b, b, unpacket_traits<Packet4Xf>::size);
420 mask = __riscv_vmand_mm_b8(mask, mask2, unpacket_traits<Packet4Xf>::size);
422 return __riscv_vfmax_vv_f32m4_tumu(mask, nans, a, b, unpacket_traits<Packet4Xf>::size);
426EIGEN_STRONG_INLINE Packet4Xf pmax<PropagateNumbers, Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
427 return __riscv_vfmax_vv_f32m4(a, b, unpacket_traits<Packet4Xf>::size);
431EIGEN_STRONG_INLINE Packet4Xf pcmp_le<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
432 PacketMask8 mask = __riscv_vmfle_vv_f32m4_b8(a, b, unpacket_traits<Packet4Xf>::size);
433 return __riscv_vmerge_vvm_f32m4(pzero<Packet4Xf>(a), ptrue<Packet4Xf>(a), mask, unpacket_traits<Packet4Xf>::size);
437EIGEN_STRONG_INLINE Packet4Xf pcmp_lt<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
438 PacketMask8 mask = __riscv_vmflt_vv_f32m4_b8(a, b, unpacket_traits<Packet4Xf>::size);
439 return __riscv_vmerge_vvm_f32m4(pzero<Packet4Xf>(a), ptrue<Packet4Xf>(a), mask, unpacket_traits<Packet4Xf>::size);
443EIGEN_STRONG_INLINE Packet4Xf pcmp_eq<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
444 PacketMask8 mask = __riscv_vmfeq_vv_f32m4_b8(a, b, unpacket_traits<Packet4Xf>::size);
445 return __riscv_vmerge_vvm_f32m4(pzero<Packet4Xf>(a), ptrue<Packet4Xf>(a), mask, unpacket_traits<Packet4Xf>::size);
449EIGEN_STRONG_INLINE Packet4Xf pcmp_lt_or_nan<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
450 PacketMask8 mask = __riscv_vmfge_vv_f32m4_b8(a, b, unpacket_traits<Packet4Xf>::size);
451 return __riscv_vfmerge_vfm_f32m4(ptrue<Packet4Xf>(a), 0.0f, mask, unpacket_traits<Packet4Xf>::size);
454EIGEN_STRONG_INLINE Packet4Xf pselect(
const PacketMask8& mask,
const Packet4Xf& a,
const Packet4Xf& b) {
455 return __riscv_vmerge_vvm_f32m4(b, a, mask, unpacket_traits<Packet4Xf>::size);
458EIGEN_STRONG_INLINE Packet4Xf pselect(
const Packet4Xf& mask,
const Packet4Xf& a,
const Packet4Xf& b) {
460 __riscv_vmsne_vx_i32m4_b8(__riscv_vreinterpret_v_f32m4_i32m4(mask), 0, unpacket_traits<Packet4Xf>::size);
461 return __riscv_vmerge_vvm_f32m4(b, a, mask2, unpacket_traits<Packet4Xf>::size);
466EIGEN_STRONG_INLINE Packet4Xf pand<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
467 return __riscv_vreinterpret_v_u32m4_f32m4(__riscv_vand_vv_u32m4(
468 __riscv_vreinterpret_v_f32m4_u32m4(a), __riscv_vreinterpret_v_f32m4_u32m4(b), unpacket_traits<Packet4Xf>::size));
472EIGEN_STRONG_INLINE Packet4Xf por<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
473 return __riscv_vreinterpret_v_u32m4_f32m4(__riscv_vor_vv_u32m4(
474 __riscv_vreinterpret_v_f32m4_u32m4(a), __riscv_vreinterpret_v_f32m4_u32m4(b), unpacket_traits<Packet4Xf>::size));
478EIGEN_STRONG_INLINE Packet4Xf pxor<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
479 return __riscv_vreinterpret_v_u32m4_f32m4(__riscv_vxor_vv_u32m4(
480 __riscv_vreinterpret_v_f32m4_u32m4(a), __riscv_vreinterpret_v_f32m4_u32m4(b), unpacket_traits<Packet4Xf>::size));
484EIGEN_STRONG_INLINE Packet4Xf pandnot<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& b) {
486 return __riscv_vreinterpret_v_u32m4_f32m4(__riscv_vand_vv_u32m4(
487 __riscv_vreinterpret_v_f32m4_u32m4(a),
488 __riscv_vnot_v_u32m4(__riscv_vreinterpret_v_f32m4_u32m4(b), unpacket_traits<Packet4Xf>::size),
489 unpacket_traits<Packet4Xf>::size));
491 return __riscv_vreinterpret_v_u32m4_f32m4(__riscv_vandn_vv_u32m4(
492 __riscv_vreinterpret_v_f32m4_u32m4(a), __riscv_vreinterpret_v_f32m4_u32m4(b), unpacket_traits<Packet4Xi>::size));
497EIGEN_STRONG_INLINE Packet4Xf pnot<Packet4Xf>(
const Packet4Xf& a) {
498 return __riscv_vreinterpret_v_u32m4_f32m4(
499 __riscv_vnot_v_u32m4(__riscv_vreinterpret_v_f32m4_u32m4(a), unpacket_traits<Packet4Xf>::size));
503EIGEN_STRONG_INLINE Packet4Xs pnot<Packet4Xs>(
const Packet4Xs& a) {
504 return __riscv_vnot_v_i16m4(a, unpacket_traits<Packet4Xs>::size);
508EIGEN_STRONG_INLINE Packet4Xf pload<Packet4Xf>(
const float* from) {
509 EIGEN_DEBUG_ALIGNED_LOAD
return __riscv_vle32_v_f32m4(from, unpacket_traits<Packet4Xf>::size);
513EIGEN_STRONG_INLINE Packet4Xf ploadu<Packet4Xf>(
const float* from) {
514 EIGEN_DEBUG_UNALIGNED_LOAD
return __riscv_vle32_v_f32m4(from, unpacket_traits<Packet4Xf>::size);
518EIGEN_STRONG_INLINE Packet4Xf ploaddup<Packet4Xf>(
const float* from) {
519 Packet4Xul data = __riscv_vwcvtu_x_x_v_u64m4(__riscv_vreinterpret_v_f32m2_u32m2(pload<Packet2Xf>(from)),
520 unpacket_traits<Packet2Xi>::size);
521 return __riscv_vreinterpret_v_u64m4_f32m4(__riscv_vadd_vv_u64m4(
522 __riscv_vsll_vx_u64m4(data, 32, unpacket_traits<Packet4Xl>::size), data, unpacket_traits<Packet4Xl>::size));
526EIGEN_STRONG_INLINE Packet4Xf ploadquad<Packet4Xf>(
const float* from) {
528 __riscv_vsrl_vx_u32m4(__riscv_vid_v_u32m4(unpacket_traits<Packet4Xf>::size), 2, unpacket_traits<Packet4Xf>::size);
529 return __riscv_vrgather_vv_f32m4(__riscv_vlmul_ext_v_f32m1_f32m4(pload<Packet1Xf>(from)), idx,
530 unpacket_traits<Packet4Xf>::size);
534EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4Xf& from) {
535 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse32_v_f32m4(to, from, unpacket_traits<Packet4Xf>::size);
539EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4Xf& from) {
540 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse32_v_f32m4(to, from, unpacket_traits<Packet4Xf>::size);
544EIGEN_DEVICE_FUNC
inline Packet4Xf pgather<float, Packet4Xf>(
const float* from, Index stride) {
545 return __riscv_vlse32_v_f32m4(from, stride *
sizeof(
float), unpacket_traits<Packet4Xf>::size);
549EIGEN_DEVICE_FUNC
inline void pscatter<float, Packet4Xf>(
float* to,
const Packet4Xf& from, Index stride) {
550 __riscv_vsse32(to, stride *
sizeof(
float), from, unpacket_traits<Packet4Xf>::size);
554EIGEN_STRONG_INLINE
float pfirst<Packet4Xf>(
const Packet4Xf& a) {
555 return __riscv_vfmv_f_s_f32m4_f32(a);
559EIGEN_STRONG_INLINE Packet4Xf psqrt(
const Packet4Xf& a) {
560 return __riscv_vfsqrt_v_f32m4(a, unpacket_traits<Packet4Xf>::size);
564EIGEN_STRONG_INLINE Packet4Xf print<Packet4Xf>(
const Packet4Xf& a) {
565 const Packet4Xf limit = pset1<Packet4Xf>(
static_cast<float>(1 << 23));
566 const Packet4Xf abs_a = pabs(a);
568 PacketMask8 mask = __riscv_vmfne_vv_f32m4_b8(a, a, unpacket_traits<Packet4Xf>::size);
569 const Packet4Xf x = __riscv_vfadd_vv_f32m4_tumu(mask, a, a, a, unpacket_traits<Packet4Xf>::size);
570 const Packet4Xf new_x = __riscv_vfcvt_f_x_v_f32m4(__riscv_vfcvt_x_f_v_i32m4(a, unpacket_traits<Packet4Xf>::size),
571 unpacket_traits<Packet4Xf>::size);
573 mask = __riscv_vmflt_vv_f32m4_b8(abs_a, limit, unpacket_traits<Packet4Xf>::size);
574 Packet4Xf signed_x = __riscv_vfsgnj_vv_f32m4(new_x, x, unpacket_traits<Packet4Xf>::size);
575 return __riscv_vmerge_vvm_f32m4(x, signed_x, mask, unpacket_traits<Packet4Xf>::size);
579EIGEN_STRONG_INLINE Packet4Xf pfloor<Packet4Xf>(
const Packet4Xf& a) {
580 Packet4Xf tmp = print<Packet4Xf>(a);
582 PacketMask8 mask = __riscv_vmflt_vv_f32m4_b8(a, tmp, unpacket_traits<Packet4Xf>::size);
583 return __riscv_vfsub_vf_f32m4_tumu(mask, tmp, tmp, 1.0f, unpacket_traits<Packet4Xf>::size);
587EIGEN_STRONG_INLINE Packet4Xf preverse(
const Packet4Xf& a) {
588 Packet4Xu idx = __riscv_vrsub_vx_u32m4(__riscv_vid_v_u32m4(unpacket_traits<Packet4Xf>::size),
589 unpacket_traits<Packet4Xf>::size - 1, unpacket_traits<Packet4Xf>::size);
590 return __riscv_vrgather_vv_f32m4(a, idx, unpacket_traits<Packet4Xf>::size);
594EIGEN_STRONG_INLINE Packet4Xf pfrexp<Packet4Xf>(
const Packet4Xf& a, Packet4Xf& exponent) {
595 return pfrexp_generic(a, exponent);
599EIGEN_STRONG_INLINE
float predux<Packet4Xf>(
const Packet4Xf& a) {
600 return __riscv_vfmv_f(__riscv_vfredusum_vs_f32m4_f32m1(
601 a, __riscv_vfmv_v_f_f32m1(0.0f, unpacket_traits<Packet4Xf>::size / 4), unpacket_traits<Packet4Xf>::size));
605EIGEN_STRONG_INLINE
bool predux_any(
const Packet4Xf& a) {
606 const PacketMask8 mask =
607 __riscv_vmsne_vx_u32m4_b8(__riscv_vreinterpret_v_f32m4_u32m4(a), 0, unpacket_traits<Packet4Xf>::size);
608 return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xf>::size) != 0;
612EIGEN_STRONG_INLINE
bool predux_all(
const Packet4Xf& a) {
613 const PacketMask8 mask = __riscv_vmfeq_vf_f32m4_b8(a, 0.0f, unpacket_traits<Packet4Xf>::size);
614 return __riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xf>::size) == 0;
618EIGEN_STRONG_INLINE Index predux_count(
const Packet4Xf& a) {
619 const PacketMask8 mask = __riscv_vmfne_vf_f32m4_b8(a, 0.0f, unpacket_traits<Packet4Xf>::size);
620 return static_cast<Index
>(__riscv_vcpop_m_b8(mask, unpacket_traits<Packet4Xf>::size));
624EIGEN_STRONG_INLINE
float predux_mul<Packet4Xf>(
const Packet4Xf& a) {
625 Packet1Xf half1 = __riscv_vfmul_vv_f32m1(__riscv_vget_v_f32m4_f32m1(a, 0), __riscv_vget_v_f32m4_f32m1(a, 1),
626 unpacket_traits<Packet1Xf>::size);
627 Packet1Xf half2 = __riscv_vfmul_vv_f32m1(__riscv_vget_v_f32m4_f32m1(a, 2), __riscv_vget_v_f32m4_f32m1(a, 3),
628 unpacket_traits<Packet1Xf>::size);
629 return predux_mul<Packet1Xf>(__riscv_vfmul_vv_f32m1(half1, half2, unpacket_traits<Packet1Xf>::size));
635EIGEN_STRONG_INLINE
float predux_min<Packet4Xf>(
const Packet4Xf& a) {
636 return __riscv_vfmv_f(
637 __riscv_vfredmin_vs_f32m4_f32m1(a, __riscv_vget_v_f32m4_f32m1(a, 0), unpacket_traits<Packet4Xf>::size));
641EIGEN_STRONG_INLINE
float predux_max<Packet4Xf>(
const Packet4Xf& a) {
642 return __riscv_vfmv_f(
643 __riscv_vfredmax_vs_f32m4_f32m1(a, __riscv_vget_v_f32m4_f32m1(a, 0), unpacket_traits<Packet4Xf>::size));
647EIGEN_STRONG_INLINE Packet2Xf predux_half(
const Packet4Xf& a) {
648 return Packet2Xf(__riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(a, 0), __riscv_vget_v_f32m4_f32m2(a, 1),
649 unpacket_traits<Packet2Xf>::size));
653EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4Xf, N>& kernel) {
654 float buffer[unpacket_traits<Packet4Xf>::size * N];
657 for (i = 0; i < N; i++) {
658 __riscv_vsse32(&buffer[i], N *
sizeof(
float), kernel.packet[i], unpacket_traits<Packet4Xf>::size);
661 for (i = 0; i < N; i++) {
663 __riscv_vle32_v_f32m4(&buffer[i * unpacket_traits<Packet4Xf>::size], unpacket_traits<Packet4Xf>::size);
668EIGEN_STRONG_INLINE Packet4Xf pldexp<Packet4Xf>(
const Packet4Xf& a,
const Packet4Xf& exponent) {
669 return pldexp_generic(a, exponent);
672template <
typename Packet = Packet4Xf>
674 std::enable_if_t<std::is_same<Packet, Packet4Xf>::value && (unpacket_traits<Packet4Xf>::size % 8) == 0, Packet2Xf>
675 predux_half(
const Packet4Xf& a) {
676 return __riscv_vfadd_vv_f32m2(__riscv_vget_v_f32m4_f32m2(a, 0), __riscv_vget_v_f32m4_f32m2(a, 1),
677 unpacket_traits<Packet2Xf>::size);
683EIGEN_STRONG_INLINE Packet4Xl pset1<Packet4Xl>(
const numext::int64_t& from) {
684 return __riscv_vmv_v_x_i64m4(from, unpacket_traits<Packet4Xl>::size);
688EIGEN_STRONG_INLINE Packet4Xl plset<Packet4Xl>(
const numext::int64_t& a) {
689 Packet4Xl idx = __riscv_vreinterpret_v_u64m4_i64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xl>::size));
690 return __riscv_vadd_vx_i64m4(idx, a, unpacket_traits<Packet4Xl>::size);
694EIGEN_STRONG_INLINE Packet4Xl pzero<Packet4Xl>(
const Packet4Xl& ) {
695 return __riscv_vmv_v_x_i64m4(0, unpacket_traits<Packet4Xl>::size);
699EIGEN_STRONG_INLINE Packet4Xl padd<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
700 return __riscv_vadd_vv_i64m4(a, b, unpacket_traits<Packet4Xl>::size);
704EIGEN_STRONG_INLINE Packet4Xl psub<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
705 return __riscv_vsub(a, b, unpacket_traits<Packet4Xl>::size);
709EIGEN_STRONG_INLINE Packet4Xl pnegate(
const Packet4Xl& a) {
710 return __riscv_vneg(a, unpacket_traits<Packet4Xl>::size);
714EIGEN_STRONG_INLINE Packet4Xl pmul<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
715 return __riscv_vmul(a, b, unpacket_traits<Packet4Xl>::size);
719EIGEN_STRONG_INLINE Packet4Xl pdiv<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
720 return __riscv_vdiv(a, b, unpacket_traits<Packet4Xl>::size);
724EIGEN_STRONG_INLINE Packet4Xl pmadd(
const Packet4Xl& a,
const Packet4Xl& b,
const Packet4Xl& c) {
725 return __riscv_vmadd(a, b, c, unpacket_traits<Packet4Xl>::size);
729EIGEN_STRONG_INLINE Packet4Xl pmsub(
const Packet4Xl& a,
const Packet4Xl& b,
const Packet4Xl& c) {
730 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet4Xl>::size);
734EIGEN_STRONG_INLINE Packet4Xl pnmadd(
const Packet4Xl& a,
const Packet4Xl& b,
const Packet4Xl& c) {
735 return __riscv_vnmsub_vv_i64m4(a, b, c, unpacket_traits<Packet4Xl>::size);
739EIGEN_STRONG_INLINE Packet4Xl pnmsub(
const Packet4Xl& a,
const Packet4Xl& b,
const Packet4Xl& c) {
740 return __riscv_vnmsub_vv_i64m4(a, b, pnegate(c), unpacket_traits<Packet4Xl>::size);
744EIGEN_STRONG_INLINE Packet4Xl pmin<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
745 return __riscv_vmin(a, b, unpacket_traits<Packet4Xl>::size);
749EIGEN_STRONG_INLINE Packet4Xl pmax<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
750 return __riscv_vmax(a, b, unpacket_traits<Packet4Xl>::size);
754EIGEN_STRONG_INLINE Packet4Xl pcmp_le<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
755 PacketMask16 mask = __riscv_vmsle_vv_i64m4_b16(a, b, unpacket_traits<Packet4Xl>::size);
756 return __riscv_vmerge_vxm_i64m4(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet4Xl>::size);
760EIGEN_STRONG_INLINE Packet4Xl pcmp_lt<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
761 PacketMask16 mask = __riscv_vmslt_vv_i64m4_b16(a, b, unpacket_traits<Packet4Xl>::size);
762 return __riscv_vmerge_vxm_i64m4(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet4Xl>::size);
766EIGEN_STRONG_INLINE Packet4Xl pcmp_eq<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
767 PacketMask16 mask = __riscv_vmseq_vv_i64m4_b16(a, b, unpacket_traits<Packet4Xl>::size);
768 return __riscv_vmerge_vxm_i64m4(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet4Xl>::size);
772EIGEN_STRONG_INLINE Packet4Xl ptrue<Packet4Xl>(
const Packet4Xl& ) {
773 return __riscv_vmv_v_x_i64m4(0xffffffffffffffffu, unpacket_traits<Packet4Xl>::size);
777EIGEN_STRONG_INLINE Packet4Xl pand<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
778 return __riscv_vand_vv_i64m4(a, b, unpacket_traits<Packet4Xl>::size);
782EIGEN_STRONG_INLINE Packet4Xl por<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
783 return __riscv_vor_vv_i64m4(a, b, unpacket_traits<Packet4Xl>::size);
787EIGEN_STRONG_INLINE Packet4Xl pnot<Packet4Xl>(
const Packet4Xl& a) {
788 return __riscv_vnot_v_i64m4(a, unpacket_traits<Packet4Xl>::size);
792EIGEN_STRONG_INLINE Packet4Xl pxor<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
793 return __riscv_vxor_vv_i64m4(a, b, unpacket_traits<Packet4Xl>::size);
797EIGEN_STRONG_INLINE Packet4Xl pandnot<Packet4Xl>(
const Packet4Xl& a,
const Packet4Xl& b) {
799 return __riscv_vand_vv_i64m4(a, __riscv_vnot_v_i64m4(b, unpacket_traits<Packet4Xl>::size),
800 unpacket_traits<Packet4Xl>::size);
802 return __riscv_vreinterpret_v_u64m4_i64m4(__riscv_vandn_vv_u64m4(
803 __riscv_vreinterpret_v_i64m4_u64m4(a), __riscv_vreinterpret_v_i64m4_u64m4(b), unpacket_traits<Packet4Xl>::size));
808EIGEN_STRONG_INLINE Packet4Xl parithmetic_shift_right(
const Packet4Xl& a) {
809 return __riscv_vsra_vx_i64m4(a, N, unpacket_traits<Packet4Xl>::size);
813EIGEN_STRONG_INLINE Packet4Xl plogical_shift_right(
const Packet4Xl& a) {
814 return __riscv_vreinterpret_i64m4(
815 __riscv_vsrl_vx_u64m4(__riscv_vreinterpret_u64m4(a), N, unpacket_traits<Packet4Xl>::size));
819EIGEN_STRONG_INLINE Packet4Xl plogical_shift_left(
const Packet4Xl& a) {
820 return __riscv_vsll_vx_i64m4(a, N, unpacket_traits<Packet4Xl>::size);
824EIGEN_STRONG_INLINE Packet4Xl pload<Packet4Xl>(
const numext::int64_t* from) {
825 EIGEN_DEBUG_ALIGNED_LOAD
return __riscv_vle64_v_i64m4(from, unpacket_traits<Packet4Xl>::size);
829EIGEN_STRONG_INLINE Packet4Xl ploadu<Packet4Xl>(
const numext::int64_t* from) {
830 EIGEN_DEBUG_UNALIGNED_LOAD
return __riscv_vle64_v_i64m4(from, unpacket_traits<Packet4Xl>::size);
834EIGEN_STRONG_INLINE Packet4Xl ploaddup<Packet4Xl>(
const numext::int64_t* from) {
836 __riscv_vsrl_vx_u64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xl>::size), 1, unpacket_traits<Packet4Xl>::size);
837 return __riscv_vrgather_vv_i64m4(__riscv_vlmul_ext_v_i64m2_i64m4(pload<Packet2Xl>(from)), idx,
838 unpacket_traits<Packet4Xl>::size);
842EIGEN_STRONG_INLINE Packet4Xl ploadquad<Packet4Xl>(
const numext::int64_t* from) {
844 __riscv_vsrl_vx_u64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xl>::size), 2, unpacket_traits<Packet4Xl>::size);
845 return __riscv_vrgather_vv_i64m4(__riscv_vlmul_ext_v_i64m1_i64m4(pload<Packet1Xl>(from)), idx,
846 unpacket_traits<Packet4Xl>::size);
850EIGEN_STRONG_INLINE
void pstore<numext::int64_t>(numext::int64_t* to,
const Packet4Xl& from) {
851 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse64_v_i64m4(to, from, unpacket_traits<Packet4Xl>::size);
855EIGEN_STRONG_INLINE
void pstoreu<numext::int64_t>(numext::int64_t* to,
const Packet4Xl& from) {
856 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse64_v_i64m4(to, from, unpacket_traits<Packet4Xl>::size);
860EIGEN_DEVICE_FUNC
inline Packet4Xl pgather<numext::int64_t, Packet4Xl>(
const numext::int64_t* from, Index stride) {
861 return __riscv_vlse64_v_i64m4(from, stride *
sizeof(numext::int64_t), unpacket_traits<Packet4Xl>::size);
865EIGEN_DEVICE_FUNC
inline void pscatter<numext::int64_t, Packet4Xl>(numext::int64_t* to,
const Packet4Xl& from,
867 __riscv_vsse64(to, stride *
sizeof(numext::int64_t), from, unpacket_traits<Packet4Xl>::size);
871EIGEN_STRONG_INLINE numext::int64_t pfirst<Packet4Xl>(
const Packet4Xl& a) {
872 return __riscv_vmv_x_s_i64m4_i64(a);
876EIGEN_STRONG_INLINE Packet4Xl preverse(
const Packet4Xl& a) {
877 Packet4Xul idx = __riscv_vrsub_vx_u64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xl>::size),
878 unpacket_traits<Packet4Xl>::size - 1, unpacket_traits<Packet4Xl>::size);
879 return __riscv_vrgather_vv_i64m4(a, idx, unpacket_traits<Packet4Xl>::size);
883EIGEN_STRONG_INLINE Packet4Xl pabs(
const Packet4Xl& a) {
884 Packet4Xl mask = __riscv_vsra_vx_i64m4(a, 63, unpacket_traits<Packet4Xl>::size);
885 return __riscv_vsub_vv_i64m4(__riscv_vxor_vv_i64m4(a, mask, unpacket_traits<Packet4Xl>::size), mask,
886 unpacket_traits<Packet4Xl>::size);
890EIGEN_STRONG_INLINE numext::int64_t predux<Packet4Xl>(
const Packet4Xl& a) {
891 return __riscv_vmv_x(__riscv_vredsum_vs_i64m4_i64m1(a, __riscv_vmv_v_x_i64m1(0, unpacket_traits<Packet4Xl>::size / 4),
892 unpacket_traits<Packet4Xl>::size));
896EIGEN_STRONG_INLINE
bool predux_all(
const Packet4Xl& a) {
897 const PacketMask16 mask = __riscv_vmseq_vx_i64m4_b16(a, 0, unpacket_traits<Packet4Xl>::size);
898 return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xl>::size) == 0;
902EIGEN_STRONG_INLINE numext::int64_t predux_mul<Packet4Xl>(
const Packet4Xl& a) {
903 Packet1Xl half1 = __riscv_vmul_vv_i64m1(__riscv_vget_v_i64m4_i64m1(a, 0), __riscv_vget_v_i64m4_i64m1(a, 1),
904 unpacket_traits<Packet1Xl>::size);
905 Packet1Xl half2 = __riscv_vmul_vv_i64m1(__riscv_vget_v_i64m4_i64m1(a, 2), __riscv_vget_v_i64m4_i64m1(a, 3),
906 unpacket_traits<Packet1Xl>::size);
907 return predux_mul<Packet1Xl>(__riscv_vmul_vv_i64m1(half1, half2, unpacket_traits<Packet1Xl>::size));
911EIGEN_STRONG_INLINE numext::int64_t predux_min<Packet4Xl>(
const Packet4Xl& a) {
912 return __riscv_vmv_x(__riscv_vredmin_vs_i64m4_i64m1(
913 a, __riscv_vmv_v_x_i64m1((std::numeric_limits<numext::int64_t>::max)(), unpacket_traits<Packet4Xl>::size / 4),
914 unpacket_traits<Packet4Xl>::size));
918EIGEN_STRONG_INLINE numext::int64_t predux_max<Packet4Xl>(
const Packet4Xl& a) {
919 return __riscv_vmv_x(__riscv_vredmax_vs_i64m4_i64m1(
920 a, __riscv_vmv_v_x_i64m1((std::numeric_limits<numext::int64_t>::min)(), unpacket_traits<Packet4Xl>::size / 4),
921 unpacket_traits<Packet4Xl>::size));
925EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4Xl, N>& kernel) {
926 numext::int64_t buffer[unpacket_traits<Packet4Xl>::size * N] = {0};
929 for (i = 0; i < N; i++) {
930 __riscv_vsse64(&buffer[i], N *
sizeof(numext::int64_t), kernel.packet[i], unpacket_traits<Packet4Xl>::size);
932 for (i = 0; i < N; i++) {
934 __riscv_vle64_v_i64m4(&buffer[i * unpacket_traits<Packet4Xl>::size], unpacket_traits<Packet4Xl>::size);
938template <
typename Packet = Packet4Xl>
940 std::enable_if_t<std::is_same<Packet, Packet4Xl>::value && (unpacket_traits<Packet4Xl>::size % 8) == 0, Packet2Xl>
941 predux_half(
const Packet4Xl& a) {
942 return __riscv_vadd_vv_i64m2(__riscv_vget_v_i64m4_i64m2(a, 0), __riscv_vget_v_i64m4_i64m2(a, 1),
943 unpacket_traits<Packet2Xl>::size);
949EIGEN_STRONG_INLINE Packet4Xd ptrue<Packet4Xd>(
const Packet4Xd& ) {
951 __riscv_vreinterpret_f64m4(__riscv_vmv_v_x_u64m4(0xffffffffffffffffu, unpacket_traits<Packet4Xd>::size));
952 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
957EIGEN_STRONG_INLINE Packet4Xd pzero<Packet4Xd>(
const Packet4Xd& ) {
958 return __riscv_vfmv_v_f_f64m4(0.0, unpacket_traits<Packet4Xd>::size);
962EIGEN_STRONG_INLINE Packet4Xd pabs(
const Packet4Xd& a) {
963 return __riscv_vfabs_v_f64m4(a, unpacket_traits<Packet4Xd>::size);
967EIGEN_STRONG_INLINE Packet4Xd pabsdiff(
const Packet4Xd& a,
const Packet4Xd& b) {
968 return __riscv_vfabs_v_f64m4(__riscv_vfsub_vv_f64m4(a, b, unpacket_traits<Packet4Xd>::size),
969 unpacket_traits<Packet4Xd>::size);
973EIGEN_STRONG_INLINE Packet4Xd pset1<Packet4Xd>(
const double& from) {
974 return __riscv_vfmv_v_f_f64m4(from, unpacket_traits<Packet4Xd>::size);
978EIGEN_STRONG_INLINE Packet4Xd pset1frombits<Packet4Xd>(numext::uint64_t from) {
979 return __riscv_vreinterpret_f64m4(__riscv_vmv_v_x_u64m4(from, unpacket_traits<Packet4Xd>::size));
983EIGEN_STRONG_INLINE Packet4Xd plset<Packet4Xd>(
const double& a) {
984 Packet4Xd idx = __riscv_vfcvt_f_x_v_f64m4(
985 __riscv_vreinterpret_v_u64m4_i64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xi>::size)),
986 unpacket_traits<Packet4Xd>::size);
987 return __riscv_vfadd_vf_f64m4(idx, a, unpacket_traits<Packet4Xd>::size);
991EIGEN_STRONG_INLINE
void pbroadcast4<Packet4Xd>(
const double* a, Packet4Xd& a0, Packet4Xd& a1, Packet4Xd& a2,
994 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
995 aa = __riscv_vlmul_ext_f64m4(__riscv_vle64_v_f64m1(a, 4));
997 aa = __riscv_vlmul_ext_f64m4(__riscv_vle64_v_f64m2(a, 4));
999 a0 = __riscv_vrgather_vx_f64m4(aa, 0, unpacket_traits<Packet4Xd>::size);
1000 a1 = __riscv_vrgather_vx_f64m4(aa, 1, unpacket_traits<Packet4Xd>::size);
1001 a2 = __riscv_vrgather_vx_f64m4(aa, 2, unpacket_traits<Packet4Xd>::size);
1002 a3 = __riscv_vrgather_vx_f64m4(aa, 3, unpacket_traits<Packet4Xd>::size);
1006EIGEN_STRONG_INLINE Packet4Xd padd<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1007 return __riscv_vfadd_vv_f64m4(a, b, unpacket_traits<Packet4Xd>::size);
1011EIGEN_STRONG_INLINE Packet4Xd psub<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1012 return __riscv_vfsub_vv_f64m4(a, b, unpacket_traits<Packet4Xd>::size);
1016EIGEN_STRONG_INLINE Packet4Xd pnegate(
const Packet4Xd& a) {
1017 return __riscv_vfneg_v_f64m4(a, unpacket_traits<Packet4Xd>::size);
1021EIGEN_STRONG_INLINE Packet4Xd psignbit(
const Packet4Xd& a) {
1022 return __riscv_vreinterpret_v_i64m4_f64m4(
1023 __riscv_vsra_vx_i64m4(__riscv_vreinterpret_v_f64m4_i64m4(a), 63, unpacket_traits<Packet4Xl>::size));
1027EIGEN_STRONG_INLINE Packet4Xd pmul<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1028 return __riscv_vfmul_vv_f64m4(a, b, unpacket_traits<Packet4Xd>::size);
1032EIGEN_STRONG_INLINE Packet4Xd pdiv<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1033 return __riscv_vfdiv_vv_f64m4(a, b, unpacket_traits<Packet4Xd>::size);
1037struct is_fused_madd<Packet4Xd> : true_type {};
1039EIGEN_STRONG_INLINE Packet4Xd pmadd(
const Packet4Xd& a,
const Packet4Xd& b,
const Packet4Xd& c) {
1040 return __riscv_vfmadd_vv_f64m4(a, b, c, unpacket_traits<Packet4Xd>::size);
1044EIGEN_STRONG_INLINE Packet4Xd pmsub(
const Packet4Xd& a,
const Packet4Xd& b,
const Packet4Xd& c) {
1045 return __riscv_vfmsub_vv_f64m4(a, b, c, unpacket_traits<Packet4Xd>::size);
1049EIGEN_STRONG_INLINE Packet4Xd pnmadd(
const Packet4Xd& a,
const Packet4Xd& b,
const Packet4Xd& c) {
1050 return __riscv_vfnmsub_vv_f64m4(a, b, c, unpacket_traits<Packet4Xd>::size);
1054EIGEN_STRONG_INLINE Packet4Xd pnmsub(
const Packet4Xd& a,
const Packet4Xd& b,
const Packet4Xd& c) {
1055 return __riscv_vfnmadd_vv_f64m4(a, b, c, unpacket_traits<Packet4Xd>::size);
1059struct pminmax_propagates_nan<Packet4Xd> : bool_constant<true> {};
1062EIGEN_STRONG_INLINE Packet4Xd pmin<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1063 Packet4Xd nans = __riscv_vfmv_v_f_f64m4((std::numeric_limits<double>::quiet_NaN)(), unpacket_traits<Packet4Xd>::size);
1064 PacketMask16 mask = __riscv_vmfeq_vv_f64m4_b16(a, a, unpacket_traits<Packet4Xd>::size);
1065 PacketMask16 mask2 = __riscv_vmfeq_vv_f64m4_b16(b, b, unpacket_traits<Packet4Xd>::size);
1066 mask = __riscv_vmand_mm_b16(mask, mask2, unpacket_traits<Packet4Xd>::size);
1068 return __riscv_vfmin_vv_f64m4_tumu(mask, nans, a, b, unpacket_traits<Packet4Xd>::size);
1072EIGEN_STRONG_INLINE Packet4Xd pmin<PropagateNumbers, Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1073 return __riscv_vfmin_vv_f64m4(a, b, unpacket_traits<Packet4Xd>::size);
1077EIGEN_STRONG_INLINE Packet4Xd pmax<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1078 Packet4Xd nans = __riscv_vfmv_v_f_f64m4((std::numeric_limits<double>::quiet_NaN)(), unpacket_traits<Packet4Xd>::size);
1079 PacketMask16 mask = __riscv_vmfeq_vv_f64m4_b16(a, a, unpacket_traits<Packet4Xd>::size);
1080 PacketMask16 mask2 = __riscv_vmfeq_vv_f64m4_b16(b, b, unpacket_traits<Packet4Xd>::size);
1081 mask = __riscv_vmand_mm_b16(mask, mask2, unpacket_traits<Packet4Xd>::size);
1083 return __riscv_vfmax_vv_f64m4_tumu(mask, nans, a, b, unpacket_traits<Packet4Xd>::size);
1087EIGEN_STRONG_INLINE Packet4Xd pmax<PropagateNumbers, Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1088 return __riscv_vfmax_vv_f64m4(a, b, unpacket_traits<Packet4Xd>::size);
1092EIGEN_STRONG_INLINE Packet4Xd pcmp_le<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1093 PacketMask16 mask = __riscv_vmfle_vv_f64m4_b16(a, b, unpacket_traits<Packet4Xd>::size);
1094 return __riscv_vmerge_vvm_f64m4(pzero<Packet4Xd>(a), ptrue<Packet4Xd>(a), mask, unpacket_traits<Packet4Xd>::size);
1098EIGEN_STRONG_INLINE Packet4Xd pcmp_lt<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1099 PacketMask16 mask = __riscv_vmflt_vv_f64m4_b16(a, b, unpacket_traits<Packet4Xd>::size);
1100 return __riscv_vmerge_vvm_f64m4(pzero<Packet4Xd>(a), ptrue<Packet4Xd>(a), mask, unpacket_traits<Packet4Xd>::size);
1104EIGEN_STRONG_INLINE Packet4Xd pcmp_eq<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1105 PacketMask16 mask = __riscv_vmfeq_vv_f64m4_b16(a, b, unpacket_traits<Packet4Xd>::size);
1106 return __riscv_vmerge_vvm_f64m4(pzero<Packet4Xd>(a), ptrue<Packet4Xd>(a), mask, unpacket_traits<Packet4Xd>::size);
1110EIGEN_STRONG_INLINE Packet4Xd pcmp_lt_or_nan<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1111 PacketMask16 mask = __riscv_vmfge_vv_f64m4_b16(a, b, unpacket_traits<Packet4Xd>::size);
1112 return __riscv_vfmerge_vfm_f64m4(ptrue<Packet4Xd>(a), 0.0, mask, unpacket_traits<Packet4Xd>::size);
1115EIGEN_STRONG_INLINE Packet4Xd pselect(
const PacketMask16& mask,
const Packet4Xd& a,
const Packet4Xd& b) {
1116 return __riscv_vmerge_vvm_f64m4(b, a, mask, unpacket_traits<Packet4Xd>::size);
1119EIGEN_STRONG_INLINE Packet4Xd pselect(
const Packet4Xd& mask,
const Packet4Xd& a,
const Packet4Xd& b) {
1120 PacketMask16 mask2 =
1121 __riscv_vmsne_vx_i64m4_b16(__riscv_vreinterpret_v_f64m4_i64m4(mask), 0, unpacket_traits<Packet4Xd>::size);
1122 return __riscv_vmerge_vvm_f64m4(b, a, mask2, unpacket_traits<Packet4Xd>::size);
1127EIGEN_STRONG_INLINE Packet4Xd pand<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1128 return __riscv_vreinterpret_v_u64m4_f64m4(__riscv_vand_vv_u64m4(
1129 __riscv_vreinterpret_v_f64m4_u64m4(a), __riscv_vreinterpret_v_f64m4_u64m4(b), unpacket_traits<Packet4Xd>::size));
1133EIGEN_STRONG_INLINE Packet4Xd por<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1134 return __riscv_vreinterpret_v_u64m4_f64m4(__riscv_vor_vv_u64m4(
1135 __riscv_vreinterpret_v_f64m4_u64m4(a), __riscv_vreinterpret_v_f64m4_u64m4(b), unpacket_traits<Packet4Xd>::size));
1139EIGEN_STRONG_INLINE Packet4Xd pxor<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1140 return __riscv_vreinterpret_v_u64m4_f64m4(__riscv_vxor_vv_u64m4(
1141 __riscv_vreinterpret_v_f64m4_u64m4(a), __riscv_vreinterpret_v_f64m4_u64m4(b), unpacket_traits<Packet4Xd>::size));
1145EIGEN_STRONG_INLINE Packet4Xd pandnot<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& b) {
1147 return __riscv_vreinterpret_v_u64m4_f64m4(__riscv_vand_vv_u64m4(
1148 __riscv_vreinterpret_v_f64m4_u64m4(a),
1149 __riscv_vnot_v_u64m4(__riscv_vreinterpret_v_f64m4_u64m4(b), unpacket_traits<Packet4Xd>::size),
1150 unpacket_traits<Packet4Xd>::size));
1152 return __riscv_vreinterpret_v_u64m4_f64m4(__riscv_vandn_vv_u64m4(
1153 __riscv_vreinterpret_v_f64m4_u64m4(a), __riscv_vreinterpret_v_f64m4_u64m4(b), unpacket_traits<Packet4Xl>::size));
1158EIGEN_STRONG_INLINE Packet4Xd pnot<Packet4Xd>(
const Packet4Xd& a) {
1159 return __riscv_vreinterpret_v_u64m4_f64m4(
1160 __riscv_vnot_v_u64m4(__riscv_vreinterpret_v_f64m4_u64m4(a), unpacket_traits<Packet4Xd>::size));
1164EIGEN_STRONG_INLINE Packet4Xd pload<Packet4Xd>(
const double* from) {
1165 EIGEN_DEBUG_ALIGNED_LOAD
return __riscv_vle64_v_f64m4(from, unpacket_traits<Packet4Xd>::size);
1169EIGEN_STRONG_INLINE Packet4Xd ploadu<Packet4Xd>(
const double* from) {
1170 EIGEN_DEBUG_UNALIGNED_LOAD
return __riscv_vle64_v_f64m4(from, unpacket_traits<Packet4Xd>::size);
1174EIGEN_STRONG_INLINE Packet4Xd ploaddup<Packet4Xd>(
const double* from) {
1176 __riscv_vsrl_vx_u64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xd>::size), 1, unpacket_traits<Packet4Xd>::size);
1177 return __riscv_vrgather_vv_f64m4(__riscv_vlmul_ext_v_f64m2_f64m4(pload<Packet2Xd>(from)), idx,
1178 unpacket_traits<Packet4Xd>::size);
1182EIGEN_STRONG_INLINE Packet4Xd ploadquad<Packet4Xd>(
const double* from) {
1184 __riscv_vsrl_vx_u64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xd>::size), 2, unpacket_traits<Packet4Xd>::size);
1185 return __riscv_vrgather_vv_f64m4(__riscv_vlmul_ext_v_f64m1_f64m4(pload<Packet1Xd>(from)), idx,
1186 unpacket_traits<Packet4Xd>::size);
1190EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet4Xd& from) {
1191 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse64_v_f64m4(to, from, unpacket_traits<Packet4Xd>::size);
1195EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet4Xd& from) {
1196 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse64_v_f64m4(to, from, unpacket_traits<Packet4Xd>::size);
1200EIGEN_DEVICE_FUNC
inline Packet4Xd pgather<double, Packet4Xd>(
const double* from, Index stride) {
1201 return __riscv_vlse64_v_f64m4(from, stride *
sizeof(
double), unpacket_traits<Packet4Xd>::size);
1205EIGEN_DEVICE_FUNC
inline void pscatter<double, Packet4Xd>(
double* to,
const Packet4Xd& from, Index stride) {
1206 __riscv_vsse64(to, stride *
sizeof(
double), from, unpacket_traits<Packet4Xd>::size);
1210EIGEN_STRONG_INLINE
double pfirst<Packet4Xd>(
const Packet4Xd& a) {
1211 return __riscv_vfmv_f_s_f64m4_f64(a);
1215EIGEN_STRONG_INLINE Packet4Xd psqrt(
const Packet4Xd& a) {
1216 return __riscv_vfsqrt_v_f64m4(a, unpacket_traits<Packet4Xd>::size);
1220EIGEN_STRONG_INLINE Packet4Xd print<Packet4Xd>(
const Packet4Xd& a) {
1221 const Packet4Xd limit = pset1<Packet4Xd>(
static_cast<double>(1ull << 52));
1222 const Packet4Xd abs_a = pabs(a);
1224 PacketMask16 mask = __riscv_vmfne_vv_f64m4_b16(a, a, unpacket_traits<Packet4Xd>::size);
1225 const Packet4Xd x = __riscv_vfadd_vv_f64m4_tumu(mask, a, a, a, unpacket_traits<Packet4Xd>::size);
1226 const Packet4Xd new_x = __riscv_vfcvt_f_x_v_f64m4(__riscv_vfcvt_x_f_v_i64m4(a, unpacket_traits<Packet4Xd>::size),
1227 unpacket_traits<Packet4Xd>::size);
1229 mask = __riscv_vmflt_vv_f64m4_b16(abs_a, limit, unpacket_traits<Packet4Xd>::size);
1230 Packet4Xd signed_x = __riscv_vfsgnj_vv_f64m4(new_x, x, unpacket_traits<Packet4Xd>::size);
1231 return __riscv_vmerge_vvm_f64m4(x, signed_x, mask, unpacket_traits<Packet4Xd>::size);
1235EIGEN_STRONG_INLINE Packet4Xd pfloor<Packet4Xd>(
const Packet4Xd& a) {
1236 Packet4Xd tmp = print<Packet4Xd>(a);
1238 PacketMask16 mask = __riscv_vmflt_vv_f64m4_b16(a, tmp, unpacket_traits<Packet4Xd>::size);
1239 return __riscv_vfsub_vf_f64m4_tumu(mask, tmp, tmp, 1.0, unpacket_traits<Packet4Xd>::size);
1243EIGEN_STRONG_INLINE Packet4Xd preverse(
const Packet4Xd& a) {
1244 Packet4Xul idx = __riscv_vrsub_vx_u64m4(__riscv_vid_v_u64m4(unpacket_traits<Packet4Xd>::size),
1245 unpacket_traits<Packet4Xd>::size - 1, unpacket_traits<Packet4Xd>::size);
1246 return __riscv_vrgather_vv_f64m4(a, idx, unpacket_traits<Packet4Xd>::size);
1250EIGEN_STRONG_INLINE Packet4Xd pfrexp<Packet4Xd>(
const Packet4Xd& a, Packet4Xd& exponent) {
1251 return pfrexp_generic(a, exponent);
1255EIGEN_STRONG_INLINE
double predux<Packet4Xd>(
const Packet4Xd& a) {
1256 return __riscv_vfmv_f(__riscv_vfredusum_vs_f64m4_f64m1(
1257 a, __riscv_vfmv_v_f_f64m1(0.0, unpacket_traits<Packet4Xd>::size / 4), unpacket_traits<Packet4Xd>::size));
1261EIGEN_STRONG_INLINE
bool predux_any(
const Packet4Xd& a) {
1262 const PacketMask16 mask =
1263 __riscv_vmsne_vx_u64m4_b16(__riscv_vreinterpret_v_f64m4_u64m4(a), 0, unpacket_traits<Packet4Xd>::size);
1264 return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xd>::size) != 0;
1268EIGEN_STRONG_INLINE
bool predux_all(
const Packet4Xd& a) {
1269 const PacketMask16 mask = __riscv_vmfeq_vf_f64m4_b16(a, 0.0, unpacket_traits<Packet4Xd>::size);
1270 return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xd>::size) == 0;
1274EIGEN_STRONG_INLINE Index predux_count(
const Packet4Xd& a) {
1275 const PacketMask16 mask = __riscv_vmfne_vf_f64m4_b16(a, 0.0, unpacket_traits<Packet4Xd>::size);
1276 return static_cast<Index
>(__riscv_vcpop_m_b16(mask, unpacket_traits<Packet4Xd>::size));
1280EIGEN_STRONG_INLINE
double predux_mul<Packet4Xd>(
const Packet4Xd& a) {
1281 Packet1Xd half1 = __riscv_vfmul_vv_f64m1(__riscv_vget_v_f64m4_f64m1(a, 0), __riscv_vget_v_f64m4_f64m1(a, 1),
1282 unpacket_traits<Packet1Xd>::size);
1283 Packet1Xd half2 = __riscv_vfmul_vv_f64m1(__riscv_vget_v_f64m4_f64m1(a, 2), __riscv_vget_v_f64m4_f64m1(a, 3),
1284 unpacket_traits<Packet1Xd>::size);
1285 return predux_mul<Packet1Xd>(__riscv_vfmul_vv_f64m1(half1, half2, unpacket_traits<Packet1Xd>::size));
1289EIGEN_STRONG_INLINE
double predux_min<Packet4Xd>(
const Packet4Xd& a) {
1290 return __riscv_vfmv_f(
1291 __riscv_vfredmin_vs_f64m4_f64m1(a, __riscv_vget_v_f64m4_f64m1(a, 0), unpacket_traits<Packet4Xd>::size));
1295EIGEN_STRONG_INLINE
double predux_max<Packet4Xd>(
const Packet4Xd& a) {
1296 return __riscv_vfmv_f(
1297 __riscv_vfredmax_vs_f64m4_f64m1(a, __riscv_vget_v_f64m4_f64m1(a, 0), unpacket_traits<Packet4Xd>::size));
1301EIGEN_STRONG_INLINE Packet2Xd predux_half(
const Packet4Xd& a) {
1302 return Packet2Xd(__riscv_vfadd_vv_f64m2(__riscv_vget_v_f64m4_f64m2(a, 0), __riscv_vget_v_f64m4_f64m2(a, 1),
1303 unpacket_traits<Packet2Xd>::size));
1307EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4Xd, N>& kernel) {
1308 double buffer[unpacket_traits<Packet4Xd>::size * N];
1311 for (i = 0; i < N; i++) {
1312 __riscv_vsse64(&buffer[i], N *
sizeof(
double), kernel.packet[i], unpacket_traits<Packet4Xd>::size);
1315 for (i = 0; i < N; i++) {
1317 __riscv_vle64_v_f64m4(&buffer[i * unpacket_traits<Packet4Xd>::size], unpacket_traits<Packet4Xd>::size);
1322EIGEN_STRONG_INLINE Packet4Xd pldexp<Packet4Xd>(
const Packet4Xd& a,
const Packet4Xd& exponent) {
1323 return pldexp_generic(a, exponent);
1326template <
typename Packet = Packet4Xd>
1328 std::enable_if_t<std::is_same<Packet, Packet4Xd>::value && (unpacket_traits<Packet4Xd>::size % 8) == 0, Packet2Xd>
1329 predux_half(
const Packet4Xd& a) {
1330 return __riscv_vfadd_vv_f64m2(__riscv_vget_v_f64m4_f64m2(a, 0), __riscv_vget_v_f64m4_f64m2(a, 1),
1331 unpacket_traits<Packet2Xd>::size);
1336EIGEN_STRONG_INLINE Packet4Xs __riscv_vreinterpret_v_u32m4_i16m4(
const Packet4Xu& a) {
1337 return __riscv_vreinterpret_v_i32m4_i16m4(__riscv_vreinterpret_v_u32m4_i32m4(a));
1341EIGEN_STRONG_INLINE Packet4Xs pset1<Packet4Xs>(
const numext::int16_t& from) {
1342 return __riscv_vmv_v_x_i16m4(from, unpacket_traits<Packet4Xs>::size);
1346EIGEN_STRONG_INLINE Packet4Xs plset<Packet4Xs>(
const numext::int16_t& a) {
1347 Packet4Xs idx = __riscv_vreinterpret_v_u16m4_i16m4(__riscv_vid_v_u16m4(unpacket_traits<Packet4Xs>::size));
1348 return __riscv_vadd_vx_i16m4(idx, a, unpacket_traits<Packet4Xs>::size);
1352EIGEN_STRONG_INLINE Packet4Xs pzero<Packet4Xs>(
const Packet4Xs& ) {
1353 return __riscv_vmv_v_x_i16m4(0, unpacket_traits<Packet4Xs>::size);
1357EIGEN_STRONG_INLINE Packet4Xs padd<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1358 return __riscv_vadd_vv_i16m4(a, b, unpacket_traits<Packet4Xs>::size);
1362EIGEN_STRONG_INLINE Packet4Xs psub<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1363 return __riscv_vsub(a, b, unpacket_traits<Packet4Xs>::size);
1367EIGEN_STRONG_INLINE Packet4Xs pnegate(
const Packet4Xs& a) {
1368 return __riscv_vneg(a, unpacket_traits<Packet4Xs>::size);
1372EIGEN_STRONG_INLINE Packet4Xs pmul<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1373 return __riscv_vmul(a, b, unpacket_traits<Packet4Xs>::size);
1377EIGEN_STRONG_INLINE Packet4Xs pdiv<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1378 return __riscv_vdiv(a, b, unpacket_traits<Packet4Xs>::size);
1382EIGEN_STRONG_INLINE Packet4Xs pmadd(
const Packet4Xs& a,
const Packet4Xs& b,
const Packet4Xs& c) {
1383 return __riscv_vmadd(a, b, c, unpacket_traits<Packet4Xs>::size);
1387EIGEN_STRONG_INLINE Packet4Xs pmsub(
const Packet4Xs& a,
const Packet4Xs& b,
const Packet4Xs& c) {
1388 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet4Xs>::size);
1392EIGEN_STRONG_INLINE Packet4Xs pnmadd(
const Packet4Xs& a,
const Packet4Xs& b,
const Packet4Xs& c) {
1393 return __riscv_vnmsub_vv_i16m4(a, b, c, unpacket_traits<Packet4Xs>::size);
1397EIGEN_STRONG_INLINE Packet4Xs pnmsub(
const Packet4Xs& a,
const Packet4Xs& b,
const Packet4Xs& c) {
1398 return __riscv_vnmsub_vv_i16m4(a, b, pnegate(c), unpacket_traits<Packet4Xs>::size);
1402EIGEN_STRONG_INLINE Packet4Xs pmin<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1403 return __riscv_vmin(a, b, unpacket_traits<Packet4Xs>::size);
1407EIGEN_STRONG_INLINE Packet4Xs pmax<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1408 return __riscv_vmax(a, b, unpacket_traits<Packet4Xs>::size);
1412EIGEN_STRONG_INLINE Packet4Xs pcmp_le<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1413 PacketMask4 mask = __riscv_vmsle_vv_i16m4_b4(a, b, unpacket_traits<Packet4Xs>::size);
1414 return __riscv_vmerge_vxm_i16m4(pzero(a),
static_cast<short>(0xffff), mask, unpacket_traits<Packet4Xs>::size);
1418EIGEN_STRONG_INLINE Packet4Xs pcmp_lt<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1419 PacketMask4 mask = __riscv_vmslt_vv_i16m4_b4(a, b, unpacket_traits<Packet4Xs>::size);
1420 return __riscv_vmerge_vxm_i16m4(pzero(a),
static_cast<short>(0xffff), mask, unpacket_traits<Packet4Xs>::size);
1424EIGEN_STRONG_INLINE Packet4Xs pcmp_eq<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1425 PacketMask4 mask = __riscv_vmseq_vv_i16m4_b4(a, b, unpacket_traits<Packet4Xs>::size);
1426 return __riscv_vmerge_vxm_i16m4(pzero(a),
static_cast<short>(0xffff), mask, unpacket_traits<Packet4Xs>::size);
1430EIGEN_STRONG_INLINE Packet4Xs ptrue<Packet4Xs>(
const Packet4Xs& ) {
1431 return __riscv_vmv_v_x_i16m4(
static_cast<unsigned short>(0xffffu), unpacket_traits<Packet4Xs>::size);
1435EIGEN_STRONG_INLINE Packet4Xs pand<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1436 return __riscv_vand_vv_i16m4(a, b, unpacket_traits<Packet4Xs>::size);
1440EIGEN_STRONG_INLINE Packet4Xs por<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1441 return __riscv_vor_vv_i16m4(a, b, unpacket_traits<Packet4Xs>::size);
1445EIGEN_STRONG_INLINE Packet4Xs pxor<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1446 return __riscv_vxor_vv_i16m4(a, b, unpacket_traits<Packet4Xs>::size);
1450EIGEN_STRONG_INLINE Packet4Xs pandnot<Packet4Xs>(
const Packet4Xs& a,
const Packet4Xs& b) {
1452 return __riscv_vand_vv_i16m4(a, __riscv_vnot_v_i16m4(b, unpacket_traits<Packet4Xs>::size),
1453 unpacket_traits<Packet4Xs>::size);
1455 return __riscv_vreinterpret_v_u16m4_i16m4(__riscv_vandn_vv_u16m4(
1456 __riscv_vreinterpret_v_i16m4_u16m4(a), __riscv_vreinterpret_v_i16m4_u16m4(b), unpacket_traits<Packet4Xs>::size));
1461EIGEN_STRONG_INLINE Packet4Xs parithmetic_shift_right(
const Packet4Xs& a) {
1462 return __riscv_vsra_vx_i16m4(a, N, unpacket_traits<Packet4Xs>::size);
1466EIGEN_STRONG_INLINE Packet4Xs plogical_shift_right(
const Packet4Xs& a) {
1467 return __riscv_vreinterpret_i16m4(
1468 __riscv_vsrl_vx_u16m4(__riscv_vreinterpret_u16m4(a), N, unpacket_traits<Packet4Xs>::size));
1472EIGEN_STRONG_INLINE Packet4Xs plogical_shift_left(
const Packet4Xs& a) {
1473 return __riscv_vsll_vx_i16m4(a, N, unpacket_traits<Packet4Xs>::size);
1477EIGEN_STRONG_INLINE Packet4Xs pload<Packet4Xs>(
const numext::int16_t* from) {
1478 EIGEN_DEBUG_ALIGNED_LOAD
return __riscv_vle16_v_i16m4(from, unpacket_traits<Packet4Xs>::size);
1482EIGEN_STRONG_INLINE Packet4Xs ploadu<Packet4Xs>(
const numext::int16_t* from) {
1483 EIGEN_DEBUG_UNALIGNED_LOAD
return __riscv_vle16_v_i16m4(from, unpacket_traits<Packet4Xs>::size);
1487EIGEN_STRONG_INLINE Packet4Xs ploaddup<Packet4Xs>(
const numext::int16_t* from) {
1488 Packet4Xu data = __riscv_vwcvtu_x_x_v_u32m4(__riscv_vreinterpret_v_i16m2_u16m2(pload<Packet2Xs>(from)),
1489 unpacket_traits<Packet2Xs>::size);
1490 return __riscv_vreinterpret_v_u32m4_i16m4(__riscv_vadd_vv_u32m4(
1491 __riscv_vsll_vx_u32m4(data, 16, unpacket_traits<Packet4Xi>::size), data, unpacket_traits<Packet4Xi>::size));
1495EIGEN_STRONG_INLINE Packet4Xs ploadquad<Packet4Xs>(
const numext::int16_t* from) {
1497 __riscv_vsrl_vx_u16m4(__riscv_vid_v_u16m4(unpacket_traits<Packet4Xs>::size), 2, unpacket_traits<Packet4Xs>::size);
1498 return __riscv_vrgather_vv_i16m4(__riscv_vlmul_ext_v_i16m1_i16m4(pload<Packet1Xs>(from)), idx,
1499 unpacket_traits<Packet4Xs>::size);
1503EIGEN_STRONG_INLINE
void pstore<numext::int16_t>(numext::int16_t* to,
const Packet4Xs& from) {
1504 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse16_v_i16m4(to, from, unpacket_traits<Packet4Xs>::size);
1508EIGEN_STRONG_INLINE
void pstoreu<numext::int16_t>(numext::int16_t* to,
const Packet4Xs& from) {
1509 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse16_v_i16m4(to, from, unpacket_traits<Packet4Xs>::size);
1513EIGEN_DEVICE_FUNC
inline Packet4Xs pgather<numext::int16_t, Packet4Xs>(
const numext::int16_t* from, Index stride) {
1514 return __riscv_vlse16_v_i16m4(from, stride *
sizeof(numext::int16_t), unpacket_traits<Packet4Xs>::size);
1518EIGEN_DEVICE_FUNC
inline void pscatter<numext::int16_t, Packet4Xs>(numext::int16_t* to,
const Packet4Xs& from,
1520 __riscv_vsse16(to, stride *
sizeof(numext::int16_t), from, unpacket_traits<Packet4Xs>::size);
1524EIGEN_STRONG_INLINE numext::int16_t pfirst<Packet4Xs>(
const Packet4Xs& a) {
1525 return __riscv_vmv_x_s_i16m4_i16(a);
1529EIGEN_STRONG_INLINE Packet4Xs preverse(
const Packet4Xs& a) {
1530 Packet4Xsu idx = __riscv_vrsub_vx_u16m4(__riscv_vid_v_u16m4(unpacket_traits<Packet4Xs>::size),
1531 unpacket_traits<Packet4Xs>::size - 1, unpacket_traits<Packet4Xs>::size);
1532 return __riscv_vrgather_vv_i16m4(a, idx, unpacket_traits<Packet4Xs>::size);
1536EIGEN_STRONG_INLINE Packet4Xs pabs(
const Packet4Xs& a) {
1537 Packet4Xs mask = __riscv_vsra_vx_i16m4(a, 15, unpacket_traits<Packet4Xs>::size);
1538 return __riscv_vsub_vv_i16m4(__riscv_vxor_vv_i16m4(a, mask, unpacket_traits<Packet4Xs>::size), mask,
1539 unpacket_traits<Packet4Xs>::size);
1543EIGEN_STRONG_INLINE numext::int16_t predux<Packet4Xs>(
const Packet4Xs& a) {
1544 return __riscv_vmv_x(__riscv_vredsum_vs_i16m4_i16m1(a, __riscv_vmv_v_x_i16m1(0, unpacket_traits<Packet4Xs>::size / 4),
1545 unpacket_traits<Packet4Xs>::size));
1549EIGEN_STRONG_INLINE
bool predux_all(
const Packet4Xs& a) {
1550 const PacketMask4 mask = __riscv_vmseq_vx_i16m4_b4(a, 0, unpacket_traits<Packet4Xs>::size);
1551 return __riscv_vcpop_m_b4(mask, unpacket_traits<Packet4Xs>::size) == 0;
1555EIGEN_STRONG_INLINE numext::int16_t predux_mul<Packet4Xs>(
const Packet4Xs& a) {
1556 Packet1Xs half1 = __riscv_vmul_vv_i16m1(__riscv_vget_v_i16m4_i16m1(a, 0), __riscv_vget_v_i16m4_i16m1(a, 1),
1557 unpacket_traits<Packet1Xs>::size);
1558 Packet1Xs half2 = __riscv_vmul_vv_i16m1(__riscv_vget_v_i16m4_i16m1(a, 2), __riscv_vget_v_i16m4_i16m1(a, 3),
1559 unpacket_traits<Packet1Xs>::size);
1560 return predux_mul<Packet1Xs>(__riscv_vmul_vv_i16m1(half1, half2, unpacket_traits<Packet1Xs>::size));
1564EIGEN_STRONG_INLINE numext::int16_t predux_min<Packet4Xs>(
const Packet4Xs& a) {
1565 return __riscv_vmv_x(__riscv_vredmin_vs_i16m4_i16m1(
1566 a, __riscv_vmv_v_x_i16m1((std::numeric_limits<numext::int16_t>::max)(), unpacket_traits<Packet4Xs>::size / 4),
1567 unpacket_traits<Packet4Xs>::size));
1571EIGEN_STRONG_INLINE numext::int16_t predux_max<Packet4Xs>(
const Packet4Xs& a) {
1572 return __riscv_vmv_x(__riscv_vredmax_vs_i16m4_i16m1(
1573 a, __riscv_vmv_v_x_i16m1((std::numeric_limits<numext::int16_t>::min)(), unpacket_traits<Packet4Xs>::size / 4),
1574 unpacket_traits<Packet4Xs>::size));
1578EIGEN_DEVICE_FUNC
inline void ptranspose(PacketBlock<Packet4Xs, N>& kernel) {
1579 numext::int16_t buffer[unpacket_traits<Packet4Xs>::size * N] = {0};
1582 for (i = 0; i < N; i++) {
1583 __riscv_vsse16(&buffer[i], N *
sizeof(numext::int16_t), kernel.packet[i], unpacket_traits<Packet4Xs>::size);
1585 for (i = 0; i < N; i++) {
1587 __riscv_vle16_v_i16m4(&buffer[i * unpacket_traits<Packet4Xs>::size], unpacket_traits<Packet4Xs>::size);
1591template <
typename Packet = Packet4Xs>
1593 std::enable_if_t<std::is_same<Packet, Packet4Xs>::value && (unpacket_traits<Packet4Xs>::size % 8) == 0, Packet2Xs>
1594 predux_half(
const Packet4Xs& a) {
1595 return __riscv_vadd_vv_i16m2(__riscv_vget_v_i16m4_i16m2(a, 0), __riscv_vget_v_i16m4_i16m2(a, 1),
1596 unpacket_traits<Packet2Xs>::size);