Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2024 Kseniya Zaytseva <kseniya.zaytseva@syntacore.com>
5// Copyright (C) 2025 Chip Kerchner <ckerchner@tenstorrent.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_PACKET_MATH_RVV10_H
13#define EIGEN_PACKET_MATH_RVV10_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19namespace internal {
20
21/********************************* Packet1Xi ************************************/
22
23EIGEN_STRONG_INLINE Packet1Xi __riscv_vreinterpret_v_u64m1_i32m1(const Packet1Xul& a) {
24 return __riscv_vreinterpret_v_i64m1_i32m1(__riscv_vreinterpret_v_u64m1_i64m1(a));
25}
26
27template <>
28EIGEN_STRONG_INLINE Packet1Xi pset1<Packet1Xi>(const numext::int32_t& from) {
29 return __riscv_vmv_v_x_i32m1(from, unpacket_traits<Packet1Xi>::size);
30}
31
32template <>
33EIGEN_STRONG_INLINE Packet1Xi plset<Packet1Xi>(const numext::int32_t& a) {
34 Packet1Xi idx = __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size));
35 return __riscv_vadd_vx_i32m1(idx, a, unpacket_traits<Packet1Xi>::size);
36}
37
38template <>
39EIGEN_STRONG_INLINE Packet1Xi pzero<Packet1Xi>(const Packet1Xi& /*a*/) {
40 return __riscv_vmv_v_x_i32m1(0, unpacket_traits<Packet1Xi>::size);
41}
42
43template <>
44EIGEN_STRONG_INLINE Packet1Xi padd<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
45 return __riscv_vadd_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
46}
47
48template <>
49EIGEN_STRONG_INLINE Packet1Xi psub<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
50 return __riscv_vsub(a, b, unpacket_traits<Packet1Xi>::size);
51}
52
53template <>
54EIGEN_STRONG_INLINE Packet1Xi pnegate(const Packet1Xi& a) {
55 return __riscv_vneg(a, unpacket_traits<Packet1Xi>::size);
56}
57
58template <>
59EIGEN_STRONG_INLINE Packet1Xi pmul<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
60 return __riscv_vmul(a, b, unpacket_traits<Packet1Xi>::size);
61}
62
63template <>
64EIGEN_STRONG_INLINE Packet1Xi pdiv<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
65 return __riscv_vdiv(a, b, unpacket_traits<Packet1Xi>::size);
66}
67
68template <>
69EIGEN_STRONG_INLINE Packet1Xi pmadd(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
70 return __riscv_vmadd(a, b, c, unpacket_traits<Packet1Xi>::size);
71}
72
73template <>
74EIGEN_STRONG_INLINE Packet1Xi pmsub(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
75 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet1Xi>::size);
76}
77
78template <>
79EIGEN_STRONG_INLINE Packet1Xi pnmadd(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
80 return __riscv_vnmsub_vv_i32m1(a, b, c, unpacket_traits<Packet1Xi>::size);
81}
82
83template <>
84EIGEN_STRONG_INLINE Packet1Xi pnmsub(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
85 return __riscv_vnmsub_vv_i32m1(a, b, pnegate(c), unpacket_traits<Packet1Xi>::size);
86}
87
88template <>
89EIGEN_STRONG_INLINE Packet1Xi pmin<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
90 return __riscv_vmin(a, b, unpacket_traits<Packet1Xi>::size);
91}
92
93template <>
94EIGEN_STRONG_INLINE Packet1Xi pmax<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
95 return __riscv_vmax(a, b, unpacket_traits<Packet1Xi>::size);
96}
97
98template <>
99EIGEN_STRONG_INLINE Packet1Xi pcmp_le<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
100 PacketMask32 mask = __riscv_vmsle_vv_i32m1_b32(a, b, unpacket_traits<Packet1Xi>::size);
101 return __riscv_vmerge_vxm_i32m1(pzero(a), 0xffffffff, mask, unpacket_traits<Packet1Xi>::size);
102}
103
104template <>
105EIGEN_STRONG_INLINE Packet1Xi pcmp_lt<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
106 PacketMask32 mask = __riscv_vmslt_vv_i32m1_b32(a, b, unpacket_traits<Packet1Xi>::size);
107 return __riscv_vmerge_vxm_i32m1(pzero(a), 0xffffffff, mask, unpacket_traits<Packet1Xi>::size);
108}
109
110template <>
111EIGEN_STRONG_INLINE Packet1Xi pcmp_eq<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
112 PacketMask32 mask = __riscv_vmseq_vv_i32m1_b32(a, b, unpacket_traits<Packet1Xi>::size);
113 return __riscv_vmerge_vxm_i32m1(pzero(a), 0xffffffff, mask, unpacket_traits<Packet1Xi>::size);
114}
115
116template <>
117EIGEN_STRONG_INLINE Packet1Xi ptrue<Packet1Xi>(const Packet1Xi& /*a*/) {
118 return __riscv_vmv_v_x_i32m1(0xffffffffu, unpacket_traits<Packet1Xi>::size);
119}
120
121template <>
122EIGEN_STRONG_INLINE Packet1Xi pand<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
123 return __riscv_vand_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
124}
125
126template <>
127EIGEN_STRONG_INLINE Packet1Xi por<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
128 return __riscv_vor_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
129}
130
131template <>
132EIGEN_STRONG_INLINE Packet1Xi pxor<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
133 return __riscv_vxor_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
134}
135
136template <>
137EIGEN_STRONG_INLINE Packet1Xi pandnot<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
138#ifndef __riscv_zvbb
139 return __riscv_vand_vv_i32m1(a, __riscv_vnot_v_i32m1(b, unpacket_traits<Packet1Xi>::size),
140 unpacket_traits<Packet1Xi>::size);
141#else
142 return __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vandn_vv_u32m1(
143 __riscv_vreinterpret_v_i32m1_u32m1(a), __riscv_vreinterpret_v_i32m1_u32m1(b), unpacket_traits<Packet1Xi>::size));
144#endif
145}
146
147template <>
148EIGEN_STRONG_INLINE Packet1Xi pnot<Packet1Xi>(const Packet1Xi& a) {
149 return __riscv_vnot_v_i32m1(a, unpacket_traits<Packet1Xi>::size);
150}
151
152template <int N>
153EIGEN_STRONG_INLINE Packet1Xi parithmetic_shift_right(const Packet1Xi& a) {
154 return __riscv_vsra_vx_i32m1(a, N, unpacket_traits<Packet1Xi>::size);
155}
156
157template <int N>
158EIGEN_STRONG_INLINE Packet1Xi plogical_shift_right(const Packet1Xi& a) {
159 return __riscv_vreinterpret_i32m1(
160 __riscv_vsrl_vx_u32m1(__riscv_vreinterpret_u32m1(a), N, unpacket_traits<Packet1Xi>::size));
161}
162
163template <int N>
164EIGEN_STRONG_INLINE Packet1Xi plogical_shift_left(const Packet1Xi& a) {
165 return __riscv_vsll_vx_i32m1(a, N, unpacket_traits<Packet1Xi>::size);
166}
167
168template <>
169EIGEN_STRONG_INLINE Packet1Xi pload<Packet1Xi>(const numext::int32_t* from) {
170 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle32_v_i32m1(from, unpacket_traits<Packet1Xi>::size);
171}
172
173template <>
174EIGEN_STRONG_INLINE Packet1Xi ploadu<Packet1Xi>(const numext::int32_t* from) {
175 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle32_v_i32m1(from, unpacket_traits<Packet1Xi>::size);
176}
177
178template <>
179EIGEN_STRONG_INLINE Packet1Xi ploaddup<Packet1Xi>(const numext::int32_t* from) {
180 Packet1Xul data = __riscv_vlmul_trunc_v_u64m2_u64m1(__riscv_vwcvtu_x_x_v_u64m2(
181 __riscv_vreinterpret_v_i32m1_u32m1(__riscv_vle32_v_i32m1(from, unpacket_traits<Packet1Xi>::size / 2)),
182 unpacket_traits<Packet1Xi>::size / 2));
183 return __riscv_vreinterpret_v_u64m1_i32m1(__riscv_vadd_vv_u64m1(
184 __riscv_vsll_vx_u64m1(data, 32, unpacket_traits<Packet1Xl>::size), data, unpacket_traits<Packet1Xl>::size));
185}
186
187template <>
188EIGEN_STRONG_INLINE Packet1Xi ploadquad<Packet1Xi>(const numext::int32_t* from) {
189 Packet1Xu idx =
190 __riscv_vsrl_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size), 2, unpacket_traits<Packet1Xi>::size);
191 return __riscv_vrgather_vv_i32m1(pload<Packet1Xi>(from), idx, unpacket_traits<Packet1Xi>::size);
192}
193
194template <>
195EIGEN_STRONG_INLINE void pstore<numext::int32_t>(numext::int32_t* to, const Packet1Xi& from) {
196 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse32_v_i32m1(to, from, unpacket_traits<Packet1Xi>::size);
197}
198
199template <>
200EIGEN_STRONG_INLINE void pstoreu<numext::int32_t>(numext::int32_t* to, const Packet1Xi& from) {
201 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse32_v_i32m1(to, from, unpacket_traits<Packet1Xi>::size);
202}
203
204template <>
205EIGEN_DEVICE_FUNC inline Packet1Xi pgather<numext::int32_t, Packet1Xi>(const numext::int32_t* from, Index stride) {
206 return __riscv_vlse32_v_i32m1(from, stride * sizeof(numext::int32_t), unpacket_traits<Packet1Xi>::size);
207}
208
209template <>
210EIGEN_DEVICE_FUNC inline void pscatter<numext::int32_t, Packet1Xi>(numext::int32_t* to, const Packet1Xi& from,
211 Index stride) {
212 __riscv_vsse32(to, stride * sizeof(numext::int32_t), from, unpacket_traits<Packet1Xi>::size);
213}
214
215template <>
216EIGEN_STRONG_INLINE numext::int32_t pfirst<Packet1Xi>(const Packet1Xi& a) {
217 return __riscv_vmv_x_s_i32m1_i32(a);
218}
219
220template <>
221EIGEN_STRONG_INLINE Packet1Xi preverse(const Packet1Xi& a) {
222 Packet1Xu idx = __riscv_vrsub_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size),
223 unpacket_traits<Packet1Xi>::size - 1, unpacket_traits<Packet1Xi>::size);
224 return __riscv_vrgather_vv_i32m1(a, idx, unpacket_traits<Packet1Xi>::size);
225}
226
227template <>
228EIGEN_STRONG_INLINE Packet1Xi pabs(const Packet1Xi& a) {
229 Packet1Xi mask = __riscv_vsra_vx_i32m1(a, 31, unpacket_traits<Packet1Xi>::size);
230 return __riscv_vsub_vv_i32m1(__riscv_vxor_vv_i32m1(a, mask, unpacket_traits<Packet1Xi>::size), mask,
231 unpacket_traits<Packet1Xi>::size);
232}
233
234template <>
235EIGEN_STRONG_INLINE numext::int32_t predux<Packet1Xi>(const Packet1Xi& a) {
236 return __riscv_vmv_x(__riscv_vredsum_vs_i32m1_i32m1(a, __riscv_vmv_v_x_i32m1(0, unpacket_traits<Packet1Xi>::size),
237 unpacket_traits<Packet1Xi>::size));
238}
239
240template <>
241EIGEN_STRONG_INLINE bool predux_all(const Packet1Xi& a) {
242 const PacketMask32 mask = __riscv_vmseq_vx_i32m1_b32(a, 0, unpacket_traits<Packet1Xi>::size);
243 return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xi>::size) == 0;
244}
245
246template <>
247EIGEN_STRONG_INLINE numext::int32_t predux_mul<Packet1Xi>(const Packet1Xi& a) {
248 // Multiply the vector by its reverse
249 Packet1Xi prod = __riscv_vmul_vv_i32m1(preverse(a), a, unpacket_traits<Packet1Xi>::size);
250 Packet1Xi half_prod;
251
252 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
253 half_prod = __riscv_vslidedown_vx_i32m1(prod, 8, unpacket_traits<Packet1Xi>::size);
254 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
255 }
256 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
257 half_prod = __riscv_vslidedown_vx_i32m1(prod, 4, unpacket_traits<Packet1Xi>::size);
258 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
259 }
260 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
261 half_prod = __riscv_vslidedown_vx_i32m1(prod, 2, unpacket_traits<Packet1Xi>::size);
262 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
263 }
264 // Last reduction
265 half_prod = __riscv_vslidedown_vx_i32m1(prod, 1, unpacket_traits<Packet1Xi>::size);
266 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
267
268 // The reduction is done to the first element.
269 return pfirst(prod);
270}
271
272template <>
273EIGEN_STRONG_INLINE numext::int32_t predux_min<Packet1Xi>(const Packet1Xi& a) {
274 return __riscv_vmv_x(__riscv_vredmin_vs_i32m1_i32m1(
275 a, __riscv_vmv_v_x_i32m1((std::numeric_limits<numext::int32_t>::max)(), unpacket_traits<Packet1Xi>::size),
276 unpacket_traits<Packet1Xi>::size));
277}
278
279template <>
280EIGEN_STRONG_INLINE numext::int32_t predux_max<Packet1Xi>(const Packet1Xi& a) {
281 return __riscv_vmv_x(__riscv_vredmax_vs_i32m1_i32m1(
282 a, __riscv_vmv_v_x_i32m1((std::numeric_limits<numext::int32_t>::min)(), unpacket_traits<Packet1Xi>::size),
283 unpacket_traits<Packet1Xi>::size));
284}
285
286template <int N>
287EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xi, N>& kernel) {
288 numext::int32_t buffer[unpacket_traits<Packet1Xi>::size * N] = {0};
289 int i = 0;
290
291 for (i = 0; i < N; i++) {
292 __riscv_vsse32(&buffer[i], N * sizeof(numext::int32_t), kernel.packet[i], unpacket_traits<Packet1Xi>::size);
293 }
294 for (i = 0; i < N; i++) {
295 kernel.packet[i] =
296 __riscv_vle32_v_i32m1(&buffer[i * unpacket_traits<Packet1Xi>::size], unpacket_traits<Packet1Xi>::size);
297 }
298}
299
300/********************************* Packet1Xf ************************************/
301
302EIGEN_STRONG_INLINE Packet1Xf __riscv_vreinterpret_v_u64m1_f32m1(const Packet1Xul& a) {
303 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vreinterpret_v_u64m1_u32m1(a));
304}
305
306EIGEN_STRONG_INLINE Packet2Xf __riscv_vreinterpret_v_u64m2_f32m2(const Packet2Xul& a) {
307 return __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vreinterpret_v_u64m2_u32m2(a));
308}
309
310template <>
311EIGEN_STRONG_INLINE Packet1Xf ptrue<Packet1Xf>(const Packet1Xf& /*a*/) {
312 Packet1Xf r = __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0xffffffffu, unpacket_traits<Packet1Xf>::size));
313 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
314 return r;
315}
316
317template <>
318EIGEN_STRONG_INLINE Packet1Xf pzero<Packet1Xf>(const Packet1Xf& /*a*/) {
319 return __riscv_vfmv_v_f_f32m1(0.0f, unpacket_traits<Packet1Xf>::size);
320}
321
322template <>
323EIGEN_STRONG_INLINE Packet1Xf pabs(const Packet1Xf& a) {
324 return __riscv_vfabs_v_f32m1(a, unpacket_traits<Packet1Xf>::size);
325}
326
327template <>
328EIGEN_STRONG_INLINE Packet1Xf pabsdiff(const Packet1Xf& a, const Packet1Xf& b) {
329 return __riscv_vfabs_v_f32m1(__riscv_vfsub_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size),
330 unpacket_traits<Packet1Xf>::size);
331}
332
333template <>
334EIGEN_STRONG_INLINE Packet1Xf pset1<Packet1Xf>(const float& from) {
335 return __riscv_vfmv_v_f_f32m1(from, unpacket_traits<Packet1Xf>::size);
336}
337
338template <>
339EIGEN_STRONG_INLINE Packet1Xf pset1frombits<Packet1Xf>(numext::uint32_t from) {
340 return __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(from, unpacket_traits<Packet1Xf>::size));
341}
342
343template <>
344EIGEN_STRONG_INLINE Packet1Xf plset<Packet1Xf>(const float& a) {
345 Packet1Xf idx = __riscv_vfcvt_f_x_v_f32m1(
346 __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size)),
347 unpacket_traits<Packet1Xf>::size);
348 return __riscv_vfadd_vf_f32m1(idx, a, unpacket_traits<Packet1Xf>::size);
349}
350
351template <>
352EIGEN_STRONG_INLINE void pbroadcast4<Packet1Xf>(const float* a, Packet1Xf& a0, Packet1Xf& a1, Packet1Xf& a2,
353 Packet1Xf& a3) {
354 Packet1Xf aa = __riscv_vle32_v_f32m1(a, 4);
355 a0 = __riscv_vrgather_vx_f32m1(aa, 0, unpacket_traits<Packet1Xf>::size);
356 a1 = __riscv_vrgather_vx_f32m1(aa, 1, unpacket_traits<Packet1Xf>::size);
357 a2 = __riscv_vrgather_vx_f32m1(aa, 2, unpacket_traits<Packet1Xf>::size);
358 a3 = __riscv_vrgather_vx_f32m1(aa, 3, unpacket_traits<Packet1Xf>::size);
359}
360
361template <>
362EIGEN_STRONG_INLINE Packet1Xf padd<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
363 return __riscv_vfadd_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
364}
365
366template <>
367EIGEN_STRONG_INLINE Packet1Xf psub<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
368 return __riscv_vfsub_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
369}
370
371template <>
372EIGEN_STRONG_INLINE Packet1Xf pnegate(const Packet1Xf& a) {
373 return __riscv_vfneg_v_f32m1(a, unpacket_traits<Packet1Xf>::size);
374}
375
376template <>
377EIGEN_STRONG_INLINE Packet1Xf psignbit(const Packet1Xf& a) {
378 return __riscv_vreinterpret_v_i32m1_f32m1(
379 __riscv_vsra_vx_i32m1(__riscv_vreinterpret_v_f32m1_i32m1(a), 31, unpacket_traits<Packet1Xi>::size));
380}
381
382template <>
383EIGEN_STRONG_INLINE Packet1Xf pmul<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
384 return __riscv_vfmul_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
385}
386
387template <>
388EIGEN_STRONG_INLINE Packet1Xf pdiv<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
389 return __riscv_vfdiv_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
390}
391
392template <>
393struct is_fused_madd<Packet1Xf> : true_type {};
394template <>
395EIGEN_STRONG_INLINE Packet1Xf pmadd(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
396 return __riscv_vfmadd_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
397}
398
399template <>
400EIGEN_STRONG_INLINE Packet1Xf pmsub(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
401 return __riscv_vfmsub_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
402}
403
404template <>
405EIGEN_STRONG_INLINE Packet1Xf pnmadd(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
406 return __riscv_vfnmsub_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
407}
408
409template <>
410EIGEN_STRONG_INLINE Packet1Xf pnmsub(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
411 return __riscv_vfnmadd_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
412}
413
414template <>
415struct pminmax_propagates_nan<Packet1Xf> : bool_constant<true> {};
416
417template <>
418EIGEN_STRONG_INLINE Packet1Xf pmin<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
419 Packet1Xf nans = __riscv_vfmv_v_f_f32m1((std::numeric_limits<float>::quiet_NaN)(), unpacket_traits<Packet1Xf>::size);
420 PacketMask32 mask = __riscv_vmfeq_vv_f32m1_b32(a, a, unpacket_traits<Packet1Xf>::size);
421 PacketMask32 mask2 = __riscv_vmfeq_vv_f32m1_b32(b, b, unpacket_traits<Packet1Xf>::size);
422 mask = __riscv_vmand_mm_b32(mask, mask2, unpacket_traits<Packet1Xf>::size);
423
424 return __riscv_vfmin_vv_f32m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xf>::size);
425}
426
427template <>
428EIGEN_STRONG_INLINE Packet1Xf pmin<PropagateNumbers, Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
429 return __riscv_vfmin_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
430}
431
432template <>
433EIGEN_STRONG_INLINE Packet1Xf pmax<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
434 Packet1Xf nans = __riscv_vfmv_v_f_f32m1((std::numeric_limits<float>::quiet_NaN)(), unpacket_traits<Packet1Xf>::size);
435 PacketMask32 mask = __riscv_vmfeq_vv_f32m1_b32(a, a, unpacket_traits<Packet1Xf>::size);
436 PacketMask32 mask2 = __riscv_vmfeq_vv_f32m1_b32(b, b, unpacket_traits<Packet1Xf>::size);
437 mask = __riscv_vmand_mm_b32(mask, mask2, unpacket_traits<Packet1Xf>::size);
438
439 return __riscv_vfmax_vv_f32m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xf>::size);
440}
441
442template <>
443EIGEN_STRONG_INLINE Packet1Xf pmax<PropagateNumbers, Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
444 return __riscv_vfmax_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
445}
446
447template <>
448EIGEN_STRONG_INLINE Packet1Xf pcmp_le<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
449 PacketMask32 mask = __riscv_vmfle_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
450 return __riscv_vmerge_vvm_f32m1(pzero<Packet1Xf>(a), ptrue<Packet1Xf>(a), mask, unpacket_traits<Packet1Xf>::size);
451}
452
453template <>
454EIGEN_STRONG_INLINE Packet1Xf pcmp_lt<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
455 PacketMask32 mask = __riscv_vmflt_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
456 return __riscv_vmerge_vvm_f32m1(pzero<Packet1Xf>(a), ptrue<Packet1Xf>(a), mask, unpacket_traits<Packet1Xf>::size);
457}
458
459template <>
460EIGEN_STRONG_INLINE Packet1Xf pcmp_eq<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
461 PacketMask32 mask = __riscv_vmfeq_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
462 return __riscv_vmerge_vvm_f32m1(pzero<Packet1Xf>(a), ptrue<Packet1Xf>(a), mask, unpacket_traits<Packet1Xf>::size);
463}
464
465template <>
466EIGEN_STRONG_INLINE Packet1Xf pcmp_lt_or_nan<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
467 PacketMask32 mask = __riscv_vmfge_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
468 return __riscv_vfmerge_vfm_f32m1(ptrue<Packet1Xf>(a), 0.0f, mask, unpacket_traits<Packet1Xf>::size);
469}
470
471// Logical Operations are not supported for float, so reinterpret casts
472template <>
473EIGEN_STRONG_INLINE Packet1Xf pand<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
474 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vand_vv_u32m1(
475 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size));
476}
477
478template <>
479EIGEN_STRONG_INLINE Packet1Xf por<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
480 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vor_vv_u32m1(
481 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size));
482}
483
484template <>
485EIGEN_STRONG_INLINE Packet1Xf pxor<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
486 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vxor_vv_u32m1(
487 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size));
488}
489
490template <>
491EIGEN_STRONG_INLINE Packet1Xf pandnot<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
492#ifndef __riscv_zvbb
493 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vand_vv_u32m1(
494 __riscv_vreinterpret_v_f32m1_u32m1(a),
495 __riscv_vnot_v_u32m1(__riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size),
496 unpacket_traits<Packet1Xf>::size));
497#else
498 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vandn_vv_u32m1(
499 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xi>::size));
500#endif
501}
502
503template <>
504EIGEN_STRONG_INLINE Packet1Xf pnot<Packet1Xf>(const Packet1Xf& a) {
505 return __riscv_vreinterpret_v_u32m1_f32m1(
506 __riscv_vnot_v_u32m1(__riscv_vreinterpret_v_f32m1_u32m1(a), unpacket_traits<Packet1Xf>::size));
507}
508
509template <>
510EIGEN_STRONG_INLINE Packet1Xf pload<Packet1Xf>(const float* from) {
511 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle32_v_f32m1(from, unpacket_traits<Packet1Xf>::size);
512}
513
514template <>
515EIGEN_STRONG_INLINE Packet1Xf ploadu<Packet1Xf>(const float* from) {
516 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle32_v_f32m1(from, unpacket_traits<Packet1Xf>::size);
517}
518
519EIGEN_STRONG_INLINE Packet2Xf pdup(const Packet1Xf& a) {
520 Packet2Xul data = __riscv_vwcvtu_x_x_v_u64m2(__riscv_vreinterpret_v_f32m1_u32m1(a), unpacket_traits<Packet1Xi>::size);
521 return __riscv_vreinterpret_v_u64m2_f32m2(__riscv_vadd_vv_u64m2(
522 __riscv_vsll_vx_u64m2(data, 32, unpacket_traits<Packet2Xl>::size), data, unpacket_traits<Packet2Xl>::size));
523}
524
525template <>
526EIGEN_STRONG_INLINE Packet1Xf ploaddup<Packet1Xf>(const float* from) {
527 Packet1Xul data = __riscv_vlmul_trunc_v_u64m2_u64m1(__riscv_vwcvtu_x_x_v_u64m2(
528 __riscv_vreinterpret_v_f32m1_u32m1(__riscv_vle32_v_f32m1(from, unpacket_traits<Packet1Xf>::size / 2)),
529 unpacket_traits<Packet1Xf>::size / 2));
530 return __riscv_vreinterpret_v_u64m1_f32m1(__riscv_vadd_vv_u64m1(
531 __riscv_vsll_vx_u64m1(data, 32, unpacket_traits<Packet1Xl>::size), data, unpacket_traits<Packet1Xl>::size));
532}
533
534template <>
535EIGEN_STRONG_INLINE Packet1Xf ploadquad<Packet1Xf>(const float* from) {
536 Packet1Xu idx =
537 __riscv_vsrl_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xf>::size), 2, unpacket_traits<Packet1Xf>::size);
538 return __riscv_vrgather_vv_f32m1(pload<Packet1Xf>(from), idx, unpacket_traits<Packet1Xf>::size);
539}
540
541template <>
542EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet1Xf& from) {
543 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse32_v_f32m1(to, from, unpacket_traits<Packet1Xf>::size);
544}
545
546template <>
547EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet1Xf& from) {
548 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse32_v_f32m1(to, from, unpacket_traits<Packet1Xf>::size);
549}
550
551template <>
552EIGEN_DEVICE_FUNC inline Packet1Xf pgather<float, Packet1Xf>(const float* from, Index stride) {
553 return __riscv_vlse32_v_f32m1(from, stride * sizeof(float), unpacket_traits<Packet1Xf>::size);
554}
555
556template <>
557EIGEN_DEVICE_FUNC inline void pscatter<float, Packet1Xf>(float* to, const Packet1Xf& from, Index stride) {
558 __riscv_vsse32(to, stride * sizeof(float), from, unpacket_traits<Packet1Xf>::size);
559}
560
561template <>
562EIGEN_STRONG_INLINE float pfirst<Packet1Xf>(const Packet1Xf& a) {
563 return __riscv_vfmv_f_s_f32m1_f32(a);
564}
565
566template <>
567EIGEN_STRONG_INLINE Packet1Xf psqrt(const Packet1Xf& a) {
568 return __riscv_vfsqrt_v_f32m1(a, unpacket_traits<Packet1Xf>::size);
569}
570
571template <>
572EIGEN_STRONG_INLINE Packet1Xf print<Packet1Xf>(const Packet1Xf& a) {
573 const Packet1Xf limit = pset1<Packet1Xf>(static_cast<float>(1 << 23));
574 const Packet1Xf abs_a = pabs(a);
575
576 PacketMask32 mask = __riscv_vmfne_vv_f32m1_b32(a, a, unpacket_traits<Packet1Xf>::size);
577 const Packet1Xf x = __riscv_vfadd_vv_f32m1_tumu(mask, a, a, a, unpacket_traits<Packet1Xf>::size);
578 const Packet1Xf new_x = __riscv_vfcvt_f_x_v_f32m1(__riscv_vfcvt_x_f_v_i32m1(a, unpacket_traits<Packet1Xf>::size),
579 unpacket_traits<Packet1Xf>::size);
580
581 mask = __riscv_vmflt_vv_f32m1_b32(abs_a, limit, unpacket_traits<Packet1Xf>::size);
582 Packet1Xf signed_x = __riscv_vfsgnj_vv_f32m1(new_x, x, unpacket_traits<Packet1Xf>::size);
583 return __riscv_vmerge_vvm_f32m1(x, signed_x, mask, unpacket_traits<Packet1Xf>::size);
584}
585
586template <>
587EIGEN_STRONG_INLINE Packet1Xf pfloor<Packet1Xf>(const Packet1Xf& a) {
588 Packet1Xf tmp = print<Packet1Xf>(a);
589 // If greater, subtract one.
590 PacketMask32 mask = __riscv_vmflt_vv_f32m1_b32(a, tmp, unpacket_traits<Packet1Xf>::size);
591 return __riscv_vfsub_vf_f32m1_tumu(mask, tmp, tmp, 1.0f, unpacket_traits<Packet1Xf>::size);
592}
593
594template <>
595EIGEN_STRONG_INLINE Packet1Xf preverse(const Packet1Xf& a) {
596 Packet1Xu idx = __riscv_vrsub_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xf>::size),
597 unpacket_traits<Packet1Xf>::size - 1, unpacket_traits<Packet1Xf>::size);
598 return __riscv_vrgather_vv_f32m1(a, idx, unpacket_traits<Packet1Xf>::size);
599}
600
601template <>
602EIGEN_STRONG_INLINE Packet1Xf pfrexp<Packet1Xf>(const Packet1Xf& a, Packet1Xf& exponent) {
603 return pfrexp_generic(a, exponent);
604}
605
606template <>
607EIGEN_STRONG_INLINE float predux<Packet1Xf>(const Packet1Xf& a) {
608 return __riscv_vfmv_f(__riscv_vfredusum_vs_f32m1_f32m1(
609 a, __riscv_vfmv_v_f_f32m1(0.0f, unpacket_traits<Packet1Xf>::size), unpacket_traits<Packet1Xf>::size));
610}
611
612template <>
613EIGEN_STRONG_INLINE bool predux_any(const Packet1Xf& a) {
614 const PacketMask32 mask =
615 __riscv_vmsne_vx_u32m1_b32(__riscv_vreinterpret_v_f32m1_u32m1(a), 0, unpacket_traits<Packet1Xf>::size);
616 return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size) != 0;
617}
618
619template <>
620EIGEN_STRONG_INLINE bool predux_all(const Packet1Xf& a) {
621 const PacketMask32 mask = __riscv_vmfeq_vf_f32m1_b32(a, 0.0f, unpacket_traits<Packet1Xf>::size);
622 return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size) == 0;
623}
624
625template <>
626EIGEN_STRONG_INLINE Index predux_count(const Packet1Xf& a) {
627 const PacketMask32 mask = __riscv_vmfne_vf_f32m1_b32(a, 0.0f, unpacket_traits<Packet1Xf>::size);
628 return static_cast<Index>(__riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size));
629}
630
631template <>
632EIGEN_STRONG_INLINE float predux_mul<Packet1Xf>(const Packet1Xf& a) {
633 // Multiply the vector by its reverse
634 Packet1Xf prod = __riscv_vfmul_vv_f32m1(preverse(a), a, unpacket_traits<Packet1Xf>::size);
635 Packet1Xf half_prod;
636
637 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
638 half_prod = __riscv_vslidedown_vx_f32m1(prod, 8, unpacket_traits<Packet1Xf>::size);
639 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
640 }
641 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
642 half_prod = __riscv_vslidedown_vx_f32m1(prod, 4, unpacket_traits<Packet1Xf>::size);
643 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
644 }
645 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
646 half_prod = __riscv_vslidedown_vx_f32m1(prod, 2, unpacket_traits<Packet1Xf>::size);
647 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
648 }
649 // Last reduction
650 half_prod = __riscv_vslidedown_vx_f32m1(prod, 1, unpacket_traits<Packet1Xf>::size);
651 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
652
653 // The reduction is done to the first element.
654 return pfirst(prod);
655}
656
657// Reusing the first lane is exact for an idempotent reduction and avoids a NaN seed that becomes poison under
658// finite fast-math.
659template <>
660EIGEN_STRONG_INLINE float predux_min<Packet1Xf>(const Packet1Xf& a) {
661 return __riscv_vfmv_f(__riscv_vfredmin_vs_f32m1_f32m1(a, a, unpacket_traits<Packet1Xf>::size));
662}
663
664template <>
665EIGEN_STRONG_INLINE float predux_max<Packet1Xf>(const Packet1Xf& a) {
666 return __riscv_vfmv_f(__riscv_vfredmax_vs_f32m1_f32m1(a, a, unpacket_traits<Packet1Xf>::size));
667}
668
669template <int N>
670EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xf, N>& kernel) {
671 float buffer[unpacket_traits<Packet1Xf>::size * N];
672 int i = 0;
673
674 for (i = 0; i < N; i++) {
675 __riscv_vsse32(&buffer[i], N * sizeof(float), kernel.packet[i], unpacket_traits<Packet1Xf>::size);
676 }
677
678 for (i = 0; i < N; i++) {
679 kernel.packet[i] =
680 __riscv_vle32_v_f32m1(&buffer[i * unpacket_traits<Packet1Xf>::size], unpacket_traits<Packet1Xf>::size);
681 }
682}
683
684template <>
685EIGEN_STRONG_INLINE Packet1Xf pldexp<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& exponent) {
686 return pldexp_generic(a, exponent);
687}
688
689template <>
690EIGEN_STRONG_INLINE PacketMask32 por(const PacketMask32& a, const PacketMask32& b) {
691 return __riscv_vmor_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
692}
693
694template <>
695EIGEN_STRONG_INLINE PacketMask32 pand(const PacketMask32& a, const PacketMask32& b) {
696 return __riscv_vmand_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
697}
698
699template <>
700EIGEN_STRONG_INLINE PacketMask32 pxor(const PacketMask32& a, const PacketMask32& b) {
701 return __riscv_vmxor_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
702}
703
704template <>
705EIGEN_STRONG_INLINE PacketMask32 pandnot(const PacketMask32& a, const PacketMask32& b) {
706 return __riscv_vmandn_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
707}
708
709template <>
710EIGEN_STRONG_INLINE PacketMask32 pnot(const PacketMask32& a) {
711 return __riscv_vmnot_m_b32(a, unpacket_traits<Packet1Xf>::size);
712}
713
714EIGEN_STRONG_INLINE PacketMask32 pcmp_eq_mask(const Packet1Xf& a, const Packet1Xf& b) {
715 return __riscv_vmfeq_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
716}
717
718EIGEN_STRONG_INLINE PacketMask32 pcmp_lt_mask(const Packet1Xf& a, const Packet1Xf& b) {
719 return __riscv_vmflt_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
720}
721
722EIGEN_STRONG_INLINE Packet1Xf pselect(const PacketMask32& mask, const Packet1Xf& a, const Packet1Xf& b) {
723 return __riscv_vmerge_vvm_f32m1(b, a, mask, unpacket_traits<Packet1Xf>::size);
724}
725
726EIGEN_STRONG_INLINE Packet1Xf pselect(const Packet1Xf& mask, const Packet1Xf& a, const Packet1Xf& b) {
727 PacketMask32 mask2 =
728 __riscv_vmsne_vx_i32m1_b32(__riscv_vreinterpret_v_f32m1_i32m1(mask), 0, unpacket_traits<Packet1Xf>::size);
729 return __riscv_vmerge_vvm_f32m1(b, a, mask2, unpacket_traits<Packet1Xf>::size);
730}
731
732/********************************* Packet1Xl ************************************/
733
734template <>
735EIGEN_STRONG_INLINE Packet1Xl pset1<Packet1Xl>(const numext::int64_t& from) {
736 return __riscv_vmv_v_x_i64m1(from, unpacket_traits<Packet1Xl>::size);
737}
738
739template <>
740EIGEN_STRONG_INLINE Packet1Xl plset<Packet1Xl>(const numext::int64_t& a) {
741 Packet1Xl idx = __riscv_vreinterpret_v_u64m1_i64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size));
742 return __riscv_vadd_vx_i64m1(idx, a, unpacket_traits<Packet1Xl>::size);
743}
744
745template <>
746EIGEN_STRONG_INLINE Packet1Xl pzero<Packet1Xl>(const Packet1Xl& /*a*/) {
747 return __riscv_vmv_v_x_i64m1(0, unpacket_traits<Packet1Xl>::size);
748}
749
750template <>
751EIGEN_STRONG_INLINE Packet1Xl padd<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
752 return __riscv_vadd_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
753}
754
755template <>
756EIGEN_STRONG_INLINE Packet1Xl psub<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
757 return __riscv_vsub(a, b, unpacket_traits<Packet1Xl>::size);
758}
759
760template <>
761EIGEN_STRONG_INLINE Packet1Xl pnegate(const Packet1Xl& a) {
762 return __riscv_vneg(a, unpacket_traits<Packet1Xl>::size);
763}
764
765template <>
766EIGEN_STRONG_INLINE Packet1Xl pmul<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
767 return __riscv_vmul(a, b, unpacket_traits<Packet1Xl>::size);
768}
769
770template <>
771EIGEN_STRONG_INLINE Packet1Xl pdiv<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
772 return __riscv_vdiv(a, b, unpacket_traits<Packet1Xl>::size);
773}
774
775template <>
776EIGEN_STRONG_INLINE Packet1Xl pmadd(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
777 return __riscv_vmadd(a, b, c, unpacket_traits<Packet1Xl>::size);
778}
779
780template <>
781EIGEN_STRONG_INLINE Packet1Xl pmsub(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
782 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet1Xl>::size);
783}
784
785template <>
786EIGEN_STRONG_INLINE Packet1Xl pnmadd(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
787 return __riscv_vnmsub_vv_i64m1(a, b, c, unpacket_traits<Packet1Xl>::size);
788}
789
790template <>
791EIGEN_STRONG_INLINE Packet1Xl pnmsub(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
792 return __riscv_vnmsub_vv_i64m1(a, b, pnegate(c), unpacket_traits<Packet1Xl>::size);
793}
794
795template <>
796EIGEN_STRONG_INLINE Packet1Xl pmin<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
797 return __riscv_vmin(a, b, unpacket_traits<Packet1Xl>::size);
798}
799
800template <>
801EIGEN_STRONG_INLINE Packet1Xl pmax<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
802 return __riscv_vmax(a, b, unpacket_traits<Packet1Xl>::size);
803}
804
805template <>
806EIGEN_STRONG_INLINE Packet1Xl pcmp_le<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
807 PacketMask64 mask = __riscv_vmsle_vv_i64m1_b64(a, b, unpacket_traits<Packet1Xl>::size);
808 return __riscv_vmerge_vxm_i64m1(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet1Xl>::size);
809}
810
811template <>
812EIGEN_STRONG_INLINE Packet1Xl pcmp_lt<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
813 PacketMask64 mask = __riscv_vmslt_vv_i64m1_b64(a, b, unpacket_traits<Packet1Xl>::size);
814 return __riscv_vmerge_vxm_i64m1(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet1Xl>::size);
815}
816
817template <>
818EIGEN_STRONG_INLINE Packet1Xl pcmp_eq<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
819 PacketMask64 mask = __riscv_vmseq_vv_i64m1_b64(a, b, unpacket_traits<Packet1Xl>::size);
820 return __riscv_vmerge_vxm_i64m1(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet1Xl>::size);
821}
822
823template <>
824EIGEN_STRONG_INLINE Packet1Xl ptrue<Packet1Xl>(const Packet1Xl& /*a*/) {
825 return __riscv_vmv_v_x_i64m1(0xffffffffffffffffu, unpacket_traits<Packet1Xl>::size);
826}
827
828template <>
829EIGEN_STRONG_INLINE Packet1Xl pand<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
830 return __riscv_vand_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
831}
832
833template <>
834EIGEN_STRONG_INLINE Packet1Xl por<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
835 return __riscv_vor_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
836}
837
838template <>
839EIGEN_STRONG_INLINE Packet1Xl pnot<Packet1Xl>(const Packet1Xl& a) {
840 return __riscv_vnot_v_i64m1(a, unpacket_traits<Packet1Xl>::size);
841}
842
843template <>
844EIGEN_STRONG_INLINE Packet1Xl pxor<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
845 return __riscv_vxor_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
846}
847
848template <>
849EIGEN_STRONG_INLINE Packet1Xl pandnot<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
850#ifndef __riscv_zvbb
851 return __riscv_vand_vv_i64m1(a, __riscv_vnot_v_i64m1(b, unpacket_traits<Packet1Xl>::size),
852 unpacket_traits<Packet1Xl>::size);
853#else
854 return __riscv_vreinterpret_v_u64m1_i64m1(__riscv_vandn_vv_u64m1(
855 __riscv_vreinterpret_v_i64m1_u64m1(a), __riscv_vreinterpret_v_i64m1_u64m1(b), unpacket_traits<Packet1Xl>::size));
856#endif
857}
858
859template <int N>
860EIGEN_STRONG_INLINE Packet1Xl parithmetic_shift_right(const Packet1Xl& a) {
861 return __riscv_vsra_vx_i64m1(a, N, unpacket_traits<Packet1Xl>::size);
862}
863
864template <int N>
865EIGEN_STRONG_INLINE Packet1Xl plogical_shift_right(const Packet1Xl& a) {
866 return __riscv_vreinterpret_i64m1(
867 __riscv_vsrl_vx_u64m1(__riscv_vreinterpret_u64m1(a), N, unpacket_traits<Packet1Xl>::size));
868}
869
870template <int N>
871EIGEN_STRONG_INLINE Packet1Xl plogical_shift_left(const Packet1Xl& a) {
872 return __riscv_vsll_vx_i64m1(a, N, unpacket_traits<Packet1Xl>::size);
873}
874
875template <>
876EIGEN_STRONG_INLINE Packet1Xl pload<Packet1Xl>(const numext::int64_t* from) {
877 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle64_v_i64m1(from, unpacket_traits<Packet1Xl>::size);
878}
879
880template <>
881EIGEN_STRONG_INLINE Packet1Xl ploadu<Packet1Xl>(const numext::int64_t* from) {
882 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle64_v_i64m1(from, unpacket_traits<Packet1Xl>::size);
883}
884
885template <>
886EIGEN_STRONG_INLINE Packet1Xl ploaddup<Packet1Xl>(const numext::int64_t* from) {
887 Packet1Xul idx =
888 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size), 1, unpacket_traits<Packet1Xl>::size);
889 return __riscv_vrgather_vv_i64m1(__riscv_vle64_v_i64m1(from, unpacket_traits<Packet1Xl>::size / 2), idx,
890 unpacket_traits<Packet1Xl>::size);
891}
892
893template <>
894EIGEN_STRONG_INLINE Packet1Xl ploadquad<Packet1Xl>(const numext::int64_t* from) {
895 Packet1Xul idx =
896 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size), 2, unpacket_traits<Packet1Xl>::size);
897 return __riscv_vrgather_vv_i64m1(pload<Packet1Xl>(from), idx, unpacket_traits<Packet1Xl>::size);
898}
899
900template <>
901EIGEN_STRONG_INLINE void pstore<numext::int64_t>(numext::int64_t* to, const Packet1Xl& from) {
902 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse64_v_i64m1(to, from, unpacket_traits<Packet1Xl>::size);
903}
904
905template <>
906EIGEN_STRONG_INLINE void pstoreu<numext::int64_t>(numext::int64_t* to, const Packet1Xl& from) {
907 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse64_v_i64m1(to, from, unpacket_traits<Packet1Xl>::size);
908}
909
910template <>
911EIGEN_DEVICE_FUNC inline Packet1Xl pgather<numext::int64_t, Packet1Xl>(const numext::int64_t* from, Index stride) {
912 return __riscv_vlse64_v_i64m1(from, stride * sizeof(numext::int64_t), unpacket_traits<Packet1Xl>::size);
913}
914
915template <>
916EIGEN_DEVICE_FUNC inline void pscatter<numext::int64_t, Packet1Xl>(numext::int64_t* to, const Packet1Xl& from,
917 Index stride) {
918 __riscv_vsse64(to, stride * sizeof(numext::int64_t), from, unpacket_traits<Packet1Xl>::size);
919}
920
921template <>
922EIGEN_STRONG_INLINE numext::int64_t pfirst<Packet1Xl>(const Packet1Xl& a) {
923 return __riscv_vmv_x_s_i64m1_i64(a);
924}
925
926template <>
927EIGEN_STRONG_INLINE Packet1Xl preverse(const Packet1Xl& a) {
928 Packet1Xul idx = __riscv_vrsub_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size),
929 unpacket_traits<Packet1Xl>::size - 1, unpacket_traits<Packet1Xl>::size);
930 return __riscv_vrgather_vv_i64m1(a, idx, unpacket_traits<Packet1Xl>::size);
931}
932
933template <>
934EIGEN_STRONG_INLINE Packet1Xl pabs(const Packet1Xl& a) {
935 Packet1Xl mask = __riscv_vsra_vx_i64m1(a, 63, unpacket_traits<Packet1Xl>::size);
936 return __riscv_vsub_vv_i64m1(__riscv_vxor_vv_i64m1(a, mask, unpacket_traits<Packet1Xl>::size), mask,
937 unpacket_traits<Packet1Xl>::size);
938}
939
940template <>
941EIGEN_STRONG_INLINE numext::int64_t predux<Packet1Xl>(const Packet1Xl& a) {
942 return __riscv_vmv_x(__riscv_vredsum_vs_i64m1_i64m1(a, __riscv_vmv_v_x_i64m1(0, unpacket_traits<Packet1Xl>::size),
943 unpacket_traits<Packet1Xl>::size));
944}
945
946template <>
947EIGEN_STRONG_INLINE bool predux_all(const Packet1Xl& a) {
948 const PacketMask64 mask = __riscv_vmseq_vx_i64m1_b64(a, 0, unpacket_traits<Packet1Xl>::size);
949 return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xl>::size) == 0;
950}
951
952template <>
953EIGEN_STRONG_INLINE numext::int64_t predux_mul<Packet1Xl>(const Packet1Xl& a) {
954 // Multiply the vector by its reverse
955 Packet1Xl prod = __riscv_vmul_vv_i64m1(preverse(a), a, unpacket_traits<Packet1Xl>::size);
956 Packet1Xl half_prod;
957
958 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
959 half_prod = __riscv_vslidedown_vx_i64m1(prod, 4, unpacket_traits<Packet1Xl>::size);
960 prod = __riscv_vmul_vv_i64m1(prod, half_prod, unpacket_traits<Packet1Xl>::size);
961 }
962 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
963 half_prod = __riscv_vslidedown_vx_i64m1(prod, 2, unpacket_traits<Packet1Xl>::size);
964 prod = __riscv_vmul_vv_i64m1(prod, half_prod, unpacket_traits<Packet1Xl>::size);
965 }
966 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
967 half_prod = __riscv_vslidedown_vx_i64m1(prod, 1, unpacket_traits<Packet1Xl>::size);
968 prod = __riscv_vmul_vv_i64m1(prod, half_prod, unpacket_traits<Packet1Xl>::size);
969 }
970
971 // The reduction is done to the first element.
972 return pfirst(prod);
973}
974
975template <>
976EIGEN_STRONG_INLINE numext::int64_t predux_min<Packet1Xl>(const Packet1Xl& a) {
977 return __riscv_vmv_x(__riscv_vredmin_vs_i64m1_i64m1(
978 a, __riscv_vmv_v_x_i64m1((std::numeric_limits<numext::int64_t>::max)(), unpacket_traits<Packet1Xl>::size),
979 unpacket_traits<Packet1Xl>::size));
980}
981
982template <>
983EIGEN_STRONG_INLINE numext::int64_t predux_max<Packet1Xl>(const Packet1Xl& a) {
984 return __riscv_vmv_x(__riscv_vredmax_vs_i64m1_i64m1(
985 a, __riscv_vmv_v_x_i64m1((std::numeric_limits<numext::int64_t>::min)(), unpacket_traits<Packet1Xl>::size),
986 unpacket_traits<Packet1Xl>::size));
987}
988
989template <int N>
990EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xl, N>& kernel) {
991 numext::int64_t buffer[unpacket_traits<Packet1Xl>::size * N] = {0};
992 int i = 0;
993
994 for (i = 0; i < N; i++) {
995 __riscv_vsse64(&buffer[i], N * sizeof(numext::int64_t), kernel.packet[i], unpacket_traits<Packet1Xl>::size);
996 }
997 for (i = 0; i < N; i++) {
998 kernel.packet[i] =
999 __riscv_vle64_v_i64m1(&buffer[i * unpacket_traits<Packet1Xl>::size], unpacket_traits<Packet1Xl>::size);
1000 }
1001}
1002
1003/********************************* Packet1Xd ************************************/
1004
1005template <>
1006EIGEN_STRONG_INLINE Packet1Xd ptrue<Packet1Xd>(const Packet1Xd& /*a*/) {
1007 Packet1Xd r =
1008 __riscv_vreinterpret_f64m1(__riscv_vmv_v_x_u64m1(0xffffffffffffffffu, unpacket_traits<Packet1Xd>::size));
1009 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1010 return r;
1011}
1012
1013template <>
1014EIGEN_STRONG_INLINE Packet1Xd pzero<Packet1Xd>(const Packet1Xd& /*a*/) {
1015 return __riscv_vfmv_v_f_f64m1(0.0, unpacket_traits<Packet1Xd>::size);
1016}
1017
1018template <>
1019EIGEN_STRONG_INLINE Packet1Xd pabs(const Packet1Xd& a) {
1020 return __riscv_vfabs_v_f64m1(a, unpacket_traits<Packet1Xd>::size);
1021}
1022
1023template <>
1024EIGEN_STRONG_INLINE Packet1Xd pabsdiff(const Packet1Xd& a, const Packet1Xd& b) {
1025 return __riscv_vfabs_v_f64m1(__riscv_vfsub_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size),
1026 unpacket_traits<Packet1Xd>::size);
1027}
1028
1029template <>
1030EIGEN_STRONG_INLINE Packet1Xd pset1<Packet1Xd>(const double& from) {
1031 return __riscv_vfmv_v_f_f64m1(from, unpacket_traits<Packet1Xd>::size);
1032}
1033
1034template <>
1035EIGEN_STRONG_INLINE Packet1Xd pset1frombits<Packet1Xd>(numext::uint64_t from) {
1036 return __riscv_vreinterpret_f64m1(__riscv_vmv_v_x_u64m1(from, unpacket_traits<Packet1Xd>::size));
1037}
1038
1039template <>
1040EIGEN_STRONG_INLINE Packet1Xd plset<Packet1Xd>(const double& a) {
1041 Packet1Xd idx = __riscv_vfcvt_f_x_v_f64m1(
1042 __riscv_vreinterpret_v_u64m1_i64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size)),
1043 unpacket_traits<Packet1Xd>::size);
1044 return __riscv_vfadd_vf_f64m1(idx, a, unpacket_traits<Packet1Xd>::size);
1045}
1046
1047template <>
1048EIGEN_STRONG_INLINE void pbroadcast4<Packet1Xd>(const double* a, Packet1Xd& a0, Packet1Xd& a1, Packet1Xd& a2,
1049 Packet1Xd& a3) {
1050 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
1051 Packet1Xd aa = __riscv_vle64_v_f64m1(a, 4);
1052 a0 = __riscv_vrgather_vx_f64m1(aa, 0, unpacket_traits<Packet1Xd>::size);
1053 a1 = __riscv_vrgather_vx_f64m1(aa, 1, unpacket_traits<Packet1Xd>::size);
1054 a2 = __riscv_vrgather_vx_f64m1(aa, 2, unpacket_traits<Packet1Xd>::size);
1055 a3 = __riscv_vrgather_vx_f64m1(aa, 3, unpacket_traits<Packet1Xd>::size);
1056 } else {
1057 Packet2Xd aa = __riscv_vle64_v_f64m2(a, 4);
1058 Packet1Xd aa0 = __riscv_vget_v_f64m2_f64m1(aa, 0);
1059 Packet1Xd aa1 = __riscv_vget_v_f64m2_f64m1(aa, 1);
1060 a0 = __riscv_vrgather_vx_f64m1(aa0, 0, unpacket_traits<Packet1Xd>::size);
1061 a1 = __riscv_vrgather_vx_f64m1(aa0, 1, unpacket_traits<Packet1Xd>::size);
1062 a2 = __riscv_vrgather_vx_f64m1(aa1, 0, unpacket_traits<Packet1Xd>::size);
1063 a3 = __riscv_vrgather_vx_f64m1(aa1, 1, unpacket_traits<Packet1Xd>::size);
1064 }
1065}
1066
1067template <>
1068EIGEN_STRONG_INLINE Packet1Xd padd<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1069 return __riscv_vfadd_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1070}
1071
1072template <>
1073EIGEN_STRONG_INLINE Packet1Xd psub<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1074 return __riscv_vfsub_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1075}
1076
1077template <>
1078EIGEN_STRONG_INLINE Packet1Xd pnegate(const Packet1Xd& a) {
1079 return __riscv_vfneg_v_f64m1(a, unpacket_traits<Packet1Xd>::size);
1080}
1081
1082template <>
1083EIGEN_STRONG_INLINE Packet1Xd psignbit(const Packet1Xd& a) {
1084 return __riscv_vreinterpret_v_i64m1_f64m1(
1085 __riscv_vsra_vx_i64m1(__riscv_vreinterpret_v_f64m1_i64m1(a), 63, unpacket_traits<Packet1Xl>::size));
1086}
1087
1088template <>
1089EIGEN_STRONG_INLINE Packet1Xd pmul<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1090 return __riscv_vfmul_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1091}
1092
1093template <>
1094EIGEN_STRONG_INLINE Packet1Xd pdiv<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1095 return __riscv_vfdiv_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1096}
1097
1098template <>
1099struct is_fused_madd<Packet1Xd> : true_type {};
1100template <>
1101EIGEN_STRONG_INLINE Packet1Xd pmadd(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1102 return __riscv_vfmadd_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1103}
1104
1105template <>
1106EIGEN_STRONG_INLINE Packet1Xd pmsub(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1107 return __riscv_vfmsub_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1108}
1109
1110template <>
1111EIGEN_STRONG_INLINE Packet1Xd pnmadd(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1112 return __riscv_vfnmsub_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1113}
1114
1115template <>
1116EIGEN_STRONG_INLINE Packet1Xd pnmsub(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1117 return __riscv_vfnmadd_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1118}
1119
1120template <>
1121struct pminmax_propagates_nan<Packet1Xd> : bool_constant<true> {};
1122
1123template <>
1124EIGEN_STRONG_INLINE Packet1Xd pmin<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1125 Packet1Xd nans = __riscv_vfmv_v_f_f64m1((std::numeric_limits<double>::quiet_NaN)(), unpacket_traits<Packet1Xd>::size);
1126 PacketMask64 mask = __riscv_vmfeq_vv_f64m1_b64(a, a, unpacket_traits<Packet1Xd>::size);
1127 PacketMask64 mask2 = __riscv_vmfeq_vv_f64m1_b64(b, b, unpacket_traits<Packet1Xd>::size);
1128 mask = __riscv_vmand_mm_b64(mask, mask2, unpacket_traits<Packet1Xd>::size);
1129
1130 return __riscv_vfmin_vv_f64m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xd>::size);
1131}
1132
1133template <>
1134EIGEN_STRONG_INLINE Packet1Xd pmin<PropagateNumbers, Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1135 return __riscv_vfmin_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1136}
1137
1138template <>
1139EIGEN_STRONG_INLINE Packet1Xd pmax<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1140 Packet1Xd nans = __riscv_vfmv_v_f_f64m1((std::numeric_limits<double>::quiet_NaN)(), unpacket_traits<Packet1Xd>::size);
1141 PacketMask64 mask = __riscv_vmfeq_vv_f64m1_b64(a, a, unpacket_traits<Packet1Xd>::size);
1142 PacketMask64 mask2 = __riscv_vmfeq_vv_f64m1_b64(b, b, unpacket_traits<Packet1Xd>::size);
1143 mask = __riscv_vmand_mm_b64(mask, mask2, unpacket_traits<Packet1Xd>::size);
1144
1145 return __riscv_vfmax_vv_f64m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xd>::size);
1146}
1147
1148template <>
1149EIGEN_STRONG_INLINE Packet1Xd pmax<PropagateNumbers, Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1150 return __riscv_vfmax_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1151}
1152
1153template <>
1154EIGEN_STRONG_INLINE Packet1Xd pcmp_le<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1155 PacketMask64 mask = __riscv_vmfle_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1156 return __riscv_vmerge_vvm_f64m1(pzero<Packet1Xd>(a), ptrue<Packet1Xd>(a), mask, unpacket_traits<Packet1Xd>::size);
1157}
1158
1159template <>
1160EIGEN_STRONG_INLINE Packet1Xd pcmp_lt<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1161 PacketMask64 mask = __riscv_vmflt_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1162 return __riscv_vmerge_vvm_f64m1(pzero<Packet1Xd>(a), ptrue<Packet1Xd>(a), mask, unpacket_traits<Packet1Xd>::size);
1163}
1164
1165template <>
1166EIGEN_STRONG_INLINE Packet1Xd pcmp_eq<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1167 PacketMask64 mask = __riscv_vmfeq_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1168 return __riscv_vmerge_vvm_f64m1(pzero<Packet1Xd>(a), ptrue<Packet1Xd>(a), mask, unpacket_traits<Packet1Xd>::size);
1169}
1170
1171template <>
1172EIGEN_STRONG_INLINE Packet1Xd pcmp_lt_or_nan<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1173 PacketMask64 mask = __riscv_vmfge_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1174 return __riscv_vfmerge_vfm_f64m1(ptrue<Packet1Xd>(a), 0.0, mask, unpacket_traits<Packet1Xd>::size);
1175}
1176
1177// Logical Operations are not supported for double, so reinterpret casts
1178template <>
1179EIGEN_STRONG_INLINE Packet1Xd pand<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1180 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vand_vv_u64m1(
1181 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size));
1182}
1183
1184template <>
1185EIGEN_STRONG_INLINE Packet1Xd por<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1186 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vor_vv_u64m1(
1187 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size));
1188}
1189
1190template <>
1191EIGEN_STRONG_INLINE Packet1Xd pxor<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1192 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vxor_vv_u64m1(
1193 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size));
1194}
1195
1196template <>
1197EIGEN_STRONG_INLINE Packet1Xd pandnot<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1198#ifndef __riscv_zvbb
1199 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vand_vv_u64m1(
1200 __riscv_vreinterpret_v_f64m1_u64m1(a),
1201 __riscv_vnot_v_u64m1(__riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size),
1202 unpacket_traits<Packet1Xd>::size));
1203#else
1204 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vandn_vv_u64m1(
1205 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xl>::size));
1206#endif
1207}
1208
1209template <>
1210EIGEN_STRONG_INLINE Packet1Xd pnot<Packet1Xd>(const Packet1Xd& a) {
1211 return __riscv_vreinterpret_v_u64m1_f64m1(
1212 __riscv_vnot_v_u64m1(__riscv_vreinterpret_v_f64m1_u64m1(a), unpacket_traits<Packet1Xd>::size));
1213}
1214
1215template <>
1216EIGEN_STRONG_INLINE Packet1Xd pload<Packet1Xd>(const double* from) {
1217 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle64_v_f64m1(from, unpacket_traits<Packet1Xd>::size);
1218}
1219
1220template <>
1221EIGEN_STRONG_INLINE Packet1Xd ploadu<Packet1Xd>(const double* from) {
1222 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle64_v_f64m1(from, unpacket_traits<Packet1Xd>::size);
1223}
1224
1225EIGEN_STRONG_INLINE Packet2Xd pdup(const Packet1Xd& a) {
1226 Packet2Xul idx =
1227 __riscv_vsrl_vx_u64m2(__riscv_vid_v_u64m2(unpacket_traits<Packet2Xd>::size), 1, unpacket_traits<Packet2Xd>::size);
1228 return __riscv_vrgather_vv_f64m2(__riscv_vlmul_ext_v_f64m1_f64m2(a), idx, unpacket_traits<Packet2Xd>::size);
1229}
1230
1231template <>
1232EIGEN_STRONG_INLINE Packet1Xd ploaddup<Packet1Xd>(const double* from) {
1233 Packet1Xul idx =
1234 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xd>::size), 1, unpacket_traits<Packet1Xd>::size);
1235 return __riscv_vrgather_vv_f64m1(__riscv_vle64_v_f64m1(from, unpacket_traits<Packet1Xd>::size / 2), idx,
1236 unpacket_traits<Packet1Xd>::size);
1237}
1238
1239template <>
1240EIGEN_STRONG_INLINE Packet1Xd ploadquad<Packet1Xd>(const double* from) {
1241 Packet1Xul idx =
1242 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xd>::size), 2, unpacket_traits<Packet1Xd>::size);
1243 return __riscv_vrgather_vv_f64m1(pload<Packet1Xd>(from), idx, unpacket_traits<Packet1Xd>::size);
1244}
1245
1246template <>
1247EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet1Xd& from) {
1248 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse64_v_f64m1(to, from, unpacket_traits<Packet1Xd>::size);
1249}
1250
1251template <>
1252EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet1Xd& from) {
1253 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse64_v_f64m1(to, from, unpacket_traits<Packet1Xd>::size);
1254}
1255
1256template <>
1257EIGEN_DEVICE_FUNC inline Packet1Xd pgather<double, Packet1Xd>(const double* from, Index stride) {
1258 return __riscv_vlse64_v_f64m1(from, stride * sizeof(double), unpacket_traits<Packet1Xd>::size);
1259}
1260
1261template <>
1262EIGEN_DEVICE_FUNC inline void pscatter<double, Packet1Xd>(double* to, const Packet1Xd& from, Index stride) {
1263 __riscv_vsse64(to, stride * sizeof(double), from, unpacket_traits<Packet1Xd>::size);
1264}
1265
1266template <>
1267EIGEN_STRONG_INLINE double pfirst<Packet1Xd>(const Packet1Xd& a) {
1268 return __riscv_vfmv_f_s_f64m1_f64(a);
1269}
1270
1271template <>
1272EIGEN_STRONG_INLINE Packet1Xd psqrt(const Packet1Xd& a) {
1273 return __riscv_vfsqrt_v_f64m1(a, unpacket_traits<Packet1Xd>::size);
1274}
1275
1276template <>
1277EIGEN_STRONG_INLINE Packet1Xd print<Packet1Xd>(const Packet1Xd& a) {
1278 const Packet1Xd limit = pset1<Packet1Xd>(static_cast<double>(1ull << 52));
1279 const Packet1Xd abs_a = pabs(a);
1280
1281 PacketMask64 mask = __riscv_vmfne_vv_f64m1_b64(a, a, unpacket_traits<Packet1Xd>::size);
1282 const Packet1Xd x = __riscv_vfadd_vv_f64m1_tumu(mask, a, a, a, unpacket_traits<Packet1Xd>::size);
1283 const Packet1Xd new_x = __riscv_vfcvt_f_x_v_f64m1(__riscv_vfcvt_x_f_v_i64m1(a, unpacket_traits<Packet1Xd>::size),
1284 unpacket_traits<Packet1Xd>::size);
1285
1286 mask = __riscv_vmflt_vv_f64m1_b64(abs_a, limit, unpacket_traits<Packet1Xd>::size);
1287 Packet1Xd signed_x = __riscv_vfsgnj_vv_f64m1(new_x, x, unpacket_traits<Packet1Xd>::size);
1288 return __riscv_vmerge_vvm_f64m1(x, signed_x, mask, unpacket_traits<Packet1Xd>::size);
1289}
1290
1291template <>
1292EIGEN_STRONG_INLINE Packet1Xd pfloor<Packet1Xd>(const Packet1Xd& a) {
1293 Packet1Xd tmp = print<Packet1Xd>(a);
1294 // If greater, subtract one.
1295 PacketMask64 mask = __riscv_vmflt_vv_f64m1_b64(a, tmp, unpacket_traits<Packet1Xd>::size);
1296 return __riscv_vfsub_vf_f64m1_tumu(mask, tmp, tmp, 1.0, unpacket_traits<Packet1Xd>::size);
1297}
1298
1299template <>
1300EIGEN_STRONG_INLINE Packet1Xd preverse(const Packet1Xd& a) {
1301 Packet1Xul idx = __riscv_vrsub_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xd>::size),
1302 unpacket_traits<Packet1Xd>::size - 1, unpacket_traits<Packet1Xd>::size);
1303 return __riscv_vrgather_vv_f64m1(a, idx, unpacket_traits<Packet1Xd>::size);
1304}
1305
1306template <>
1307EIGEN_STRONG_INLINE Packet1Xd pfrexp<Packet1Xd>(const Packet1Xd& a, Packet1Xd& exponent) {
1308 return pfrexp_generic(a, exponent);
1309}
1310
1311template <>
1312EIGEN_STRONG_INLINE double predux<Packet1Xd>(const Packet1Xd& a) {
1313 return __riscv_vfmv_f(__riscv_vfredusum_vs_f64m1_f64m1(
1314 a, __riscv_vfmv_v_f_f64m1(0.0, unpacket_traits<Packet1Xd>::size), unpacket_traits<Packet1Xd>::size));
1315}
1316
1317template <>
1318EIGEN_STRONG_INLINE bool predux_any(const Packet1Xd& a) {
1319 const PacketMask64 mask =
1320 __riscv_vmsne_vx_u64m1_b64(__riscv_vreinterpret_v_f64m1_u64m1(a), 0, unpacket_traits<Packet1Xd>::size);
1321 return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size) != 0;
1322}
1323
1324template <>
1325EIGEN_STRONG_INLINE bool predux_all(const Packet1Xd& a) {
1326 const PacketMask64 mask = __riscv_vmfeq_vf_f64m1_b64(a, 0.0, unpacket_traits<Packet1Xd>::size);
1327 return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size) == 0;
1328}
1329
1330template <>
1331EIGEN_STRONG_INLINE Index predux_count(const Packet1Xd& a) {
1332 const PacketMask64 mask = __riscv_vmfne_vf_f64m1_b64(a, 0.0, unpacket_traits<Packet1Xd>::size);
1333 return static_cast<Index>(__riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size));
1334}
1335
1336template <>
1337EIGEN_STRONG_INLINE double predux_mul<Packet1Xd>(const Packet1Xd& a) {
1338 // Multiply the vector by its reverse
1339 Packet1Xd prod = __riscv_vfmul_vv_f64m1(preverse(a), a, unpacket_traits<Packet1Xd>::size);
1340 Packet1Xd half_prod;
1341
1342 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
1343 half_prod = __riscv_vslidedown_vx_f64m1(prod, 4, unpacket_traits<Packet1Xd>::size);
1344 prod = __riscv_vfmul_vv_f64m1(prod, half_prod, unpacket_traits<Packet1Xd>::size);
1345 }
1346 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
1347 half_prod = __riscv_vslidedown_vx_f64m1(prod, 2, unpacket_traits<Packet1Xd>::size);
1348 prod = __riscv_vfmul_vv_f64m1(prod, half_prod, unpacket_traits<Packet1Xd>::size);
1349 }
1350 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
1351 half_prod = __riscv_vslidedown_vx_f64m1(prod, 1, unpacket_traits<Packet1Xd>::size);
1352 prod = __riscv_vfmul_vv_f64m1(prod, half_prod, unpacket_traits<Packet1Xd>::size);
1353 }
1354
1355 // The reduction is done to the first element.
1356 return pfirst(prod);
1357}
1358
1359template <>
1360EIGEN_STRONG_INLINE double predux_min<Packet1Xd>(const Packet1Xd& a) {
1361 return __riscv_vfmv_f(__riscv_vfredmin_vs_f64m1_f64m1(a, a, unpacket_traits<Packet1Xd>::size));
1362}
1363
1364template <>
1365EIGEN_STRONG_INLINE double predux_max<Packet1Xd>(const Packet1Xd& a) {
1366 return __riscv_vfmv_f(__riscv_vfredmax_vs_f64m1_f64m1(a, a, unpacket_traits<Packet1Xd>::size));
1367}
1368
1369template <int N>
1370EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xd, N>& kernel) {
1371 double buffer[unpacket_traits<Packet1Xd>::size * N];
1372 int i = 0;
1373
1374 for (i = 0; i < N; i++) {
1375 __riscv_vsse64(&buffer[i], N * sizeof(double), kernel.packet[i], unpacket_traits<Packet1Xd>::size);
1376 }
1377
1378 for (i = 0; i < N; i++) {
1379 kernel.packet[i] =
1380 __riscv_vle64_v_f64m1(&buffer[i * unpacket_traits<Packet1Xd>::size], unpacket_traits<Packet1Xd>::size);
1381 }
1382}
1383
1384template <>
1385EIGEN_STRONG_INLINE Packet1Xd pldexp<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& exponent) {
1386 return pldexp_generic(a, exponent);
1387}
1388
1389template <>
1390EIGEN_STRONG_INLINE PacketMask64 por(const PacketMask64& a, const PacketMask64& b) {
1391 return __riscv_vmor_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1392}
1393
1394template <>
1395EIGEN_STRONG_INLINE PacketMask64 pandnot(const PacketMask64& a, const PacketMask64& b) {
1396 return __riscv_vmnand_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1397}
1398
1399template <>
1400EIGEN_STRONG_INLINE PacketMask64 pand(const PacketMask64& a, const PacketMask64& b) {
1401 return __riscv_vmand_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1402}
1403
1404template <>
1405EIGEN_STRONG_INLINE PacketMask64 pxor(const PacketMask64& a, const PacketMask64& b) {
1406 return __riscv_vmxor_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1407}
1408
1409template <>
1410EIGEN_STRONG_INLINE PacketMask64 pnot(const PacketMask64& a) {
1411 return __riscv_vmnot_m_b64(a, unpacket_traits<Packet1Xd>::size);
1412}
1413
1414EIGEN_STRONG_INLINE PacketMask64 pcmp_eq_mask(const Packet1Xd& a, const Packet1Xd& b) {
1415 return __riscv_vmfeq_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1416}
1417
1418EIGEN_STRONG_INLINE PacketMask64 pcmp_lt_mask(const Packet1Xd& a, const Packet1Xd& b) {
1419 return __riscv_vmflt_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1420}
1421
1422EIGEN_STRONG_INLINE Packet1Xd pselect(const PacketMask64& mask, const Packet1Xd& a, const Packet1Xd& b) {
1423 return __riscv_vmerge_vvm_f64m1(b, a, mask, unpacket_traits<Packet1Xd>::size);
1424}
1425
1426EIGEN_STRONG_INLINE Packet1Xd pselect(const Packet1Xd& mask, const Packet1Xd& a, const Packet1Xd& b) {
1427 PacketMask64 mask2 =
1428 __riscv_vmsne_vx_i64m1_b64(__riscv_vreinterpret_v_f64m1_i64m1(mask), 0, unpacket_traits<Packet1Xd>::size);
1429 return __riscv_vmerge_vvm_f64m1(b, a, mask2, unpacket_traits<Packet1Xd>::size);
1430}
1431
1432/********************************* Packet1Xs ************************************/
1433
1434EIGEN_STRONG_INLINE Packet1Xs __riscv_vreinterpret_v_u32m1_i16m1(const Packet1Xu& a) {
1435 return __riscv_vreinterpret_v_i32m1_i16m1(__riscv_vreinterpret_v_u32m1_i32m1(a));
1436}
1437
1438template <>
1439EIGEN_STRONG_INLINE Packet1Xs pset1<Packet1Xs>(const numext::int16_t& from) {
1440 return __riscv_vmv_v_x_i16m1(from, unpacket_traits<Packet1Xs>::size);
1441}
1442
1443template <>
1444EIGEN_STRONG_INLINE Packet1Xs plset<Packet1Xs>(const numext::int16_t& a) {
1445 Packet1Xs idx = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vid_v_u16m1(unpacket_traits<Packet1Xs>::size));
1446 return __riscv_vadd_vx_i16m1(idx, a, unpacket_traits<Packet1Xs>::size);
1447}
1448
1449template <>
1450EIGEN_STRONG_INLINE Packet1Xs pzero<Packet1Xs>(const Packet1Xs& /*a*/) {
1451 return __riscv_vmv_v_x_i16m1(0, unpacket_traits<Packet1Xs>::size);
1452}
1453
1454template <>
1455EIGEN_STRONG_INLINE Packet1Xs padd<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1456 return __riscv_vadd_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1457}
1458
1459template <>
1460EIGEN_STRONG_INLINE Packet1Xs psub<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1461 return __riscv_vsub(a, b, unpacket_traits<Packet1Xs>::size);
1462}
1463
1464template <>
1465EIGEN_STRONG_INLINE Packet1Xs pnegate(const Packet1Xs& a) {
1466 return __riscv_vneg(a, unpacket_traits<Packet1Xs>::size);
1467}
1468
1469template <>
1470EIGEN_STRONG_INLINE Packet1Xs pmul<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1471 return __riscv_vmul(a, b, unpacket_traits<Packet1Xs>::size);
1472}
1473
1474template <>
1475EIGEN_STRONG_INLINE Packet1Xs pdiv<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1476 return __riscv_vdiv(a, b, unpacket_traits<Packet1Xs>::size);
1477}
1478
1479template <>
1480EIGEN_STRONG_INLINE Packet1Xs pmadd(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1481 return __riscv_vmadd(a, b, c, unpacket_traits<Packet1Xs>::size);
1482}
1483
1484template <>
1485EIGEN_STRONG_INLINE Packet1Xs pmsub(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1486 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet1Xs>::size);
1487}
1488
1489template <>
1490EIGEN_STRONG_INLINE Packet1Xs pnmadd(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1491 return __riscv_vnmsub_vv_i16m1(a, b, c, unpacket_traits<Packet1Xs>::size);
1492}
1493
1494template <>
1495EIGEN_STRONG_INLINE Packet1Xs pnmsub(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1496 return __riscv_vnmsub_vv_i16m1(a, b, pnegate(c), unpacket_traits<Packet1Xs>::size);
1497}
1498
1499template <>
1500EIGEN_STRONG_INLINE Packet1Xs pmin<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1501 return __riscv_vmin(a, b, unpacket_traits<Packet1Xs>::size);
1502}
1503
1504template <>
1505EIGEN_STRONG_INLINE Packet1Xs pmax<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1506 return __riscv_vmax(a, b, unpacket_traits<Packet1Xs>::size);
1507}
1508
1509template <>
1510EIGEN_STRONG_INLINE Packet1Xs pcmp_le<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1511 PacketMask16 mask = __riscv_vmsle_vv_i16m1_b16(a, b, unpacket_traits<Packet1Xs>::size);
1512 return __riscv_vmerge_vxm_i16m1(pzero(a), static_cast<short>(0xffff), mask, unpacket_traits<Packet1Xs>::size);
1513}
1514
1515template <>
1516EIGEN_STRONG_INLINE Packet1Xs pcmp_lt<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1517 PacketMask16 mask = __riscv_vmslt_vv_i16m1_b16(a, b, unpacket_traits<Packet1Xs>::size);
1518 return __riscv_vmerge_vxm_i16m1(pzero(a), static_cast<short>(0xffff), mask, unpacket_traits<Packet1Xs>::size);
1519}
1520
1521template <>
1522EIGEN_STRONG_INLINE Packet1Xs pcmp_eq<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1523 PacketMask16 mask = __riscv_vmseq_vv_i16m1_b16(a, b, unpacket_traits<Packet1Xs>::size);
1524 return __riscv_vmerge_vxm_i16m1(pzero(a), static_cast<short>(0xffff), mask, unpacket_traits<Packet1Xs>::size);
1525}
1526
1527template <>
1528EIGEN_STRONG_INLINE Packet1Xs ptrue<Packet1Xs>(const Packet1Xs& /*a*/) {
1529 return __riscv_vmv_v_x_i16m1(static_cast<unsigned short>(0xffffu), unpacket_traits<Packet1Xs>::size);
1530}
1531
1532template <>
1533EIGEN_STRONG_INLINE Packet1Xs pand<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1534 return __riscv_vand_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1535}
1536
1537template <>
1538EIGEN_STRONG_INLINE Packet1Xs por<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1539 return __riscv_vor_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1540}
1541
1542template <>
1543EIGEN_STRONG_INLINE Packet1Xs pxor<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1544 return __riscv_vxor_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1545}
1546
1547template <>
1548EIGEN_STRONG_INLINE Packet1Xs pnot<Packet1Xs>(const Packet1Xs& a) {
1549 return __riscv_vnot_v_i16m1(a, unpacket_traits<Packet1Xs>::size);
1550}
1551
1552template <>
1553EIGEN_STRONG_INLINE Packet1Xs pandnot<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1554#ifndef __riscv_zvbb
1555 return __riscv_vand_vv_i16m1(a, __riscv_vnot_v_i16m1(b, unpacket_traits<Packet1Xs>::size),
1556 unpacket_traits<Packet1Xs>::size);
1557#else
1558 return __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vandn_vv_u16m1(
1559 __riscv_vreinterpret_v_i16m1_u16m1(a), __riscv_vreinterpret_v_i16m1_u16m1(b), unpacket_traits<Packet1Xs>::size));
1560#endif
1561}
1562
1563template <int N>
1564EIGEN_STRONG_INLINE Packet1Xs parithmetic_shift_right(const Packet1Xs& a) {
1565 return __riscv_vsra_vx_i16m1(a, N, unpacket_traits<Packet1Xs>::size);
1566}
1567
1568template <int N>
1569EIGEN_STRONG_INLINE Packet1Xs plogical_shift_right(const Packet1Xs& a) {
1570 return __riscv_vreinterpret_i16m1(
1571 __riscv_vsrl_vx_u16m1(__riscv_vreinterpret_u16m1(a), N, unpacket_traits<Packet1Xs>::size));
1572}
1573
1574template <int N>
1575EIGEN_STRONG_INLINE Packet1Xs plogical_shift_left(const Packet1Xs& a) {
1576 return __riscv_vsll_vx_i16m1(a, N, unpacket_traits<Packet1Xs>::size);
1577}
1578
1579template <>
1580EIGEN_STRONG_INLINE Packet1Xs pload<Packet1Xs>(const numext::int16_t* from) {
1581 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle16_v_i16m1(from, unpacket_traits<Packet1Xs>::size);
1582}
1583
1584template <>
1585EIGEN_STRONG_INLINE Packet1Xs ploadu<Packet1Xs>(const numext::int16_t* from) {
1586 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle16_v_i16m1(from, unpacket_traits<Packet1Xs>::size);
1587}
1588
1589template <>
1590EIGEN_STRONG_INLINE Packet1Xs ploaddup<Packet1Xs>(const numext::int16_t* from) {
1591 Packet1Xu data = __riscv_vlmul_trunc_v_u32m2_u32m1(__riscv_vwcvtu_x_x_v_u32m2(
1592 __riscv_vreinterpret_v_i16m1_u16m1(__riscv_vle16_v_i16m1(from, unpacket_traits<Packet1Xs>::size / 2)),
1593 unpacket_traits<Packet1Xs>::size / 2));
1594 return __riscv_vreinterpret_v_u32m1_i16m1(__riscv_vadd_vv_u32m1(
1595 __riscv_vsll_vx_u32m1(data, 16, unpacket_traits<Packet1Xi>::size), data, unpacket_traits<Packet1Xi>::size));
1596}
1597
1598template <>
1599EIGEN_STRONG_INLINE Packet1Xs ploadquad<Packet1Xs>(const numext::int16_t* from) {
1600 Packet1Xsu idx =
1601 __riscv_vsrl_vx_u16m1(__riscv_vid_v_u16m1(unpacket_traits<Packet1Xs>::size), 2, unpacket_traits<Packet1Xs>::size);
1602 return __riscv_vrgather_vv_i16m1(pload<Packet1Xs>(from), idx, unpacket_traits<Packet1Xs>::size);
1603}
1604
1605template <>
1606EIGEN_STRONG_INLINE void pstore<numext::int16_t>(numext::int16_t* to, const Packet1Xs& from) {
1607 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse16_v_i16m1(to, from, unpacket_traits<Packet1Xs>::size);
1608}
1609
1610template <>
1611EIGEN_STRONG_INLINE void pstoreu<numext::int16_t>(numext::int16_t* to, const Packet1Xs& from) {
1612 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse16_v_i16m1(to, from, unpacket_traits<Packet1Xs>::size);
1613}
1614
1615template <>
1616EIGEN_DEVICE_FUNC inline Packet1Xs pgather<numext::int16_t, Packet1Xs>(const numext::int16_t* from, Index stride) {
1617 return __riscv_vlse16_v_i16m1(from, stride * sizeof(numext::int16_t), unpacket_traits<Packet1Xs>::size);
1618}
1619
1620template <>
1621EIGEN_DEVICE_FUNC inline void pscatter<numext::int16_t, Packet1Xs>(numext::int16_t* to, const Packet1Xs& from,
1622 Index stride) {
1623 __riscv_vsse16(to, stride * sizeof(numext::int16_t), from, unpacket_traits<Packet1Xs>::size);
1624}
1625
1626template <>
1627EIGEN_STRONG_INLINE numext::int16_t pfirst<Packet1Xs>(const Packet1Xs& a) {
1628 return __riscv_vmv_x_s_i16m1_i16(a);
1629}
1630
1631template <>
1632EIGEN_STRONG_INLINE Packet1Xs preverse(const Packet1Xs& a) {
1633 Packet1Xsu idx = __riscv_vrsub_vx_u16m1(__riscv_vid_v_u16m1(unpacket_traits<Packet1Xs>::size),
1634 unpacket_traits<Packet1Xs>::size - 1, unpacket_traits<Packet1Xs>::size);
1635 return __riscv_vrgather_vv_i16m1(a, idx, unpacket_traits<Packet1Xs>::size);
1636}
1637
1638template <>
1639EIGEN_STRONG_INLINE Packet1Xs pabs(const Packet1Xs& a) {
1640 Packet1Xs mask = __riscv_vsra_vx_i16m1(a, 15, unpacket_traits<Packet1Xs>::size);
1641 return __riscv_vsub_vv_i16m1(__riscv_vxor_vv_i16m1(a, mask, unpacket_traits<Packet1Xs>::size), mask,
1642 unpacket_traits<Packet1Xs>::size);
1643}
1644
1645template <>
1646EIGEN_STRONG_INLINE numext::int16_t predux<Packet1Xs>(const Packet1Xs& a) {
1647 return __riscv_vmv_x(__riscv_vredsum_vs_i16m1_i16m1(a, __riscv_vmv_v_x_i16m1(0, unpacket_traits<Packet1Xs>::size),
1648 unpacket_traits<Packet1Xs>::size));
1649}
1650
1651template <>
1652EIGEN_STRONG_INLINE bool predux_all(const Packet1Xs& a) {
1653 const PacketMask16 mask = __riscv_vmseq_vx_i16m1_b16(a, 0, unpacket_traits<Packet1Xs>::size);
1654 return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet1Xs>::size) == 0;
1655}
1656
1657template <>
1658EIGEN_STRONG_INLINE numext::int16_t predux_mul<Packet1Xs>(const Packet1Xs& a) {
1659 // Multiply the vector by its reverse
1660 Packet1Xs prod = __riscv_vmul_vv_i16m1(preverse(a), a, unpacket_traits<Packet1Xs>::size);
1661 Packet1Xs half_prod;
1662
1663 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
1664 half_prod = __riscv_vslidedown_vx_i16m1(prod, 16, unpacket_traits<Packet1Xs>::size);
1665 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1666 }
1667 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
1668 half_prod = __riscv_vslidedown_vx_i16m1(prod, 8, unpacket_traits<Packet1Xs>::size);
1669 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1670 }
1671 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
1672 half_prod = __riscv_vslidedown_vx_i16m1(prod, 4, unpacket_traits<Packet1Xs>::size);
1673 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1674 }
1675 // Last reduction
1676 half_prod = __riscv_vslidedown_vx_i16m1(prod, 2, unpacket_traits<Packet1Xs>::size);
1677 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1678
1679 half_prod = __riscv_vslidedown_vx_i16m1(prod, 1, unpacket_traits<Packet1Xs>::size);
1680 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1681
1682 // The reduction is done to the first element.
1683 return pfirst(prod);
1684}
1685
1686template <>
1687EIGEN_STRONG_INLINE numext::int16_t predux_min<Packet1Xs>(const Packet1Xs& a) {
1688 return __riscv_vmv_x(__riscv_vredmin_vs_i16m1_i16m1(
1689 a, __riscv_vmv_v_x_i16m1((std::numeric_limits<numext::int16_t>::max)(), unpacket_traits<Packet1Xs>::size),
1690 unpacket_traits<Packet1Xs>::size));
1691}
1692
1693template <>
1694EIGEN_STRONG_INLINE numext::int16_t predux_max<Packet1Xs>(const Packet1Xs& a) {
1695 return __riscv_vmv_x(__riscv_vredmax_vs_i16m1_i16m1(
1696 a, __riscv_vmv_v_x_i16m1((std::numeric_limits<numext::int16_t>::min)(), unpacket_traits<Packet1Xs>::size),
1697 unpacket_traits<Packet1Xs>::size));
1698}
1699
1700template <int N>
1701EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xs, N>& kernel) {
1702 numext::int16_t buffer[unpacket_traits<Packet1Xs>::size * N] = {0};
1703 int i = 0;
1704
1705 for (i = 0; i < N; i++) {
1706 __riscv_vsse16(&buffer[i], N * sizeof(numext::int16_t), kernel.packet[i], unpacket_traits<Packet1Xs>::size);
1707 }
1708 for (i = 0; i < N; i++) {
1709 kernel.packet[i] =
1710 __riscv_vle16_v_i16m1(&buffer[i * unpacket_traits<Packet1Xs>::size], unpacket_traits<Packet1Xs>::size);
1711 }
1712}
1713
1714template <>
1715EIGEN_STRONG_INLINE PacketMask16 por(const PacketMask16& a, const PacketMask16& b) {
1716 return __riscv_vmor_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1717}
1718
1719template <>
1720EIGEN_STRONG_INLINE PacketMask16 pand(const PacketMask16& a, const PacketMask16& b) {
1721 return __riscv_vmand_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1722}
1723
1724template <>
1725EIGEN_STRONG_INLINE PacketMask16 pxor(const PacketMask16& a, const PacketMask16& b) {
1726 return __riscv_vmxor_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1727}
1728
1729template <>
1730EIGEN_STRONG_INLINE PacketMask16 pandnot(const PacketMask16& a, const PacketMask16& b) {
1731 return __riscv_vmandn_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1732}
1733
1734template <>
1735EIGEN_STRONG_INLINE PacketMask16 pnot(const PacketMask16& a) {
1736 return __riscv_vmnot_m_b16(a, unpacket_traits<Packet1Xs>::size);
1737}
1738
1739} // namespace internal
1740} // namespace Eigen
1741
1742#endif // EIGEN_PACKET_MATH_RVV10_H