Eigen  5.0.1
 
Loading...
Searching...
No Matches
PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2024 Kseniya Zaytseva <kseniya.zaytseva@syntacore.com>
5// Copyright (C) 2025 Chip Kerchner <ckerchner@tenstorrent.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_PACKET_MATH_RVV10_H
13#define EIGEN_PACKET_MATH_RVV10_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19namespace internal {
20
21/********************************* Packet1Xi ************************************/
22
23EIGEN_STRONG_INLINE Packet1Xi __riscv_vreinterpret_v_u64m1_i32m1(const Packet1Xul& a) {
24 return __riscv_vreinterpret_v_i64m1_i32m1(__riscv_vreinterpret_v_u64m1_i64m1(a));
25}
26
27template <>
28EIGEN_STRONG_INLINE Packet1Xi pset1<Packet1Xi>(const numext::int32_t& from) {
29 return __riscv_vmv_v_x_i32m1(from, unpacket_traits<Packet1Xi>::size);
30}
31
32template <>
33EIGEN_STRONG_INLINE Packet1Xi plset<Packet1Xi>(const numext::int32_t& a) {
34 Packet1Xi idx = __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size));
35 return __riscv_vadd_vx_i32m1(idx, a, unpacket_traits<Packet1Xi>::size);
36}
37
38template <>
39EIGEN_STRONG_INLINE Packet1Xi pzero<Packet1Xi>(const Packet1Xi& /*a*/) {
40 return __riscv_vmv_v_x_i32m1(0, unpacket_traits<Packet1Xi>::size);
41}
42
43template <>
44EIGEN_STRONG_INLINE Packet1Xi padd<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
45 return __riscv_vadd_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
46}
47
48template <>
49EIGEN_STRONG_INLINE Packet1Xi psub<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
50 return __riscv_vsub(a, b, unpacket_traits<Packet1Xi>::size);
51}
52
53template <>
54EIGEN_STRONG_INLINE Packet1Xi pnegate(const Packet1Xi& a) {
55 return __riscv_vneg(a, unpacket_traits<Packet1Xi>::size);
56}
57
58template <>
59EIGEN_STRONG_INLINE Packet1Xi pmul<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
60 return __riscv_vmul(a, b, unpacket_traits<Packet1Xi>::size);
61}
62
63template <>
64EIGEN_STRONG_INLINE Packet1Xi pdiv<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
65 return __riscv_vdiv(a, b, unpacket_traits<Packet1Xi>::size);
66}
67
68template <>
69EIGEN_STRONG_INLINE Packet1Xi pmadd(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
70 return __riscv_vmadd(a, b, c, unpacket_traits<Packet1Xi>::size);
71}
72
73template <>
74EIGEN_STRONG_INLINE Packet1Xi pmsub(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
75 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet1Xi>::size);
76}
77
78template <>
79EIGEN_STRONG_INLINE Packet1Xi pnmadd(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
80 return __riscv_vnmsub_vv_i32m1(a, b, c, unpacket_traits<Packet1Xi>::size);
81}
82
83template <>
84EIGEN_STRONG_INLINE Packet1Xi pnmsub(const Packet1Xi& a, const Packet1Xi& b, const Packet1Xi& c) {
85 return __riscv_vnmsub_vv_i32m1(a, b, pnegate(c), unpacket_traits<Packet1Xi>::size);
86}
87
88template <>
89EIGEN_STRONG_INLINE Packet1Xi pmin<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
90 return __riscv_vmin(a, b, unpacket_traits<Packet1Xi>::size);
91}
92
93template <>
94EIGEN_STRONG_INLINE Packet1Xi pmax<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
95 return __riscv_vmax(a, b, unpacket_traits<Packet1Xi>::size);
96}
97
98template <>
99EIGEN_STRONG_INLINE Packet1Xi pcmp_le<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
100 PacketMask32 mask = __riscv_vmsle_vv_i32m1_b32(a, b, unpacket_traits<Packet1Xi>::size);
101 return __riscv_vmerge_vxm_i32m1(pzero(a), 0xffffffff, mask, unpacket_traits<Packet1Xi>::size);
102}
103
104template <>
105EIGEN_STRONG_INLINE Packet1Xi pcmp_lt<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
106 PacketMask32 mask = __riscv_vmslt_vv_i32m1_b32(a, b, unpacket_traits<Packet1Xi>::size);
107 return __riscv_vmerge_vxm_i32m1(pzero(a), 0xffffffff, mask, unpacket_traits<Packet1Xi>::size);
108}
109
110template <>
111EIGEN_STRONG_INLINE Packet1Xi pcmp_eq<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
112 PacketMask32 mask = __riscv_vmseq_vv_i32m1_b32(a, b, unpacket_traits<Packet1Xi>::size);
113 return __riscv_vmerge_vxm_i32m1(pzero(a), 0xffffffff, mask, unpacket_traits<Packet1Xi>::size);
114}
115
116template <>
117EIGEN_STRONG_INLINE Packet1Xi ptrue<Packet1Xi>(const Packet1Xi& /*a*/) {
118 return __riscv_vmv_v_x_i32m1(0xffffffffu, unpacket_traits<Packet1Xi>::size);
119}
120
121template <>
122EIGEN_STRONG_INLINE Packet1Xi pand<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
123 return __riscv_vand_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
124}
125
126template <>
127EIGEN_STRONG_INLINE Packet1Xi por<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
128 return __riscv_vor_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
129}
130
131template <>
132EIGEN_STRONG_INLINE Packet1Xi pxor<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
133 return __riscv_vxor_vv_i32m1(a, b, unpacket_traits<Packet1Xi>::size);
134}
135
136template <>
137EIGEN_STRONG_INLINE Packet1Xi pandnot<Packet1Xi>(const Packet1Xi& a, const Packet1Xi& b) {
138#ifndef __riscv_zvbb
139 return __riscv_vand_vv_i32m1(a, __riscv_vnot_v_i32m1(b, unpacket_traits<Packet1Xi>::size),
140 unpacket_traits<Packet1Xi>::size);
141#else
142 return __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vandn_vv_u32m1(
143 __riscv_vreinterpret_v_i32m1_u32m1(a), __riscv_vreinterpret_v_i32m1_u32m1(b), unpacket_traits<Packet1Xi>::size));
144#endif
145}
146
147template <>
148EIGEN_STRONG_INLINE Packet1Xi pnot<Packet1Xi>(const Packet1Xi& a) {
149 return __riscv_vnot_v_i32m1(a, unpacket_traits<Packet1Xi>::size);
150}
151
152template <int N>
153EIGEN_STRONG_INLINE Packet1Xi parithmetic_shift_right(const Packet1Xi& a) {
154 return __riscv_vsra_vx_i32m1(a, N, unpacket_traits<Packet1Xi>::size);
155}
156
157template <int N>
158EIGEN_STRONG_INLINE Packet1Xi plogical_shift_right(const Packet1Xi& a) {
159 return __riscv_vreinterpret_i32m1(
160 __riscv_vsrl_vx_u32m1(__riscv_vreinterpret_u32m1(a), N, unpacket_traits<Packet1Xi>::size));
161}
162
163template <int N>
164EIGEN_STRONG_INLINE Packet1Xi plogical_shift_left(const Packet1Xi& a) {
165 return __riscv_vsll_vx_i32m1(a, N, unpacket_traits<Packet1Xi>::size);
166}
167
168template <>
169EIGEN_STRONG_INLINE Packet1Xi pload<Packet1Xi>(const numext::int32_t* from) {
170 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle32_v_i32m1(from, unpacket_traits<Packet1Xi>::size);
171}
172
173template <>
174EIGEN_STRONG_INLINE Packet1Xi ploadu<Packet1Xi>(const numext::int32_t* from) {
175 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle32_v_i32m1(from, unpacket_traits<Packet1Xi>::size);
176}
177
178template <>
179EIGEN_STRONG_INLINE Packet1Xi ploaddup<Packet1Xi>(const numext::int32_t* from) {
180 Packet1Xul data = __riscv_vlmul_trunc_v_u64m2_u64m1(__riscv_vwcvtu_x_x_v_u64m2(
181 __riscv_vreinterpret_v_i32m1_u32m1(pload<Packet1Xi>(from)), unpacket_traits<Packet1Xi>::size));
182 return __riscv_vreinterpret_v_u64m1_i32m1(__riscv_vadd_vv_u64m1(
183 __riscv_vsll_vx_u64m1(data, 32, unpacket_traits<Packet1Xl>::size), data, unpacket_traits<Packet1Xl>::size));
184}
185
186template <>
187EIGEN_STRONG_INLINE Packet1Xi ploadquad<Packet1Xi>(const numext::int32_t* from) {
188 Packet1Xu idx =
189 __riscv_vsrl_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size), 2, unpacket_traits<Packet1Xi>::size);
190 return __riscv_vrgather_vv_i32m1(pload<Packet1Xi>(from), idx, unpacket_traits<Packet1Xi>::size);
191}
192
193template <>
194EIGEN_STRONG_INLINE void pstore<numext::int32_t>(numext::int32_t* to, const Packet1Xi& from) {
195 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse32_v_i32m1(to, from, unpacket_traits<Packet1Xi>::size);
196}
197
198template <>
199EIGEN_STRONG_INLINE void pstoreu<numext::int32_t>(numext::int32_t* to, const Packet1Xi& from) {
200 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse32_v_i32m1(to, from, unpacket_traits<Packet1Xi>::size);
201}
202
203template <>
204EIGEN_DEVICE_FUNC inline Packet1Xi pgather<numext::int32_t, Packet1Xi>(const numext::int32_t* from, Index stride) {
205 return __riscv_vlse32_v_i32m1(from, stride * sizeof(numext::int32_t), unpacket_traits<Packet1Xi>::size);
206}
207
208template <>
209EIGEN_DEVICE_FUNC inline void pscatter<numext::int32_t, Packet1Xi>(numext::int32_t* to, const Packet1Xi& from,
210 Index stride) {
211 __riscv_vsse32(to, stride * sizeof(numext::int32_t), from, unpacket_traits<Packet1Xi>::size);
212}
213
214template <>
215EIGEN_STRONG_INLINE numext::int32_t pfirst<Packet1Xi>(const Packet1Xi& a) {
216 return __riscv_vmv_x_s_i32m1_i32(a);
217}
218
219template <>
220EIGEN_STRONG_INLINE Packet1Xi preverse(const Packet1Xi& a) {
221 Packet1Xu idx = __riscv_vrsub_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size),
222 unpacket_traits<Packet1Xi>::size - 1, unpacket_traits<Packet1Xi>::size);
223 return __riscv_vrgather_vv_i32m1(a, idx, unpacket_traits<Packet1Xi>::size);
224}
225
226template <>
227EIGEN_STRONG_INLINE Packet1Xi pabs(const Packet1Xi& a) {
228 Packet1Xi mask = __riscv_vsra_vx_i32m1(a, 31, unpacket_traits<Packet1Xi>::size);
229 return __riscv_vsub_vv_i32m1(__riscv_vxor_vv_i32m1(a, mask, unpacket_traits<Packet1Xi>::size), mask,
230 unpacket_traits<Packet1Xi>::size);
231}
232
233template <>
234EIGEN_STRONG_INLINE numext::int32_t predux<Packet1Xi>(const Packet1Xi& a) {
235 return __riscv_vmv_x(__riscv_vredsum_vs_i32m1_i32m1(a, __riscv_vmv_v_x_i32m1(0, unpacket_traits<Packet1Xi>::size),
236 unpacket_traits<Packet1Xi>::size));
237}
238
239template <>
240EIGEN_STRONG_INLINE bool predux_all(const Packet1Xi& a) {
241 const PacketMask32 mask = __riscv_vmseq_vx_i32m1_b32(a, 0, unpacket_traits<Packet1Xi>::size);
242 return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xi>::size) == 0;
243}
244
245template <>
246EIGEN_STRONG_INLINE numext::int32_t predux_mul<Packet1Xi>(const Packet1Xi& a) {
247 // Multiply the vector by its reverse
248 Packet1Xi prod = __riscv_vmul_vv_i32m1(preverse(a), a, unpacket_traits<Packet1Xi>::size);
249 Packet1Xi half_prod;
250
251 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
252 half_prod = __riscv_vslidedown_vx_i32m1(prod, 8, unpacket_traits<Packet1Xi>::size);
253 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
254 }
255 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
256 half_prod = __riscv_vslidedown_vx_i32m1(prod, 4, unpacket_traits<Packet1Xi>::size);
257 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
258 }
259 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
260 half_prod = __riscv_vslidedown_vx_i32m1(prod, 2, unpacket_traits<Packet1Xi>::size);
261 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
262 }
263 // Last reduction
264 half_prod = __riscv_vslidedown_vx_i32m1(prod, 1, unpacket_traits<Packet1Xi>::size);
265 prod = __riscv_vmul_vv_i32m1(prod, half_prod, unpacket_traits<Packet1Xi>::size);
266
267 // The reduction is done to the first element.
268 return pfirst(prod);
269}
270
271template <>
272EIGEN_STRONG_INLINE numext::int32_t predux_min<Packet1Xi>(const Packet1Xi& a) {
273 return __riscv_vmv_x(__riscv_vredmin_vs_i32m1_i32m1(
274 a, __riscv_vmv_v_x_i32m1((std::numeric_limits<numext::int32_t>::max)(), unpacket_traits<Packet1Xi>::size),
275 unpacket_traits<Packet1Xi>::size));
276}
277
278template <>
279EIGEN_STRONG_INLINE numext::int32_t predux_max<Packet1Xi>(const Packet1Xi& a) {
280 return __riscv_vmv_x(__riscv_vredmax_vs_i32m1_i32m1(
281 a, __riscv_vmv_v_x_i32m1((std::numeric_limits<numext::int32_t>::min)(), unpacket_traits<Packet1Xi>::size),
282 unpacket_traits<Packet1Xi>::size));
283}
284
285template <int N>
286EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xi, N>& kernel) {
287 numext::int32_t buffer[unpacket_traits<Packet1Xi>::size * N] = {0};
288 int i = 0;
289
290 for (i = 0; i < N; i++) {
291 __riscv_vsse32(&buffer[i], N * sizeof(numext::int32_t), kernel.packet[i], unpacket_traits<Packet1Xi>::size);
292 }
293 for (i = 0; i < N; i++) {
294 kernel.packet[i] =
295 __riscv_vle32_v_i32m1(&buffer[i * unpacket_traits<Packet1Xi>::size], unpacket_traits<Packet1Xi>::size);
296 }
297}
298
299/********************************* Packet1Xf ************************************/
300
301EIGEN_STRONG_INLINE Packet1Xf __riscv_vreinterpret_v_u64m1_f32m1(const Packet1Xul& a) {
302 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vreinterpret_v_u64m1_u32m1(a));
303}
304
305EIGEN_STRONG_INLINE Packet2Xf __riscv_vreinterpret_v_u64m2_f32m2(const Packet2Xul& a) {
306 return __riscv_vreinterpret_v_u32m2_f32m2(__riscv_vreinterpret_v_u64m2_u32m2(a));
307}
308
309template <>
310EIGEN_STRONG_INLINE Packet1Xf ptrue<Packet1Xf>(const Packet1Xf& /*a*/) {
311 Packet1Xf r = __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(0xffffffffu, unpacket_traits<Packet1Xf>::size));
312 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
313 return r;
314}
315
316template <>
317EIGEN_STRONG_INLINE Packet1Xf pzero<Packet1Xf>(const Packet1Xf& /*a*/) {
318 return __riscv_vfmv_v_f_f32m1(0.0f, unpacket_traits<Packet1Xf>::size);
319}
320
321template <>
322EIGEN_STRONG_INLINE Packet1Xf pabs(const Packet1Xf& a) {
323 return __riscv_vfabs_v_f32m1(a, unpacket_traits<Packet1Xf>::size);
324}
325
326template <>
327EIGEN_STRONG_INLINE Packet1Xf pabsdiff(const Packet1Xf& a, const Packet1Xf& b) {
328 return __riscv_vfabs_v_f32m1(__riscv_vfsub_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size),
329 unpacket_traits<Packet1Xf>::size);
330}
331
332template <>
333EIGEN_STRONG_INLINE Packet1Xf pset1<Packet1Xf>(const float& from) {
334 return __riscv_vfmv_v_f_f32m1(from, unpacket_traits<Packet1Xf>::size);
335}
336
337template <>
338EIGEN_STRONG_INLINE Packet1Xf pset1frombits<Packet1Xf>(numext::uint32_t from) {
339 return __riscv_vreinterpret_f32m1(__riscv_vmv_v_x_u32m1(from, unpacket_traits<Packet1Xf>::size));
340}
341
342template <>
343EIGEN_STRONG_INLINE Packet1Xf plset<Packet1Xf>(const float& a) {
344 Packet1Xf idx = __riscv_vfcvt_f_x_v_f32m1(
345 __riscv_vreinterpret_v_u32m1_i32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xi>::size)),
346 unpacket_traits<Packet1Xf>::size);
347 return __riscv_vfadd_vf_f32m1(idx, a, unpacket_traits<Packet1Xf>::size);
348}
349
350template <>
351EIGEN_STRONG_INLINE void pbroadcast4<Packet1Xf>(const float* a, Packet1Xf& a0, Packet1Xf& a1, Packet1Xf& a2,
352 Packet1Xf& a3) {
353 Packet1Xf aa = __riscv_vle32_v_f32m1(a, 4);
354 a0 = __riscv_vrgather_vx_f32m1(aa, 0, unpacket_traits<Packet1Xf>::size);
355 a1 = __riscv_vrgather_vx_f32m1(aa, 1, unpacket_traits<Packet1Xf>::size);
356 a2 = __riscv_vrgather_vx_f32m1(aa, 2, unpacket_traits<Packet1Xf>::size);
357 a3 = __riscv_vrgather_vx_f32m1(aa, 3, unpacket_traits<Packet1Xf>::size);
358}
359
360template <>
361EIGEN_STRONG_INLINE Packet1Xf padd<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
362 return __riscv_vfadd_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
363}
364
365template <>
366EIGEN_STRONG_INLINE Packet1Xf psub<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
367 return __riscv_vfsub_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
368}
369
370template <>
371EIGEN_STRONG_INLINE Packet1Xf pnegate(const Packet1Xf& a) {
372 return __riscv_vfneg_v_f32m1(a, unpacket_traits<Packet1Xf>::size);
373}
374
375template <>
376EIGEN_STRONG_INLINE Packet1Xf psignbit(const Packet1Xf& a) {
377 return __riscv_vreinterpret_v_i32m1_f32m1(
378 __riscv_vsra_vx_i32m1(__riscv_vreinterpret_v_f32m1_i32m1(a), 31, unpacket_traits<Packet1Xi>::size));
379}
380
381template <>
382EIGEN_STRONG_INLINE Packet1Xf pmul<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
383 return __riscv_vfmul_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
384}
385
386template <>
387EIGEN_STRONG_INLINE Packet1Xf pdiv<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
388 return __riscv_vfdiv_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
389}
390
391template <>
392EIGEN_STRONG_INLINE Packet1Xf pmadd(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
393 return __riscv_vfmadd_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
394}
395
396template <>
397EIGEN_STRONG_INLINE Packet1Xf pmsub(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
398 return __riscv_vfmsub_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
399}
400
401template <>
402EIGEN_STRONG_INLINE Packet1Xf pnmadd(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
403 return __riscv_vfnmsub_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
404}
405
406template <>
407EIGEN_STRONG_INLINE Packet1Xf pnmsub(const Packet1Xf& a, const Packet1Xf& b, const Packet1Xf& c) {
408 return __riscv_vfnmadd_vv_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
409}
410
411template <>
412struct pminmax_propagates_nan<Packet1Xf> : bool_constant<true> {};
413
414template <>
415EIGEN_STRONG_INLINE Packet1Xf pmin<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
416 Packet1Xf nans = __riscv_vfmv_v_f_f32m1((std::numeric_limits<float>::quiet_NaN)(), unpacket_traits<Packet1Xf>::size);
417 PacketMask32 mask = __riscv_vmfeq_vv_f32m1_b32(a, a, unpacket_traits<Packet1Xf>::size);
418 PacketMask32 mask2 = __riscv_vmfeq_vv_f32m1_b32(b, b, unpacket_traits<Packet1Xf>::size);
419 mask = __riscv_vmand_mm_b32(mask, mask2, unpacket_traits<Packet1Xf>::size);
420
421 return __riscv_vfmin_vv_f32m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xf>::size);
422}
423
424template <>
425EIGEN_STRONG_INLINE Packet1Xf pmin<PropagateNumbers, Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
426 return __riscv_vfmin_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
427}
428
429template <>
430EIGEN_STRONG_INLINE Packet1Xf pmax<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
431 Packet1Xf nans = __riscv_vfmv_v_f_f32m1((std::numeric_limits<float>::quiet_NaN)(), unpacket_traits<Packet1Xf>::size);
432 PacketMask32 mask = __riscv_vmfeq_vv_f32m1_b32(a, a, unpacket_traits<Packet1Xf>::size);
433 PacketMask32 mask2 = __riscv_vmfeq_vv_f32m1_b32(b, b, unpacket_traits<Packet1Xf>::size);
434 mask = __riscv_vmand_mm_b32(mask, mask2, unpacket_traits<Packet1Xf>::size);
435
436 return __riscv_vfmax_vv_f32m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xf>::size);
437}
438
439template <>
440EIGEN_STRONG_INLINE Packet1Xf pmax<PropagateNumbers, Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
441 return __riscv_vfmax_vv_f32m1(a, b, unpacket_traits<Packet1Xf>::size);
442}
443
444template <>
445EIGEN_STRONG_INLINE Packet1Xf pcmp_le<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
446 PacketMask32 mask = __riscv_vmfle_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
447 return __riscv_vmerge_vvm_f32m1(pzero<Packet1Xf>(a), ptrue<Packet1Xf>(a), mask, unpacket_traits<Packet1Xf>::size);
448}
449
450template <>
451EIGEN_STRONG_INLINE Packet1Xf pcmp_lt<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
452 PacketMask32 mask = __riscv_vmflt_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
453 return __riscv_vmerge_vvm_f32m1(pzero<Packet1Xf>(a), ptrue<Packet1Xf>(a), mask, unpacket_traits<Packet1Xf>::size);
454}
455
456template <>
457EIGEN_STRONG_INLINE Packet1Xf pcmp_eq<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
458 PacketMask32 mask = __riscv_vmfeq_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
459 return __riscv_vmerge_vvm_f32m1(pzero<Packet1Xf>(a), ptrue<Packet1Xf>(a), mask, unpacket_traits<Packet1Xf>::size);
460}
461
462template <>
463EIGEN_STRONG_INLINE Packet1Xf pcmp_lt_or_nan<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
464 PacketMask32 mask = __riscv_vmfge_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
465 return __riscv_vfmerge_vfm_f32m1(ptrue<Packet1Xf>(a), 0.0f, mask, unpacket_traits<Packet1Xf>::size);
466}
467
468// Logical Operations are not supported for float, so reinterpret casts
469template <>
470EIGEN_STRONG_INLINE Packet1Xf pand<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
471 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vand_vv_u32m1(
472 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size));
473}
474
475template <>
476EIGEN_STRONG_INLINE Packet1Xf por<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
477 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vor_vv_u32m1(
478 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size));
479}
480
481template <>
482EIGEN_STRONG_INLINE Packet1Xf pxor<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
483 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vxor_vv_u32m1(
484 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size));
485}
486
487template <>
488EIGEN_STRONG_INLINE Packet1Xf pandnot<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& b) {
489#ifndef __riscv_zvbb
490 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vand_vv_u32m1(
491 __riscv_vreinterpret_v_f32m1_u32m1(a),
492 __riscv_vnot_v_u32m1(__riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xf>::size),
493 unpacket_traits<Packet1Xf>::size));
494#else
495 return __riscv_vreinterpret_v_u32m1_f32m1(__riscv_vandn_vv_u32m1(
496 __riscv_vreinterpret_v_f32m1_u32m1(a), __riscv_vreinterpret_v_f32m1_u32m1(b), unpacket_traits<Packet1Xi>::size));
497#endif
498}
499
500template <>
501EIGEN_STRONG_INLINE Packet1Xf pnot<Packet1Xf>(const Packet1Xf& a) {
502 return __riscv_vreinterpret_v_u32m1_f32m1(
503 __riscv_vnot_v_u32m1(__riscv_vreinterpret_v_f32m1_u32m1(a), unpacket_traits<Packet1Xf>::size));
504}
505
506template <>
507EIGEN_STRONG_INLINE Packet1Xf pload<Packet1Xf>(const float* from) {
508 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle32_v_f32m1(from, unpacket_traits<Packet1Xf>::size);
509}
510
511template <>
512EIGEN_STRONG_INLINE Packet1Xf ploadu<Packet1Xf>(const float* from) {
513 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle32_v_f32m1(from, unpacket_traits<Packet1Xf>::size);
514}
515
516EIGEN_STRONG_INLINE Packet2Xf pdup(const Packet1Xf& a) {
517 Packet2Xul data = __riscv_vwcvtu_x_x_v_u64m2(__riscv_vreinterpret_v_f32m1_u32m1(a), unpacket_traits<Packet1Xi>::size);
518 return __riscv_vreinterpret_v_u64m2_f32m2(__riscv_vadd_vv_u64m2(
519 __riscv_vsll_vx_u64m2(data, 32, unpacket_traits<Packet2Xl>::size), data, unpacket_traits<Packet2Xl>::size));
520}
521
522template <>
523EIGEN_STRONG_INLINE Packet1Xf ploaddup<Packet1Xf>(const float* from) {
524 Packet1Xul data = __riscv_vlmul_trunc_v_u64m2_u64m1(__riscv_vwcvtu_x_x_v_u64m2(
525 __riscv_vreinterpret_v_f32m1_u32m1(pload<Packet1Xf>(from)), unpacket_traits<Packet1Xi>::size));
526 return __riscv_vreinterpret_v_u64m1_f32m1(__riscv_vadd_vv_u64m1(
527 __riscv_vsll_vx_u64m1(data, 32, unpacket_traits<Packet1Xl>::size), data, unpacket_traits<Packet1Xl>::size));
528}
529
530template <>
531EIGEN_STRONG_INLINE Packet1Xf ploadquad<Packet1Xf>(const float* from) {
532 Packet1Xu idx =
533 __riscv_vsrl_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xf>::size), 2, unpacket_traits<Packet1Xf>::size);
534 return __riscv_vrgather_vv_f32m1(pload<Packet1Xf>(from), idx, unpacket_traits<Packet1Xf>::size);
535}
536
537template <>
538EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet1Xf& from) {
539 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse32_v_f32m1(to, from, unpacket_traits<Packet1Xf>::size);
540}
541
542template <>
543EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet1Xf& from) {
544 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse32_v_f32m1(to, from, unpacket_traits<Packet1Xf>::size);
545}
546
547template <>
548EIGEN_DEVICE_FUNC inline Packet1Xf pgather<float, Packet1Xf>(const float* from, Index stride) {
549 return __riscv_vlse32_v_f32m1(from, stride * sizeof(float), unpacket_traits<Packet1Xf>::size);
550}
551
552template <>
553EIGEN_DEVICE_FUNC inline void pscatter<float, Packet1Xf>(float* to, const Packet1Xf& from, Index stride) {
554 __riscv_vsse32(to, stride * sizeof(float), from, unpacket_traits<Packet1Xf>::size);
555}
556
557template <>
558EIGEN_STRONG_INLINE float pfirst<Packet1Xf>(const Packet1Xf& a) {
559 return __riscv_vfmv_f_s_f32m1_f32(a);
560}
561
562template <>
563EIGEN_STRONG_INLINE Packet1Xf psqrt(const Packet1Xf& a) {
564 return __riscv_vfsqrt_v_f32m1(a, unpacket_traits<Packet1Xf>::size);
565}
566
567template <>
568EIGEN_STRONG_INLINE Packet1Xf print<Packet1Xf>(const Packet1Xf& a) {
569 const Packet1Xf limit = pset1<Packet1Xf>(static_cast<float>(1 << 23));
570 const Packet1Xf abs_a = pabs(a);
571
572 PacketMask32 mask = __riscv_vmfne_vv_f32m1_b32(a, a, unpacket_traits<Packet1Xf>::size);
573 const Packet1Xf x = __riscv_vfadd_vv_f32m1_tumu(mask, a, a, a, unpacket_traits<Packet1Xf>::size);
574 const Packet1Xf new_x = __riscv_vfcvt_f_x_v_f32m1(__riscv_vfcvt_x_f_v_i32m1(a, unpacket_traits<Packet1Xf>::size),
575 unpacket_traits<Packet1Xf>::size);
576
577 mask = __riscv_vmflt_vv_f32m1_b32(abs_a, limit, unpacket_traits<Packet1Xf>::size);
578 Packet1Xf signed_x = __riscv_vfsgnj_vv_f32m1(new_x, x, unpacket_traits<Packet1Xf>::size);
579 return __riscv_vmerge_vvm_f32m1(x, signed_x, mask, unpacket_traits<Packet1Xf>::size);
580}
581
582template <>
583EIGEN_STRONG_INLINE Packet1Xf pfloor<Packet1Xf>(const Packet1Xf& a) {
584 Packet1Xf tmp = print<Packet1Xf>(a);
585 // If greater, subtract one.
586 PacketMask32 mask = __riscv_vmflt_vv_f32m1_b32(a, tmp, unpacket_traits<Packet1Xf>::size);
587 return __riscv_vfsub_vf_f32m1_tumu(mask, tmp, tmp, 1.0f, unpacket_traits<Packet1Xf>::size);
588}
589
590template <>
591EIGEN_STRONG_INLINE Packet1Xf preverse(const Packet1Xf& a) {
592 Packet1Xu idx = __riscv_vrsub_vx_u32m1(__riscv_vid_v_u32m1(unpacket_traits<Packet1Xf>::size),
593 unpacket_traits<Packet1Xf>::size - 1, unpacket_traits<Packet1Xf>::size);
594 return __riscv_vrgather_vv_f32m1(a, idx, unpacket_traits<Packet1Xf>::size);
595}
596
597template <>
598EIGEN_STRONG_INLINE Packet1Xf pfrexp<Packet1Xf>(const Packet1Xf& a, Packet1Xf& exponent) {
599 return pfrexp_generic(a, exponent);
600}
601
602template <>
603EIGEN_STRONG_INLINE float predux<Packet1Xf>(const Packet1Xf& a) {
604 return __riscv_vfmv_f(__riscv_vfredusum_vs_f32m1_f32m1(
605 a, __riscv_vfmv_v_f_f32m1(0.0f, unpacket_traits<Packet1Xf>::size), unpacket_traits<Packet1Xf>::size));
606}
607
608template <>
609EIGEN_STRONG_INLINE bool predux_any(const Packet1Xf& a) {
610 const PacketMask32 mask =
611 __riscv_vmsne_vx_u32m1_b32(__riscv_vreinterpret_v_f32m1_u32m1(a), 0, unpacket_traits<Packet1Xf>::size);
612 return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size) != 0;
613}
614
615template <>
616EIGEN_STRONG_INLINE bool predux_all(const Packet1Xf& a) {
617 const PacketMask32 mask = __riscv_vmfeq_vf_f32m1_b32(a, 0.0f, unpacket_traits<Packet1Xf>::size);
618 return __riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size) == 0;
619}
620
621template <>
622EIGEN_STRONG_INLINE Index predux_count(const Packet1Xf& a) {
623 const PacketMask32 mask = __riscv_vmfne_vf_f32m1_b32(a, 0.0f, unpacket_traits<Packet1Xf>::size);
624 return static_cast<Index>(__riscv_vcpop_m_b32(mask, unpacket_traits<Packet1Xf>::size));
625}
626
627template <>
628EIGEN_STRONG_INLINE float predux_mul<Packet1Xf>(const Packet1Xf& a) {
629 // Multiply the vector by its reverse
630 Packet1Xf prod = __riscv_vfmul_vv_f32m1(preverse(a), a, unpacket_traits<Packet1Xf>::size);
631 Packet1Xf half_prod;
632
633 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
634 half_prod = __riscv_vslidedown_vx_f32m1(prod, 8, unpacket_traits<Packet1Xf>::size);
635 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
636 }
637 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
638 half_prod = __riscv_vslidedown_vx_f32m1(prod, 4, unpacket_traits<Packet1Xf>::size);
639 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
640 }
641 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
642 half_prod = __riscv_vslidedown_vx_f32m1(prod, 2, unpacket_traits<Packet1Xf>::size);
643 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
644 }
645 // Last reduction
646 half_prod = __riscv_vslidedown_vx_f32m1(prod, 1, unpacket_traits<Packet1Xf>::size);
647 prod = __riscv_vfmul_vv_f32m1(prod, half_prod, unpacket_traits<Packet1Xf>::size);
648
649 // The reduction is done to the first element.
650 return pfirst(prod);
651}
652
653// Reusing the first lane is exact for an idempotent reduction and avoids a NaN seed that becomes poison under
654// finite fast-math.
655template <>
656EIGEN_STRONG_INLINE float predux_min<Packet1Xf>(const Packet1Xf& a) {
657 return __riscv_vfmv_f(__riscv_vfredmin_vs_f32m1_f32m1(a, a, unpacket_traits<Packet1Xf>::size));
658}
659
660template <>
661EIGEN_STRONG_INLINE float predux_max<Packet1Xf>(const Packet1Xf& a) {
662 return __riscv_vfmv_f(__riscv_vfredmax_vs_f32m1_f32m1(a, a, unpacket_traits<Packet1Xf>::size));
663}
664
665template <int N>
666EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xf, N>& kernel) {
667 float buffer[unpacket_traits<Packet1Xf>::size * N];
668 int i = 0;
669
670 for (i = 0; i < N; i++) {
671 __riscv_vsse32(&buffer[i], N * sizeof(float), kernel.packet[i], unpacket_traits<Packet1Xf>::size);
672 }
673
674 for (i = 0; i < N; i++) {
675 kernel.packet[i] =
676 __riscv_vle32_v_f32m1(&buffer[i * unpacket_traits<Packet1Xf>::size], unpacket_traits<Packet1Xf>::size);
677 }
678}
679
680template <>
681EIGEN_STRONG_INLINE Packet1Xf pldexp<Packet1Xf>(const Packet1Xf& a, const Packet1Xf& exponent) {
682 return pldexp_generic(a, exponent);
683}
684
685template <>
686EIGEN_STRONG_INLINE PacketMask32 por(const PacketMask32& a, const PacketMask32& b) {
687 return __riscv_vmor_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
688}
689
690template <>
691EIGEN_STRONG_INLINE PacketMask32 pand(const PacketMask32& a, const PacketMask32& b) {
692 return __riscv_vmand_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
693}
694
695template <>
696EIGEN_STRONG_INLINE PacketMask32 pxor(const PacketMask32& a, const PacketMask32& b) {
697 return __riscv_vmxor_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
698}
699
700template <>
701EIGEN_STRONG_INLINE PacketMask32 pandnot(const PacketMask32& a, const PacketMask32& b) {
702 return __riscv_vmandn_mm_b32(a, b, unpacket_traits<Packet1Xf>::size);
703}
704
705template <>
706EIGEN_STRONG_INLINE PacketMask32 pnot(const PacketMask32& a) {
707 return __riscv_vmnot_m_b32(a, unpacket_traits<Packet1Xf>::size);
708}
709
710EIGEN_STRONG_INLINE PacketMask32 pcmp_eq_mask(const Packet1Xf& a, const Packet1Xf& b) {
711 return __riscv_vmfeq_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
712}
713
714EIGEN_STRONG_INLINE PacketMask32 pcmp_lt_mask(const Packet1Xf& a, const Packet1Xf& b) {
715 return __riscv_vmflt_vv_f32m1_b32(a, b, unpacket_traits<Packet1Xf>::size);
716}
717
718EIGEN_STRONG_INLINE Packet1Xf pselect(const PacketMask32& mask, const Packet1Xf& a, const Packet1Xf& b) {
719 return __riscv_vmerge_vvm_f32m1(b, a, mask, unpacket_traits<Packet1Xf>::size);
720}
721
722EIGEN_STRONG_INLINE Packet1Xf pselect(const Packet1Xf& mask, const Packet1Xf& a, const Packet1Xf& b) {
723 PacketMask32 mask2 =
724 __riscv_vmsne_vx_i32m1_b32(__riscv_vreinterpret_v_f32m1_i32m1(mask), 0, unpacket_traits<Packet1Xf>::size);
725 return __riscv_vmerge_vvm_f32m1(b, a, mask2, unpacket_traits<Packet1Xf>::size);
726}
727
728/********************************* Packet1Xl ************************************/
729
730template <>
731EIGEN_STRONG_INLINE Packet1Xl pset1<Packet1Xl>(const numext::int64_t& from) {
732 return __riscv_vmv_v_x_i64m1(from, unpacket_traits<Packet1Xl>::size);
733}
734
735template <>
736EIGEN_STRONG_INLINE Packet1Xl plset<Packet1Xl>(const numext::int64_t& a) {
737 Packet1Xl idx = __riscv_vreinterpret_v_u64m1_i64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size));
738 return __riscv_vadd_vx_i64m1(idx, a, unpacket_traits<Packet1Xl>::size);
739}
740
741template <>
742EIGEN_STRONG_INLINE Packet1Xl pzero<Packet1Xl>(const Packet1Xl& /*a*/) {
743 return __riscv_vmv_v_x_i64m1(0, unpacket_traits<Packet1Xl>::size);
744}
745
746template <>
747EIGEN_STRONG_INLINE Packet1Xl padd<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
748 return __riscv_vadd_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
749}
750
751template <>
752EIGEN_STRONG_INLINE Packet1Xl psub<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
753 return __riscv_vsub(a, b, unpacket_traits<Packet1Xl>::size);
754}
755
756template <>
757EIGEN_STRONG_INLINE Packet1Xl pnegate(const Packet1Xl& a) {
758 return __riscv_vneg(a, unpacket_traits<Packet1Xl>::size);
759}
760
761template <>
762EIGEN_STRONG_INLINE Packet1Xl pmul<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
763 return __riscv_vmul(a, b, unpacket_traits<Packet1Xl>::size);
764}
765
766template <>
767EIGEN_STRONG_INLINE Packet1Xl pdiv<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
768 return __riscv_vdiv(a, b, unpacket_traits<Packet1Xl>::size);
769}
770
771template <>
772EIGEN_STRONG_INLINE Packet1Xl pmadd(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
773 return __riscv_vmadd(a, b, c, unpacket_traits<Packet1Xl>::size);
774}
775
776template <>
777EIGEN_STRONG_INLINE Packet1Xl pmsub(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
778 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet1Xl>::size);
779}
780
781template <>
782EIGEN_STRONG_INLINE Packet1Xl pnmadd(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
783 return __riscv_vnmsub_vv_i64m1(a, b, c, unpacket_traits<Packet1Xl>::size);
784}
785
786template <>
787EIGEN_STRONG_INLINE Packet1Xl pnmsub(const Packet1Xl& a, const Packet1Xl& b, const Packet1Xl& c) {
788 return __riscv_vnmsub_vv_i64m1(a, b, pnegate(c), unpacket_traits<Packet1Xl>::size);
789}
790
791template <>
792EIGEN_STRONG_INLINE Packet1Xl pmin<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
793 return __riscv_vmin(a, b, unpacket_traits<Packet1Xl>::size);
794}
795
796template <>
797EIGEN_STRONG_INLINE Packet1Xl pmax<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
798 return __riscv_vmax(a, b, unpacket_traits<Packet1Xl>::size);
799}
800
801template <>
802EIGEN_STRONG_INLINE Packet1Xl pcmp_le<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
803 PacketMask64 mask = __riscv_vmsle_vv_i64m1_b64(a, b, unpacket_traits<Packet1Xl>::size);
804 return __riscv_vmerge_vxm_i64m1(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet1Xl>::size);
805}
806
807template <>
808EIGEN_STRONG_INLINE Packet1Xl pcmp_lt<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
809 PacketMask64 mask = __riscv_vmslt_vv_i64m1_b64(a, b, unpacket_traits<Packet1Xl>::size);
810 return __riscv_vmerge_vxm_i64m1(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet1Xl>::size);
811}
812
813template <>
814EIGEN_STRONG_INLINE Packet1Xl pcmp_eq<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
815 PacketMask64 mask = __riscv_vmseq_vv_i64m1_b64(a, b, unpacket_traits<Packet1Xl>::size);
816 return __riscv_vmerge_vxm_i64m1(pzero(a), 0xffffffffffffffff, mask, unpacket_traits<Packet1Xl>::size);
817}
818
819template <>
820EIGEN_STRONG_INLINE Packet1Xl ptrue<Packet1Xl>(const Packet1Xl& /*a*/) {
821 return __riscv_vmv_v_x_i64m1(0xffffffffffffffffu, unpacket_traits<Packet1Xl>::size);
822}
823
824template <>
825EIGEN_STRONG_INLINE Packet1Xl pand<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
826 return __riscv_vand_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
827}
828
829template <>
830EIGEN_STRONG_INLINE Packet1Xl por<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
831 return __riscv_vor_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
832}
833
834template <>
835EIGEN_STRONG_INLINE Packet1Xl pnot<Packet1Xl>(const Packet1Xl& a) {
836 return __riscv_vnot_v_i64m1(a, unpacket_traits<Packet1Xl>::size);
837}
838
839template <>
840EIGEN_STRONG_INLINE Packet1Xl pxor<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
841 return __riscv_vxor_vv_i64m1(a, b, unpacket_traits<Packet1Xl>::size);
842}
843
844template <>
845EIGEN_STRONG_INLINE Packet1Xl pandnot<Packet1Xl>(const Packet1Xl& a, const Packet1Xl& b) {
846#ifndef __riscv_zvbb
847 return __riscv_vand_vv_i64m1(a, __riscv_vnot_v_i64m1(b, unpacket_traits<Packet1Xl>::size),
848 unpacket_traits<Packet1Xl>::size);
849#else
850 return __riscv_vreinterpret_v_u64m1_i64m1(__riscv_vandn_vv_u64m1(
851 __riscv_vreinterpret_v_i64m1_u64m1(a), __riscv_vreinterpret_v_i64m1_u64m1(b), unpacket_traits<Packet1Xl>::size));
852#endif
853}
854
855template <int N>
856EIGEN_STRONG_INLINE Packet1Xl parithmetic_shift_right(const Packet1Xl& a) {
857 return __riscv_vsra_vx_i64m1(a, N, unpacket_traits<Packet1Xl>::size);
858}
859
860template <int N>
861EIGEN_STRONG_INLINE Packet1Xl plogical_shift_right(const Packet1Xl& a) {
862 return __riscv_vreinterpret_i64m1(
863 __riscv_vsrl_vx_u64m1(__riscv_vreinterpret_u64m1(a), N, unpacket_traits<Packet1Xl>::size));
864}
865
866template <int N>
867EIGEN_STRONG_INLINE Packet1Xl plogical_shift_left(const Packet1Xl& a) {
868 return __riscv_vsll_vx_i64m1(a, N, unpacket_traits<Packet1Xl>::size);
869}
870
871template <>
872EIGEN_STRONG_INLINE Packet1Xl pload<Packet1Xl>(const numext::int64_t* from) {
873 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle64_v_i64m1(from, unpacket_traits<Packet1Xl>::size);
874}
875
876template <>
877EIGEN_STRONG_INLINE Packet1Xl ploadu<Packet1Xl>(const numext::int64_t* from) {
878 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle64_v_i64m1(from, unpacket_traits<Packet1Xl>::size);
879}
880
881template <>
882EIGEN_STRONG_INLINE Packet1Xl ploaddup<Packet1Xl>(const numext::int64_t* from) {
883 Packet1Xul idx =
884 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size), 1, unpacket_traits<Packet1Xl>::size);
885 return __riscv_vrgather_vv_i64m1(pload<Packet1Xl>(from), idx, unpacket_traits<Packet1Xl>::size);
886}
887
888template <>
889EIGEN_STRONG_INLINE Packet1Xl ploadquad<Packet1Xl>(const numext::int64_t* from) {
890 Packet1Xul idx =
891 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size), 2, unpacket_traits<Packet1Xl>::size);
892 return __riscv_vrgather_vv_i64m1(pload<Packet1Xl>(from), idx, unpacket_traits<Packet1Xl>::size);
893}
894
895template <>
896EIGEN_STRONG_INLINE void pstore<numext::int64_t>(numext::int64_t* to, const Packet1Xl& from) {
897 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse64_v_i64m1(to, from, unpacket_traits<Packet1Xl>::size);
898}
899
900template <>
901EIGEN_STRONG_INLINE void pstoreu<numext::int64_t>(numext::int64_t* to, const Packet1Xl& from) {
902 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse64_v_i64m1(to, from, unpacket_traits<Packet1Xl>::size);
903}
904
905template <>
906EIGEN_DEVICE_FUNC inline Packet1Xl pgather<numext::int64_t, Packet1Xl>(const numext::int64_t* from, Index stride) {
907 return __riscv_vlse64_v_i64m1(from, stride * sizeof(numext::int64_t), unpacket_traits<Packet1Xl>::size);
908}
909
910template <>
911EIGEN_DEVICE_FUNC inline void pscatter<numext::int64_t, Packet1Xl>(numext::int64_t* to, const Packet1Xl& from,
912 Index stride) {
913 __riscv_vsse64(to, stride * sizeof(numext::int64_t), from, unpacket_traits<Packet1Xl>::size);
914}
915
916template <>
917EIGEN_STRONG_INLINE numext::int64_t pfirst<Packet1Xl>(const Packet1Xl& a) {
918 return __riscv_vmv_x_s_i64m1_i64(a);
919}
920
921template <>
922EIGEN_STRONG_INLINE Packet1Xl preverse(const Packet1Xl& a) {
923 Packet1Xul idx = __riscv_vrsub_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size),
924 unpacket_traits<Packet1Xl>::size - 1, unpacket_traits<Packet1Xl>::size);
925 return __riscv_vrgather_vv_i64m1(a, idx, unpacket_traits<Packet1Xl>::size);
926}
927
928template <>
929EIGEN_STRONG_INLINE Packet1Xl pabs(const Packet1Xl& a) {
930 Packet1Xl mask = __riscv_vsra_vx_i64m1(a, 63, unpacket_traits<Packet1Xl>::size);
931 return __riscv_vsub_vv_i64m1(__riscv_vxor_vv_i64m1(a, mask, unpacket_traits<Packet1Xl>::size), mask,
932 unpacket_traits<Packet1Xl>::size);
933}
934
935template <>
936EIGEN_STRONG_INLINE numext::int64_t predux<Packet1Xl>(const Packet1Xl& a) {
937 return __riscv_vmv_x(__riscv_vredsum_vs_i64m1_i64m1(a, __riscv_vmv_v_x_i64m1(0, unpacket_traits<Packet1Xl>::size),
938 unpacket_traits<Packet1Xl>::size));
939}
940
941template <>
942EIGEN_STRONG_INLINE bool predux_all(const Packet1Xl& a) {
943 const PacketMask64 mask = __riscv_vmseq_vx_i64m1_b64(a, 0, unpacket_traits<Packet1Xl>::size);
944 return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xl>::size) == 0;
945}
946
947template <>
948EIGEN_STRONG_INLINE numext::int64_t predux_mul<Packet1Xl>(const Packet1Xl& a) {
949 // Multiply the vector by its reverse
950 Packet1Xl prod = __riscv_vmul_vv_i64m1(preverse(a), a, unpacket_traits<Packet1Xl>::size);
951 Packet1Xl half_prod;
952
953 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
954 half_prod = __riscv_vslidedown_vx_i64m1(prod, 4, unpacket_traits<Packet1Xl>::size);
955 prod = __riscv_vmul_vv_i64m1(prod, half_prod, unpacket_traits<Packet1Xl>::size);
956 }
957 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
958 half_prod = __riscv_vslidedown_vx_i64m1(prod, 2, unpacket_traits<Packet1Xl>::size);
959 prod = __riscv_vmul_vv_i64m1(prod, half_prod, unpacket_traits<Packet1Xl>::size);
960 }
961 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
962 half_prod = __riscv_vslidedown_vx_i64m1(prod, 1, unpacket_traits<Packet1Xl>::size);
963 prod = __riscv_vmul_vv_i64m1(prod, half_prod, unpacket_traits<Packet1Xl>::size);
964 }
965
966 // The reduction is done to the first element.
967 return pfirst(prod);
968}
969
970template <>
971EIGEN_STRONG_INLINE numext::int64_t predux_min<Packet1Xl>(const Packet1Xl& a) {
972 return __riscv_vmv_x(__riscv_vredmin_vs_i64m1_i64m1(
973 a, __riscv_vmv_v_x_i64m1((std::numeric_limits<numext::int64_t>::max)(), unpacket_traits<Packet1Xl>::size),
974 unpacket_traits<Packet1Xl>::size));
975}
976
977template <>
978EIGEN_STRONG_INLINE numext::int64_t predux_max<Packet1Xl>(const Packet1Xl& a) {
979 return __riscv_vmv_x(__riscv_vredmax_vs_i64m1_i64m1(
980 a, __riscv_vmv_v_x_i64m1((std::numeric_limits<numext::int64_t>::min)(), unpacket_traits<Packet1Xl>::size),
981 unpacket_traits<Packet1Xl>::size));
982}
983
984template <int N>
985EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xl, N>& kernel) {
986 numext::int64_t buffer[unpacket_traits<Packet1Xl>::size * N] = {0};
987 int i = 0;
988
989 for (i = 0; i < N; i++) {
990 __riscv_vsse64(&buffer[i], N * sizeof(numext::int64_t), kernel.packet[i], unpacket_traits<Packet1Xl>::size);
991 }
992 for (i = 0; i < N; i++) {
993 kernel.packet[i] =
994 __riscv_vle64_v_i64m1(&buffer[i * unpacket_traits<Packet1Xl>::size], unpacket_traits<Packet1Xl>::size);
995 }
996}
997
998/********************************* Packet1Xd ************************************/
999
1000template <>
1001EIGEN_STRONG_INLINE Packet1Xd ptrue<Packet1Xd>(const Packet1Xd& /*a*/) {
1002 Packet1Xd r =
1003 __riscv_vreinterpret_f64m1(__riscv_vmv_v_x_u64m1(0xffffffffffffffffu, unpacket_traits<Packet1Xd>::size));
1004 EIGEN_FAST_MATH_CONSTANT_BARRIER(r);
1005 return r;
1006}
1007
1008template <>
1009EIGEN_STRONG_INLINE Packet1Xd pzero<Packet1Xd>(const Packet1Xd& /*a*/) {
1010 return __riscv_vfmv_v_f_f64m1(0.0, unpacket_traits<Packet1Xd>::size);
1011}
1012
1013template <>
1014EIGEN_STRONG_INLINE Packet1Xd pabs(const Packet1Xd& a) {
1015 return __riscv_vfabs_v_f64m1(a, unpacket_traits<Packet1Xd>::size);
1016}
1017
1018template <>
1019EIGEN_STRONG_INLINE Packet1Xd pabsdiff(const Packet1Xd& a, const Packet1Xd& b) {
1020 return __riscv_vfabs_v_f64m1(__riscv_vfsub_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size),
1021 unpacket_traits<Packet1Xd>::size);
1022}
1023
1024template <>
1025EIGEN_STRONG_INLINE Packet1Xd pset1<Packet1Xd>(const double& from) {
1026 return __riscv_vfmv_v_f_f64m1(from, unpacket_traits<Packet1Xd>::size);
1027}
1028
1029template <>
1030EIGEN_STRONG_INLINE Packet1Xd pset1frombits<Packet1Xd>(numext::uint64_t from) {
1031 return __riscv_vreinterpret_f64m1(__riscv_vmv_v_x_u64m1(from, unpacket_traits<Packet1Xd>::size));
1032}
1033
1034template <>
1035EIGEN_STRONG_INLINE Packet1Xd plset<Packet1Xd>(const double& a) {
1036 Packet1Xd idx = __riscv_vfcvt_f_x_v_f64m1(
1037 __riscv_vreinterpret_v_u64m1_i64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xl>::size)),
1038 unpacket_traits<Packet1Xd>::size);
1039 return __riscv_vfadd_vf_f64m1(idx, a, unpacket_traits<Packet1Xd>::size);
1040}
1041
1042template <>
1043EIGEN_STRONG_INLINE void pbroadcast4<Packet1Xd>(const double* a, Packet1Xd& a0, Packet1Xd& a1, Packet1Xd& a2,
1044 Packet1Xd& a3) {
1045 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
1046 Packet1Xd aa = __riscv_vle64_v_f64m1(a, 4);
1047 a0 = __riscv_vrgather_vx_f64m1(aa, 0, unpacket_traits<Packet1Xd>::size);
1048 a1 = __riscv_vrgather_vx_f64m1(aa, 1, unpacket_traits<Packet1Xd>::size);
1049 a2 = __riscv_vrgather_vx_f64m1(aa, 2, unpacket_traits<Packet1Xd>::size);
1050 a3 = __riscv_vrgather_vx_f64m1(aa, 3, unpacket_traits<Packet1Xd>::size);
1051 } else {
1052 Packet2Xd aa = __riscv_vle64_v_f64m2(a, 4);
1053 Packet1Xd aa0 = __riscv_vget_v_f64m2_f64m1(aa, 0);
1054 Packet1Xd aa1 = __riscv_vget_v_f64m2_f64m1(aa, 1);
1055 a0 = __riscv_vrgather_vx_f64m1(aa0, 0, unpacket_traits<Packet1Xd>::size);
1056 a1 = __riscv_vrgather_vx_f64m1(aa0, 1, unpacket_traits<Packet1Xd>::size);
1057 a2 = __riscv_vrgather_vx_f64m1(aa1, 0, unpacket_traits<Packet1Xd>::size);
1058 a3 = __riscv_vrgather_vx_f64m1(aa1, 1, unpacket_traits<Packet1Xd>::size);
1059 }
1060}
1061
1062template <>
1063EIGEN_STRONG_INLINE Packet1Xd padd<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1064 return __riscv_vfadd_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1065}
1066
1067template <>
1068EIGEN_STRONG_INLINE Packet1Xd psub<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1069 return __riscv_vfsub_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1070}
1071
1072template <>
1073EIGEN_STRONG_INLINE Packet1Xd pnegate(const Packet1Xd& a) {
1074 return __riscv_vfneg_v_f64m1(a, unpacket_traits<Packet1Xd>::size);
1075}
1076
1077template <>
1078EIGEN_STRONG_INLINE Packet1Xd psignbit(const Packet1Xd& a) {
1079 return __riscv_vreinterpret_v_i64m1_f64m1(
1080 __riscv_vsra_vx_i64m1(__riscv_vreinterpret_v_f64m1_i64m1(a), 63, unpacket_traits<Packet1Xl>::size));
1081}
1082
1083template <>
1084EIGEN_STRONG_INLINE Packet1Xd pmul<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1085 return __riscv_vfmul_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1086}
1087
1088template <>
1089EIGEN_STRONG_INLINE Packet1Xd pdiv<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1090 return __riscv_vfdiv_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1091}
1092
1093template <>
1094EIGEN_STRONG_INLINE Packet1Xd pmadd(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1095 return __riscv_vfmadd_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1096}
1097
1098template <>
1099EIGEN_STRONG_INLINE Packet1Xd pmsub(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1100 return __riscv_vfmsub_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1101}
1102
1103template <>
1104EIGEN_STRONG_INLINE Packet1Xd pnmadd(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1105 return __riscv_vfnmsub_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1106}
1107
1108template <>
1109EIGEN_STRONG_INLINE Packet1Xd pnmsub(const Packet1Xd& a, const Packet1Xd& b, const Packet1Xd& c) {
1110 return __riscv_vfnmadd_vv_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
1111}
1112
1113template <>
1114struct pminmax_propagates_nan<Packet1Xd> : bool_constant<true> {};
1115
1116template <>
1117EIGEN_STRONG_INLINE Packet1Xd pmin<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1118 Packet1Xd nans = __riscv_vfmv_v_f_f64m1((std::numeric_limits<double>::quiet_NaN)(), unpacket_traits<Packet1Xd>::size);
1119 PacketMask64 mask = __riscv_vmfeq_vv_f64m1_b64(a, a, unpacket_traits<Packet1Xd>::size);
1120 PacketMask64 mask2 = __riscv_vmfeq_vv_f64m1_b64(b, b, unpacket_traits<Packet1Xd>::size);
1121 mask = __riscv_vmand_mm_b64(mask, mask2, unpacket_traits<Packet1Xd>::size);
1122
1123 return __riscv_vfmin_vv_f64m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xd>::size);
1124}
1125
1126template <>
1127EIGEN_STRONG_INLINE Packet1Xd pmin<PropagateNumbers, Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1128 return __riscv_vfmin_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1129}
1130
1131template <>
1132EIGEN_STRONG_INLINE Packet1Xd pmax<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1133 Packet1Xd nans = __riscv_vfmv_v_f_f64m1((std::numeric_limits<double>::quiet_NaN)(), unpacket_traits<Packet1Xd>::size);
1134 PacketMask64 mask = __riscv_vmfeq_vv_f64m1_b64(a, a, unpacket_traits<Packet1Xd>::size);
1135 PacketMask64 mask2 = __riscv_vmfeq_vv_f64m1_b64(b, b, unpacket_traits<Packet1Xd>::size);
1136 mask = __riscv_vmand_mm_b64(mask, mask2, unpacket_traits<Packet1Xd>::size);
1137
1138 return __riscv_vfmax_vv_f64m1_tumu(mask, nans, a, b, unpacket_traits<Packet1Xd>::size);
1139}
1140
1141template <>
1142EIGEN_STRONG_INLINE Packet1Xd pmax<PropagateNumbers, Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1143 return __riscv_vfmax_vv_f64m1(a, b, unpacket_traits<Packet1Xd>::size);
1144}
1145
1146template <>
1147EIGEN_STRONG_INLINE Packet1Xd pcmp_le<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1148 PacketMask64 mask = __riscv_vmfle_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1149 return __riscv_vmerge_vvm_f64m1(pzero<Packet1Xd>(a), ptrue<Packet1Xd>(a), mask, unpacket_traits<Packet1Xd>::size);
1150}
1151
1152template <>
1153EIGEN_STRONG_INLINE Packet1Xd pcmp_lt<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1154 PacketMask64 mask = __riscv_vmflt_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1155 return __riscv_vmerge_vvm_f64m1(pzero<Packet1Xd>(a), ptrue<Packet1Xd>(a), mask, unpacket_traits<Packet1Xd>::size);
1156}
1157
1158template <>
1159EIGEN_STRONG_INLINE Packet1Xd pcmp_eq<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1160 PacketMask64 mask = __riscv_vmfeq_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1161 return __riscv_vmerge_vvm_f64m1(pzero<Packet1Xd>(a), ptrue<Packet1Xd>(a), mask, unpacket_traits<Packet1Xd>::size);
1162}
1163
1164template <>
1165EIGEN_STRONG_INLINE Packet1Xd pcmp_lt_or_nan<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1166 PacketMask64 mask = __riscv_vmfge_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1167 return __riscv_vfmerge_vfm_f64m1(ptrue<Packet1Xd>(a), 0.0, mask, unpacket_traits<Packet1Xd>::size);
1168}
1169
1170// Logical Operations are not supported for double, so reinterpret casts
1171template <>
1172EIGEN_STRONG_INLINE Packet1Xd pand<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1173 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vand_vv_u64m1(
1174 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size));
1175}
1176
1177template <>
1178EIGEN_STRONG_INLINE Packet1Xd por<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1179 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vor_vv_u64m1(
1180 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size));
1181}
1182
1183template <>
1184EIGEN_STRONG_INLINE Packet1Xd pxor<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1185 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vxor_vv_u64m1(
1186 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size));
1187}
1188
1189template <>
1190EIGEN_STRONG_INLINE Packet1Xd pandnot<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& b) {
1191#ifndef __riscv_zvbb
1192 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vand_vv_u64m1(
1193 __riscv_vreinterpret_v_f64m1_u64m1(a),
1194 __riscv_vnot_v_u64m1(__riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xd>::size),
1195 unpacket_traits<Packet1Xd>::size));
1196#else
1197 return __riscv_vreinterpret_v_u64m1_f64m1(__riscv_vandn_vv_u64m1(
1198 __riscv_vreinterpret_v_f64m1_u64m1(a), __riscv_vreinterpret_v_f64m1_u64m1(b), unpacket_traits<Packet1Xl>::size));
1199#endif
1200}
1201
1202template <>
1203EIGEN_STRONG_INLINE Packet1Xd pnot<Packet1Xd>(const Packet1Xd& a) {
1204 return __riscv_vreinterpret_v_u64m1_f64m1(
1205 __riscv_vnot_v_u64m1(__riscv_vreinterpret_v_f64m1_u64m1(a), unpacket_traits<Packet1Xd>::size));
1206}
1207
1208template <>
1209EIGEN_STRONG_INLINE Packet1Xd pload<Packet1Xd>(const double* from) {
1210 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle64_v_f64m1(from, unpacket_traits<Packet1Xd>::size);
1211}
1212
1213template <>
1214EIGEN_STRONG_INLINE Packet1Xd ploadu<Packet1Xd>(const double* from) {
1215 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle64_v_f64m1(from, unpacket_traits<Packet1Xd>::size);
1216}
1217
1218EIGEN_STRONG_INLINE Packet2Xd pdup(const Packet1Xd& a) {
1219 Packet2Xul idx =
1220 __riscv_vsrl_vx_u64m2(__riscv_vid_v_u64m2(unpacket_traits<Packet2Xd>::size), 1, unpacket_traits<Packet2Xd>::size);
1221 return __riscv_vrgather_vv_f64m2(__riscv_vlmul_ext_v_f64m1_f64m2(a), idx, unpacket_traits<Packet2Xd>::size);
1222}
1223
1224template <>
1225EIGEN_STRONG_INLINE Packet1Xd ploaddup<Packet1Xd>(const double* from) {
1226 Packet1Xul idx =
1227 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xd>::size), 1, unpacket_traits<Packet1Xd>::size);
1228 return __riscv_vrgather_vv_f64m1(pload<Packet1Xd>(from), idx, unpacket_traits<Packet1Xd>::size);
1229}
1230
1231template <>
1232EIGEN_STRONG_INLINE Packet1Xd ploadquad<Packet1Xd>(const double* from) {
1233 Packet1Xul idx =
1234 __riscv_vsrl_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xd>::size), 2, unpacket_traits<Packet1Xd>::size);
1235 return __riscv_vrgather_vv_f64m1(pload<Packet1Xd>(from), idx, unpacket_traits<Packet1Xd>::size);
1236}
1237
1238template <>
1239EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet1Xd& from) {
1240 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse64_v_f64m1(to, from, unpacket_traits<Packet1Xd>::size);
1241}
1242
1243template <>
1244EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet1Xd& from) {
1245 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse64_v_f64m1(to, from, unpacket_traits<Packet1Xd>::size);
1246}
1247
1248template <>
1249EIGEN_DEVICE_FUNC inline Packet1Xd pgather<double, Packet1Xd>(const double* from, Index stride) {
1250 return __riscv_vlse64_v_f64m1(from, stride * sizeof(double), unpacket_traits<Packet1Xd>::size);
1251}
1252
1253template <>
1254EIGEN_DEVICE_FUNC inline void pscatter<double, Packet1Xd>(double* to, const Packet1Xd& from, Index stride) {
1255 __riscv_vsse64(to, stride * sizeof(double), from, unpacket_traits<Packet1Xd>::size);
1256}
1257
1258template <>
1259EIGEN_STRONG_INLINE double pfirst<Packet1Xd>(const Packet1Xd& a) {
1260 return __riscv_vfmv_f_s_f64m1_f64(a);
1261}
1262
1263template <>
1264EIGEN_STRONG_INLINE Packet1Xd psqrt(const Packet1Xd& a) {
1265 return __riscv_vfsqrt_v_f64m1(a, unpacket_traits<Packet1Xd>::size);
1266}
1267
1268template <>
1269EIGEN_STRONG_INLINE Packet1Xd print<Packet1Xd>(const Packet1Xd& a) {
1270 const Packet1Xd limit = pset1<Packet1Xd>(static_cast<double>(1ull << 52));
1271 const Packet1Xd abs_a = pabs(a);
1272
1273 PacketMask64 mask = __riscv_vmfne_vv_f64m1_b64(a, a, unpacket_traits<Packet1Xd>::size);
1274 const Packet1Xd x = __riscv_vfadd_vv_f64m1_tumu(mask, a, a, a, unpacket_traits<Packet1Xd>::size);
1275 const Packet1Xd new_x = __riscv_vfcvt_f_x_v_f64m1(__riscv_vfcvt_x_f_v_i64m1(a, unpacket_traits<Packet1Xd>::size),
1276 unpacket_traits<Packet1Xd>::size);
1277
1278 mask = __riscv_vmflt_vv_f64m1_b64(abs_a, limit, unpacket_traits<Packet1Xd>::size);
1279 Packet1Xd signed_x = __riscv_vfsgnj_vv_f64m1(new_x, x, unpacket_traits<Packet1Xd>::size);
1280 return __riscv_vmerge_vvm_f64m1(x, signed_x, mask, unpacket_traits<Packet1Xd>::size);
1281}
1282
1283template <>
1284EIGEN_STRONG_INLINE Packet1Xd pfloor<Packet1Xd>(const Packet1Xd& a) {
1285 Packet1Xd tmp = print<Packet1Xd>(a);
1286 // If greater, subtract one.
1287 PacketMask64 mask = __riscv_vmflt_vv_f64m1_b64(a, tmp, unpacket_traits<Packet1Xd>::size);
1288 return __riscv_vfsub_vf_f64m1_tumu(mask, tmp, tmp, 1.0, unpacket_traits<Packet1Xd>::size);
1289}
1290
1291template <>
1292EIGEN_STRONG_INLINE Packet1Xd preverse(const Packet1Xd& a) {
1293 Packet1Xul idx = __riscv_vrsub_vx_u64m1(__riscv_vid_v_u64m1(unpacket_traits<Packet1Xd>::size),
1294 unpacket_traits<Packet1Xd>::size - 1, unpacket_traits<Packet1Xd>::size);
1295 return __riscv_vrgather_vv_f64m1(a, idx, unpacket_traits<Packet1Xd>::size);
1296}
1297
1298template <>
1299EIGEN_STRONG_INLINE Packet1Xd pfrexp<Packet1Xd>(const Packet1Xd& a, Packet1Xd& exponent) {
1300 return pfrexp_generic(a, exponent);
1301}
1302
1303template <>
1304EIGEN_STRONG_INLINE double predux<Packet1Xd>(const Packet1Xd& a) {
1305 return __riscv_vfmv_f(__riscv_vfredusum_vs_f64m1_f64m1(
1306 a, __riscv_vfmv_v_f_f64m1(0.0, unpacket_traits<Packet1Xd>::size), unpacket_traits<Packet1Xd>::size));
1307}
1308
1309template <>
1310EIGEN_STRONG_INLINE bool predux_any(const Packet1Xd& a) {
1311 const PacketMask64 mask =
1312 __riscv_vmsne_vx_u64m1_b64(__riscv_vreinterpret_v_f64m1_u64m1(a), 0, unpacket_traits<Packet1Xd>::size);
1313 return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size) != 0;
1314}
1315
1316template <>
1317EIGEN_STRONG_INLINE bool predux_all(const Packet1Xd& a) {
1318 const PacketMask64 mask = __riscv_vmfeq_vf_f64m1_b64(a, 0.0, unpacket_traits<Packet1Xd>::size);
1319 return __riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size) == 0;
1320}
1321
1322template <>
1323EIGEN_STRONG_INLINE Index predux_count(const Packet1Xd& a) {
1324 const PacketMask64 mask = __riscv_vmfne_vf_f64m1_b64(a, 0.0, unpacket_traits<Packet1Xd>::size);
1325 return static_cast<Index>(__riscv_vcpop_m_b64(mask, unpacket_traits<Packet1Xd>::size));
1326}
1327
1328template <>
1329EIGEN_STRONG_INLINE double predux_mul<Packet1Xd>(const Packet1Xd& a) {
1330 // Multiply the vector by its reverse
1331 Packet1Xd prod = __riscv_vfmul_vv_f64m1(preverse(a), a, unpacket_traits<Packet1Xd>::size);
1332 Packet1Xd half_prod;
1333
1334 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
1335 half_prod = __riscv_vslidedown_vx_f64m1(prod, 4, unpacket_traits<Packet1Xd>::size);
1336 prod = __riscv_vfmul_vv_f64m1(prod, half_prod, unpacket_traits<Packet1Xd>::size);
1337 }
1338 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
1339 half_prod = __riscv_vslidedown_vx_f64m1(prod, 2, unpacket_traits<Packet1Xd>::size);
1340 prod = __riscv_vfmul_vv_f64m1(prod, half_prod, unpacket_traits<Packet1Xd>::size);
1341 }
1342 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
1343 half_prod = __riscv_vslidedown_vx_f64m1(prod, 1, unpacket_traits<Packet1Xd>::size);
1344 prod = __riscv_vfmul_vv_f64m1(prod, half_prod, unpacket_traits<Packet1Xd>::size);
1345 }
1346
1347 // The reduction is done to the first element.
1348 return pfirst(prod);
1349}
1350
1351template <>
1352EIGEN_STRONG_INLINE double predux_min<Packet1Xd>(const Packet1Xd& a) {
1353 return __riscv_vfmv_f(__riscv_vfredmin_vs_f64m1_f64m1(a, a, unpacket_traits<Packet1Xd>::size));
1354}
1355
1356template <>
1357EIGEN_STRONG_INLINE double predux_max<Packet1Xd>(const Packet1Xd& a) {
1358 return __riscv_vfmv_f(__riscv_vfredmax_vs_f64m1_f64m1(a, a, unpacket_traits<Packet1Xd>::size));
1359}
1360
1361template <int N>
1362EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xd, N>& kernel) {
1363 double buffer[unpacket_traits<Packet1Xd>::size * N];
1364 int i = 0;
1365
1366 for (i = 0; i < N; i++) {
1367 __riscv_vsse64(&buffer[i], N * sizeof(double), kernel.packet[i], unpacket_traits<Packet1Xd>::size);
1368 }
1369
1370 for (i = 0; i < N; i++) {
1371 kernel.packet[i] =
1372 __riscv_vle64_v_f64m1(&buffer[i * unpacket_traits<Packet1Xd>::size], unpacket_traits<Packet1Xd>::size);
1373 }
1374}
1375
1376template <>
1377EIGEN_STRONG_INLINE Packet1Xd pldexp<Packet1Xd>(const Packet1Xd& a, const Packet1Xd& exponent) {
1378 return pldexp_generic(a, exponent);
1379}
1380
1381template <>
1382EIGEN_STRONG_INLINE PacketMask64 por(const PacketMask64& a, const PacketMask64& b) {
1383 return __riscv_vmor_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1384}
1385
1386template <>
1387EIGEN_STRONG_INLINE PacketMask64 pandnot(const PacketMask64& a, const PacketMask64& b) {
1388 return __riscv_vmnand_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1389}
1390
1391template <>
1392EIGEN_STRONG_INLINE PacketMask64 pand(const PacketMask64& a, const PacketMask64& b) {
1393 return __riscv_vmand_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1394}
1395
1396template <>
1397EIGEN_STRONG_INLINE PacketMask64 pxor(const PacketMask64& a, const PacketMask64& b) {
1398 return __riscv_vmxor_mm_b64(a, b, unpacket_traits<Packet1Xd>::size);
1399}
1400
1401template <>
1402EIGEN_STRONG_INLINE PacketMask64 pnot(const PacketMask64& a) {
1403 return __riscv_vmnot_m_b64(a, unpacket_traits<Packet1Xd>::size);
1404}
1405
1406EIGEN_STRONG_INLINE PacketMask64 pcmp_eq_mask(const Packet1Xd& a, const Packet1Xd& b) {
1407 return __riscv_vmfeq_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1408}
1409
1410EIGEN_STRONG_INLINE PacketMask64 pcmp_lt_mask(const Packet1Xd& a, const Packet1Xd& b) {
1411 return __riscv_vmflt_vv_f64m1_b64(a, b, unpacket_traits<Packet1Xd>::size);
1412}
1413
1414EIGEN_STRONG_INLINE Packet1Xd pselect(const PacketMask64& mask, const Packet1Xd& a, const Packet1Xd& b) {
1415 return __riscv_vmerge_vvm_f64m1(b, a, mask, unpacket_traits<Packet1Xd>::size);
1416}
1417
1418EIGEN_STRONG_INLINE Packet1Xd pselect(const Packet1Xd& mask, const Packet1Xd& a, const Packet1Xd& b) {
1419 PacketMask64 mask2 =
1420 __riscv_vmsne_vx_i64m1_b64(__riscv_vreinterpret_v_f64m1_i64m1(mask), 0, unpacket_traits<Packet1Xd>::size);
1421 return __riscv_vmerge_vvm_f64m1(b, a, mask2, unpacket_traits<Packet1Xd>::size);
1422}
1423
1424/********************************* Packet1Xs ************************************/
1425
1426EIGEN_STRONG_INLINE Packet1Xs __riscv_vreinterpret_v_u32m1_i16m1(const Packet1Xu& a) {
1427 return __riscv_vreinterpret_v_i32m1_i16m1(__riscv_vreinterpret_v_u32m1_i32m1(a));
1428}
1429
1430template <>
1431EIGEN_STRONG_INLINE Packet1Xs pset1<Packet1Xs>(const numext::int16_t& from) {
1432 return __riscv_vmv_v_x_i16m1(from, unpacket_traits<Packet1Xs>::size);
1433}
1434
1435template <>
1436EIGEN_STRONG_INLINE Packet1Xs plset<Packet1Xs>(const numext::int16_t& a) {
1437 Packet1Xs idx = __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vid_v_u16m1(unpacket_traits<Packet1Xs>::size));
1438 return __riscv_vadd_vx_i16m1(idx, a, unpacket_traits<Packet1Xs>::size);
1439}
1440
1441template <>
1442EIGEN_STRONG_INLINE Packet1Xs pzero<Packet1Xs>(const Packet1Xs& /*a*/) {
1443 return __riscv_vmv_v_x_i16m1(0, unpacket_traits<Packet1Xs>::size);
1444}
1445
1446template <>
1447EIGEN_STRONG_INLINE Packet1Xs padd<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1448 return __riscv_vadd_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1449}
1450
1451template <>
1452EIGEN_STRONG_INLINE Packet1Xs psub<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1453 return __riscv_vsub(a, b, unpacket_traits<Packet1Xs>::size);
1454}
1455
1456template <>
1457EIGEN_STRONG_INLINE Packet1Xs pnegate(const Packet1Xs& a) {
1458 return __riscv_vneg(a, unpacket_traits<Packet1Xs>::size);
1459}
1460
1461template <>
1462EIGEN_STRONG_INLINE Packet1Xs pmul<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1463 return __riscv_vmul(a, b, unpacket_traits<Packet1Xs>::size);
1464}
1465
1466template <>
1467EIGEN_STRONG_INLINE Packet1Xs pdiv<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1468 return __riscv_vdiv(a, b, unpacket_traits<Packet1Xs>::size);
1469}
1470
1471template <>
1472EIGEN_STRONG_INLINE Packet1Xs pmadd(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1473 return __riscv_vmadd(a, b, c, unpacket_traits<Packet1Xs>::size);
1474}
1475
1476template <>
1477EIGEN_STRONG_INLINE Packet1Xs pmsub(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1478 return __riscv_vmadd(a, b, pnegate(c), unpacket_traits<Packet1Xs>::size);
1479}
1480
1481template <>
1482EIGEN_STRONG_INLINE Packet1Xs pnmadd(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1483 return __riscv_vnmsub_vv_i16m1(a, b, c, unpacket_traits<Packet1Xs>::size);
1484}
1485
1486template <>
1487EIGEN_STRONG_INLINE Packet1Xs pnmsub(const Packet1Xs& a, const Packet1Xs& b, const Packet1Xs& c) {
1488 return __riscv_vnmsub_vv_i16m1(a, b, pnegate(c), unpacket_traits<Packet1Xs>::size);
1489}
1490
1491template <>
1492EIGEN_STRONG_INLINE Packet1Xs pmin<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1493 return __riscv_vmin(a, b, unpacket_traits<Packet1Xs>::size);
1494}
1495
1496template <>
1497EIGEN_STRONG_INLINE Packet1Xs pmax<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1498 return __riscv_vmax(a, b, unpacket_traits<Packet1Xs>::size);
1499}
1500
1501template <>
1502EIGEN_STRONG_INLINE Packet1Xs pcmp_le<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1503 PacketMask16 mask = __riscv_vmsle_vv_i16m1_b16(a, b, unpacket_traits<Packet1Xs>::size);
1504 return __riscv_vmerge_vxm_i16m1(pzero(a), static_cast<short>(0xffff), mask, unpacket_traits<Packet1Xs>::size);
1505}
1506
1507template <>
1508EIGEN_STRONG_INLINE Packet1Xs pcmp_lt<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1509 PacketMask16 mask = __riscv_vmslt_vv_i16m1_b16(a, b, unpacket_traits<Packet1Xs>::size);
1510 return __riscv_vmerge_vxm_i16m1(pzero(a), static_cast<short>(0xffff), mask, unpacket_traits<Packet1Xs>::size);
1511}
1512
1513template <>
1514EIGEN_STRONG_INLINE Packet1Xs pcmp_eq<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1515 PacketMask16 mask = __riscv_vmseq_vv_i16m1_b16(a, b, unpacket_traits<Packet1Xs>::size);
1516 return __riscv_vmerge_vxm_i16m1(pzero(a), static_cast<short>(0xffff), mask, unpacket_traits<Packet1Xs>::size);
1517}
1518
1519template <>
1520EIGEN_STRONG_INLINE Packet1Xs ptrue<Packet1Xs>(const Packet1Xs& /*a*/) {
1521 return __riscv_vmv_v_x_i16m1(static_cast<unsigned short>(0xffffu), unpacket_traits<Packet1Xs>::size);
1522}
1523
1524template <>
1525EIGEN_STRONG_INLINE Packet1Xs pand<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1526 return __riscv_vand_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1527}
1528
1529template <>
1530EIGEN_STRONG_INLINE Packet1Xs por<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1531 return __riscv_vor_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1532}
1533
1534template <>
1535EIGEN_STRONG_INLINE Packet1Xs pxor<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1536 return __riscv_vxor_vv_i16m1(a, b, unpacket_traits<Packet1Xs>::size);
1537}
1538
1539template <>
1540EIGEN_STRONG_INLINE Packet1Xs pnot<Packet1Xs>(const Packet1Xs& a) {
1541 return __riscv_vnot_v_i16m1(a, unpacket_traits<Packet1Xs>::size);
1542}
1543
1544template <>
1545EIGEN_STRONG_INLINE Packet1Xs pandnot<Packet1Xs>(const Packet1Xs& a, const Packet1Xs& b) {
1546#ifndef __riscv_zvbb
1547 return __riscv_vand_vv_i16m1(a, __riscv_vnot_v_i16m1(b, unpacket_traits<Packet1Xs>::size),
1548 unpacket_traits<Packet1Xs>::size);
1549#else
1550 return __riscv_vreinterpret_v_u16m1_i16m1(__riscv_vandn_vv_u16m1(
1551 __riscv_vreinterpret_v_i16m1_u16m1(a), __riscv_vreinterpret_v_i16m1_u16m1(b), unpacket_traits<Packet1Xs>::size));
1552#endif
1553}
1554
1555template <int N>
1556EIGEN_STRONG_INLINE Packet1Xs parithmetic_shift_right(const Packet1Xs& a) {
1557 return __riscv_vsra_vx_i16m1(a, N, unpacket_traits<Packet1Xs>::size);
1558}
1559
1560template <int N>
1561EIGEN_STRONG_INLINE Packet1Xs plogical_shift_right(const Packet1Xs& a) {
1562 return __riscv_vreinterpret_i16m1(
1563 __riscv_vsrl_vx_u16m1(__riscv_vreinterpret_u16m1(a), N, unpacket_traits<Packet1Xs>::size));
1564}
1565
1566template <int N>
1567EIGEN_STRONG_INLINE Packet1Xs plogical_shift_left(const Packet1Xs& a) {
1568 return __riscv_vsll_vx_i16m1(a, N, unpacket_traits<Packet1Xs>::size);
1569}
1570
1571template <>
1572EIGEN_STRONG_INLINE Packet1Xs pload<Packet1Xs>(const numext::int16_t* from) {
1573 EIGEN_DEBUG_ALIGNED_LOAD return __riscv_vle16_v_i16m1(from, unpacket_traits<Packet1Xs>::size);
1574}
1575
1576template <>
1577EIGEN_STRONG_INLINE Packet1Xs ploadu<Packet1Xs>(const numext::int16_t* from) {
1578 EIGEN_DEBUG_UNALIGNED_LOAD return __riscv_vle16_v_i16m1(from, unpacket_traits<Packet1Xs>::size);
1579}
1580
1581template <>
1582EIGEN_STRONG_INLINE Packet1Xs ploaddup<Packet1Xs>(const numext::int16_t* from) {
1583 Packet1Xu data = __riscv_vlmul_trunc_v_u32m2_u32m1(__riscv_vwcvtu_x_x_v_u32m2(
1584 __riscv_vreinterpret_v_i16m1_u16m1(pload<Packet1Xs>(from)), unpacket_traits<Packet1Xs>::size));
1585 return __riscv_vreinterpret_v_u32m1_i16m1(__riscv_vadd_vv_u32m1(
1586 __riscv_vsll_vx_u32m1(data, 16, unpacket_traits<Packet1Xi>::size), data, unpacket_traits<Packet1Xi>::size));
1587}
1588
1589template <>
1590EIGEN_STRONG_INLINE Packet1Xs ploadquad<Packet1Xs>(const numext::int16_t* from) {
1591 Packet1Xsu idx =
1592 __riscv_vsrl_vx_u16m1(__riscv_vid_v_u16m1(unpacket_traits<Packet1Xs>::size), 2, unpacket_traits<Packet1Xs>::size);
1593 return __riscv_vrgather_vv_i16m1(pload<Packet1Xs>(from), idx, unpacket_traits<Packet1Xs>::size);
1594}
1595
1596template <>
1597EIGEN_STRONG_INLINE void pstore<numext::int16_t>(numext::int16_t* to, const Packet1Xs& from) {
1598 EIGEN_DEBUG_ALIGNED_STORE __riscv_vse16_v_i16m1(to, from, unpacket_traits<Packet1Xs>::size);
1599}
1600
1601template <>
1602EIGEN_STRONG_INLINE void pstoreu<numext::int16_t>(numext::int16_t* to, const Packet1Xs& from) {
1603 EIGEN_DEBUG_UNALIGNED_STORE __riscv_vse16_v_i16m1(to, from, unpacket_traits<Packet1Xs>::size);
1604}
1605
1606template <>
1607EIGEN_DEVICE_FUNC inline Packet1Xs pgather<numext::int16_t, Packet1Xs>(const numext::int16_t* from, Index stride) {
1608 return __riscv_vlse16_v_i16m1(from, stride * sizeof(numext::int16_t), unpacket_traits<Packet1Xs>::size);
1609}
1610
1611template <>
1612EIGEN_DEVICE_FUNC inline void pscatter<numext::int16_t, Packet1Xs>(numext::int16_t* to, const Packet1Xs& from,
1613 Index stride) {
1614 __riscv_vsse16(to, stride * sizeof(numext::int16_t), from, unpacket_traits<Packet1Xs>::size);
1615}
1616
1617template <>
1618EIGEN_STRONG_INLINE numext::int16_t pfirst<Packet1Xs>(const Packet1Xs& a) {
1619 return __riscv_vmv_x_s_i16m1_i16(a);
1620}
1621
1622template <>
1623EIGEN_STRONG_INLINE Packet1Xs preverse(const Packet1Xs& a) {
1624 Packet1Xsu idx = __riscv_vrsub_vx_u16m1(__riscv_vid_v_u16m1(unpacket_traits<Packet1Xs>::size),
1625 unpacket_traits<Packet1Xs>::size - 1, unpacket_traits<Packet1Xs>::size);
1626 return __riscv_vrgather_vv_i16m1(a, idx, unpacket_traits<Packet1Xs>::size);
1627}
1628
1629template <>
1630EIGEN_STRONG_INLINE Packet1Xs pabs(const Packet1Xs& a) {
1631 Packet1Xs mask = __riscv_vsra_vx_i16m1(a, 15, unpacket_traits<Packet1Xs>::size);
1632 return __riscv_vsub_vv_i16m1(__riscv_vxor_vv_i16m1(a, mask, unpacket_traits<Packet1Xs>::size), mask,
1633 unpacket_traits<Packet1Xs>::size);
1634}
1635
1636template <>
1637EIGEN_STRONG_INLINE numext::int16_t predux<Packet1Xs>(const Packet1Xs& a) {
1638 return __riscv_vmv_x(__riscv_vredsum_vs_i16m1_i16m1(a, __riscv_vmv_v_x_i16m1(0, unpacket_traits<Packet1Xs>::size),
1639 unpacket_traits<Packet1Xs>::size));
1640}
1641
1642template <>
1643EIGEN_STRONG_INLINE bool predux_all(const Packet1Xs& a) {
1644 const PacketMask16 mask = __riscv_vmseq_vx_i16m1_b16(a, 0, unpacket_traits<Packet1Xs>::size);
1645 return __riscv_vcpop_m_b16(mask, unpacket_traits<Packet1Xs>::size) == 0;
1646}
1647
1648template <>
1649EIGEN_STRONG_INLINE numext::int16_t predux_mul<Packet1Xs>(const Packet1Xs& a) {
1650 // Multiply the vector by its reverse
1651 Packet1Xs prod = __riscv_vmul_vv_i16m1(preverse(a), a, unpacket_traits<Packet1Xs>::size);
1652 Packet1Xs half_prod;
1653
1654 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 1024) {
1655 half_prod = __riscv_vslidedown_vx_i16m1(prod, 16, unpacket_traits<Packet1Xs>::size);
1656 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1657 }
1658 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 512) {
1659 half_prod = __riscv_vslidedown_vx_i16m1(prod, 8, unpacket_traits<Packet1Xs>::size);
1660 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1661 }
1662 EIGEN_IF_CONSTEXPR (EIGEN_RISCV64_RVV_VL >= 256) {
1663 half_prod = __riscv_vslidedown_vx_i16m1(prod, 4, unpacket_traits<Packet1Xs>::size);
1664 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1665 }
1666 // Last reduction
1667 half_prod = __riscv_vslidedown_vx_i16m1(prod, 2, unpacket_traits<Packet1Xs>::size);
1668 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1669
1670 half_prod = __riscv_vslidedown_vx_i16m1(prod, 1, unpacket_traits<Packet1Xs>::size);
1671 prod = __riscv_vmul_vv_i16m1(prod, half_prod, unpacket_traits<Packet1Xs>::size);
1672
1673 // The reduction is done to the first element.
1674 return pfirst(prod);
1675}
1676
1677template <>
1678EIGEN_STRONG_INLINE numext::int16_t predux_min<Packet1Xs>(const Packet1Xs& a) {
1679 return __riscv_vmv_x(__riscv_vredmin_vs_i16m1_i16m1(
1680 a, __riscv_vmv_v_x_i16m1((std::numeric_limits<numext::int16_t>::max)(), unpacket_traits<Packet1Xs>::size),
1681 unpacket_traits<Packet1Xs>::size));
1682}
1683
1684template <>
1685EIGEN_STRONG_INLINE numext::int16_t predux_max<Packet1Xs>(const Packet1Xs& a) {
1686 return __riscv_vmv_x(__riscv_vredmax_vs_i16m1_i16m1(
1687 a, __riscv_vmv_v_x_i16m1((std::numeric_limits<numext::int16_t>::min)(), unpacket_traits<Packet1Xs>::size),
1688 unpacket_traits<Packet1Xs>::size));
1689}
1690
1691template <int N>
1692EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet1Xs, N>& kernel) {
1693 numext::int16_t buffer[unpacket_traits<Packet1Xs>::size * N] = {0};
1694 int i = 0;
1695
1696 for (i = 0; i < N; i++) {
1697 __riscv_vsse16(&buffer[i], N * sizeof(numext::int16_t), kernel.packet[i], unpacket_traits<Packet1Xs>::size);
1698 }
1699 for (i = 0; i < N; i++) {
1700 kernel.packet[i] =
1701 __riscv_vle16_v_i16m1(&buffer[i * unpacket_traits<Packet1Xs>::size], unpacket_traits<Packet1Xs>::size);
1702 }
1703}
1704
1705template <>
1706EIGEN_STRONG_INLINE PacketMask16 por(const PacketMask16& a, const PacketMask16& b) {
1707 return __riscv_vmor_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1708}
1709
1710template <>
1711EIGEN_STRONG_INLINE PacketMask16 pand(const PacketMask16& a, const PacketMask16& b) {
1712 return __riscv_vmand_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1713}
1714
1715template <>
1716EIGEN_STRONG_INLINE PacketMask16 pxor(const PacketMask16& a, const PacketMask16& b) {
1717 return __riscv_vmxor_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1718}
1719
1720template <>
1721EIGEN_STRONG_INLINE PacketMask16 pandnot(const PacketMask16& a, const PacketMask16& b) {
1722 return __riscv_vmandn_mm_b16(a, b, unpacket_traits<Packet1Xs>::size);
1723}
1724
1725template <>
1726EIGEN_STRONG_INLINE PacketMask16 pnot(const PacketMask16& a) {
1727 return __riscv_vmnot_m_b16(a, unpacket_traits<Packet1Xs>::size);
1728}
1729
1730} // namespace internal
1731} // namespace Eigen
1732
1733#endif // EIGEN_PACKET_MATH_RVV10_H