12#ifndef EIGEN_RVV10_GENERAL_BLOCK_KERNEL_H
13#define EIGEN_RVV10_GENERAL_BLOCK_KERNEL_H
16#include "../../InternalHeaderCheck.h"
24struct gebp_traits<float, float, false, false, Architecture::RVV10, GEBPPacketFull>
25 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
26 using RhsPacket = float;
27 using RhsPacketx4 = QuadPacket<float>;
28 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = pset1<RhsPacket>(*b); }
29 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
30 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
33 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacket& dest)
const { loadRhs(b, dest); }
35 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
37 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { dest = ploadquad<RhsPacket>(b); }
39 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, AccPacket& c, RhsPacket& ,
40 const FixedInt<0>&)
const {
41#if EIGEN_RISCV64_DEFAULT_LMUL == 1
42 c = __riscv_vfmadd_vf_f32m1(a, b, c, unpacket_traits<AccPacket>::size);
43#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
44 c = __riscv_vfmadd_vf_f32m2(a, b, c, unpacket_traits<AccPacket>::size);
45#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
46 c = __riscv_vfmadd_vf_f32m4(a, b, c, unpacket_traits<AccPacket>::size);
50#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
51 EIGEN_STRONG_INLINE
void madd(
const Packet1Xf& a,
const RhsPacket& b, Packet1Xf& c, RhsPacket& ,
52 const FixedInt<0>&)
const {
53 c = __riscv_vfmadd_vf_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
56#if EIGEN_RISCV64_DEFAULT_LMUL == 4
57 EIGEN_STRONG_INLINE
void madd(
const Packet2Xf& a,
const RhsPacket& b, Packet2Xf& c, RhsPacket& ,
58 const FixedInt<0>&)
const {
59 c = __riscv_vfmadd_vf_f32m2(a, b, c, unpacket_traits<Packet2Xf>::size);
63 template <
typename LaneIdType>
64 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
65 const LaneIdType& lane)
const {
66#if EIGEN_RISCV64_DEFAULT_LMUL == 1
67 c = __riscv_vfmadd_vf_f32m1(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
68#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
69 c = __riscv_vfmadd_vf_f32m2(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
70#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
71 c = __riscv_vfmadd_vf_f32m4(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
77struct gebp_traits<double, double, false, false, Architecture::RVV10, GEBPPacketFull>
78 : gebp_traits<double, double, false, false, Architecture::Generic, GEBPPacketFull> {
79 using RhsPacket = double;
80 using RhsPacketx4 = QuadPacket<double>;
81 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = pset1<RhsPacket>(*b); }
82 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
83 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
86 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacket& dest)
const { loadRhs(b, dest); }
88 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
90 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { dest = ploadquad<RhsPacket>(b); }
92 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, AccPacket& c, RhsPacket& ,
93 const FixedInt<0>&)
const {
94#if EIGEN_RISCV64_DEFAULT_LMUL == 1
95 c = __riscv_vfmadd_vf_f64m1(a, b, c, unpacket_traits<AccPacket>::size);
96#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
97 c = __riscv_vfmadd_vf_f64m2(a, b, c, unpacket_traits<AccPacket>::size);
98#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
99 c = __riscv_vfmadd_vf_f64m4(a, b, c, unpacket_traits<AccPacket>::size);
103#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
104 EIGEN_STRONG_INLINE
void madd(
const Packet1Xd& a,
const RhsPacket& b, Packet1Xd& c, RhsPacket& ,
105 const FixedInt<0>&)
const {
106 c = __riscv_vfmadd_vf_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
109#if EIGEN_RISCV64_DEFAULT_LMUL == 4
110 EIGEN_STRONG_INLINE
void madd(
const Packet2Xd& a,
const RhsPacket& b, Packet2Xd& c, RhsPacket& ,
111 const FixedInt<0>&)
const {
112 c = __riscv_vfmadd_vf_f64m2(a, b, c, unpacket_traits<Packet2Xd>::size);
116 template <
typename LaneIdType>
117 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
118 const LaneIdType& lane)
const {
119#if EIGEN_RISCV64_DEFAULT_LMUL == 1
120 c = __riscv_vfmadd_vf_f64m1(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
121#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
122 c = __riscv_vfmadd_vf_f64m2(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
123#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
124 c = __riscv_vfmadd_vf_f64m4(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
129#if defined(EIGEN_VECTORIZE_RVV10FP16)
132struct gebp_traits<half, half, false, false, Architecture::RVV10>
133 : gebp_traits<half, half, false, false, Architecture::Generic> {
134 using RhsPacket = half;
135 using LhsPacket = PacketXh;
136 using AccPacket = PacketXh;
137 using RhsPacketx4 = QuadPacket<half>;
139 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = pset1<RhsPacket>(*b); }
140 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
141 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
144 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacket& dest)
const { loadRhs(b, dest); }
146 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
148 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { dest = pload<RhsPacket>(b); }
150 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, AccPacket& c, RhsPacket& ,
151 const FixedInt<0>&)
const {
152#if EIGEN_RISCV64_DEFAULT_LMUL == 1
153 c = __riscv_vfmadd_vf_f16m1(a, numext::bit_cast<_Float16>(b), c, unpacket_traits<AccPacket>::size);
155 c = __riscv_vfmadd_vf_f16m2(a, numext::bit_cast<_Float16>(b), c, unpacket_traits<AccPacket>::size);
159#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
160 EIGEN_STRONG_INLINE
void madd(
const Packet1Xh& a,
const RhsPacket& b, Packet1Xh& c, RhsPacket& ,
161 const FixedInt<0>&)
const {
162 c = __riscv_vfmadd_vf_f16m1(a, numext::bit_cast<_Float16>(b), c, unpacket_traits<Packet1Xh>::size);
166 template <
typename LaneIdType>
167 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
168 const LaneIdType& lane)
const {
169#if EIGEN_RISCV64_DEFAULT_LMUL == 1
170 c = __riscv_vfmadd_vf_f16m1(a, numext::bit_cast<_Float16>(b.get(lane)), c, unpacket_traits<AccPacket>::size);
172 c = __riscv_vfmadd_vf_f16m2(a, numext::bit_cast<_Float16>(b.get(lane)), c, unpacket_traits<AccPacket>::size);
179#if defined(EIGEN_VECTORIZE_RVV10BF16)
182struct gebp_traits<bfloat16, bfloat16, false, false, Architecture::RVV10>
183 : gebp_traits<bfloat16, bfloat16, false, false, Architecture::Generic> {
184 using RhsPacket = bfloat16;
185 using LhsPacket = PacketXbf;
186 using AccPacket = PacketXbf;
187 using RhsPacketx4 = QuadPacket<bfloat16>;
189 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacket& dest)
const { dest = pset1<RhsPacket>(*b); }
190 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
191 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
194 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacket& dest)
const { loadRhs(b, dest); }
196 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
198 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { dest = pload<RhsPacket>(b); }
200 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, AccPacket& c, RhsPacket& ,
201 const FixedInt<0>&)
const {
202#if EIGEN_RISCV64_DEFAULT_LMUL == 1
204 __riscv_vfwmaccbf16_vf_f32m2(Bf16ToF32(c), numext::bit_cast<__bf16>(b), a, unpacket_traits<AccPacket>::size));
207 __riscv_vfwmaccbf16_vf_f32m4(Bf16ToF32(c), numext::bit_cast<__bf16>(b), a, unpacket_traits<AccPacket>::size));
211#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
212 EIGEN_STRONG_INLINE
void madd(
const Packet1Xbf& a,
const RhsPacket& b, Packet1Xbf& c, RhsPacket& ,
213 const FixedInt<0>&)
const {
215 __riscv_vfwmaccbf16_vf_f32m2(Bf16ToF32(c), numext::bit_cast<__bf16>(b), a, unpacket_traits<Packet1Xbf>::size));
219 template <
typename LaneIdType>
220 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacketx4& b, AccPacket& c, RhsPacket& ,
221 const LaneIdType& lane)
const {
222#if EIGEN_RISCV64_DEFAULT_LMUL == 1
223 c = F32ToBf16(__riscv_vfwmaccbf16_vf_f32m2(Bf16ToF32(c), numext::bit_cast<__bf16>(b.get(lane)), a,
224 unpacket_traits<AccPacket>::size));
226 c = F32ToBf16(__riscv_vfwmaccbf16_vf_f32m4(Bf16ToF32(c), numext::bit_cast<__bf16>(b.get(lane)), a,
227 unpacket_traits<AccPacket>::size));
234#if EIGEN_RISCV64_DEFAULT_LMUL == 1
237#define PACKET_DECL_COND_POSTFIX(postfix, name, packet_size) \
238 using name##Packet##postfix = \
239 typename packet_conditional<packet_size, typename packet_traits<name##Scalar>::type, \
240 typename packet_traits<name##Scalar>::half, \
241 typename unpacket_traits<typename packet_traits<name##Scalar>::half>::half>::type
243#define RISCV_COMPLEX_PACKET_DECL_COND_SCALAR(packet_size) \
244 using ScalarPacket = \
245 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
246 typename packet_traits<Scalar>::half, \
247 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
249template <
typename RealScalar,
bool ConjLhs_,
bool ConjRhs_,
int PacketSize_>
250struct gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, ConjLhs_, ConjRhs_, Architecture::RVV10,
251 PacketSize_> : gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, ConjLhs_, ConjRhs_,
252 Architecture::Generic, PacketSize_> {
253 using Scalar = std::complex<RealScalar>;
254 using LhsScalar = std::complex<RealScalar>;
255 using RhsScalar = std::complex<RealScalar>;
256 using ResScalar = std::complex<RealScalar>;
257 using RealPacket =
typename packet_traits<std::complex<RealScalar>>::type;
259 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
260 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
261 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
262 RISCV_COMPLEX_PACKET_DECL_COND_SCALAR(PacketSize_);
263#undef RISCV_COMPLEX_PACKET_DECL_COND_SCALAR
268 Vectorizable = unpacket_traits<RealPacket>::vectorizable && unpacket_traits<ScalarPacket>::vectorizable,
269 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
270 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
271 RhsPacketSize = Vectorizable ? unpacket_traits<RhsScalar>::size : 1,
272 RealPacketSize = Vectorizable ? unpacket_traits<RealPacket>::size : 1,
277 LhsProgress = ResPacketSize,
281 using DoublePacketType = DoublePacket<RealPacket>;
283 using LhsPacket4Packing = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
284 using LhsPacket = std::conditional_t<Vectorizable, RealPacket, Scalar>;
285 using RhsPacket = std::conditional_t<Vectorizable, DoublePacket<RealScalar>, Scalar>;
286 using ResPacket = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
287 using AccPacket = std::conditional_t<Vectorizable, DoublePacketType, Scalar>;
289 using RhsPacketx4 = QuadPacket<RhsPacket>;
291 EIGEN_STRONG_INLINE
void initAcc(Scalar& p) { p = Scalar(0); }
293 EIGEN_STRONG_INLINE
void initAcc(DoublePacketType& p) {
294 p.first = pset1<RealPacket>(RealScalar(0));
295 p.second = pset1<RealPacket>(RealScalar(0));
299 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, ScalarPacket& dest)
const { dest = pset1<ScalarPacket>(*b); }
302 template <
typename RealPacketType>
303 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, DoublePacket<RealPacketType>& dest)
const {
304 dest.first = pset1<RealPacketType>(numext::real(*b));
305 dest.second = pset1<RealPacketType>(numext::imag(*b));
308 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
309 loadRhs(b, dest.B_0);
310 loadRhs(b + 1, dest.B1);
311 loadRhs(b + 2, dest.B2);
312 loadRhs(b + 3, dest.B3);
316 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, ScalarPacket& dest)
const { loadRhs(b, dest); }
319 template <
typename RealPacketType>
320 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, DoublePacket<RealPacketType>& dest)
const {
324 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
326 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, ResPacket& dest)
const { loadRhs(b, dest); }
327 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, DoublePacket<RealScalar>& dest)
const {
328 loadQuadToDoublePacket(b, dest);
332 EIGEN_STRONG_INLINE
void loadLhs(
const LhsScalar* a, LhsPacket& dest)
const {
333 dest = pload<LhsPacket>((
const typename unpacket_traits<LhsPacket>::type*)(a));
336 template <
typename LhsPacketType>
337 EIGEN_STRONG_INLINE
void loadLhsUnaligned(
const LhsScalar* a, LhsPacketType& dest)
const {
338 dest = ploadu<LhsPacketType>((
const typename unpacket_traits<LhsPacketType>::type*)(a));
341 EIGEN_STRONG_INLINE Packet1Xcf pmadd_scalar(
const Packet1Xcf& a,
float b,
const Packet1Xcf& c)
const {
342 return Packet1Xcf(__riscv_vfmadd_vf_f32m1(a.v, b, c.v, unpacket_traits<Packet1Xf>::size));
345 EIGEN_STRONG_INLINE Packet1Xcd pmadd_scalar(
const Packet1Xcd& a,
double b,
const Packet1Xcd& c)
const {
346 return Packet1Xcd(__riscv_vfmadd_vf_f64m1(a.v, b, c.v, unpacket_traits<Packet1Xd>::size));
349#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
350 EIGEN_STRONG_INLINE Packet2Xcf pmadd_scalar(
const Packet2Xcf& a,
float b,
const Packet2Xcf& c)
const {
351 return Packet2Xcf(__riscv_vfmadd_vf_f32m2(a.v, b, c.v, unpacket_traits<Packet2Xf>::size));
354 EIGEN_STRONG_INLINE Packet2Xcd pmadd_scalar(
const Packet2Xcd& a,
double b,
const Packet2Xcd& c)
const {
355 return Packet2Xcd(__riscv_vfmadd_vf_f64m2(a.v, b, c.v, unpacket_traits<Packet2Xd>::size));
359#if EIGEN_RISCV64_DEFAULT_LMUL == 4
360 EIGEN_STRONG_INLINE Packet4Xcf pmadd_scalar(
const Packet4Xcf& a,
float b,
const Packet4Xcf& c)
const {
361 return Packet4Xcf(__riscv_vfmadd_vf_f32m4(a.v, b, c.v, unpacket_traits<Packet4Xf>::size));
364 EIGEN_STRONG_INLINE Packet4Xcd pmadd_scalar(
const Packet4Xcd& a,
double b,
const Packet4Xcd& c)
const {
365 return Packet4Xcd(__riscv_vfmadd_vf_f64m4(a.v, b, c.v, unpacket_traits<Packet4Xd>::size));
369 template <
typename LhsPacketType,
typename RhsPacketType,
typename ResPacketType,
typename TmpType,
371 EIGEN_STRONG_INLINE std::enable_if_t<!is_same<RhsPacketType, RhsPacketx4>::value> madd(
const LhsPacketType& a,
372 const RhsPacketType& b,
373 DoublePacket<ResPacketType>& c,
375 const LaneIdType&)
const {
376 c.first = pmadd_scalar(a, b.first, c.first);
377 c.second = pmadd_scalar(a, b.second, c.second);
380 template <
typename LhsPacketType,
typename AccPacketType,
typename LaneIdType>
381 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
382 const LaneIdType& lane)
const {
383 madd(a, b.get(lane), c, tmp, lane);
386 template <
typename LaneIdType>
387 EIGEN_STRONG_INLINE
void madd(
const LhsPacket& a,
const RhsPacket& b, ResPacket& c, RhsPacket& ,
388 const LaneIdType&)
const {
389 c = cj.pmadd(a, b, c);
392 template <
typename RealPacketType,
typename ResPacketType>
393 EIGEN_STRONG_INLINE
void acc(
const DoublePacket<RealPacketType>& c,
const ResPacketType& alpha,
394 ResPacketType& r)
const {
397 EIGEN_IF_CONSTEXPR ((!ConjLhs) && (!ConjRhs)) {
398 tmp = pcplxflip(pconj(ResPacketType(c.second)));
399 tmp = padd(ResPacketType(c.first), tmp);
400 }
else EIGEN_IF_CONSTEXPR ((!ConjLhs) && (ConjRhs)) {
401 tmp = pconj(pcplxflip(ResPacketType(c.second)));
402 tmp = padd(ResPacketType(c.first), tmp);
403 }
else EIGEN_IF_CONSTEXPR ((ConjLhs) && (!ConjRhs)) {
404 tmp = pcplxflip(ResPacketType(c.second));
405 tmp = padd(pconj(ResPacketType(c.first)), tmp);
406 }
else EIGEN_IF_CONSTEXPR ((ConjLhs) && (ConjRhs)) {
407 tmp = pcplxflip(ResPacketType(c.second));
408 tmp = psub(pconj(ResPacketType(c.first)), tmp);
411 r = pmadd(tmp, alpha, r);
415 EIGEN_STRONG_INLINE
void acc(
const Scalar& c,
const Scalar& alpha, Scalar& r)
const { r += alpha * c; }
418 conj_helper<LhsScalar, RhsScalar, ConjLhs, ConjRhs> cj;
421#define PACKET_DECL_COND_SCALAR_POSTFIX(postfix, packet_size) \
422 using ScalarPacket##postfix = \
423 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
424 typename packet_traits<Scalar>::half, \
425 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
427template <
typename RealScalar,
bool ConjRhs_,
int PacketSize_>
428class gebp_traits<RealScalar, std::complex<RealScalar>, false, ConjRhs_, Architecture::RVV10, PacketSize_>
429 :
public gebp_traits<RealScalar, std::complex<RealScalar>, false, ConjRhs_, Architecture::Generic, PacketSize_> {
431 using Scalar = std::complex<RealScalar>;
432 using LhsScalar = RealScalar;
433 using RhsScalar = Scalar;
434 using ResScalar = Scalar;
435 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
436 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
437 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
438 PACKET_DECL_COND_POSTFIX(_, Real, PacketSize_);
439 PACKET_DECL_COND_SCALAR_POSTFIX(_, PacketSize_);
440#undef PACKET_DECL_COND_SCALAR_POSTFIX
445 Vectorizable = unpacket_traits<RealPacket_>::vectorizable && unpacket_traits<ScalarPacket_>::vectorizable,
446 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
447 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
448 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
450 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
453 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * ResPacketSize,
455 LhsProgress = ResPacketSize,
459 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
460 using RhsPacket = RhsScalar;
461 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
462 using LhsPacket4Packing = LhsPacket;
463 using RhsPacketx4 = QuadPacket<RhsPacket>;
464 using AccPacket = ResPacket;
466 EIGEN_STRONG_INLINE
void initAcc(AccPacket& p)
const { p = pset1<ResPacket>(ResScalar(0)); }
468 template <
typename RhsPacketType>
469 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
470 dest = pset1<RhsPacketType>(*b);
473 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
474 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
477 template <
typename RhsPacketType>
478 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
482 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
484 EIGEN_STRONG_INLINE
void loadLhs(
const LhsScalar* a, LhsPacket& dest)
const { dest = ploaddup<LhsPacket>(a); }
486 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const { dest = ploadquad<RhsPacket>(b); }
488 template <
typename LhsPacketType>
489 EIGEN_STRONG_INLINE
void loadLhsUnaligned(
const LhsScalar* a, LhsPacketType& dest)
const {
490 dest = ploaddup<LhsPacketType>(a);
493 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType,
typename LaneIdType>
494 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
495 const LaneIdType&)
const {
496 madd_impl(a, b, c, tmp, std::conditional_t<Vectorizable, std::true_type, std::false_type>());
499 EIGEN_STRONG_INLINE Packet1Xcf pmadd_scalar(
const Packet1Xf& a, std::complex<float> b,
const Packet1Xcf& c)
const {
500 Packet1Xcf bp = pset1<Packet1Xcf>(b);
501 return Packet1Xcf(__riscv_vfmadd_vv_f32m1(a, bp.v, c.v, unpacket_traits<Packet1Xf>::size));
504 EIGEN_STRONG_INLINE Packet1Xcd pmadd_scalar(
const Packet1Xd& a, std::complex<double> b,
const Packet1Xcd& c)
const {
505 Packet1Xcd bp = pset1<Packet1Xcd>(b);
506 return Packet1Xcd(__riscv_vfmadd_vv_f64m1(a, bp.v, c.v, unpacket_traits<Packet1Xd>::size));
509#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
510 EIGEN_STRONG_INLINE Packet2Xcf pmadd_scalar(
const Packet2Xf& a, std::complex<float> b,
const Packet2Xcf& c)
const {
511 Packet2Xcf bp = pset1<Packet2Xcf>(b);
512 return Packet2Xcf(__riscv_vfmadd_vv_f32m2(a, bp.v, c.v, unpacket_traits<Packet2Xf>::size));
515 EIGEN_STRONG_INLINE Packet2Xcd pmadd_scalar(
const Packet2Xd& a, std::complex<double> b,
const Packet2Xcd& c)
const {
516 Packet2Xcd bp = pset1<Packet2Xcd>(b);
517 return Packet2Xcd(__riscv_vfmadd_vv_f64m2(a, bp.v, c.v, unpacket_traits<Packet2Xd>::size));
521#if EIGEN_RISCV64_DEFAULT_LMUL == 4
522 EIGEN_STRONG_INLINE Packet4Xcf pmadd_scalar(
const Packet4Xf& a, std::complex<float> b,
const Packet4Xcf& c)
const {
523 Packet4Xcf bp = pset1<Packet4Xcf>(b);
524 return Packet4Xcf(__riscv_vfmadd_vv_f32m4(a, bp.v, c.v, unpacket_traits<Packet4Xf>::size));
527 EIGEN_STRONG_INLINE Packet4Xcd pmadd_scalar(
const Packet4Xd& a, std::complex<double> b,
const Packet4Xcd& c)
const {
528 Packet4Xcd bp = pset1<Packet4Xcd>(b);
529 return Packet4Xcd(__riscv_vfmadd_vv_f64m4(a, bp.v, c.v, unpacket_traits<Packet4Xd>::size));
533 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType>
534 EIGEN_STRONG_INLINE
void madd_impl(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c,
535 RhsPacketType& tmp,
const std::true_type&)
const {
536 EIGEN_UNUSED_VARIABLE(tmp);
537 c = pmadd_scalar(a, b, c);
540 EIGEN_STRONG_INLINE
void madd_impl(
const LhsScalar& a,
const RhsScalar& b, ResScalar& c, RhsScalar& ,
541 const std::false_type&)
const {
545 template <
typename LhsPacketType,
typename AccPacketType,
typename LaneIdType>
546 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
547 const LaneIdType& lane)
const {
548 madd(a, b.get(lane), c, tmp, lane);
551 template <
typename ResPacketType,
typename AccPacketType>
552 EIGEN_STRONG_INLINE
void acc(
const AccPacketType& c,
const ResPacketType& alpha, ResPacketType& r)
const {
553 conj_helper<ResPacketType, ResPacketType, false, ConjRhs> cj;
554 r = cj.pmadd(alpha, c, r);
558template <
typename RealScalar,
bool ConjLhs_,
int PacketSize_>
559class gebp_traits<std::complex<RealScalar>, RealScalar, ConjLhs_, false, Architecture::RVV10, PacketSize_>
560 :
public gebp_traits<RealScalar, std::complex<RealScalar>, ConjLhs_, false, Architecture::Generic, PacketSize_> {
562 using LhsScalar = std::complex<RealScalar>;
563 using RhsScalar = RealScalar;
564 using ResScalar =
typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
566 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
567 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
568 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
569#undef PACKET_DECL_COND_POSTFIX
574 Vectorizable = unpacket_traits<LhsPacket_>::vectorizable && unpacket_traits<RhsPacket_>::vectorizable,
575 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
576 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
577 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
580 mr = 3 * LhsPacketSize,
582 LhsProgress = LhsPacketSize,
586 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
587 using RhsPacket = RhsScalar;
588 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
589 using LhsPacket4Packing = LhsPacket;
591 using RhsPacketx4 = QuadPacket<RhsPacket>;
593 using AccPacket = ResPacket;
595 EIGEN_STRONG_INLINE
void initAcc(AccPacket& p) { p = pset1<ResPacket>(ResScalar(0)); }
597 template <
typename RhsPacketType>
598 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
599 dest = pset1<RhsPacketType>(*b);
602 EIGEN_STRONG_INLINE
void loadRhs(
const RhsScalar* b, RhsPacketx4& dest)
const {
603 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
606 template <
typename RhsPacketType>
607 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar* b, RhsPacketType& dest)
const {
611 EIGEN_STRONG_INLINE
void updateRhs(
const RhsScalar*, RhsPacketx4&)
const {}
613 EIGEN_STRONG_INLINE
void loadRhsQuad(
const RhsScalar* b, RhsPacket& dest)
const {
614 loadRhsQuad_impl(b, dest, std::conditional_t<RhsPacketSize == 16, std::true_type, std::false_type>());
617 EIGEN_STRONG_INLINE
void loadRhsQuad_impl(
const RhsScalar* b, RhsPacket& dest,
const std::true_type&)
const {
620 RhsScalar tmp[4] = {b[0], b[0], b[1], b[1]};
621 dest = ploadquad<RhsPacket>(tmp);
624 EIGEN_STRONG_INLINE
void loadRhsQuad_impl(
const RhsScalar* b, RhsPacket& dest,
const std::false_type&)
const {
625 eigen_internal_assert(RhsPacketSize <= 8);
626 dest = pset1<RhsPacket>(*b);
629 EIGEN_STRONG_INLINE
void loadLhs(
const LhsScalar* a, LhsPacket& dest)
const { dest = pload<LhsPacket>(a); }
631 template <
typename LhsPacketType>
632 EIGEN_STRONG_INLINE
void loadLhsUnaligned(
const LhsScalar* a, LhsPacketType& dest)
const {
633 dest = ploadu<LhsPacketType>(a);
636 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType,
typename LaneIdType>
637 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
638 const LaneIdType&)
const {
639 madd_impl(a, b, c, tmp, std::conditional_t<Vectorizable, std::true_type, std::false_type>());
642 EIGEN_STRONG_INLINE Packet1Xcf pmadd_scalar(
const Packet1Xcf& a,
float b,
const Packet1Xcf& c)
const {
643 return Packet1Xcf(__riscv_vfmadd_vf_f32m1(a.v, b, c.v, unpacket_traits<Packet1Xf>::size));
646 EIGEN_STRONG_INLINE Packet1Xcd pmadd_scalar(
const Packet1Xcd& a,
double b,
const Packet1Xcd& c)
const {
647 return Packet1Xcd(__riscv_vfmadd_vf_f64m1(a.v, b, c.v, unpacket_traits<Packet1Xd>::size));
650#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
651 EIGEN_STRONG_INLINE Packet2Xcf pmadd_scalar(
const Packet2Xcf& a,
float b,
const Packet2Xcf& c)
const {
652 return Packet2Xcf(__riscv_vfmadd_vf_f32m2(a.v, b, c.v, unpacket_traits<Packet2Xf>::size));
655 EIGEN_STRONG_INLINE Packet2Xcd pmadd_scalar(
const Packet2Xcd& a,
double b,
const Packet2Xcd& c)
const {
656 return Packet2Xcd(__riscv_vfmadd_vf_f64m2(a.v, b, c.v, unpacket_traits<Packet2Xd>::size));
660#if EIGEN_RISCV64_DEFAULT_LMUL == 4
661 EIGEN_STRONG_INLINE Packet4Xcf pmadd_scalar(
const Packet4Xcf& a,
float b,
const Packet4Xcf& c)
const {
662 return Packet4Xcf(__riscv_vfmadd_vf_f32m4(a.v, b, c.v, unpacket_traits<Packet4Xf>::size));
665 EIGEN_STRONG_INLINE Packet4Xcd pmadd_scalar(
const Packet4Xcd& a,
double b,
const Packet4Xcd& c)
const {
666 return Packet4Xcd(__riscv_vfmadd_vf_f64m4(a.v, b, c.v, unpacket_traits<Packet4Xd>::size));
670 template <
typename LhsPacketType,
typename RhsPacketType,
typename AccPacketType>
671 EIGEN_STRONG_INLINE
void madd_impl(
const LhsPacketType& a,
const RhsPacketType& b, AccPacketType& c,
672 RhsPacketType& tmp,
const std::true_type&)
const {
673 EIGEN_UNUSED_VARIABLE(tmp);
674 c = pmadd_scalar(a, b, c);
677 EIGEN_STRONG_INLINE
void madd_impl(
const LhsScalar& a,
const RhsScalar& b, ResScalar& c, RhsScalar& ,
678 const std::false_type&)
const {
682 template <
typename LhsPacketType,
typename AccPacketType,
typename LaneIdType>
683 EIGEN_STRONG_INLINE
void madd(
const LhsPacketType& a,
const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
684 const LaneIdType& lane)
const {
685 madd(a, b.get(lane), c, tmp, lane);
688 template <
typename ResPacketType,
typename AccPacketType>
689 EIGEN_STRONG_INLINE
void acc(
const AccPacketType& c,
const ResPacketType& alpha, ResPacketType& r)
const {
690 conj_helper<ResPacketType, ResPacketType, ConjLhs, false> cj;
691 r = cj.pmadd(c, alpha, r);