Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GeneralBlockPanelKernel.h
1// This file is part of Eigen, a lightweight C template library
2// for linear algebra.
3//
4// Copyright (C) 2024 Kseniya Zaytseva <kseniya.zaytseva@syntacore.com>
5// Copyright (C) 2025 Chip Kerchner <ckerchner@tenstorrent.com>
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_RVV10_GENERAL_BLOCK_KERNEL_H
13#define EIGEN_RVV10_GENERAL_BLOCK_KERNEL_H
14
15// IWYU pragma: private
16#include "../../InternalHeaderCheck.h"
17
18namespace Eigen {
19namespace internal {
20
21/********************************* real ************************************/
22
23template <>
24struct gebp_traits<float, float, false, false, Architecture::RVV10, GEBPPacketFull>
25 : gebp_traits<float, float, false, false, Architecture::Generic, GEBPPacketFull> {
26 using RhsPacket = float;
27 using RhsPacketx4 = QuadPacket<float>;
28 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = pset1<RhsPacket>(*b); }
29 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
30 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
31 }
32
33 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
34
35 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
36
37 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = ploadquad<RhsPacket>(b); }
38
39 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
40 const FixedInt<0>&) const {
41#if EIGEN_RISCV64_DEFAULT_LMUL == 1
42 c = __riscv_vfmadd_vf_f32m1(a, b, c, unpacket_traits<AccPacket>::size);
43#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
44 c = __riscv_vfmadd_vf_f32m2(a, b, c, unpacket_traits<AccPacket>::size);
45#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
46 c = __riscv_vfmadd_vf_f32m4(a, b, c, unpacket_traits<AccPacket>::size);
47#endif
48 }
49
50#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
51 EIGEN_STRONG_INLINE void madd(const Packet1Xf& a, const RhsPacket& b, Packet1Xf& c, RhsPacket& /*tmp*/,
52 const FixedInt<0>&) const {
53 c = __riscv_vfmadd_vf_f32m1(a, b, c, unpacket_traits<Packet1Xf>::size);
54 }
55#endif
56#if EIGEN_RISCV64_DEFAULT_LMUL == 4
57 EIGEN_STRONG_INLINE void madd(const Packet2Xf& a, const RhsPacket& b, Packet2Xf& c, RhsPacket& /*tmp*/,
58 const FixedInt<0>&) const {
59 c = __riscv_vfmadd_vf_f32m2(a, b, c, unpacket_traits<Packet2Xf>::size);
60 }
61#endif
62
63 template <typename LaneIdType>
64 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
65 const LaneIdType& lane) const {
66#if EIGEN_RISCV64_DEFAULT_LMUL == 1
67 c = __riscv_vfmadd_vf_f32m1(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
68#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
69 c = __riscv_vfmadd_vf_f32m2(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
70#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
71 c = __riscv_vfmadd_vf_f32m4(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
72#endif
73 }
74};
75
76template <>
77struct gebp_traits<double, double, false, false, Architecture::RVV10, GEBPPacketFull>
78 : gebp_traits<double, double, false, false, Architecture::Generic, GEBPPacketFull> {
79 using RhsPacket = double;
80 using RhsPacketx4 = QuadPacket<double>;
81 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = pset1<RhsPacket>(*b); }
82 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
83 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
84 }
85
86 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
87
88 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
89
90 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = ploadquad<RhsPacket>(b); }
91
92 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
93 const FixedInt<0>&) const {
94#if EIGEN_RISCV64_DEFAULT_LMUL == 1
95 c = __riscv_vfmadd_vf_f64m1(a, b, c, unpacket_traits<AccPacket>::size);
96#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
97 c = __riscv_vfmadd_vf_f64m2(a, b, c, unpacket_traits<AccPacket>::size);
98#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
99 c = __riscv_vfmadd_vf_f64m4(a, b, c, unpacket_traits<AccPacket>::size);
100#endif
101 }
102
103#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
104 EIGEN_STRONG_INLINE void madd(const Packet1Xd& a, const RhsPacket& b, Packet1Xd& c, RhsPacket& /*tmp*/,
105 const FixedInt<0>&) const {
106 c = __riscv_vfmadd_vf_f64m1(a, b, c, unpacket_traits<Packet1Xd>::size);
107 }
108#endif
109#if EIGEN_RISCV64_DEFAULT_LMUL == 4
110 EIGEN_STRONG_INLINE void madd(const Packet2Xd& a, const RhsPacket& b, Packet2Xd& c, RhsPacket& /*tmp*/,
111 const FixedInt<0>&) const {
112 c = __riscv_vfmadd_vf_f64m2(a, b, c, unpacket_traits<Packet2Xd>::size);
113 }
114#endif
115
116 template <typename LaneIdType>
117 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
118 const LaneIdType& lane) const {
119#if EIGEN_RISCV64_DEFAULT_LMUL == 1
120 c = __riscv_vfmadd_vf_f64m1(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
121#elif EIGEN_RISCV64_DEFAULT_LMUL == 2
122 c = __riscv_vfmadd_vf_f64m2(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
123#elif EIGEN_RISCV64_DEFAULT_LMUL == 4
124 c = __riscv_vfmadd_vf_f64m4(a, b.get(lane), c, unpacket_traits<AccPacket>::size);
125#endif
126 }
127};
128
129#if defined(EIGEN_VECTORIZE_RVV10FP16)
130
131template <>
132struct gebp_traits<half, half, false, false, Architecture::RVV10>
133 : gebp_traits<half, half, false, false, Architecture::Generic> {
134 using RhsPacket = half;
135 using LhsPacket = PacketXh;
136 using AccPacket = PacketXh;
137 using RhsPacketx4 = QuadPacket<half>;
138
139 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = pset1<RhsPacket>(*b); }
140 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
141 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
142 }
143
144 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
145
146 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
147
148 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = pload<RhsPacket>(b); }
149
150 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
151 const FixedInt<0>&) const {
152#if EIGEN_RISCV64_DEFAULT_LMUL == 1
153 c = __riscv_vfmadd_vf_f16m1(a, numext::bit_cast<_Float16>(b), c, unpacket_traits<AccPacket>::size);
154#else
155 c = __riscv_vfmadd_vf_f16m2(a, numext::bit_cast<_Float16>(b), c, unpacket_traits<AccPacket>::size);
156#endif
157 }
158
159#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
160 EIGEN_STRONG_INLINE void madd(const Packet1Xh& a, const RhsPacket& b, Packet1Xh& c, RhsPacket& /*tmp*/,
161 const FixedInt<0>&) const {
162 c = __riscv_vfmadd_vf_f16m1(a, numext::bit_cast<_Float16>(b), c, unpacket_traits<Packet1Xh>::size);
163 }
164#endif
165
166 template <typename LaneIdType>
167 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
168 const LaneIdType& lane) const {
169#if EIGEN_RISCV64_DEFAULT_LMUL == 1
170 c = __riscv_vfmadd_vf_f16m1(a, numext::bit_cast<_Float16>(b.get(lane)), c, unpacket_traits<AccPacket>::size);
171#else
172 c = __riscv_vfmadd_vf_f16m2(a, numext::bit_cast<_Float16>(b.get(lane)), c, unpacket_traits<AccPacket>::size);
173#endif
174 }
175};
176
177#endif
178
179#if defined(EIGEN_VECTORIZE_RVV10BF16)
180
181template <>
182struct gebp_traits<bfloat16, bfloat16, false, false, Architecture::RVV10>
183 : gebp_traits<bfloat16, bfloat16, false, false, Architecture::Generic> {
184 using RhsPacket = bfloat16;
185 using LhsPacket = PacketXbf;
186 using AccPacket = PacketXbf;
187 using RhsPacketx4 = QuadPacket<bfloat16>;
188
189 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacket& dest) const { dest = pset1<RhsPacket>(*b); }
190 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
191 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
192 }
193
194 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacket& dest) const { loadRhs(b, dest); }
195
196 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
197
198 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = pload<RhsPacket>(b); }
199
200 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, AccPacket& c, RhsPacket& /*tmp*/,
201 const FixedInt<0>&) const {
202#if EIGEN_RISCV64_DEFAULT_LMUL == 1
203 c = F32ToBf16(
204 __riscv_vfwmaccbf16_vf_f32m2(Bf16ToF32(c), numext::bit_cast<__bf16>(b), a, unpacket_traits<AccPacket>::size));
205#else
206 c = F32ToBf16(
207 __riscv_vfwmaccbf16_vf_f32m4(Bf16ToF32(c), numext::bit_cast<__bf16>(b), a, unpacket_traits<AccPacket>::size));
208#endif
209 }
210
211#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
212 EIGEN_STRONG_INLINE void madd(const Packet1Xbf& a, const RhsPacket& b, Packet1Xbf& c, RhsPacket& /*tmp*/,
213 const FixedInt<0>&) const {
214 c = F32ToBf16(
215 __riscv_vfwmaccbf16_vf_f32m2(Bf16ToF32(c), numext::bit_cast<__bf16>(b), a, unpacket_traits<Packet1Xbf>::size));
216 }
217#endif
218
219 template <typename LaneIdType>
220 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacketx4& b, AccPacket& c, RhsPacket& /*tmp*/,
221 const LaneIdType& lane) const {
222#if EIGEN_RISCV64_DEFAULT_LMUL == 1
223 c = F32ToBf16(__riscv_vfwmaccbf16_vf_f32m2(Bf16ToF32(c), numext::bit_cast<__bf16>(b.get(lane)), a,
224 unpacket_traits<AccPacket>::size));
225#else
226 c = F32ToBf16(__riscv_vfwmaccbf16_vf_f32m4(Bf16ToF32(c), numext::bit_cast<__bf16>(b.get(lane)), a,
227 unpacket_traits<AccPacket>::size));
228#endif
229 }
230};
231
232#endif
233
234#if EIGEN_RISCV64_DEFAULT_LMUL == 1
235/********************************* complex ************************************/
236
237#define PACKET_DECL_COND_POSTFIX(postfix, name, packet_size) \
238 using name##Packet##postfix = \
239 typename packet_conditional<packet_size, typename packet_traits<name##Scalar>::type, \
240 typename packet_traits<name##Scalar>::half, \
241 typename unpacket_traits<typename packet_traits<name##Scalar>::half>::half>::type
242
243#define RISCV_COMPLEX_PACKET_DECL_COND_SCALAR(packet_size) \
244 using ScalarPacket = \
245 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
246 typename packet_traits<Scalar>::half, \
247 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
248
249template <typename RealScalar, bool ConjLhs_, bool ConjRhs_, int PacketSize_>
250struct gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, ConjLhs_, ConjRhs_, Architecture::RVV10,
251 PacketSize_> : gebp_traits<std::complex<RealScalar>, std::complex<RealScalar>, ConjLhs_, ConjRhs_,
252 Architecture::Generic, PacketSize_> {
253 using Scalar = std::complex<RealScalar>;
254 using LhsScalar = std::complex<RealScalar>;
255 using RhsScalar = std::complex<RealScalar>;
256 using ResScalar = std::complex<RealScalar>;
257 using RealPacket = typename packet_traits<std::complex<RealScalar>>::type;
258
259 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
260 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
261 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
262 RISCV_COMPLEX_PACKET_DECL_COND_SCALAR(PacketSize_);
263#undef RISCV_COMPLEX_PACKET_DECL_COND_SCALAR
264
265 enum {
266 ConjLhs = ConjLhs_,
267 ConjRhs = ConjRhs_,
268 Vectorizable = unpacket_traits<RealPacket>::vectorizable && unpacket_traits<ScalarPacket>::vectorizable,
269 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
270 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
271 RhsPacketSize = Vectorizable ? unpacket_traits<RhsScalar>::size : 1,
272 RealPacketSize = Vectorizable ? unpacket_traits<RealPacket>::size : 1,
273
274 nr = 4,
275 mr = ResPacketSize,
276
277 LhsProgress = ResPacketSize,
278 RhsProgress = 1
279 };
280
281 using DoublePacketType = DoublePacket<RealPacket>;
282
283 using LhsPacket4Packing = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
284 using LhsPacket = std::conditional_t<Vectorizable, RealPacket, Scalar>;
285 using RhsPacket = std::conditional_t<Vectorizable, DoublePacket<RealScalar>, Scalar>;
286 using ResPacket = std::conditional_t<Vectorizable, ScalarPacket, Scalar>;
287 using AccPacket = std::conditional_t<Vectorizable, DoublePacketType, Scalar>;
288
289 using RhsPacketx4 = QuadPacket<RhsPacket>;
290
291 EIGEN_STRONG_INLINE void initAcc(Scalar& p) { p = Scalar(0); }
292
293 EIGEN_STRONG_INLINE void initAcc(DoublePacketType& p) {
294 p.first = pset1<RealPacket>(RealScalar(0));
295 p.second = pset1<RealPacket>(RealScalar(0));
296 }
297
298 // Scalar path
299 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, ScalarPacket& dest) const { dest = pset1<ScalarPacket>(*b); }
300
301 // Vectorized path
302 template <typename RealPacketType>
303 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, DoublePacket<RealPacketType>& dest) const {
304 dest.first = pset1<RealPacketType>(numext::real(*b));
305 dest.second = pset1<RealPacketType>(numext::imag(*b));
306 }
307
308 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
309 loadRhs(b, dest.B_0);
310 loadRhs(b + 1, dest.B1);
311 loadRhs(b + 2, dest.B2);
312 loadRhs(b + 3, dest.B3);
313 }
314
315 // Scalar path
316 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, ScalarPacket& dest) const { loadRhs(b, dest); }
317
318 // Vectorized path
319 template <typename RealPacketType>
320 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, DoublePacket<RealPacketType>& dest) const {
321 loadRhs(b, dest);
322 }
323
324 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
325
326 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, ResPacket& dest) const { loadRhs(b, dest); }
327 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, DoublePacket<RealScalar>& dest) const {
328 loadQuadToDoublePacket(b, dest);
329 }
330
331 // nothing special here
332 EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacket& dest) const {
333 dest = pload<LhsPacket>((const typename unpacket_traits<LhsPacket>::type*)(a));
334 }
335
336 template <typename LhsPacketType>
337 EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const {
338 dest = ploadu<LhsPacketType>((const typename unpacket_traits<LhsPacketType>::type*)(a));
339 }
340
341 EIGEN_STRONG_INLINE Packet1Xcf pmadd_scalar(const Packet1Xcf& a, float b, const Packet1Xcf& c) const {
342 return Packet1Xcf(__riscv_vfmadd_vf_f32m1(a.v, b, c.v, unpacket_traits<Packet1Xf>::size));
343 }
344
345 EIGEN_STRONG_INLINE Packet1Xcd pmadd_scalar(const Packet1Xcd& a, double b, const Packet1Xcd& c) const {
346 return Packet1Xcd(__riscv_vfmadd_vf_f64m1(a.v, b, c.v, unpacket_traits<Packet1Xd>::size));
347 }
348
349#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
350 EIGEN_STRONG_INLINE Packet2Xcf pmadd_scalar(const Packet2Xcf& a, float b, const Packet2Xcf& c) const {
351 return Packet2Xcf(__riscv_vfmadd_vf_f32m2(a.v, b, c.v, unpacket_traits<Packet2Xf>::size));
352 }
353
354 EIGEN_STRONG_INLINE Packet2Xcd pmadd_scalar(const Packet2Xcd& a, double b, const Packet2Xcd& c) const {
355 return Packet2Xcd(__riscv_vfmadd_vf_f64m2(a.v, b, c.v, unpacket_traits<Packet2Xd>::size));
356 }
357#endif
358
359#if EIGEN_RISCV64_DEFAULT_LMUL == 4
360 EIGEN_STRONG_INLINE Packet4Xcf pmadd_scalar(const Packet4Xcf& a, float b, const Packet4Xcf& c) const {
361 return Packet4Xcf(__riscv_vfmadd_vf_f32m4(a.v, b, c.v, unpacket_traits<Packet4Xf>::size));
362 }
363
364 EIGEN_STRONG_INLINE Packet4Xcd pmadd_scalar(const Packet4Xcd& a, double b, const Packet4Xcd& c) const {
365 return Packet4Xcd(__riscv_vfmadd_vf_f64m4(a.v, b, c.v, unpacket_traits<Packet4Xd>::size));
366 }
367#endif
368
369 template <typename LhsPacketType, typename RhsPacketType, typename ResPacketType, typename TmpType,
370 typename LaneIdType>
371 EIGEN_STRONG_INLINE std::enable_if_t<!is_same<RhsPacketType, RhsPacketx4>::value> madd(const LhsPacketType& a,
372 const RhsPacketType& b,
373 DoublePacket<ResPacketType>& c,
374 TmpType& /*tmp*/,
375 const LaneIdType&) const {
376 c.first = pmadd_scalar(a, b.first, c.first);
377 c.second = pmadd_scalar(a, b.second, c.second);
378 }
379
380 template <typename LhsPacketType, typename AccPacketType, typename LaneIdType>
381 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
382 const LaneIdType& lane) const {
383 madd(a, b.get(lane), c, tmp, lane);
384 }
385
386 template <typename LaneIdType>
387 EIGEN_STRONG_INLINE void madd(const LhsPacket& a, const RhsPacket& b, ResPacket& c, RhsPacket& /*tmp*/,
388 const LaneIdType&) const {
389 c = cj.pmadd(a, b, c);
390 }
391
392 template <typename RealPacketType, typename ResPacketType>
393 EIGEN_STRONG_INLINE void acc(const DoublePacket<RealPacketType>& c, const ResPacketType& alpha,
394 ResPacketType& r) const {
395 // assemble c
396 ResPacketType tmp;
397 EIGEN_IF_CONSTEXPR ((!ConjLhs) && (!ConjRhs)) {
398 tmp = pcplxflip(pconj(ResPacketType(c.second)));
399 tmp = padd(ResPacketType(c.first), tmp);
400 } else EIGEN_IF_CONSTEXPR ((!ConjLhs) && (ConjRhs)) {
401 tmp = pconj(pcplxflip(ResPacketType(c.second)));
402 tmp = padd(ResPacketType(c.first), tmp);
403 } else EIGEN_IF_CONSTEXPR ((ConjLhs) && (!ConjRhs)) {
404 tmp = pcplxflip(ResPacketType(c.second));
405 tmp = padd(pconj(ResPacketType(c.first)), tmp);
406 } else EIGEN_IF_CONSTEXPR ((ConjLhs) && (ConjRhs)) {
407 tmp = pcplxflip(ResPacketType(c.second));
408 tmp = psub(pconj(ResPacketType(c.first)), tmp);
409 }
410
411 r = pmadd(tmp, alpha, r);
412 }
413
414 // madd() already applied the operand conjugations; neither c nor alpha is conjugated here.
415 EIGEN_STRONG_INLINE void acc(const Scalar& c, const Scalar& alpha, Scalar& r) const { r += alpha * c; }
416
417 protected:
418 conj_helper<LhsScalar, RhsScalar, ConjLhs, ConjRhs> cj;
419};
420
421#define PACKET_DECL_COND_SCALAR_POSTFIX(postfix, packet_size) \
422 using ScalarPacket##postfix = \
423 typename packet_conditional<packet_size, typename packet_traits<Scalar>::type, \
424 typename packet_traits<Scalar>::half, \
425 typename unpacket_traits<typename packet_traits<Scalar>::half>::half>::type
426
427template <typename RealScalar, bool ConjRhs_, int PacketSize_>
428class gebp_traits<RealScalar, std::complex<RealScalar>, false, ConjRhs_, Architecture::RVV10, PacketSize_>
429 : public gebp_traits<RealScalar, std::complex<RealScalar>, false, ConjRhs_, Architecture::Generic, PacketSize_> {
430 public:
431 using Scalar = std::complex<RealScalar>;
432 using LhsScalar = RealScalar;
433 using RhsScalar = Scalar;
434 using ResScalar = Scalar;
435 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
436 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
437 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
438 PACKET_DECL_COND_POSTFIX(_, Real, PacketSize_);
439 PACKET_DECL_COND_SCALAR_POSTFIX(_, PacketSize_);
440#undef PACKET_DECL_COND_SCALAR_POSTFIX
441
442 enum {
443 ConjLhs = false,
444 ConjRhs = ConjRhs_,
445 Vectorizable = unpacket_traits<RealPacket_>::vectorizable && unpacket_traits<ScalarPacket_>::vectorizable,
446 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
447 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
448 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
449
450 NumberOfRegisters = EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS,
451 // FIXME: should depend on NumberOfRegisters
452 nr = 4,
453 mr = (plain_enum_min(16, NumberOfRegisters) / 2 / nr) * ResPacketSize,
454
455 LhsProgress = ResPacketSize,
456 RhsProgress = 1
457 };
458
459 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
460 using RhsPacket = RhsScalar;
461 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
462 using LhsPacket4Packing = LhsPacket;
463 using RhsPacketx4 = QuadPacket<RhsPacket>;
464 using AccPacket = ResPacket;
465
466 EIGEN_STRONG_INLINE void initAcc(AccPacket& p) const { p = pset1<ResPacket>(ResScalar(0)); }
467
468 template <typename RhsPacketType>
469 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const {
470 dest = pset1<RhsPacketType>(*b);
471 }
472
473 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
474 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
475 }
476
477 template <typename RhsPacketType>
478 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const {
479 loadRhs(b, dest);
480 }
481
482 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
483
484 EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacket& dest) const { dest = ploaddup<LhsPacket>(a); }
485
486 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const { dest = ploadquad<RhsPacket>(b); }
487
488 template <typename LhsPacketType>
489 EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const {
490 dest = ploaddup<LhsPacketType>(a);
491 }
492
493 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType>
494 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
495 const LaneIdType&) const {
496 madd_impl(a, b, c, tmp, std::conditional_t<Vectorizable, std::true_type, std::false_type>());
497 }
498
499 EIGEN_STRONG_INLINE Packet1Xcf pmadd_scalar(const Packet1Xf& a, std::complex<float> b, const Packet1Xcf& c) const {
500 Packet1Xcf bp = pset1<Packet1Xcf>(b);
501 return Packet1Xcf(__riscv_vfmadd_vv_f32m1(a, bp.v, c.v, unpacket_traits<Packet1Xf>::size));
502 }
503
504 EIGEN_STRONG_INLINE Packet1Xcd pmadd_scalar(const Packet1Xd& a, std::complex<double> b, const Packet1Xcd& c) const {
505 Packet1Xcd bp = pset1<Packet1Xcd>(b);
506 return Packet1Xcd(__riscv_vfmadd_vv_f64m1(a, bp.v, c.v, unpacket_traits<Packet1Xd>::size));
507 }
508
509#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
510 EIGEN_STRONG_INLINE Packet2Xcf pmadd_scalar(const Packet2Xf& a, std::complex<float> b, const Packet2Xcf& c) const {
511 Packet2Xcf bp = pset1<Packet2Xcf>(b);
512 return Packet2Xcf(__riscv_vfmadd_vv_f32m2(a, bp.v, c.v, unpacket_traits<Packet2Xf>::size));
513 }
514
515 EIGEN_STRONG_INLINE Packet2Xcd pmadd_scalar(const Packet2Xd& a, std::complex<double> b, const Packet2Xcd& c) const {
516 Packet2Xcd bp = pset1<Packet2Xcd>(b);
517 return Packet2Xcd(__riscv_vfmadd_vv_f64m2(a, bp.v, c.v, unpacket_traits<Packet2Xd>::size));
518 }
519#endif
520
521#if EIGEN_RISCV64_DEFAULT_LMUL == 4
522 EIGEN_STRONG_INLINE Packet4Xcf pmadd_scalar(const Packet4Xf& a, std::complex<float> b, const Packet4Xcf& c) const {
523 Packet4Xcf bp = pset1<Packet4Xcf>(b);
524 return Packet4Xcf(__riscv_vfmadd_vv_f32m4(a, bp.v, c.v, unpacket_traits<Packet4Xf>::size));
525 }
526
527 EIGEN_STRONG_INLINE Packet4Xcd pmadd_scalar(const Packet4Xd& a, std::complex<double> b, const Packet4Xcd& c) const {
528 Packet4Xcd bp = pset1<Packet4Xcd>(b);
529 return Packet4Xcd(__riscv_vfmadd_vv_f64m4(a, bp.v, c.v, unpacket_traits<Packet4Xd>::size));
530 }
531#endif
532
533 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType>
534 EIGEN_STRONG_INLINE void madd_impl(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c,
535 RhsPacketType& tmp, const std::true_type&) const {
536 EIGEN_UNUSED_VARIABLE(tmp);
537 c = pmadd_scalar(a, b, c);
538 }
539
540 EIGEN_STRONG_INLINE void madd_impl(const LhsScalar& a, const RhsScalar& b, ResScalar& c, RhsScalar& /*tmp*/,
541 const std::false_type&) const {
542 c += a * b;
543 }
544
545 template <typename LhsPacketType, typename AccPacketType, typename LaneIdType>
546 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
547 const LaneIdType& lane) const {
548 madd(a, b.get(lane), c, tmp, lane);
549 }
550
551 template <typename ResPacketType, typename AccPacketType>
552 EIGEN_STRONG_INLINE void acc(const AccPacketType& c, const ResPacketType& alpha, ResPacketType& r) const {
553 conj_helper<ResPacketType, ResPacketType, false, ConjRhs> cj;
554 r = cj.pmadd(alpha, c, r);
555 }
556};
557
558template <typename RealScalar, bool ConjLhs_, int PacketSize_>
559class gebp_traits<std::complex<RealScalar>, RealScalar, ConjLhs_, false, Architecture::RVV10, PacketSize_>
560 : public gebp_traits<RealScalar, std::complex<RealScalar>, ConjLhs_, false, Architecture::Generic, PacketSize_> {
561 public:
562 using LhsScalar = std::complex<RealScalar>;
563 using RhsScalar = RealScalar;
564 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType;
565
566 PACKET_DECL_COND_POSTFIX(_, Lhs, PacketSize_);
567 PACKET_DECL_COND_POSTFIX(_, Rhs, PacketSize_);
568 PACKET_DECL_COND_POSTFIX(_, Res, PacketSize_);
569#undef PACKET_DECL_COND_POSTFIX
570
571 enum {
572 ConjLhs = ConjLhs_,
573 ConjRhs = false,
574 Vectorizable = unpacket_traits<LhsPacket_>::vectorizable && unpacket_traits<RhsPacket_>::vectorizable,
575 LhsPacketSize = Vectorizable ? unpacket_traits<LhsPacket_>::size : 1,
576 RhsPacketSize = Vectorizable ? unpacket_traits<RhsPacket_>::size : 1,
577 ResPacketSize = Vectorizable ? unpacket_traits<ResPacket_>::size : 1,
578
579 nr = 4,
580 mr = 3 * LhsPacketSize,
581
582 LhsProgress = LhsPacketSize,
583 RhsProgress = 1
584 };
585
586 using LhsPacket = std::conditional_t<Vectorizable, LhsPacket_, LhsScalar>;
587 using RhsPacket = RhsScalar;
588 using ResPacket = std::conditional_t<Vectorizable, ResPacket_, ResScalar>;
589 using LhsPacket4Packing = LhsPacket;
590
591 using RhsPacketx4 = QuadPacket<RhsPacket>;
592
593 using AccPacket = ResPacket;
594
595 EIGEN_STRONG_INLINE void initAcc(AccPacket& p) { p = pset1<ResPacket>(ResScalar(0)); }
596
597 template <typename RhsPacketType>
598 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketType& dest) const {
599 dest = pset1<RhsPacketType>(*b);
600 }
601
602 EIGEN_STRONG_INLINE void loadRhs(const RhsScalar* b, RhsPacketx4& dest) const {
603 pbroadcast4(b, dest.B_0, dest.B1, dest.B2, dest.B3);
604 }
605
606 template <typename RhsPacketType>
607 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar* b, RhsPacketType& dest) const {
608 loadRhs(b, dest);
609 }
610
611 EIGEN_STRONG_INLINE void updateRhs(const RhsScalar*, RhsPacketx4&) const {}
612
613 EIGEN_STRONG_INLINE void loadRhsQuad(const RhsScalar* b, RhsPacket& dest) const {
614 loadRhsQuad_impl(b, dest, std::conditional_t<RhsPacketSize == 16, std::true_type, std::false_type>());
615 }
616
617 EIGEN_STRONG_INLINE void loadRhsQuad_impl(const RhsScalar* b, RhsPacket& dest, const std::true_type&) const {
618 // FIXME we can do better!
619 // what we want here is a ploadheight
620 RhsScalar tmp[4] = {b[0], b[0], b[1], b[1]};
621 dest = ploadquad<RhsPacket>(tmp);
622 }
623
624 EIGEN_STRONG_INLINE void loadRhsQuad_impl(const RhsScalar* b, RhsPacket& dest, const std::false_type&) const {
625 eigen_internal_assert(RhsPacketSize <= 8);
626 dest = pset1<RhsPacket>(*b);
627 }
628
629 EIGEN_STRONG_INLINE void loadLhs(const LhsScalar* a, LhsPacket& dest) const { dest = pload<LhsPacket>(a); }
630
631 template <typename LhsPacketType>
632 EIGEN_STRONG_INLINE void loadLhsUnaligned(const LhsScalar* a, LhsPacketType& dest) const {
633 dest = ploadu<LhsPacketType>(a);
634 }
635
636 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType, typename LaneIdType>
637 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c, RhsPacketType& tmp,
638 const LaneIdType&) const {
639 madd_impl(a, b, c, tmp, std::conditional_t<Vectorizable, std::true_type, std::false_type>());
640 }
641
642 EIGEN_STRONG_INLINE Packet1Xcf pmadd_scalar(const Packet1Xcf& a, float b, const Packet1Xcf& c) const {
643 return Packet1Xcf(__riscv_vfmadd_vf_f32m1(a.v, b, c.v, unpacket_traits<Packet1Xf>::size));
644 }
645
646 EIGEN_STRONG_INLINE Packet1Xcd pmadd_scalar(const Packet1Xcd& a, double b, const Packet1Xcd& c) const {
647 return Packet1Xcd(__riscv_vfmadd_vf_f64m1(a.v, b, c.v, unpacket_traits<Packet1Xd>::size));
648 }
649
650#if EIGEN_RISCV64_DEFAULT_LMUL >= 2
651 EIGEN_STRONG_INLINE Packet2Xcf pmadd_scalar(const Packet2Xcf& a, float b, const Packet2Xcf& c) const {
652 return Packet2Xcf(__riscv_vfmadd_vf_f32m2(a.v, b, c.v, unpacket_traits<Packet2Xf>::size));
653 }
654
655 EIGEN_STRONG_INLINE Packet2Xcd pmadd_scalar(const Packet2Xcd& a, double b, const Packet2Xcd& c) const {
656 return Packet2Xcd(__riscv_vfmadd_vf_f64m2(a.v, b, c.v, unpacket_traits<Packet2Xd>::size));
657 }
658#endif
659
660#if EIGEN_RISCV64_DEFAULT_LMUL == 4
661 EIGEN_STRONG_INLINE Packet4Xcf pmadd_scalar(const Packet4Xcf& a, float b, const Packet4Xcf& c) const {
662 return Packet4Xcf(__riscv_vfmadd_vf_f32m4(a.v, b, c.v, unpacket_traits<Packet4Xf>::size));
663 }
664
665 EIGEN_STRONG_INLINE Packet4Xcd pmadd_scalar(const Packet4Xcd& a, double b, const Packet4Xcd& c) const {
666 return Packet4Xcd(__riscv_vfmadd_vf_f64m4(a.v, b, c.v, unpacket_traits<Packet4Xd>::size));
667 }
668#endif
669
670 template <typename LhsPacketType, typename RhsPacketType, typename AccPacketType>
671 EIGEN_STRONG_INLINE void madd_impl(const LhsPacketType& a, const RhsPacketType& b, AccPacketType& c,
672 RhsPacketType& tmp, const std::true_type&) const {
673 EIGEN_UNUSED_VARIABLE(tmp);
674 c = pmadd_scalar(a, b, c);
675 }
676
677 EIGEN_STRONG_INLINE void madd_impl(const LhsScalar& a, const RhsScalar& b, ResScalar& c, RhsScalar& /*tmp*/,
678 const std::false_type&) const {
679 c += a * b;
680 }
681
682 template <typename LhsPacketType, typename AccPacketType, typename LaneIdType>
683 EIGEN_STRONG_INLINE void madd(const LhsPacketType& a, const RhsPacketx4& b, AccPacketType& c, RhsPacket& tmp,
684 const LaneIdType& lane) const {
685 madd(a, b.get(lane), c, tmp, lane);
686 }
687
688 template <typename ResPacketType, typename AccPacketType>
689 EIGEN_STRONG_INLINE void acc(const AccPacketType& c, const ResPacketType& alpha, ResPacketType& r) const {
690 conj_helper<ResPacketType, ResPacketType, ConjLhs, false> cj;
691 r = cj.pmadd(c, alpha, r);
692 }
693};
694#endif
695
696} // namespace internal
697} // namespace Eigen
698
699#endif // EIGEN_RVV10_GENERAL_BLOCK_KERNEL_H