11#ifndef EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H
12#define EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H
15#include "../../InternalHeaderCheck.h"
17#if defined(__MMA__) && !EIGEN_ALTIVEC_DISABLE_MMA
18#if EIGEN_COMP_LLVM || (__GNUC__ > 10 || __GNUC_MINOR__ >= 3)
22#if !EIGEN_COMP_LLVM && (__GNUC__ < 11)
24#define GCC_ONE_VECTORPAIR_BUG
32#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
33#define EIGEN_POWER_GEMV_PREFETCH(p) prefetch(p)
35#define EIGEN_POWER_GEMV_PREFETCH(p)
39#if !__has_builtin(__builtin_vsx_assemble_pair)
40#define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair
42#if !__has_builtin(__builtin_vsx_disassemble_pair)
43#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair
48#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
49 __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1)
52#if (__GNUC_MINOR__ > 3)
53#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
54 __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1)
56#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
57 __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2)
60#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
61 __builtin_vsx_build_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2)
65#define GEMV_IS_COMPLEX_COMPLEX ((sizeof(LhsPacket) == 16) && (sizeof(RhsPacket) == 16))
66#define GEMV_IS_FLOAT (ResPacketSize == (16 / sizeof(float)))
67#define GEMV_IS_SCALAR (sizeof(ResPacket) != 16)
68#define GEMV_IS_COMPLEX_FLOAT (ResPacketSize == (16 / sizeof(std::complex<float>)))
71template <
typename ResPacket,
typename ResScalar>
72EIGEN_ALWAYS_INLINE
void storeMaddData(ResScalar* res,
const ResPacket& palpha,
const ResPacket& data) {
73 pstoreu(res, pmadd(data, palpha, ploadu<ResPacket>(res)));
76template <
typename ResScalar>
77EIGEN_ALWAYS_INLINE
void storeMaddData(ResScalar* res, ResScalar& alpha, ResScalar& data) {
78 *res += (alpha * data);
81#define GEMV_UNROLL(func, N) func(0, N) func(1, N) func(2, N) func(3, N) func(4, N) func(5, N) func(6, N) func(7, N)
83#define GEMV_UNROLL_HALF(func, N) func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N)
85#define GEMV_GETN(N) (((N) * ResPacketSize) >> 2)
87#define GEMV_LOADPACKET_COL(iter) lhs.template load<LhsPacket, LhsAlignment>(i + ((iter) * LhsPacketSize), j)
90#define GEMV_UNROLL3(func, N, which) \
91 func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) func(4, N, which) func(5, N, which) \
92 func(6, N, which) func(7, N, which)
94#define GEMV_UNUSED_VAR(iter, N, which) \
95 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= iter) { \
96 EIGEN_UNUSED_VARIABLE(which##iter); \
99#define GEMV_UNUSED_EXTRA_VAR(iter, N, which) \
100 EIGEN_IF_CONSTEXPR (N <= iter) { \
101 EIGEN_UNUSED_VARIABLE(which##iter); \
104#define GEMV_UNUSED_EXTRA(N, which) GEMV_UNROLL3(GEMV_UNUSED_EXTRA_VAR, N, which)
106#define GEMV_UNUSED(N, which) GEMV_UNROLL3(GEMV_UNUSED_VAR, N, which)
108#define GEMV_INIT_MMA(iter, N) \
109 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
110 __builtin_mma_xxsetaccz(&e##iter); \
114#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \
115 GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_COL(iter2), GEMV_LOADPACKET_COL((iter2) + 1));
117#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \
118 const LhsScalar& src##iter1 = lhs(i + ((iter1 * 32) / sizeof(LhsScalar)), j); \
119 b##iter1 = *reinterpret_cast<__vector_pair*>(const_cast<LhsScalar*>(&src##iter1));
122#define GEMV_LOAD1A_COL_MMA(iter, N) \
123 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
124 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
125 g##iter = GEMV_LOADPACKET_COL(iter); \
126 EIGEN_UNUSED_VARIABLE(b##iter); \
128 GEMV_LOADPAIR_COL_MMA(iter, iter << 1) \
129 EIGEN_UNUSED_VARIABLE(g##iter); \
132 EIGEN_UNUSED_VARIABLE(b##iter); \
133 EIGEN_UNUSED_VARIABLE(g##iter); \
136#define GEMV_WORK1A_COL_MMA(iter, N) \
137 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
138 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
139 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter, a0, g##iter); \
141 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter, b##iter, a0); \
145#define GEMV_LOAD1B_COL_MMA(iter1, iter2, iter3, N) \
146 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
147 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
148 GEMV_LOADPAIR_COL_MMA(iter2, iter2) \
149 EIGEN_UNUSED_VARIABLE(b##iter3); \
151 GEMV_LOADPAIR_COL_MMA(iter2, iter2 << 1) \
152 GEMV_LOADPAIR_COL_MMA(iter3, iter3 << 1) \
155 EIGEN_UNUSED_VARIABLE(b##iter2); \
156 EIGEN_UNUSED_VARIABLE(b##iter3); \
158 EIGEN_UNUSED_VARIABLE(g##iter2); \
159 EIGEN_UNUSED_VARIABLE(g##iter3);
161#define GEMV_WORK1B_COL_MMA(iter1, iter2, iter3, N) \
162 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
163 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
165 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(h), &b##iter2); \
166 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter2, a0, h[0]); \
167 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter3, a0, h[1]); \
169 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter2, b##iter2, a0); \
170 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter3, b##iter3, a0); \
175#define GEMV_LOAD_COL_MMA(N) \
176 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > 1) { \
177 GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) \
179 GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \
182#define GEMV_WORK_COL_MMA(N) \
183 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > 1) { \
184 GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) \
186 GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \
189#define GEMV_LOAD_COL_MMA(N) GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N)
191#define GEMV_WORK_COL_MMA(N) GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N)
194#define GEMV_DISASSEMBLE_MMA(iter, N) \
195 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
196 __builtin_mma_disassemble_acc(&result##iter.packet, &e##iter); \
197 EIGEN_IF_CONSTEXPR (!GEMV_IS_FLOAT) { \
198 result##iter.packet[0][1] = result##iter.packet[1][0]; \
199 result##iter.packet[2][1] = result##iter.packet[3][0]; \
203#define GEMV_LOADPAIR2_COL_MMA(iter1, iter2) \
204 b##iter1 = *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize));
206#define GEMV_LOAD2_COL_MMA(iter1, iter2, iter3, N) \
207 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
208 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
209 GEMV_LOADPAIR2_COL_MMA(iter2, iter2); \
210 EIGEN_UNUSED_VARIABLE(b##iter3); \
212 GEMV_LOADPAIR2_COL_MMA(iter2, iter2 << 1); \
213 GEMV_LOADPAIR2_COL_MMA(iter3, iter3 << 1); \
216 EIGEN_UNUSED_VARIABLE(b##iter2); \
217 EIGEN_UNUSED_VARIABLE(b##iter3); \
221#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \
222 ResPacket f##iter2[2]; \
223 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(f##iter2), &b##iter2); \
224 f##iter2[0] = pmadd(result##iter2.packet[0], palpha, f##iter2[0]); \
225 f##iter2[1] = pmadd(result##iter3.packet[(iter2 == iter3) ? 2 : 0], palpha, f##iter2[1]); \
226 GEMV_BUILDPAIR_MMA(b##iter2, f##iter2[0], f##iter2[1]);
228#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \
229 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
230 __asm__("xvmaddasp %0,%x1,%x3\n\txvmaddasp %L0,%x2,%x3" \
232 : "wa"(result##iter3.packet[0]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \
234 __asm__("xvmaddadp %0,%x1,%x3\n\txvmaddadp %L0,%x2,%x3" \
236 : "wa"(result##iter2.packet[2]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \
240#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \
241 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
242 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
243 GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); \
245 GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \
246 GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \
250#define GEMV_STOREPAIR2_COL_MMA(iter1, iter2) \
251 *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)) = b##iter1;
253#define GEMV_STORE_COL_MMA(iter, N) \
254 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
255 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
256 storeMaddData<ResPacket, ResScalar>(res + i + (iter * ResPacketSize), palpha, result##iter.packet[0]); \
258 GEMV_LOADPAIR2_COL_MMA(iter, iter << 1) \
259 GEMV_WORKPAIR2_COL_MMA(iter, iter, iter << 1) \
260 GEMV_STOREPAIR2_COL_MMA(iter, iter << 1) \
264#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \
265 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
266 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
267 GEMV_STOREPAIR2_COL_MMA(iter2, iter2); \
269 GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \
270 GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \
274#define GEMV_PROCESS_COL_ONE_MMA(N) \
275 GEMV_UNROLL(GEMV_INIT_MMA, N) \
277 __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \
279 LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \
280 RhsPacket a0 = pset1<RhsPacket>(rhs2(j, 0)); \
281 GEMV_UNROLL(GEMV_PREFETCH, N) \
282 GEMV_LOAD_COL_MMA(N) \
283 GEMV_WORK_COL_MMA(N) \
284 } while (++j < jend); \
285 GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \
286 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= 1) { \
287 GEMV_UNROLL(GEMV_STORE_COL_MMA, N) \
289 GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) \
290 GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1)) \
292 i += (ResPacketSize * N);
295#define GEMV_INIT(iter, N) \
296 EIGEN_IF_CONSTEXPR (N > iter) { \
297 c##iter = pset1<ResPacket>(ResScalar(0)); \
299 EIGEN_UNUSED_VARIABLE(c##iter); \
302#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
303#define GEMV_PREFETCH(iter, N) \
304 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \
305 lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \
308#define GEMV_PREFETCH(iter, N)
311#define GEMV_WORK_COL(iter, N) \
312 EIGEN_IF_CONSTEXPR (N > iter) { \
313 c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); \
316#define GEMV_STORE_COL(iter, N) \
317 EIGEN_IF_CONSTEXPR (N > iter) { \
318 pstoreu(res + i + (iter * ResPacketSize), \
319 pmadd(c##iter, palpha, ploadu<ResPacket>(res + i + (iter * ResPacketSize)))); \
323#define GEMV_PROCESS_COL_ONE(N) \
324 GEMV_UNROLL(GEMV_INIT, N) \
327 RhsPacket a0 = pset1<RhsPacket>(rhs2(j, 0)); \
328 GEMV_UNROLL(GEMV_PREFETCH, N) \
329 GEMV_UNROLL(GEMV_WORK_COL, N) \
330 } while (++j < jend); \
331 GEMV_UNROLL(GEMV_STORE_COL, N) \
332 i += (ResPacketSize * N);
335#define GEMV_PROCESS_COL(N) GEMV_PROCESS_COL_ONE_MMA(N)
337#define GEMV_PROCESS_COL(N) GEMV_PROCESS_COL_ONE(N)
342template <
typename LhsPacket,
typename RhsPacket,
bool accumulate>
343EIGEN_ALWAYS_INLINE
void pger_vecMMA_acc(__vector_quad* acc,
const RhsPacket& a,
const LhsPacket& b) {
345 __builtin_mma_xvf32gerpp(acc, (__vector
unsigned char)a, (__vector
unsigned char)b);
347 __builtin_mma_xvf32ger(acc, (__vector
unsigned char)a, (__vector
unsigned char)b);
352template <
typename LhsPacket,
typename RhsPacket,
bool accumulate>
353EIGEN_ALWAYS_INLINE
void pger_vecMMA_acc(__vector_quad* acc, __vector_pair& a,
const LhsPacket& b) {
355 __builtin_mma_xvf64gerpp(acc, a, (__vector
unsigned char)b);
357 __builtin_mma_xvf64ger(acc, a, (__vector
unsigned char)b);
362template <
typename LhsScalar,
typename LhsMapper,
typename RhsScalar,
typename RhsMapper,
typename ResScalar>
363EIGEN_STRONG_INLINE
void gemv_col(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols,
const LhsMapper& alhs,
364 const RhsMapper& rhs, ResScalar* res, mapper_index_t<LhsMapper> resIncr,
366 using Index = mapper_index_t<LhsMapper>;
367 using Traits = gemv_traits<LhsScalar, RhsScalar>;
369 using LhsPacket =
typename Traits::LhsPacket;
370 using RhsPacket =
typename Traits::RhsPacket;
371 using ResPacket =
typename Traits::ResPacket;
373 EIGEN_UNUSED_VARIABLE(resIncr);
374 eigen_internal_assert(resIncr == 1);
381 conj_helper<LhsScalar, RhsScalar, false, false> cj;
382 conj_helper<LhsPacket, RhsPacket, false, false> pcj;
384 const Index lhsStride = lhs.stride();
389 ResPacketSize = Traits::ResPacketSize,
390 LhsPacketSize = Traits::LhsPacketSize,
391 RhsPacketSize = Traits::RhsPacketSize,
394#ifndef GCC_ONE_VECTORPAIR_BUG
395 const Index n8 = rows - 8 * ResPacketSize + 1;
396 const Index n4 = rows - 4 * ResPacketSize + 1;
397 const Index n2 = rows - 2 * ResPacketSize + 1;
399 const Index n1 = rows - 1 * ResPacketSize + 1;
400#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
401 const Index prefetch_dist = 64 * LhsPacketSize;
405 const Index block_cols = cols < 128 ? cols : (lhsStride *
sizeof(LhsScalar) < 16000 ? 16 : 8);
406 ResPacket palpha = pset1<ResPacket>(alpha);
408 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
409 Index jend = numext::mini(j2 + block_cols, cols);
411 ResPacket c0, c1, c2, c3, c4, c5, c6, c7;
413 __vector_quad e0, e1, e2, e3, e4, e5, e6, e7;
414 PacketBlock<ResPacket, 4> result0, result1, result2, result3, result4, result5, result6, result7;
416 GEMV_UNUSED(8, result)
417 GEMV_UNUSED_EXTRA(1, c)
419#ifndef GCC_ONE_VECTORPAIR_BUG
434 GEMV_PROCESS_COL_ONE(1)
436 for (; i < rows; ++i) {
440 d0 += cj.pmul(lhs(i, j), rhs2(j, 0));
441 }
while (++j < jend);
442 res[i] += alpha * d0;
447template <
bool extraRows>
448EIGEN_ALWAYS_INLINE
void outputVecCol(
const Packet4f& acc,
float* result,
const Packet4f& pAlpha, Index extra_rows) {
449 Packet4f d0 = ploadu<Packet4f>(result);
450 d0 = pmadd(acc, pAlpha, d0);
451 EIGEN_IF_CONSTEXPR (extraRows) {
452 pstoreu_partial(result, d0, extra_rows);
458template <Index num_acc,
bool extraRows, Index size>
459EIGEN_ALWAYS_INLINE
void outputVecColResults(Packet4f (&acc)[num_acc][size],
float* result,
const Packet4f& pAlpha,
461 constexpr Index real_acc = (num_acc - (extraRows ? 1 : 0));
462 for (Index k = 0; k < real_acc; k++) {
463 outputVecCol<false>(acc[k][0], result + k * 4, pAlpha, extra_rows);
465 EIGEN_IF_CONSTEXPR (extraRows) {
466 outputVecCol<true>(acc[real_acc][0], result + real_acc * 4, pAlpha, extra_rows);
470static Packet16uc p16uc_MERGE16_32_V1 = {0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17};
471static Packet16uc p16uc_MERGE16_32_V2 = {2, 3, 18, 19, 2, 3, 18, 19, 2, 3, 18, 19, 2, 3, 18, 19};
473template <Index num_acc,
typename LhsMapper,
bool zero>
474EIGEN_ALWAYS_INLINE
void loadVecLoopVSX(Index k, LhsMapper& lhs, Packet4f (&a0)[num_acc][2]) {
475 const auto row = convert_index<mapper_index_t<LhsMapper>>(k * 4);
476 Packet8bf c0 = lhs.template loadPacket<Packet8bf>(row, 0);
478 EIGEN_IF_CONSTEXPR (!zero) {
479 b1 = lhs.template loadPacket<Packet8bf>(row, 1);
481 a0[k + 0][1] = oneConvertBF16Hi(b1.m_val);
483 a0[k + 0][0] = oneConvertBF16Hi(c0.m_val);
485 if (num_acc > (k + 1)) {
486 a0[k + 1][0] = oneConvertBF16Lo(c0.m_val);
487 EIGEN_IF_CONSTEXPR (!zero) {
488 a0[k + 1][1] = oneConvertBF16Lo(b1.m_val);
493template <Index num_acc,
bool zero>
494EIGEN_ALWAYS_INLINE
void multVecVSX(Packet4f (&acc)[num_acc][2], Packet4f (&a0)[num_acc][2], Packet4f (&b0)[2]) {
495 for (Index k = 0; k < num_acc; k++) {
496 for (Index i = 0; i < (zero ? 1 : 2); i++) {
497 acc[k][i] = pmadd(b0[i], a0[k][i], acc[k][i]);
502template <
typename RhsMapper,
bool linear>
503struct loadColData_impl {
505 template <
typename Index>
506 static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) {
507 const Index n = unpacket_traits<Packet8bf>::size;
508 EIGEN_ALIGN16 bfloat16 to[n];
510 for (Index i = 0; i < n; i++) {
511 to[i] = rhs(j + i, 0);
513 return pload<Packet8bf>(to);
517template <
typename RhsMapper>
518struct loadColData_impl<RhsMapper, true> {
520 template <
typename Index>
521 static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) {
522 return rhs.template loadPacket<Packet8bf>(j + 0, 0);
526template <
typename RhsMapper,
bool linear,
typename Index>
527EIGEN_ALWAYS_INLINE Packet8bf loadColData(RhsMapper& rhs, Index j) {
528 return loadColData_impl<RhsMapper, linear>::run(rhs, j);
531template <Index num_acc,
typename LhsMapper,
typename RhsMapper,
bool zero,
bool linear>
532EIGEN_ALWAYS_INLINE
void vecColLoopVSX(Index j, LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2]) {
533 Packet4f a0[num_acc][2], b0[2];
534 const auto col = convert_index<mapper_index_t<LhsMapper>>(j);
535 Packet8bf b2 = loadColData<RhsMapper, linear>(rhs, col);
537 b0[0] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V1);
538 EIGEN_IF_CONSTEXPR (!zero) {
539 b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2);
542 using LhsSubMapper =
typename LhsMapper::SubMapper;
544 LhsSubMapper lhs2 = lhs.getSubMapper(0, col);
545 for (Index k = 0; k < num_acc; k += 2) {
546 loadVecLoopVSX<num_acc, LhsSubMapper, zero>(k, lhs2, a0);
549 multVecVSX<num_acc, zero>(acc, a0, b0);
552template <Index num_acc>
553EIGEN_ALWAYS_INLINE
void addResultsVSX(Packet4f (&acc)[num_acc][2]) {
554 for (Index i = 0; i < num_acc; i++) {
555 acc[i][0] = acc[i][0] + acc[i][1];
560#define MAX_BFLOAT16_VEC_ACC_VSX 8
562template <const Index num_acc,
typename LhsMapper,
typename RhsMapper,
bool extraRows,
bool linear>
563void colVSXVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
const Packet4f& pAlpha,
565 constexpr Index step = (num_acc * 4);
566 const Index extra_rows = (extraRows) ? (rows & 3) : 0;
567 constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC_VSX);
570 Packet4f acc[num_acc][2];
572 zeroAccumulators<num_acc, 2>(acc);
574 using LhsSubMapper =
typename LhsMapper::SubMapper;
576 LhsSubMapper lhs2 = lhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(row), 0);
577 for (Index j = 0; j + 2 <= cend; j += 2) {
578 vecColLoopVSX<num_acc, LhsSubMapper, RhsMapper, false, linear>(j, lhs2, rhs, acc);
581 vecColLoopVSX<num_acc, LhsSubMapper, RhsMapper, true, linear>(cend - 1, lhs2, rhs, acc);
584 addResultsVSX<num_acc>(acc);
586 outputVecColResults<num_acc, extraRows, 2>(acc, result, pAlpha, extra_rows);
589 }
while (multiIters && (step <= rows - (row += step)));
592template <const Index num_acc,
typename LhsMapper,
typename RhsMapper,
bool extraRows,
bool linear>
593EIGEN_ALWAYS_INLINE
void colVSXVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
594 const Packet4f& pAlpha,
float* result) {
595 EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) {
596 colVSXVecColLoopBody<num_acc + (extraRows ? 1 : 0), LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs,
597 rhs, pAlpha, result);
601template <
typename LhsMapper,
typename RhsMapper,
bool extraRows,
bool linear>
602EIGEN_ALWAYS_INLINE
void colVSXVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
603 const Packet4f& pAlpha,
float* result) {
604 switch ((rows - row) >> 2) {
606 colVSXVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
609 colVSXVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
612 colVSXVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
615 colVSXVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
618 colVSXVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
621 colVSXVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
624 colVSXVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
627 EIGEN_IF_CONSTEXPR (extraRows) {
628 colVSXVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
634template <
typename LhsMapper,
typename RhsMapper,
bool linear>
635EIGEN_ALWAYS_INLINE
void calcVSXVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
636 const Packet4f& pAlpha,
float* result) {
638 if (rows >= (MAX_BFLOAT16_VEC_ACC_VSX * 4)) {
639 colVSXVecColLoopBody<MAX_BFLOAT16_VEC_ACC_VSX, LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs,
644 colVSXVecColLoopBodyExtra<LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
646 colVSXVecColLoopBodyExtra<LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
650template <const Index size,
bool inc, Index delta>
651EIGEN_ALWAYS_INLINE
void storeBF16fromResult(bfloat16* dst,
const Packet8bf& data, Index resInc, Index extra) {
653 EIGEN_IF_CONSTEXPR (size < 8) {
654 pscatter_partial(dst + delta * resInc, data, resInc, extra);
656 pscatter(dst + delta * resInc, data, resInc);
659 EIGEN_IF_CONSTEXPR (size < 8) {
660 pstoreu_partial(dst + delta, data, extra);
662 pstoreu(dst + delta, data);
667template <const Index size,
bool inc = false>
668EIGEN_ALWAYS_INLINE
void convertPointerF32toBF16VSX(Index& i,
float* result, Index rows, bfloat16*& dst,
670 constexpr Index extra = ((size < 8) ? 8 : size);
671 while (i + size <= rows) {
672 PacketBlock<Packet8bf, (size + 7) / 8> r32;
673 r32.packet[0] = convertF32toBF16VSX(result + i + 0);
674 EIGEN_IF_CONSTEXPR (size >= 16) {
675 r32.packet[1] = convertF32toBF16VSX(result + i + 8);
677 EIGEN_IF_CONSTEXPR (size >= 32) {
678 r32.packet[2] = convertF32toBF16VSX(result + i + 16);
679 r32.packet[3] = convertF32toBF16VSX(result + i + 24);
681 storeBF16fromResult<size, inc, 0>(dst, r32.packet[0], resInc, rows & 7);
682 EIGEN_IF_CONSTEXPR (size >= 16) {
683 storeBF16fromResult<size, inc, 8>(dst, r32.packet[1], resInc);
685 EIGEN_IF_CONSTEXPR (size >= 32) {
686 storeBF16fromResult<size, inc, 16>(dst, r32.packet[2], resInc);
687 storeBF16fromResult<size, inc, 24>(dst, r32.packet[3], resInc);
690 dst += extra * resInc;
691 EIGEN_IF_CONSTEXPR (size != 32) break;
695template <
bool inc = false>
696EIGEN_ALWAYS_INLINE
void convertArrayPointerF32toBF16VSX(
float* result, Index rows, bfloat16* dst, Index resInc = 1) {
698 convertPointerF32toBF16VSX<32, inc>(i, result, rows, dst, resInc);
699 convertPointerF32toBF16VSX<16, inc>(i, result, rows, dst, resInc);
700 convertPointerF32toBF16VSX<8, inc>(i, result, rows, dst, resInc);
701 convertPointerF32toBF16VSX<1, inc>(i, result, rows, dst, resInc);
704template <
typename RhsMapper,
typename LhsMapper,
typename =
void>
705struct UseStride : std::false_type {
706 static EIGEN_ALWAYS_INLINE
void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
707 const Packet4f& pAlpha,
float* result) {
708 using RhsSubMapper =
typename RhsMapper::SubMapper;
710 RhsSubMapper rhs2 = rhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(j2), 0);
711 calcVSXVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
715template <
typename RhsMapper,
typename LhsMapper>
716struct UseStride<RhsMapper, LhsMapper,
717 std::enable_if_t<std::is_member_function_pointer<decltype(&RhsMapper::stride)>::value>>
719 static EIGEN_ALWAYS_INLINE
void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
720 const Packet4f& pAlpha,
float* result) {
721 using RhsSubMapper =
typename RhsMapper::SubMapper;
723 RhsSubMapper rhs2 = rhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(j2), 0);
724 if (rhs.stride() == 1) {
725 calcVSXVecColLoops<LhsMapper, RhsSubMapper, true>(jend - j2, rows, lhs, rhs2, pAlpha, result);
727 calcVSXVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
732template <
typename LhsMapper,
typename RhsMapper>
733void gemv_bfloat16_col(Index rows, Index cols,
const LhsMapper& alhs,
const RhsMapper& rhs, bfloat16* res,
734 Index resIncr, bfloat16 alpha) {
735 EIGEN_UNUSED_VARIABLE(resIncr);
736 eigen_internal_assert(resIncr == 1);
743 const Index lhsStride = lhs.stride();
746 const Index block_cols = cols < 128 ? cols : (lhsStride *
sizeof(bfloat16) < 16000 ? 16 : 8);
747 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
748 Packet4f pAlpha = pset1<Packet4f>(falpha);
750 ei_declare_aligned_stack_constructed_variable(
float, result, rows, 0);
752 convertArrayPointerBF16toF32(result, 1, rows, res);
754 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
755 Index jend = numext::mini(j2 + block_cols, cols);
757 using LhsSubMapper =
typename LhsMapper::SubMapper;
759 LhsSubMapper lhs2 = lhs.getSubMapper(0, convert_index<mapper_index_t<LhsMapper>>(j2));
760 UseStride<RhsMapper, LhsSubMapper>::run(j2, jend, rows, lhs2, rhs2, pAlpha, result);
763 convertArrayPointerF32toBF16VSX(result, rows, res);
766template <Index num_acc, Index size>
767EIGEN_ALWAYS_INLINE
void outputVecResults(Packet4f (&acc)[num_acc][size],
float* result,
const Packet4f& pAlpha) {
768 constexpr Index extra = num_acc & 3;
770 for (Index k = 0; k < num_acc; k += 4) {
771 Packet4f d0 = ploadu<Packet4f>(result + k);
772 d0 = pmadd(acc[k + 0][0], pAlpha, d0);
774 if (num_acc > (k + 3)) {
775 pstoreu(result + k, d0);
778 pstoreu_partial(result + k, d0, extra);
780 memcpy((
void*)(result + k), (
void*)(&d0),
sizeof(
float) * extra);
786template <Index num_acc>
787EIGEN_ALWAYS_INLINE
void preduxVecResults2VSX(Packet4f (&acc)[num_acc][2], Index k) {
788 if (num_acc > (k + 1)) {
789 acc[k][1] = vec_mergel(acc[k + 0][0], acc[k + 1][0]);
790 acc[k][0] = vec_mergeh(acc[k + 0][0], acc[k + 1][0]);
791 acc[k][0] = acc[k][0] + acc[k][1];
792 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8);
794 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8);
796 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12);
798 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4);
803template <Index num_acc>
804EIGEN_ALWAYS_INLINE
void preduxVecResultsVSX(Packet4f (&acc)[num_acc][2]) {
805 for (Index k = 0; k < num_acc; k += 4) {
806 preduxVecResults2VSX<num_acc>(acc, k + 0);
807 if (num_acc > (k + 2)) {
808 preduxVecResults2VSX<num_acc>(acc, k + 2);
809#ifdef EIGEN_VECTORIZE_VSX
810 acc[k + 0][0] =
reinterpret_cast<Packet4f
>(
811 vec_mergeh(
reinterpret_cast<Packet2ul
>(acc[k + 0][0]),
reinterpret_cast<Packet2ul
>(acc[k + 2][0])));
813 acc[k + 0][0] =
reinterpret_cast<Packet4f
>(vec_perm(acc[k + 0][0], acc[k + 2][0], p16uc_TRANSPOSE64_HI));
820EIGEN_ALWAYS_INLINE Packet8us loadPacketPartialZero(
const Packet8us& data, Index extra_cols) {
821 Packet16uc shift = pset1<Packet16uc>(8 * 2 * (8 - extra_cols));
823 return reinterpret_cast<Packet8us
>(vec_slo(vec_sro(
reinterpret_cast<Packet16uc
>(data), shift), shift));
825 return reinterpret_cast<Packet8us
>(vec_sro(vec_slo(
reinterpret_cast<Packet16uc
>(data), shift), shift));
830template <Index num_acc,
typename LhsMapper,
typename RhsMapper,
bool extra>
831EIGEN_ALWAYS_INLINE
void multVSXVecLoop(Packet4f (&acc)[num_acc][2],
const LhsMapper& lhs, RhsMapper& rhs, Index j,
833 Packet4f a0[num_acc][2], b0[2];
835 using MapperIndex = mapper_index_t<LhsMapper>;
836 const MapperIndex col = convert_index<MapperIndex>(j);
837 const MapperIndex extraCols = convert_index<MapperIndex>(extra_cols);
840 b1 = rhs.template loadPacketPartial<Packet8bf>(col, extraCols);
842 b1 = loadPacketPartialZero(b1.m_val, extra_cols);
845 b1 = rhs.template loadPacket<Packet8bf>(col);
847 b0[0] = oneConvertBF16Hi(b1.m_val);
848 b0[1] = oneConvertBF16Lo(b1.m_val);
850 const LhsMapper lhs2 = lhs.getSubMapper(0, col);
851 for (
int k = 0; k < num_acc; k++) {
853 a1 = lhs2.template loadPacketPartial<Packet8bf>(k, 0, extraCols);
855 a1 = loadPacketPartialZero(a1.m_val, extra_cols);
858 a1 = lhs2.template loadPacket<Packet8bf>(k, 0);
860 a0[k][0] = oneConvertBF16Hi(a1.m_val);
861 a0[k][1] = oneConvertBF16Lo(a1.m_val);
864 multVecVSX<num_acc, false>(acc, a0, b0);
867template <Index num_acc,
typename LhsMapper,
typename RhsMapper>
868EIGEN_ALWAYS_INLINE
void vecVSXLoop(Index cols,
const LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2],
871 for (; j + 8 <= cols; j += 8) {
872 multVSXVecLoop<num_acc, LhsMapper, RhsMapper, false>(acc, lhs, rhs, j, extra_cols);
876 multVSXVecLoop<num_acc, LhsMapper, RhsMapper, true>(acc, lhs, rhs, j, extra_cols);
880template <const Index num_acc,
typename LhsMapper,
typename RhsMapper>
881void colVSXVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
const Packet4f& pAlpha,
883 constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC_VSX);
884 const Index extra_cols = (cols & 7);
887 Packet4f acc[num_acc][2];
889 zeroAccumulators<num_acc, 2>(acc);
891 const LhsMapper lhs2 = lhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(row), 0);
892 vecVSXLoop<num_acc, LhsMapper, RhsMapper>(cols, lhs2, rhs, acc, extra_cols);
894 addResultsVSX<num_acc>(acc);
896 preduxVecResultsVSX<num_acc>(acc);
898 outputVecResults<num_acc, 2>(acc, result, pAlpha);
901 }
while (multiIters && (num_acc <= rows - (row += num_acc)));
904template <const Index num_acc,
typename LhsMapper,
typename RhsMapper>
905EIGEN_ALWAYS_INLINE
void colVSXVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
906 const Packet4f& pAlpha,
float* result) {
907 EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) {
908 colVSXVecLoopBody<num_acc, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
912template <
typename LhsMapper,
typename RhsMapper>
913EIGEN_ALWAYS_INLINE
void colVSXVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
914 const Packet4f& pAlpha,
float* result) {
915 switch (rows - row) {
917 colVSXVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
920 colVSXVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
923 colVSXVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
926 colVSXVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
929 colVSXVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
932 colVSXVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
935 colVSXVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
940template <
typename LhsMapper,
typename RhsMapper>
941EIGEN_ALWAYS_INLINE
void calcVSXVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
const Packet4f& pAlpha,
944 if (rows >= MAX_BFLOAT16_VEC_ACC_VSX) {
945 colVSXVecLoopBody<MAX_BFLOAT16_VEC_ACC_VSX, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
948 colVSXVecLoopBodyExtra<LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
951template <
typename LhsMapper,
typename RhsMapper>
952EIGEN_STRONG_INLINE
void gemv_bfloat16_row(Index rows, Index cols,
const LhsMapper& alhs,
const RhsMapper& rhs,
953 bfloat16* res, Index resIncr, bfloat16 alpha) {
954 using LinearMapper =
typename RhsMapper::LinearMapper;
959 LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
961 eigen_internal_assert(rhs.stride() == 1);
963 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
964 const Packet4f pAlpha = pset1<Packet4f>(falpha);
966 ei_declare_aligned_stack_constructed_variable(
float, result, rows, 0);
968 convertArrayPointerBF16toF32(result, 1, rows, res);
970 convertArrayPointerBF16toF32<true>(result, 1, rows, res, resIncr);
972 calcVSXVecLoops<LhsMapper, LinearMapper>(cols, rows, lhs, rhs2, pAlpha, result);
974 convertArrayPointerF32toBF16VSX(result, rows, res);
976 convertArrayPointerF32toBF16VSX<true>(result, rows, res, resIncr);
980#undef MAX_BFLOAT16_VEC_ACC_VSX
982const Packet16uc p16uc_COMPLEX32_XORFLIP = {0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33,
983 0xcc, 0xdd, 0xee, 0xff, 0x88, 0x99, 0xaa, 0xbb};
984const Packet16uc p16uc_COMPLEX64_XORFLIP = {0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
985 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77};
988const Packet16uc p16uc_COMPLEX32_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
989 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00};
990const Packet16uc p16uc_COMPLEX64_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
991 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
992const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
993 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
994const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
995 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
996const Packet16uc p16uc_COMPLEX32_NEGATE = {0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
997 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00};
998const Packet16uc p16uc_COMPLEX64_NEGATE = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
999 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
1001const Packet16uc p16uc_COMPLEX32_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
1002 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80};
1003const Packet16uc p16uc_COMPLEX64_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1004 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80};
1005const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = {0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
1006 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00};
1007const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
1008 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
1009const Packet16uc p16uc_COMPLEX32_NEGATE = {0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
1010 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80};
1011const Packet16uc p16uc_COMPLEX64_NEGATE = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
1012 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80};
1016#define COMPLEX_DELTA 0
1018#define COMPLEX_DELTA 2
1022EIGEN_ALWAYS_INLINE Packet2cf pconj2(
const Packet2cf& a) {
1023 return Packet2cf(pxor(a.v,
reinterpret_cast<Packet4f
>(p16uc_COMPLEX32_CONJ_XOR)));
1026EIGEN_ALWAYS_INLINE Packet1cd pconj2(
const Packet1cd& a) {
1027 return Packet1cd(pxor(a.v,
reinterpret_cast<Packet2d
>(p16uc_COMPLEX64_CONJ_XOR)));
1031EIGEN_ALWAYS_INLINE Packet2cf pconjinv(
const Packet2cf& a) {
1032#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1033 return Packet2cf(Packet4f(vec_neg(Packet2d(a.v))));
1035 return Packet2cf(pxor(a.v,
reinterpret_cast<Packet4f
>(p16uc_COMPLEX32_CONJ_XOR2)));
1039EIGEN_ALWAYS_INLINE Packet1cd pconjinv(
const Packet1cd& a) {
1040 return Packet1cd(pxor(a.v,
reinterpret_cast<Packet2d
>(p16uc_COMPLEX64_CONJ_XOR2)));
1043#if defined(_ARCH_PWR8) && (!EIGEN_COMP_LLVM || __clang_major__ >= 12)
1048EIGEN_ALWAYS_INLINE Packet2cf pcplxflipconj(
const Packet2cf& a) {
1050 return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR, p16uc_COMPLEX32_XORFLIP)));
1052 return pcplxflip(pconj2(a));
1056EIGEN_ALWAYS_INLINE Packet1cd pcplxflipconj(
const Packet1cd& a) {
1058 return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR, p16uc_COMPLEX64_XORFLIP)));
1060 return pcplxflip(pconj2(a));
1065EIGEN_ALWAYS_INLINE Packet2cf pcplxconjflip(
const Packet2cf& a) {
1067 return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR2, p16uc_COMPLEX32_XORFLIP)));
1069 return pconj2(pcplxflip(a));
1073EIGEN_ALWAYS_INLINE Packet1cd pcplxconjflip(
const Packet1cd& a) {
1075 return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR2, p16uc_COMPLEX64_XORFLIP)));
1077 return pconj2(pcplxflip(a));
1082EIGEN_ALWAYS_INLINE Packet2cf pnegate2(
const Packet2cf& a) {
1083#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1084 return Packet2cf(vec_neg(a.v));
1086 return Packet2cf(pxor(a.v,
reinterpret_cast<Packet4f
>(p16uc_COMPLEX32_NEGATE)));
1090EIGEN_ALWAYS_INLINE Packet1cd pnegate2(
const Packet1cd& a) {
1091#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1092 return Packet1cd(vec_neg(a.v));
1094 return Packet1cd(pxor(a.v,
reinterpret_cast<Packet2d
>(p16uc_COMPLEX64_NEGATE)));
1099EIGEN_ALWAYS_INLINE Packet2cf pcplxflipnegate(
const Packet2cf& a) {
1101 return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_NEGATE, p16uc_COMPLEX32_XORFLIP)));
1103 return pcplxflip(pnegate2(a));
1107EIGEN_ALWAYS_INLINE Packet1cd pcplxflipnegate(
const Packet1cd& a) {
1109 return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_NEGATE, p16uc_COMPLEX64_XORFLIP)));
1111 return pcplxflip(pnegate2(a));
1116EIGEN_ALWAYS_INLINE Packet2cf pcplxflip2(
const Packet2cf& a) {
1117 return Packet2cf(Packet4f(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX32_XORFLIP)));
1120EIGEN_ALWAYS_INLINE Packet1cd pcplxflip2(
const Packet1cd& a) {
1121#ifdef EIGEN_VECTORIZE_VSX
1122 return Packet1cd(__builtin_vsx_xxpermdi(a.v, a.v, 2));
1124 return Packet1cd(Packet2d(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX64_XORFLIP)));
1129EIGEN_ALWAYS_INLINE Packet4f pload_complex_half(std::complex<float>* src) {
1131#ifdef EIGEN_VECTORIZE_VSX
1133 __asm__(
"lxsdx %x0,%y1" :
"=wa"(t) :
"Z"(*src));
1135 *
reinterpret_cast<std::complex<float>*
>(
reinterpret_cast<float*
>(&t) + COMPLEX_DELTA) = *src;
1141template <
typename RhsScalar>
1142EIGEN_ALWAYS_INLINE
void pload_realimag(RhsScalar* src, Packet4f& r, Packet4f& i) {
1144 __asm__(
"lxvwsx %x0,%y1" :
"=wa"(r) :
"Z"(*(
reinterpret_cast<float*
>(src) + 0)));
1145 __asm__(
"lxvwsx %x0,%y1" :
"=wa"(i) :
"Z"(*(
reinterpret_cast<float*
>(src) + 1)));
1147 Packet4f t = pload_complex_half(src);
1148 r = vec_splat(t, COMPLEX_DELTA + 0);
1149 i = vec_splat(t, COMPLEX_DELTA + 1);
1153template <
typename RhsScalar>
1154EIGEN_ALWAYS_INLINE
void pload_realimag(RhsScalar* src, Packet2d& r, Packet2d& i) {
1155#ifdef EIGEN_VECTORIZE_VSX
1156 __asm__(
"lxvdsx %x0,%y1" :
"=wa"(r) :
"Z"(*(
reinterpret_cast<double*
>(src) + 0)));
1157 __asm__(
"lxvdsx %x0,%y1" :
"=wa"(i) :
"Z"(*(
reinterpret_cast<double*
>(src) + 1)));
1159 Packet2d t = ploadu<Packet2d>(
reinterpret_cast<double*
>(src));
1160 r = vec_splat(t, 0);
1161 i = vec_splat(t, 1);
1165#ifndef EIGEN_VECTORIZE_POWER8_VECTOR
1166const Packet16uc p16uc_MERGEE = {0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
1167 0x08, 0x09, 0x0A, 0x0B, 0x18, 0x19, 0x1A, 0x1B};
1169const Packet16uc p16uc_MERGEO = {0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17,
1170 0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F};
1174template <
typename RhsScalar>
1175EIGEN_ALWAYS_INLINE
void pload_realimag_row(RhsScalar* src, Packet4f& r, Packet4f& i) {
1176 Packet4f t = ploadu<Packet4f>(
reinterpret_cast<float*
>(src));
1177#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1178 r = vec_mergee(t, t);
1179 i = vec_mergeo(t, t);
1181 r = vec_perm(t, t, p16uc_MERGEE);
1182 i = vec_perm(t, t, p16uc_MERGEO);
1186template <
typename RhsScalar>
1187EIGEN_ALWAYS_INLINE
void pload_realimag_row(RhsScalar* src, Packet2d& r, Packet2d& i) {
1188 return pload_realimag(src, r, i);
1192EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine(std::complex<float>* src) {
1193#ifdef EIGEN_VECTORIZE_VSX
1195 __asm__(
"lxvdsx %x0,%y1" :
"=wa"(ret) :
"Z"(*(
reinterpret_cast<double*
>(src) + 0)));
1198 return Packet4f(ploaddup<Packet2d>(
reinterpret_cast<double*
>(src)));
1202EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine(std::complex<double>* src) {
return ploadu<Packet1cd>(src).v; }
1205EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine_row(std::complex<float>* src) {
return ploadu<Packet2cf>(src).v; }
1207EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine_row(std::complex<double>* src) {
return ploadu<Packet1cd>(src).v; }
1210template <
typename ResPacket>
1211EIGEN_ALWAYS_INLINE Packet4f pload_complex(std::complex<float>* src) {
1212 EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) {
1213 return pload_complex_half(src);
1215 return ploadu<Packet4f>(
reinterpret_cast<float*
>(src));
1219template <
typename ResPacket>
1220EIGEN_ALWAYS_INLINE Packet2d pload_complex(std::complex<double>* src) {
1221 return ploadu<Packet2d>(
reinterpret_cast<double*
>(src));
1225template <
typename ResPacket>
1226EIGEN_ALWAYS_INLINE Packet4f pload_complex(Packet2cf* src) {
1230template <
typename ResPacket>
1231EIGEN_ALWAYS_INLINE Packet2d pload_complex(Packet1cd* src) {
1236EIGEN_ALWAYS_INLINE Packet4f pload_complex_full(std::complex<float>* src) {
1237 return Packet4f(ploaddup<Packet2d>(
reinterpret_cast<double*
>(src)));
1240EIGEN_ALWAYS_INLINE Packet2d pload_complex_full(std::complex<double>* src) {
return ploadu<Packet1cd>(src).v; }
1243EIGEN_ALWAYS_INLINE Packet4f pload_complex_full_row(std::complex<float>* src) {
return ploadu<Packet2cf>(src).v; }
1245EIGEN_ALWAYS_INLINE Packet2d pload_complex_full_row(std::complex<double>* src) {
return pload_complex_full(src); }
1248EIGEN_ALWAYS_INLINE Packet4f pload_real(
float* src) {
return pset1<Packet4f>(*src); }
1250EIGEN_ALWAYS_INLINE Packet2d pload_real(
double* src) {
return pset1<Packet2d>(*src); }
1252EIGEN_ALWAYS_INLINE Packet4f pload_real(
const Packet4f& src) {
return src; }
1254EIGEN_ALWAYS_INLINE Packet2d pload_real(
const Packet2d& src) {
return src; }
1257EIGEN_ALWAYS_INLINE Packet4f pload_real_full(
float* src) {
return ploaddup<Packet4f>(src); }
1259EIGEN_ALWAYS_INLINE Packet2d pload_real_full(
double* src) {
return pload_real(src); }
1261EIGEN_ALWAYS_INLINE Packet4f pload_real_full(std::complex<float>* src) {
1262 return pload_complex_full(src);
1265EIGEN_ALWAYS_INLINE Packet2d pload_real_full(std::complex<double>* src) {
1266 return pload_complex_full(src);
1270template <
typename ResPacket>
1271EIGEN_ALWAYS_INLINE Packet4f pload_real_row(
float* src) {
1272 EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) {
1273 return pload_real_full(src);
1275 return ploadu<Packet4f>(src);
1279template <
typename ResPacket>
1280EIGEN_ALWAYS_INLINE Packet2d pload_real_row(
double* src) {
1281 return pload_real(src);
1288EIGEN_ALWAYS_INLINE Packet2cf padd(
const Packet2cf& a,
const std::complex<float>& b) {
1289 EIGEN_UNUSED_VARIABLE(b);
1293EIGEN_ALWAYS_INLINE Packet1cd padd(
const Packet1cd& a,
const std::complex<double>& b) {
1294 EIGEN_UNUSED_VARIABLE(b);
1299template <
typename Scalar,
typename ResScalar>
1300EIGEN_ALWAYS_INLINE Scalar pset1_realimag(ResScalar& alpha,
int which,
int conj) {
1301 return (which) ? ((conj) ? -alpha.real() : alpha.real()) : ((conj) ? -alpha.imag() : alpha.imag());
1305template <
typename Scalar,
typename ResScalar,
typename ResPacket,
int which>
1306EIGEN_ALWAYS_INLINE Packet2cf pset1_complex(std::complex<float>& alpha) {
1308 ret.v[COMPLEX_DELTA + 0] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x01), (which & 0x04));
1309 ret.v[COMPLEX_DELTA + 1] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x02), (which & 0x08));
1310 ret.v[2 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 0];
1311 ret.v[3 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 1];
1315template <
typename Scalar,
typename ResScalar,
typename ResPacket,
int which>
1316EIGEN_ALWAYS_INLINE Packet1cd pset1_complex(std::complex<double>& alpha) {
1318 ret.v[0] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x01), (which & 0x04));
1319 ret.v[1] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x02), (which & 0x08));
1324template <
typename Packet>
1325EIGEN_ALWAYS_INLINE Packet pset_zero() {
1326 return pset1<Packet>(__UNPACK_TYPE__(Packet)(0));
1330EIGEN_ALWAYS_INLINE Packet2cf pset_zero<Packet2cf>() {
1331 return Packet2cf(pset1<Packet4f>(
float(0)));
1335EIGEN_ALWAYS_INLINE Packet1cd pset_zero<Packet1cd>() {
1336 return Packet1cd(pset1<Packet2d>(
double(0)));
1340template <
typename Packet,
typename LhsPacket,
typename RhsPacket>
1341EIGEN_ALWAYS_INLINE Packet pset_init(
const Packet& c1) {
1342 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
1343 EIGEN_UNUSED_VARIABLE(c1);
1344 return pset_zero<Packet>();
1350template <
typename PResPacket,
typename ResPacket,
typename ResScalar,
typename Scalar>
1352 alpha_store(ResScalar& alpha) {
1353 separate.r = pset1_complex<Scalar, ResScalar, ResPacket, 0x3>(alpha);
1354 separate.i = pset1_complex<Scalar, ResScalar, ResPacket, 0x0>(alpha);
1363template <
typename ScalarPacket,
typename AlphaData>
1364EIGEN_ALWAYS_INLINE ScalarPacket pmadd_complex(
const ScalarPacket& c0,
const ScalarPacket& c2,
const ScalarPacket& c4,
1366 return pmadd(c2, b0.separate.i.v, pmadd(c0, b0.separate.r.v, c4));
1370template <
typename Scalar,
typename ScalarPacket,
typename PResPacket,
typename ResPacket,
typename ResScalar,
1372EIGEN_ALWAYS_INLINE
void pstoreu_pmadd_complex(
const PResPacket& c0, AlphaData& b0, ResScalar* res) {
1373 PResPacket c2 = pcplxflipconj(c0);
1374 EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) {
1375 ScalarPacket c4 = ploadu<ScalarPacket>(
reinterpret_cast<Scalar*
>(res));
1376 ScalarPacket c3 = pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c4, b0);
1377 pstoreu(
reinterpret_cast<Scalar*
>(res), c3);
1379 ScalarPacket c4 = pload_complex<ResPacket>(res);
1380 PResPacket c3 = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c4, b0));
1385template <
typename ScalarPacket,
typename PResPacket,
typename ResPacket,
typename ResScalar,
typename AlphaData,
1386 Index ResPacketSize, Index iter2>
1387EIGEN_ALWAYS_INLINE
void pstoreu_pmadd_complex(
const PResPacket& c0,
const PResPacket& c1, AlphaData& b0,
1389 PResPacket c2 = pcplxflipconj(c0);
1390 PResPacket c3 = pcplxflipconj(c1);
1391#if !defined(_ARCH_PWR10)
1392 ScalarPacket c4 = pload_complex<ResPacket>(res + (iter2 * ResPacketSize));
1393 ScalarPacket c5 = pload_complex<ResPacket>(res + ((iter2 + 1) * ResPacketSize));
1394 PResPacket c6 = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c4, b0));
1395 PResPacket c7 = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c1.v, c3.v, c5, b0));
1396 pstoreu(res + (iter2 * ResPacketSize), c6);
1397 pstoreu(res + ((iter2 + 1) * ResPacketSize), c7);
1399 __vector_pair a = *
reinterpret_cast<__vector_pair*
>(res + (iter2 * ResPacketSize));
1402 __builtin_vsx_disassemble_pair(
reinterpret_cast<void*
>(c6), &a);
1403 c6[0] = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c6[0].v, b0));
1404 c6[1] = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c1.v, c3.v, c6[1].v, b0));
1405 GEMV_BUILDPAIR_MMA(a, c6[0].v, c6[1].v);
1407 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) {
1408 __asm__(
"xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" :
"+&d"(a) :
"wa"(b0.separate.r.v),
"wa"(c0.v),
"wa"(c1.v));
1409 __asm__(
"xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" :
"+&d"(a) :
"wa"(b0.separate.i.v),
"wa"(c2.v),
"wa"(c3.v));
1411 __asm__(
"xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" :
"+&d"(a) :
"wa"(b0.separate.r.v),
"wa"(c0.v),
"wa"(c1.v));
1412 __asm__(
"xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" :
"+&d"(a) :
"wa"(b0.separate.i.v),
"wa"(c2.v),
"wa"(c3.v));
1415 *
reinterpret_cast<__vector_pair*
>(res + (iter2 * ResPacketSize)) = a;
1420template <
typename Scalar,
typename LhsScalar,
typename LhsMapper,
typename LhsPacket>
1421EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, mapper_index_t<LhsMapper> i, mapper_index_t<LhsMapper> j) {
1422 EIGEN_IF_CONSTEXPR (
sizeof(Scalar) ==
sizeof(LhsScalar)) {
1423 const LhsScalar& src = lhs(i + 0, j);
1424 return LhsPacket(pload_real_full(
const_cast<LhsScalar*
>(&src)));
1426 return lhs.template load<LhsPacket, Unaligned>(i + 0, j);
1430template <
typename ComplexPacket,
typename RealPacket,
bool ConjugateLhs,
bool ConjugateRhs,
bool Negate>
1431EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(
const RealPacket& a,
const RealPacket& b,
const RealPacket& c) {
1432 EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) {
1433 return vec_madd(a, pconj2(ComplexPacket(b)).v, c);
1434 }
else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) {
1435 return vec_nmsub(a, b, c);
1437 return vec_madd(a, b, c);
1442template <
typename ComplexPacket,
typename RealPacket,
bool Conjugate>
1443EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_real(
const RealPacket& a,
const RealPacket& b,
const RealPacket& c) {
1444 EIGEN_IF_CONSTEXPR (Conjugate) {
1445 return vec_madd(a, pconj2(ComplexPacket(b)).v, c);
1447 return vec_madd(a, b, c);
1451template <
typename LhsPacket,
typename RhsScalar,
typename RhsPacket,
typename PResPacket,
bool ConjugateLhs,
1452 bool ConjugateRhs,
int StorageOrder>
1453EIGEN_ALWAYS_INLINE
void gemv_mult_generic(
const LhsPacket& a0, RhsScalar* b, PResPacket& c0) {
1454 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs> pcj;
1456 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1457 b0 = pset1<RhsPacket>(*b);
1459 b0 = ploadu<RhsPacket>(b);
1461 c0 = pcj.pmadd(a0, b0, c0);
1465template <
typename ScalarPacket,
typename LhsPacket,
typename RhsScalar,
typename RhsPacket,
typename PResPacket,
1466 typename ResPacket,
bool ConjugateLhs,
bool ConjugateRhs,
int StorageOrder>
1467EIGEN_ALWAYS_INLINE
void gemv_mult_complex_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0, ResPacket& c1) {
1468 ScalarPacket br, bi;
1469 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1470 pload_realimag<RhsScalar>(b, br, bi);
1472 pload_realimag_row<RhsScalar>(b, br, bi);
1474 EIGEN_IF_CONSTEXPR (ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0);
1475 LhsPacket a1 = pcplxflipconj(a0);
1476 ScalarPacket cr = pmadd_complex_complex<LhsPacket, ScalarPacket, ConjugateLhs, ConjugateRhs, false>(a0.v, br, c0.v);
1477 ScalarPacket ci = pmadd_complex_complex<LhsPacket, ScalarPacket, ConjugateLhs, ConjugateRhs, true>(a1.v, bi, c1.v);
1479 c0 = PResPacket(cr);
1483template <
typename ScalarPacket,
typename LhsPacket,
typename RhsScalar,
typename RhsPacket,
typename PResPacket,
1484 typename ResPacket,
bool ConjugateLhs,
bool ConjugateRhs,
int StorageOrder>
1485EIGEN_ALWAYS_INLINE
void gemv_mult_real_complex(
const LhsPacket& a0, RhsScalar* b, PResPacket& c0) {
1487 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1488 b0 = pload_complex_full(b);
1490 b0 = pload_complex_full_row(b);
1492 ScalarPacket cri = pmadd_complex_real<PResPacket, ScalarPacket, ConjugateRhs>(a0, b0, c0.v);
1493 c0 = PResPacket(cri);
1497template <
typename ScalarPacket,
typename LhsPacket,
typename RhsScalar,
typename RhsPacket,
typename PResPacket,
1498 typename ResPacket,
bool ConjugateLhs,
bool ConjugateRhs,
int StorageOrder>
1499EIGEN_ALWAYS_INLINE
void gemv_mult_complex_real(LhsPacket& a0, RhsScalar* b, PResPacket& c0) {
1500 ScalarPacket a1 = pload_complex<ResPacket>(&a0);
1502 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1505 b0 = pload_real_row<ResPacket>(b);
1507 ScalarPacket cri = pmadd_complex_real<PResPacket, ScalarPacket, ConjugateLhs>(a1, b0, c0.v);
1508 c0 = PResPacket(cri);
1511#define GEMV_MULT_COMPLEX_COMPLEX(LhsType, RhsType, ResType) \
1512 template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket, \
1513 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1514 EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, ResType& c1) { \
1515 gemv_mult_complex_complex<ScalarPacket, LhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
1516 ConjugateRhs, StorageOrder>(a0, b, c0, c1); \
1519GEMV_MULT_COMPLEX_COMPLEX(Packet2cf, std::complex<float>, Packet2cf)
1520GEMV_MULT_COMPLEX_COMPLEX(Packet1cd, std::complex<double>, Packet1cd)
1522#define GEMV_MULT_REAL_COMPLEX(LhsType, RhsType, ResType) \
1523 template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket, \
1524 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1525 EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, RhsType&) { \
1526 gemv_mult_real_complex<ScalarPacket, LhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
1527 ConjugateRhs, StorageOrder>(a0, b, c0); \
1530GEMV_MULT_REAL_COMPLEX(
float, std::complex<float>, Packet2cf)
1531GEMV_MULT_REAL_COMPLEX(
double, std::complex<double>, Packet1cd)
1532GEMV_MULT_REAL_COMPLEX(Packet4f, std::complex<float>, Packet2cf)
1533GEMV_MULT_REAL_COMPLEX(Packet2d, std::complex<double>, Packet1cd)
1535#define GEMV_MULT_COMPLEX_REAL(LhsType, RhsType, ResType1, ResType2) \
1536 template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket, \
1537 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1538 EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType1& c0, ResType2&) { \
1539 gemv_mult_complex_real<ScalarPacket, LhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
1540 ConjugateRhs, StorageOrder>(a0, b, c0); \
1543GEMV_MULT_COMPLEX_REAL(Packet2cf,
float, Packet2cf, std::complex<float>)
1544GEMV_MULT_COMPLEX_REAL(Packet1cd,
double, Packet1cd, std::complex<double>)
1545GEMV_MULT_COMPLEX_REAL(std::complex<float>,
float, Packet2cf, std::complex<float>)
1546GEMV_MULT_COMPLEX_REAL(std::complex<double>,
double, Packet1cd, std::complex<double>)
1550template <
typename T>
1551EIGEN_ALWAYS_INLINE T convertReal(T a) {
1555EIGEN_ALWAYS_INLINE Packet4f convertReal(
const Packet2cf& a) {
return a.v; }
1557EIGEN_ALWAYS_INLINE Packet2d convertReal(
const Packet1cd& a) {
return a.v; }
1560template <
typename T>
1561EIGEN_ALWAYS_INLINE T convertComplex(T a) {
1565EIGEN_ALWAYS_INLINE Packet2cf convertComplex(
const Packet4f& a) {
return Packet2cf(a); }
1567EIGEN_ALWAYS_INLINE Packet1cd convertComplex(
const Packet2d& a) {
return Packet1cd(a); }
1570template <
typename ScalarPacket,
typename LhsPacket,
typename SLhsPacket,
typename ResPacket>
1571EIGEN_ALWAYS_INLINE
void pload_complex_MMA(SLhsPacket& a) {
1572 a = SLhsPacket(pload_complex<ResPacket>(&a));
1575template <
typename ScalarPacket,
typename LhsPacket,
typename SLhsPacket,
typename ResPacket>
1576EIGEN_ALWAYS_INLINE
void pload_complex_MMA(__vector_pair&) {
1581template <
typename LhsPacket,
typename RhsPacket,
bool NegativeAccumulate>
1582EIGEN_ALWAYS_INLINE
void pger_vecMMA(__vector_quad* acc,
const RhsPacket& a,
const LhsPacket& b) {
1583 EIGEN_IF_CONSTEXPR (NegativeAccumulate) {
1584 __builtin_mma_xvf32gernp(acc, (__vector
unsigned char)a, (__vector
unsigned char)b);
1586 __builtin_mma_xvf32gerpp(acc, (__vector
unsigned char)a, (__vector
unsigned char)b);
1591template <
typename LhsPacket,
typename RhsPacket,
bool NegativeAccumulate>
1592EIGEN_ALWAYS_INLINE
void pger_vecMMA(__vector_quad* acc, __vector_pair& a,
const Packet2d& b) {
1593 EIGEN_IF_CONSTEXPR (NegativeAccumulate) {
1594 __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector
unsigned char)b);
1596 __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector
unsigned char)b);
1600template <
typename LhsPacket,
typename RhsPacket,
bool NegativeAccumulate>
1601EIGEN_ALWAYS_INLINE
void pger_vecMMA(__vector_quad*, __vector_pair&,
const Packet4f&) {
1606template <
typename RealPacket,
typename LhsPacket,
bool ConjugateLhs,
bool ConjugateRhs,
bool Negate>
1607EIGEN_ALWAYS_INLINE
void pmadd_complex_complex_MMA(
const LhsPacket& a,
const RealPacket& b, __vector_quad* c) {
1608 EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) {
1609 RealPacket b2 = pconj2(convertComplex(b)).v;
1610 return pger_vecMMA<RealPacket, RealPacket, false>(c, b2, a.v);
1611 }
else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) {
1612 return pger_vecMMA<RealPacket, RealPacket, true>(c, b, a.v);
1614 return pger_vecMMA<RealPacket, RealPacket, false>(c, b, a.v);
1618template <
typename RealPacket,
typename LhsPacket,
bool ConjugateLhs,
bool ConjugateRhs,
bool Negate>
1619EIGEN_ALWAYS_INLINE
void pmadd_complex_complex_MMA(__vector_pair& a,
const RealPacket& b, __vector_quad* c) {
1620 EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) {
1621 RealPacket b2 = pconj2(convertComplex(b)).v;
1622 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b2);
1623 }
else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) {
1624 return pger_vecMMA<RealPacket, __vector_pair, true>(c, a, b);
1626 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b);
1631template <
typename RealPacket,
typename LhsPacket,
bool Conjugate,
int StorageOrder>
1632EIGEN_ALWAYS_INLINE
void pmadd_complex_real_MMA(
const LhsPacket& a,
const RealPacket& b, __vector_quad* c) {
1633 RealPacket a2 = convertReal(a);
1634 EIGEN_IF_CONSTEXPR (Conjugate) {
1635 RealPacket b2 = pconj2(convertComplex(b)).v;
1636 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1637 return pger_vecMMA<RealPacket, RealPacket, false>(c, b2, a2);
1639 return pger_vecMMA<RealPacket, RealPacket, false>(c, a2, b2);
1642 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1643 return pger_vecMMA<RealPacket, RealPacket, false>(c, b, a2);
1645 return pger_vecMMA<RealPacket, RealPacket, false>(c, a2, b);
1651template <
typename RealPacket,
typename LhsPacket,
bool Conjugate,
int StorageOrder>
1652EIGEN_ALWAYS_INLINE
void pmadd_complex_real_MMA(__vector_pair& a,
const RealPacket& b, __vector_quad* c) {
1653 EIGEN_IF_CONSTEXPR (Conjugate) {
1654 RealPacket b2 = pconj2(convertComplex(b)).v;
1655 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b2);
1657 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b);
1662template <
typename ScalarPacket,
typename LhsPacket,
typename SLhsPacket,
typename RhsScalar,
typename ResPacket,
1663 bool ConjugateLhs,
bool ConjugateRhs,
int StorageOrder>
1664EIGEN_ALWAYS_INLINE
void gemv_mult_complex_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) {
1666 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1667 b0 = pload_realimag_combine(b);
1669 b0 = pload_realimag_combine_row(b);
1671 pmadd_complex_complex_MMA<ScalarPacket, LhsPacket, ConjugateLhs, ConjugateRhs, false>(a0, b0, c0);
1675template <
typename ScalarPacket,
typename LhsPacket,
typename SLhsPacket,
typename RhsScalar,
typename ResPacket,
1676 bool ConjugateLhs,
bool ConjugateRhs,
int StorageOrder>
1677EIGEN_ALWAYS_INLINE
void gemv_mult_complex_real_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) {
1678 pload_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, ResPacket>(a0);
1680 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1683 b0 = pload_real_row<ResPacket>(b);
1685 pmadd_complex_real_MMA<ScalarPacket, LhsPacket, ConjugateLhs, ColMajor>(a0, b0, c0);
1689template <
typename ScalarPacket,
typename LhsPacket,
typename SLhsPacket,
typename RhsScalar,
typename ResPacket,
1690 bool ConjugateLhs,
bool ConjugateRhs,
int StorageOrder>
1691EIGEN_ALWAYS_INLINE
void gemv_mult_real_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) {
1693 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1694 b0 = pload_complex_full(b);
1696 b0 = pload_complex_full_row(b);
1698 pmadd_complex_real_MMA<ScalarPacket, LhsPacket, ConjugateRhs,
1699 (
sizeof(RhsScalar) ==
sizeof(std::complex<float>)) ? StorageOrder :
ColMajor>(a0, b0, c0);
1702#define GEMV_MULT_COMPLEX_COMPLEX_MMA(LhsType, RhsType) \
1703 template <typename ScalarPacket, typename LhsScalar, typename LhsPacket, typename SLhsPacket, typename RhsScalar, \
1704 typename RhsPacket, typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1705 EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \
1706 gemv_mult_complex_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, \
1707 ConjugateRhs, StorageOrder>(a0, b, c0); \
1710GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet2cf, std::complex<float>)
1711GEMV_MULT_COMPLEX_COMPLEX_MMA(__vector_pair, std::complex<float>)
1712GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet1cd, std::complex<double>)
1715template <
typename ScalarPacket,
typename LhsScalar,
typename LhsPacket,
typename SLhsPacket,
typename RhsScalar,
1716 typename RhsPacket,
typename ResPacket,
bool ConjugateLhs,
bool ConjugateRhs,
int StorageOrder>
1717EIGEN_ALWAYS_INLINE
void gemv_mult_complex_MMA(__vector_pair& a0, std::complex<double>* b, __vector_quad* c0) {
1718 EIGEN_IF_CONSTEXPR (
sizeof(LhsScalar) == 16) {
1719 gemv_mult_complex_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs,
1720 StorageOrder>(a0, b, c0);
1722 gemv_mult_real_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs,
1723 StorageOrder>(a0, b, c0);
1727#define GEMV_MULT_REAL_COMPLEX_MMA(LhsType, RhsType) \
1728 template <typename ScalarPacket, typename LhsScalar, typename LhsPacket, typename SLhsPacket, typename RhsScalar, \
1729 typename RhsPacket, typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1730 EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \
1731 gemv_mult_real_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs, \
1732 StorageOrder>(a0, b, c0); \
1735GEMV_MULT_REAL_COMPLEX_MMA(Packet4f, std::complex<float>)
1736GEMV_MULT_REAL_COMPLEX_MMA(Packet2d, std::complex<double>)
1738#define GEMV_MULT_COMPLEX_REAL_MMA(LhsType, RhsType) \
1739 template <typename ScalarPacket, typename LhsScalar, typename LhsPacket, typename SLhsPacket, typename RhsScalar, \
1740 typename RhsPacket, typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1741 EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \
1742 gemv_mult_complex_real_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs, \
1743 StorageOrder>(a0, b, c0); \
1746GEMV_MULT_COMPLEX_REAL_MMA(Packet2cf,
float)
1747GEMV_MULT_COMPLEX_REAL_MMA(Packet1cd,
double)
1748GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair,
float)
1749GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair,
double)
1752template <
typename Scalar,
typename ScalarPacket,
typename LhsPacket,
typename RhsPacket,
bool ConjugateLhs,
1754EIGEN_ALWAYS_INLINE
void disassembleResults2(__vector_quad* c0, PacketBlock<ScalarPacket, 4>& result0) {
1755 __builtin_mma_disassemble_acc(&result0.packet, c0);
1756 EIGEN_IF_CONSTEXPR (
sizeof(LhsPacket) == 16) {
1757 EIGEN_IF_CONSTEXPR (
sizeof(RhsPacket) == 16) {
1758 ScalarPacket tmp0, tmp2;
1759 tmp2 = vec_mergeh(result0.packet[2], result0.packet[3]);
1760 tmp0 = vec_mergeh(result0.packet[0], result0.packet[1]);
1761 result0.packet[3] = vec_mergel(result0.packet[3], result0.packet[2]);
1762 result0.packet[1] = vec_mergel(result0.packet[1], result0.packet[0]);
1763 result0.packet[2] = tmp2;
1764 result0.packet[0] = tmp0;
1766 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
1767 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
1768 result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v;
1769 }
else EIGEN_IF_CONSTEXPR (ConjugateRhs) {
1770 result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v;
1771 result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v;
1773 result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v;
1774 result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v;
1776 result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]);
1777 result0.packet[2] = vec_add(result0.packet[2], result0.packet[3]);
1779 result0.packet[0][1] = result0.packet[1][1];
1780 result0.packet[2][1] = result0.packet[3][1];
1785template <
typename Scalar,
typename ScalarPacket,
typename LhsPacket,
typename RhsPacket,
bool ConjugateLhs,
1787EIGEN_ALWAYS_INLINE
void disassembleResults4(__vector_quad* c0, PacketBlock<ScalarPacket, 4>& result0) {
1788 __builtin_mma_disassemble_acc(&result0.packet, c0);
1789 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
1790 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
1791 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
1792 result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v;
1794 EIGEN_IF_CONSTEXPR (ConjugateRhs) {
1795 result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v;
1797 result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v;
1800 result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]);
1801 }
else EIGEN_IF_CONSTEXPR (
sizeof(LhsPacket) ==
sizeof(std::complex<float>)) {
1802 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
1803 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
1806 result0.packet[0] = vec_mergee(result0.packet[0], result0.packet[1]);
1810template <
typename Scalar,
typename ScalarPacket,
int ResPacketSize,
typename LhsPacket,
typename RhsPacket,
1811 bool ConjugateLhs,
bool ConjugateRhs>
1812EIGEN_ALWAYS_INLINE
void disassembleResults(__vector_quad* c0, PacketBlock<ScalarPacket, 4>& result0) {
1813 EIGEN_IF_CONSTEXPR (!GEMV_IS_COMPLEX_FLOAT) {
1814 disassembleResults2<Scalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(c0, result0);
1816 disassembleResults4<Scalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(c0, result0);
1821#define GEMV_GETN_COMPLEX(N) (((N) * ResPacketSize) >> 1)
1823#define GEMV_LOADPACKET_COL_COMPLEX(iter) \
1824 loadLhsPacket<Scalar, LhsScalar, LhsMapper, PLhsPacket>(lhs, i + ((iter) * ResPacketSize), j)
1826#define GEMV_LOADPACKET_COL_COMPLEX_DATA(iter) convertReal(GEMV_LOADPACKET_COL_COMPLEX(iter))
1829#define GEMV_INIT_COL_COMPLEX_MMA(iter, N) \
1830 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1831 __builtin_mma_xxsetaccz(&e0##iter); \
1835#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \
1836 GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), \
1837 GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1)); \
1838 EIGEN_UNUSED_VARIABLE(f##iter1);
1840#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \
1841 EIGEN_IF_CONSTEXPR (sizeof(LhsPacket) == 16) { \
1842 const LhsScalar& src = lhs(i + ((32 * iter1) / sizeof(LhsScalar)), j); \
1843 a##iter1 = *reinterpret_cast<__vector_pair*>(const_cast<LhsScalar*>(&src)); \
1844 EIGEN_UNUSED_VARIABLE(f##iter1); \
1846 f##iter1 = lhs.template load<PLhsPacket, Unaligned>(i + ((iter2) * ResPacketSize), j); \
1847 GEMV_BUILDPAIR_MMA(a##iter1, vec_splat(convertReal(f##iter1), 0), vec_splat(convertReal(f##iter1), 1)); \
1851#define GEMV_LOAD1_COL_COMPLEX_MMA(iter, N) \
1852 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1853 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1854 f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \
1855 EIGEN_UNUSED_VARIABLE(a##iter); \
1857 GEMV_LOADPAIR_COL_COMPLEX_MMA(iter, iter << 1) \
1860 EIGEN_UNUSED_VARIABLE(a##iter); \
1861 EIGEN_UNUSED_VARIABLE(f##iter); \
1864#define GEMV_WORK1_COL_COMPLEX_MMA(iter, N) \
1865 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1866 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1867 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
1868 ConjugateLhs, ConjugateRhs, ColMajor>(f##iter, b, &e0##iter); \
1870 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
1871 ConjugateLhs, ConjugateRhs, ColMajor>(a##iter, b, &e0##iter); \
1875#define GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter1, iter2) \
1876 GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1));
1878#define GEMV_LOAD2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \
1879 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \
1880 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1881 GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2); \
1882 EIGEN_UNUSED_VARIABLE(a##iter3); \
1884 GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2 << 1); \
1885 GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter3, iter3 << 1); \
1888 EIGEN_UNUSED_VARIABLE(a##iter2); \
1889 EIGEN_UNUSED_VARIABLE(a##iter3); \
1891 EIGEN_UNUSED_VARIABLE(f##iter2); \
1892 EIGEN_UNUSED_VARIABLE(f##iter3);
1894#define GEMV_WORK2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \
1895 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \
1896 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1898 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(g), &a##iter2); \
1899 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
1900 ConjugateLhs, ConjugateRhs, ColMajor>(g[0], b, &e0##iter2); \
1901 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
1902 ConjugateLhs, ConjugateRhs, ColMajor>(g[1], b, &e0##iter3); \
1904 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
1905 ConjugateLhs, ConjugateRhs, ColMajor>(a##iter2, b, &e0##iter2); \
1906 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
1907 ConjugateLhs, ConjugateRhs, ColMajor>(a##iter3, b, &e0##iter3); \
1912#define GEMV_LOAD_COL_COMPLEX_MMA(N) \
1913 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > 1) { \
1914 GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) \
1916 GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \
1919#define GEMV_WORK_COL_COMPLEX_MMA(N) \
1920 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > 1) { \
1921 GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) \
1923 GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \
1926#define GEMV_LOAD_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N)
1928#define GEMV_WORK_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N)
1931#define GEMV_DISASSEMBLE_COMPLEX_MMA(iter) \
1932 disassembleResults<Scalar, ScalarPacket, ResPacketSize, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>( \
1933 &e0##iter, result0##iter);
1935#define GEMV_STORE_COL_COMPLEX_MMA(iter, N) \
1936 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1937 GEMV_DISASSEMBLE_COMPLEX_MMA(iter); \
1938 c0##iter = PResPacket(result0##iter.packet[0]); \
1939 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1940 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
1941 c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \
1943 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
1944 c0##iter, alpha_data, res + i + ((iter << 1) * ResPacketSize)); \
1945 c0##iter = PResPacket(result0##iter.packet[2]); \
1946 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
1947 c0##iter, alpha_data, res + i + (((iter << 1) + 1) * ResPacketSize)); \
1951#define GEMV_STORE2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \
1952 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \
1953 GEMV_DISASSEMBLE_COMPLEX_MMA(iter2); \
1954 GEMV_DISASSEMBLE_COMPLEX_MMA(iter3); \
1955 c0##iter2 = PResPacket(result0##iter2.packet[0]); \
1956 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1957 c0##iter3 = PResPacket(result0##iter3.packet[0]); \
1958 pstoreu_pmadd_complex<ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData, ResPacketSize, iter2>( \
1959 c0##iter2, c0##iter3, alpha_data, res + i); \
1961 c0##iter3 = PResPacket(result0##iter2.packet[2]); \
1962 pstoreu_pmadd_complex<ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData, ResPacketSize, iter2 << 1>( \
1963 c0##iter2, c0##iter3, alpha_data, res + i); \
1964 c0##iter2 = PResPacket(result0##iter3.packet[0]); \
1965 c0##iter3 = PResPacket(result0##iter3.packet[2]); \
1966 pstoreu_pmadd_complex<ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData, ResPacketSize, iter3 << 1>( \
1967 c0##iter2, c0##iter3, alpha_data, res + i); \
1971#define GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \
1972 GEMV_UNROLL(GEMV_INIT_COL_COMPLEX_MMA, N) \
1975 const RhsScalar& b1 = rhs2(j, 0); \
1976 RhsScalar* b = const_cast<RhsScalar*>(&b1); \
1977 GEMV_UNROLL(GEMV_PREFETCH, N) \
1978 GEMV_LOAD_COL_COMPLEX_MMA(N) \
1979 GEMV_WORK_COL_COMPLEX_MMA(N) \
1980 } while (++j < jend); \
1981 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= 2) { \
1982 GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) \
1984 GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1)) \
1986 i += (ResPacketSize * N);
1989#define GEMV_INIT_COMPLEX(iter, N) \
1990 EIGEN_IF_CONSTEXPR (N > iter) { \
1991 c0##iter = pset_zero<PResPacket>(); \
1992 c1##iter = pset_init<ResPacket, LhsPacket, RhsPacket>(c1##iter); \
1994 EIGEN_UNUSED_VARIABLE(c0##iter); \
1995 EIGEN_UNUSED_VARIABLE(c1##iter); \
1998#define GEMV_WORK_COL_COMPLEX(iter, N) \
1999 EIGEN_IF_CONSTEXPR (N > iter) { \
2000 f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \
2001 gemv_mult_complex<ScalarPacket, PLhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
2002 ConjugateRhs, ColMajor>(f##iter, b, c0##iter, c1##iter); \
2004 EIGEN_UNUSED_VARIABLE(f##iter); \
2007#define GEMV_STORE_COL_COMPLEX(iter, N) \
2008 EIGEN_IF_CONSTEXPR (N > iter) { \
2009 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { \
2010 c0##iter = padd(c0##iter, c1##iter); \
2012 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
2013 c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \
2017#define GEMV_PROCESS_COL_COMPLEX_ONE(N) \
2018 GEMV_UNROLL(GEMV_INIT_COMPLEX, N) \
2021 const RhsScalar& b1 = rhs2(j, 0); \
2022 RhsScalar* b = const_cast<RhsScalar*>(&b1); \
2023 GEMV_UNROLL(GEMV_PREFETCH, N) \
2024 GEMV_UNROLL(GEMV_WORK_COL_COMPLEX, N) \
2025 } while (++j < jend); \
2026 GEMV_UNROLL(GEMV_STORE_COL_COMPLEX, N) \
2027 i += (ResPacketSize * N);
2029#if defined(USE_GEMV_MMA) && (EIGEN_COMP_LLVM || defined(USE_SLOWER_GEMV_MMA))
2030#define USE_GEMV_COL_COMPLEX_MMA
2033#ifdef USE_GEMV_COL_COMPLEX_MMA
2034#define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N)
2036#if defined(USE_GEMV_MMA) && (__GNUC__ > 10)
2037#define GEMV_PROCESS_COL_COMPLEX(N) \
2038 EIGEN_IF_CONSTEXPR (sizeof(Scalar) != sizeof(LhsPacket)) { \
2039 GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \
2041 GEMV_PROCESS_COL_COMPLEX_ONE(N) \
2044#define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE(N)
2048template <
typename Scalar,
typename LhsScalar,
typename LhsMapper,
bool ConjugateLhs,
bool LhsIsReal,
2049 typename RhsScalar,
typename RhsMapper,
bool ConjugateRhs,
bool RhsIsReal,
typename ResScalar>
2050EIGEN_STRONG_INLINE
void gemv_complex_col(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols,
2051 const LhsMapper& alhs,
const RhsMapper& rhs, ResScalar* res,
2052 mapper_index_t<LhsMapper> resIncr, ResScalar alpha) {
2053 using Index = mapper_index_t<LhsMapper>;
2055 using Traits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketFull, false>;
2057 using LhsPacket =
typename Traits::LhsPacket;
2058 using RhsPacket =
typename Traits::RhsPacket;
2059 using ResPacket =
typename Traits::ResPacket;
2061 using ScalarPacket =
typename packet_traits<Scalar>::type;
2062 using PLhsPacket =
typename packet_traits<LhsScalar>::type;
2063 using PResPacket =
typename packet_traits<ResScalar>::type;
2064 using PTraits = gemv_traits<ResPacket, ResPacket>;
2066 EIGEN_UNUSED_VARIABLE(resIncr);
2067 eigen_internal_assert(resIncr == 1);
2071 LhsMapper lhs(alhs);
2072 RhsMapper rhs2(rhs);
2074 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
2076 const Index lhsStride = lhs.stride();
2081 ResPacketSize = PTraits::ResPacketSize,
2082 LhsPacketSize = PTraits::LhsPacketSize,
2083 RhsPacketSize = PTraits::RhsPacketSize,
2085#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
2086 const Index prefetch_dist = 64 * LhsPacketSize;
2089#ifndef GCC_ONE_VECTORPAIR_BUG
2090 const Index n8 = rows - 8 * ResPacketSize + 1;
2091 const Index n4 = rows - 4 * ResPacketSize + 1;
2092 const Index n2 = rows - 2 * ResPacketSize + 1;
2094 const Index n1 = rows - 1 * ResPacketSize + 1;
2097 const Index block_cols = cols < 128 ? cols : (lhsStride *
sizeof(LhsScalar) < 16000 ? 16 : 8);
2099 using AlphaData = alpha_store<PResPacket, ResPacket, ResScalar, Scalar>;
2100 AlphaData alpha_data(alpha);
2102 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
2103 Index jend = numext::mini(j2 + block_cols, cols);
2105 PResPacket c00, c01, c02, c03, c04, c05, c06, c07;
2106 ResPacket c10, c11, c12, c13, c14, c15, c16, c17;
2107 PLhsPacket f0, f1, f2, f3, f4, f5, f6, f7;
2109 __vector_quad e00, e01, e02, e03, e04, e05, e06, e07;
2110 __vector_pair a0, a1, a2, a3, a4, a5, a6, a7;
2111 PacketBlock<ScalarPacket, 4> result00, result01, result02, result03, result04, result05, result06, result07;
2113 GEMV_UNUSED(8, result0)
2116#if !defined(GCC_ONE_VECTORPAIR_BUG) && defined(USE_GEMV_COL_COMPLEX_MMA)
2117 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT)
2120#ifndef GCC_ONE_VECTORPAIR_BUG
2123 GEMV_PROCESS_COL_COMPLEX(8)
2127 GEMV_PROCESS_COL_COMPLEX(4)
2130 GEMV_PROCESS_COL_COMPLEX(2)
2137 GEMV_PROCESS_COL_COMPLEX_ONE(1)
2139 for (; i < rows; ++i) {
2143 d0 += cj.pmul(lhs(i, j), rhs2(j, 0));
2144 }
while (++j < jend);
2145 res[i] += alpha * d0;
2150template <
typename Scalar,
int N>
2156static Packet16uc p16uc_ELEMENT_3 = {0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f,
2157 0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f};
2160template <
typename ResScalar,
typename ResPacket>
2161EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_real(__vector_quad* acc0, __vector_quad* acc1) {
2162 PacketBlock<ResPacket, 4> result0, result1;
2163 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2164 __builtin_mma_disassemble_acc(&result1.packet, acc1);
2165 result0.packet[0] = vec_mergeh(result0.packet[0], result1.packet[0]);
2166 result0.packet[1] = vec_mergeo(result0.packet[1], result1.packet[1]);
2167 result0.packet[2] = vec_mergel(result0.packet[2], result1.packet[2]);
2168 result0.packet[3] = vec_perm(result0.packet[3], result1.packet[3], p16uc_ELEMENT_3);
2170 vec_add(vec_add(result0.packet[0], result0.packet[2]), vec_add(result0.packet[1], result0.packet[3]));
2171 return *
reinterpret_cast<ScalarBlock<ResScalar, 2>*
>(&result0.packet[0]);
2175EIGEN_ALWAYS_INLINE ScalarBlock<double, 2> predux_real<double, Packet2d>(__vector_quad* acc0, __vector_quad* acc1) {
2176 PacketBlock<Packet2d, 4> result0, result1;
2177 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2178 __builtin_mma_disassemble_acc(&result1.packet, acc1);
2180 vec_add(vec_mergeh(result0.packet[0], result1.packet[0]), vec_mergel(result0.packet[1], result1.packet[1]));
2181 return *
reinterpret_cast<ScalarBlock<double, 2>*
>(&result0.packet[0]);
2185template <
typename LhsPacket,
typename RhsPacket,
bool ConjugateLhs,
bool ConjugateRhs>
2186EIGEN_ALWAYS_INLINE ScalarBlock<std::complex<float>, 2> addComplexResults(PacketBlock<Packet4f, 4>& result0,
2187 const PacketBlock<Packet4f, 4>& result1) {
2188 ScalarBlock<std::complex<float>, 2> cc0;
2189 result0.packet[0] =
reinterpret_cast<Packet4f
>(
2190 vec_mergeh(
reinterpret_cast<Packet2d
>(result0.packet[0]),
reinterpret_cast<Packet2d
>(result1.packet[0])));
2191 result0.packet[2] =
reinterpret_cast<Packet4f
>(
2192 vec_mergel(
reinterpret_cast<Packet2d
>(result0.packet[2]),
reinterpret_cast<Packet2d
>(result1.packet[2])));
2193 result0.packet[0] = vec_add(result0.packet[0], result0.packet[2]);
2194 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
2195 result0.packet[1] =
reinterpret_cast<Packet4f
>(
2196 vec_mergeh(
reinterpret_cast<Packet2d
>(result0.packet[1]),
reinterpret_cast<Packet2d
>(result1.packet[1])));
2197 result0.packet[3] =
reinterpret_cast<Packet4f
>(
2198 vec_mergel(
reinterpret_cast<Packet2d
>(result0.packet[3]),
reinterpret_cast<Packet2d
>(result1.packet[3])));
2199 result0.packet[1] = vec_add(result0.packet[1], result0.packet[3]);
2200 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
2201 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
2202 result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v;
2203 }
else EIGEN_IF_CONSTEXPR (ConjugateRhs) {
2204 result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v;
2206 result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v;
2208 result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]);
2210 EIGEN_IF_CONSTEXPR (ConjugateLhs && (
sizeof(LhsPacket) ==
sizeof(std::complex<float>))) {
2211 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
2214 cc0.scalar[0].real(result0.packet[0][0]);
2215 cc0.scalar[0].imag(result0.packet[0][1]);
2216 cc0.scalar[1].real(result0.packet[0][2]);
2217 cc0.scalar[1].imag(result0.packet[0][3]);
2221template <
typename LhsPacket,
typename RhsPacket,
bool ConjugateLhs,
bool ConjugateRhs>
2222EIGEN_ALWAYS_INLINE ScalarBlock<std::complex<double>, 2> addComplexResults(PacketBlock<Packet2d, 4>&,
2223 const PacketBlock<Packet2d, 4>&) {
2224 ScalarBlock<std::complex<double>, 2> cc0;
2225 EIGEN_UNUSED_VARIABLE(cc0);
2230template <
typename ResScalar,
typename ResPacket,
typename LhsPacket,
typename RhsPacket,
bool ConjugateLhs,
2232EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(__vector_quad* acc0, __vector_quad* acc1) {
2233 PacketBlock<ResPacket, 4> result0, result1;
2234 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2235 __builtin_mma_disassemble_acc(&result1.packet, acc1);
2236 return addComplexResults<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(result0, result1);
2239template <
typename ResScalar,
typename ResPacket>
2240EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_real(__vector_quad* acc0) {
2241 PacketBlock<ResPacket, 4> result0;
2242 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2244 vec_add(vec_mergeh(result0.packet[0], result0.packet[2]), vec_mergel(result0.packet[1], result0.packet[3]));
2245 return *
reinterpret_cast<ScalarBlock<ResScalar, 2>*
>(&result0.packet[0]);
2248template <
typename ResScalar,
typename ResPacket,
typename LhsPacket,
typename RhsPacket,
bool ConjugateLhs,
2250EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(__vector_quad* acc0) {
2251 ScalarBlock<ResScalar, 2> cc0;
2252 PacketBlock<ResPacket, 4> result0;
2253 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2254 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
2255 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
2256 result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v;
2257 result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v;
2258 }
else EIGEN_IF_CONSTEXPR (ConjugateRhs) {
2259 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
2260 result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v;
2262 result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v;
2263 result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v;
2265 result0.packet[0] = vec_add(result0.packet[0], __builtin_vsx_xxpermdi(result0.packet[1], result0.packet[1], 2));
2266 result0.packet[2] = vec_add(result0.packet[2], __builtin_vsx_xxpermdi(result0.packet[3], result0.packet[3], 2));
2268 result0.packet[0] = __builtin_vsx_xxpermdi(result0.packet[0], result0.packet[1], 1);
2269 result0.packet[2] = __builtin_vsx_xxpermdi(result0.packet[2], result0.packet[3], 1);
2271 cc0.scalar[0].real(result0.packet[0][0]);
2272 cc0.scalar[0].imag(result0.packet[0][1]);
2273 cc0.scalar[1].real(result0.packet[2][0]);
2274 cc0.scalar[1].imag(result0.packet[2][1]);
2279template <
typename ResScalar,
typename ResPacket>
2280EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_real(
const ResPacket& a,
const ResPacket& b) {
2281 ScalarBlock<ResScalar, 2> cc0;
2282 cc0.scalar[0] = predux(a);
2283 cc0.scalar[1] = predux(b);
2287template <
typename ResScalar,
typename ResPacket>
2288EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(
const ResPacket& a,
const ResPacket& b) {
2289 return predux_real<ResScalar, ResPacket>(a, b);
2292#define GEMV_UNROLL_ROW(func, N) func(0, N) func(1, N) func(2, N) func(3, N) func(4, N) func(5, N) func(6, N) func(7, N)
2294#define GEMV_UNROLL_ROW_HALF(func, N) func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N)
2296#define GEMV_LOADPACKET_ROW(iter) lhs.template load<LhsPacket, Unaligned>(i + (iter), j)
2299#define GEMV_UNROLL3_ROW(func, N, which) \
2300 func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) func(4, N, which) func(5, N, which) \
2301 func(6, N, which) func(7, N, which)
2303#define GEMV_UNUSED_ROW(N, which) GEMV_UNROLL3_ROW(GEMV_UNUSED_VAR, N, which)
2305#define GEMV_INIT_ROW(iter, N) \
2306 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
2307 __builtin_mma_xxsetaccz(&c##iter); \
2310#define GEMV_LOADPAIR_ROW(iter1, iter2) \
2311 GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_ROW(iter2), GEMV_LOADPACKET_ROW((iter2) + 1));
2313#define GEMV_WORK_ROW(iter, N) \
2314 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
2315 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
2316 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&c##iter, a0, GEMV_LOADPACKET_ROW(iter)); \
2318 __vector_pair b##iter; \
2319 GEMV_LOADPAIR_ROW(iter, iter << 1) \
2320 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&c##iter, b##iter, a0); \
2324#define GEMV_PREDUX2(iter1, iter2, iter3, N) \
2325 EIGEN_IF_CONSTEXPR (N > iter1) { \
2326 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
2327 cc##iter1 = predux_real<ResScalar, ResPacket>(&c##iter2, &c##iter3); \
2329 cc##iter1 = predux_real<ResScalar, ResPacket>(&c##iter1); \
2332 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2335#define GEMV_INIT_ROW(iter, N) \
2336 EIGEN_IF_CONSTEXPR (N > iter) { \
2337 c##iter = pset1<ResPacket>(ResScalar(0)); \
2339 EIGEN_UNUSED_VARIABLE(c##iter); \
2342#define GEMV_WORK_ROW(iter, N) \
2343 EIGEN_IF_CONSTEXPR (N > iter) { \
2344 c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); \
2347#define GEMV_PREDUX2(iter1, iter2, iter3, N) \
2348 EIGEN_IF_CONSTEXPR (N > iter1) { \
2349 cc##iter1 = predux_real<ResScalar, ResPacket>(c##iter2, c##iter3); \
2351 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2355#define GEMV_MULT(iter1, iter2, iter3, N) \
2356 EIGEN_IF_CONSTEXPR (N > iter1) { \
2357 cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), a0); \
2358 cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), a0); \
2361#define GEMV_STORE_ROW(iter1, iter2, iter3, N) \
2362 EIGEN_IF_CONSTEXPR (N > iter1) { \
2363 storeMaddData<ResScalar>(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \
2364 storeMaddData<ResScalar>(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \
2368#define GEMV_PROCESS_ROW(N) \
2369 for (; i < n##N; i += N) { \
2370 GEMV_UNROLL_ROW(GEMV_INIT_ROW, N) \
2372 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \
2373 RhsPacket a0 = rhs2.template load<RhsPacket, Unaligned>(j); \
2374 GEMV_UNROLL_ROW(GEMV_WORK_ROW, N) \
2376 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX2, (N >> 1)) \
2377 for (; j < cols; ++j) { \
2378 RhsScalar a0 = rhs2(j); \
2379 GEMV_UNROLL_ROW_HALF(GEMV_MULT, (N >> 1)) \
2381 GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW, (N >> 1)) \
2384template <
typename LhsScalar,
typename LhsMapper,
typename RhsScalar,
typename RhsMapper,
typename ResScalar>
2385EIGEN_STRONG_INLINE
void gemv_row(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols,
const LhsMapper& alhs,
2386 const RhsMapper& rhs, ResScalar* res, mapper_index_t<LhsMapper> resIncr,
2388 using Index = mapper_index_t<LhsMapper>;
2389 using Traits = gemv_traits<LhsScalar, RhsScalar>;
2391 using LhsPacket =
typename Traits::LhsPacket;
2392 using RhsPacket =
typename Traits::RhsPacket;
2393 using ResPacket =
typename Traits::ResPacket;
2397 LhsMapper lhs(alhs);
2398 typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
2400 eigen_internal_assert(rhs.stride() == 1);
2401 conj_helper<LhsScalar, RhsScalar, false, false> cj;
2402 conj_helper<LhsPacket, RhsPacket, false, false> pcj;
2406#ifndef GCC_ONE_VECTORPAIR_BUG
2407 const Index n8 = lhs.stride() *
sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7);
2408 const Index n4 = rows - 3;
2409 const Index n2 = rows - 1;
2416 ResPacketSize = Traits::ResPacketSize,
2417 LhsPacketSize = Traits::LhsPacketSize,
2418 RhsPacketSize = Traits::RhsPacketSize,
2423 __vector_quad c0, c1, c2, c3, c4, c5, c6, c7;
2424 GEMV_UNUSED_ROW(8, c)
2426 ResPacket c0, c1, c2, c3, c4, c5, c6, c7;
2428#ifndef GCC_ONE_VECTORPAIR_BUG
2429 ScalarBlock<ResScalar, 2> cc0, cc1, cc2, cc3;
2434 for (; i < rows; ++i) {
2435 ResPacket d0 = pset1<ResPacket>(ResScalar(0));
2437 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) {
2438 RhsPacket b0 = rhs2.template load<RhsPacket, Unaligned>(j);
2440 d0 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 0, j), b0, d0);
2442 ResScalar dd0 = predux(d0);
2443 for (; j < cols; ++j) {
2444 dd0 += cj.pmul(lhs(i, j), rhs2(j));
2446 res[i * resIncr] += alpha * dd0;
2450#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(Scalar) \
2451 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2452 struct general_matrix_vector_product<Index, Scalar, LhsMapper, ColMajor, ConjugateLhs, Scalar, RhsMapper, \
2453 ConjugateRhs, Version> { \
2454 using ResScalar = typename ScalarBinaryOpTraits<Scalar, Scalar>::ReturnType; \
2456 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2457 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2458 ResScalar alpha) { \
2459 if (numext::is_exactly_zero(alpha)) return; \
2460 gemv_col<Scalar, LhsMapper, Scalar, RhsMapper, ResScalar>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2464#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(Scalar) \
2465 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2466 struct general_matrix_vector_product<Index, Scalar, LhsMapper, RowMajor, ConjugateLhs, Scalar, RhsMapper, \
2467 ConjugateRhs, Version> { \
2468 using ResScalar = typename ScalarBinaryOpTraits<Scalar, Scalar>::ReturnType; \
2470 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2471 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2472 ResScalar alpha) { \
2473 if (numext::is_exactly_zero(alpha)) return; \
2474 gemv_row<Scalar, LhsMapper, Scalar, RhsMapper, ResScalar>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2478EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(
float)
2479EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(
double)
2480EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(
float)
2481EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(
double)
2484#define gemv_bf16_col gemvMMA_bfloat16_col
2485#define gemv_bf16_row gemvMMA_bfloat16_row
2487#define gemv_bf16_col gemv_bfloat16_col
2488#define gemv_bf16_row gemv_bfloat16_row
2491#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16() \
2492 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2493 struct general_matrix_vector_product<Index, bfloat16, LhsMapper, ColMajor, ConjugateLhs, bfloat16, RhsMapper, \
2494 ConjugateRhs, Version> { \
2495 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2496 const RhsMapper& rhs, bfloat16* res, Index resIncr, \
2498 if (numext::is_exactly_zero(alpha)) return; \
2499 gemv_bf16_col<LhsMapper, RhsMapper>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2503#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() \
2504 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2505 struct general_matrix_vector_product<Index, bfloat16, LhsMapper, RowMajor, ConjugateLhs, bfloat16, RhsMapper, \
2506 ConjugateRhs, Version> { \
2507 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2508 const RhsMapper& rhs, bfloat16* res, Index resIncr, \
2510 if (numext::is_exactly_zero(alpha)) return; \
2511 gemv_bf16_row<LhsMapper, RhsMapper>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2515EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16()
2516EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16()
2518template <typename ResScalar, typename PResPacket, typename ResPacket, typename LhsPacket, typename RhsPacket>
2519EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(PResPacket& a0, PResPacket& b0, const ResPacket& a1,
2520 const ResPacket& b1) {
2521 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
2525 return predux_complex<ResScalar, PResPacket>(a0, b0);
2528#define GEMV_LOADPACKET_ROW_COMPLEX(iter) loadLhsPacket<Scalar, LhsScalar, LhsMapper, PLhsPacket>(lhs, i + (iter), j)
2530#define GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter) convertReal(GEMV_LOADPACKET_ROW_COMPLEX(iter))
2532#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(which, N) \
2534 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \
2535 const RhsScalar& b1 = rhs2(j); \
2536 RhsScalar* b = const_cast<RhsScalar*>(&b1); \
2537 GEMV_UNROLL_ROW(which, N) \
2540#define GEMV_PROCESS_END_ROW_COMPLEX(N) \
2541 for (; j < cols; ++j) { \
2542 RhsScalar b0 = rhs2(j); \
2543 GEMV_UNROLL_ROW_HALF(GEMV_MULT_COMPLEX, (N >> 1)) \
2545 GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW_COMPLEX, (N >> 1))
2548#define GEMV_INIT_ROW_COMPLEX_MMA(iter, N) \
2549 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
2550 __builtin_mma_xxsetaccz(&e0##iter); \
2553#define GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter1, iter2) \
2554 GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter2), GEMV_LOADPACKET_ROW_COMPLEX_DATA((iter2) + 1));
2556#define GEMV_WORK_ROW_COMPLEX_MMA(iter, N) \
2557 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
2558 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
2559 PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \
2560 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
2561 ConjugateLhs, ConjugateRhs, RowMajor>(a##iter, b, &e0##iter); \
2563 __vector_pair a##iter; \
2564 GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter, iter << 1) \
2565 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
2566 ConjugateLhs, ConjugateRhs, RowMajor>(a##iter, b, &e0##iter); \
2570#define GEMV_PREDUX4_COMPLEX_MMA(iter1, iter2, iter3, N) \
2571 EIGEN_IF_CONSTEXPR (N > iter1) { \
2572 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
2573 cc##iter1 = predux_complex<ResScalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>( \
2574 &e0##iter2, &e0##iter3); \
2577 predux_complex<ResScalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(&e0##iter1); \
2580 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2583#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \
2584 GEMV_UNROLL_ROW(GEMV_INIT_ROW_COMPLEX_MMA, N) \
2585 GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX_MMA, N)
2587#define GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N) \
2588 for (; i < n##N; i += N) { \
2589 GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \
2590 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_MMA, (N >> 1)) \
2591 GEMV_PROCESS_END_ROW_COMPLEX(N); \
2595#define GEMV_WORK_ROW_COMPLEX(iter, N) \
2596 EIGEN_IF_CONSTEXPR (N > iter) { \
2597 PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \
2598 gemv_mult_complex<ScalarPacket, PLhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
2599 ConjugateRhs, RowMajor>(a##iter, b, c0##iter, c1##iter); \
2602#define GEMV_PREDUX4_COMPLEX(iter1, iter2, iter3, N) \
2603 EIGEN_IF_CONSTEXPR (N > iter1) { \
2604 cc##iter1 = predux_complex<ResScalar, PResPacket, ResPacket, LhsPacket, RhsPacket>(c0##iter2, c0##iter3, \
2605 c1##iter2, c1##iter3); \
2607 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2610#define GEMV_MULT_COMPLEX(iter1, iter2, iter3, N) \
2611 EIGEN_IF_CONSTEXPR (N > iter1) { \
2612 cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), b0); \
2613 cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), b0); \
2616#define GEMV_STORE_ROW_COMPLEX(iter1, iter2, iter3, N) \
2617 EIGEN_IF_CONSTEXPR (N > iter1) { \
2618 storeMaddData<ResScalar>(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \
2619 storeMaddData<ResScalar>(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \
2622#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \
2623 GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX, N) \
2624 GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX, N)
2628#define GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \
2629 for (; i < n##N; i += N) { \
2630 GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \
2631 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX, (N >> 1)) \
2632 GEMV_PROCESS_END_ROW_COMPLEX(N); \
2635#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \
2636 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { \
2637 c0##iter = padd(c0##iter, c1##iter); \
2639 dd0 = predux(c0##iter);
2642#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N)
2644#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N)
2646#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter)
2651#define GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter) lhs.template load<LhsPacket, LhsAlignment>(i + (iter), j)
2653#define GEMV_INIT_COMPLEX_OLD(iter, N) \
2654 EIGEN_UNUSED_VARIABLE(c0##iter); \
2655 EIGEN_IF_CONSTEXPR (N > iter) { \
2656 c1##iter = pset_zero<ResPacket>(); \
2658 EIGEN_UNUSED_VARIABLE(c1##iter); \
2661#define GEMV_WORK_ROW_COMPLEX_OLD(iter, N) \
2662 EIGEN_IF_CONSTEXPR (N > iter) { \
2663 LhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter); \
2664 c1##iter = pcj.pmadd(a##iter, b0, c1##iter); \
2667#define GEMV_PREDUX4_COMPLEX_OLD(iter1, iter2, iter3, N) \
2668 EIGEN_IF_CONSTEXPR (N > iter1) { \
2669 cc##iter1.scalar[0] = predux(c1##iter2); \
2670 cc##iter1.scalar[1] = predux(c1##iter3); \
2672 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2675#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \
2676 GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX_OLD, N) \
2678 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \
2679 RhsPacket b0 = rhs2.template load<RhsPacket, Unaligned>(j); \
2680 GEMV_UNROLL_ROW(GEMV_WORK_ROW_COMPLEX_OLD, N) \
2683#define GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \
2684 for (; i < n##N; i += N) { \
2685 GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \
2686 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_OLD, (N >> 1)) \
2687 GEMV_PROCESS_END_ROW_COMPLEX(N) \
2690#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) dd0 = predux(c1##iter);
2693#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW 1
2695#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW (sizeof(Scalar) == sizeof(float)) || GEMV_IS_COMPLEX_COMPLEX
2698#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) \
2699 if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \
2700 GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \
2702 GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \
2705#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) \
2706 if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \
2707 GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \
2709 GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \
2712#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) \
2713 if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \
2714 GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \
2716 GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) \
2721#define GEMV_PROCESS_ROW_COMPLEX(N) GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N)
2723#define GEMV_PROCESS_ROW_COMPLEX(N) GEMV_PROCESS_ROW_COMPLEX_ONE(N)
2726template <
typename Scalar,
typename LhsScalar,
typename LhsMapper,
bool ConjugateLhs,
bool LhsIsReal,
2727 typename RhsScalar,
typename RhsMapper,
bool ConjugateRhs,
bool RhsIsReal,
typename ResScalar>
2728EIGEN_STRONG_INLINE
void gemv_complex_row(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols,
2729 const LhsMapper& alhs,
const RhsMapper& rhs, ResScalar* res,
2730 mapper_index_t<LhsMapper> resIncr, ResScalar alpha) {
2731 using Index = mapper_index_t<LhsMapper>;
2733 using Traits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketFull, false>;
2735 using LhsPacket =
typename Traits::LhsPacket;
2736 using RhsPacket =
typename Traits::RhsPacket;
2737 using ResPacket =
typename Traits::ResPacket;
2739 using ScalarPacket =
typename packet_traits<Scalar>::type;
2740 using PLhsPacket =
typename packet_traits<LhsScalar>::type;
2741 using PResPacket =
typename packet_traits<ResScalar>::type;
2742 using PTraits = gemv_traits<ResPacket, ResPacket>;
2746 LhsMapper lhs(alhs);
2747 typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
2749 eigen_internal_assert(rhs.stride() == 1);
2750 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
2752 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs> pcj;
2757#ifndef GCC_ONE_VECTORPAIR_BUG
2758 const Index n8 = lhs.stride() *
sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7);
2759 const Index n4 = rows - 3;
2760 const Index n2 = rows - 1;
2767 ResPacketSize = PTraits::ResPacketSize,
2768 LhsPacketSize = PTraits::LhsPacketSize,
2769 RhsPacketSize = PTraits::RhsPacketSize,
2773 PResPacket c00, c01, c02, c03, c04, c05, c06, c07;
2774 ResPacket c10, c11, c12, c13, c14, c15, c16, c17;
2776 __vector_quad e00, e01, e02, e03, e04, e05, e06, e07;
2777 GEMV_UNUSED_ROW(8, e0)
2778 GEMV_UNUSED_EXTRA(1, c0)
2779 GEMV_UNUSED_EXTRA(1, c1)
2782#ifndef GCC_ONE_VECTORPAIR_BUG
2783 ScalarBlock<ResScalar, 2> cc0, cc1, cc2, cc3;
2785 EIGEN_IF_CONSTEXPR (!GEMV_IS_COMPLEX_COMPLEX)
2788 GEMV_PROCESS_ROW_COMPLEX(8)
2790 GEMV_PROCESS_ROW_COMPLEX(4)
2791 GEMV_PROCESS_ROW_COMPLEX(2)
2793 for (; i < rows; ++i) {
2794 GEMV_PROCESS_ROW_COMPLEX_SINGLE(1)
2795 GEMV_PROCESS_ROW_COMPLEX_PREDUX(0)
2796 for (; j < cols; ++j) {
2797 dd0 += cj.pmul(lhs(i, j), rhs2(j));
2799 res[i * resIncr] += alpha * dd0;
2803#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(Scalar, LhsScalar, RhsScalar) \
2804 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2805 struct general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, \
2806 ConjugateRhs, Version> { \
2807 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType; \
2809 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2810 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2811 ResScalar alpha) { \
2812 if (numext::is_exactly_zero(alpha)) return; \
2813 gemv_complex_col<Scalar, LhsScalar, LhsMapper, ConjugateLhs, sizeof(Scalar) == sizeof(LhsScalar), RhsScalar, \
2814 RhsMapper, ConjugateRhs, sizeof(Scalar) == sizeof(RhsScalar), ResScalar>(rows, cols, lhs, rhs, \
2815 res, resIncr, alpha); \
2819#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(Scalar, LhsScalar, RhsScalar) \
2820 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2821 struct general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper, \
2822 ConjugateRhs, Version> { \
2823 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType; \
2825 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2826 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2827 ResScalar alpha) { \
2828 if (numext::is_exactly_zero(alpha)) return; \
2829 gemv_complex_row<Scalar, LhsScalar, LhsMapper, ConjugateLhs, sizeof(Scalar) == sizeof(LhsScalar), RhsScalar, \
2830 RhsMapper, ConjugateRhs, sizeof(Scalar) == sizeof(RhsScalar), ResScalar>(rows, cols, lhs, rhs, \
2831 res, resIncr, alpha); \
2835EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(
float,
float, std::complex<float>)
2836EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(
float, std::complex<float>,
float)
2837EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(
float, std::complex<float>, std::complex<float>)
2838EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(
double,
double, std::complex<double>)
2839EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(
double, std::complex<double>,
double)
2840EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(
double, std::complex<double>, std::complex<double>)
2841EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(
float,
float, std::complex<float>)
2842EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(
float, std::complex<float>,
float)
2843EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(
float, std::complex<float>, std::complex<float>)
2844EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(
double,
double, std::complex<double>)
2845EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(
double, std::complex<double>,
double)
2846EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(
double, std::complex<double>, std::complex<double>)
@ Unaligned
Definition Constants.h:236
@ ColMajor
Definition Constants.h:319