Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
MatrixVectorProduct.inc
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H
12#define EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H
13
14// IWYU pragma: private
15#include "../../InternalHeaderCheck.h"
16
17#if defined(__MMA__) && !EIGEN_ALTIVEC_DISABLE_MMA
18#if EIGEN_COMP_LLVM || (__GNUC__ > 10 || __GNUC_MINOR__ >= 3)
19#define USE_GEMV_MMA
20#endif
21
22#if !EIGEN_COMP_LLVM && (__GNUC__ < 11)
23// Only allow one vector_pair in buggy gcc - gcc 10.x has a bug
24#define GCC_ONE_VECTORPAIR_BUG
25#endif
26#endif
27
28// #define USE_SLOWER_GEMV_MMA // MMA is currently not as fast as VSX in complex double GEMV (revisit when gcc is
29// improved)
30
31// #define EIGEN_POWER_USE_GEMV_PREFETCH
32#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
33#define EIGEN_POWER_GEMV_PREFETCH(p) prefetch(p)
34#else
35#define EIGEN_POWER_GEMV_PREFETCH(p)
36#endif
37
38#ifdef __has_builtin
39#if !__has_builtin(__builtin_vsx_assemble_pair)
40#define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair
41#endif
42#if !__has_builtin(__builtin_vsx_disassemble_pair)
43#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair
44#endif
45#endif
46
47#if EIGEN_COMP_LLVM
48#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
49 __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1)
50#else
51#if (__GNUC__ <= 10)
52#if (__GNUC_MINOR__ > 3)
53#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
54 __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1)
55#else
56#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
57 __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2)
58#endif
59#else
60#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \
61 __builtin_vsx_build_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2)
62#endif
63#endif
64
65#define GEMV_IS_COMPLEX_COMPLEX ((sizeof(LhsPacket) == 16) && (sizeof(RhsPacket) == 16))
66#define GEMV_IS_FLOAT (ResPacketSize == (16 / sizeof(float)))
67#define GEMV_IS_SCALAR (sizeof(ResPacket) != 16)
68#define GEMV_IS_COMPLEX_FLOAT (ResPacketSize == (16 / sizeof(std::complex<float>)))
69
71template <typename ResPacket, typename ResScalar>
72EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, const ResPacket& palpha, const ResPacket& data) {
73 pstoreu(res, pmadd(data, palpha, ploadu<ResPacket>(res)));
74}
75
76template <typename ResScalar>
77EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScalar& data) {
78 *res += (alpha * data);
79}
80
81#define GEMV_UNROLL(func, N) func(0, N) func(1, N) func(2, N) func(3, N) func(4, N) func(5, N) func(6, N) func(7, N)
82
83#define GEMV_UNROLL_HALF(func, N) func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N)
84
85#define GEMV_GETN(N) (((N) * ResPacketSize) >> 2)
86
87#define GEMV_LOADPACKET_COL(iter) lhs.template load<LhsPacket, LhsAlignment>(i + ((iter) * LhsPacketSize), j)
88
89#ifdef USE_GEMV_MMA
90#define GEMV_UNROLL3(func, N, which) \
91 func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) func(4, N, which) func(5, N, which) \
92 func(6, N, which) func(7, N, which)
93
94#define GEMV_UNUSED_VAR(iter, N, which) \
95 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= iter) { \
96 EIGEN_UNUSED_VARIABLE(which##iter); \
97 }
98
99#define GEMV_UNUSED_EXTRA_VAR(iter, N, which) \
100 EIGEN_IF_CONSTEXPR (N <= iter) { \
101 EIGEN_UNUSED_VARIABLE(which##iter); \
102 }
103
104#define GEMV_UNUSED_EXTRA(N, which) GEMV_UNROLL3(GEMV_UNUSED_EXTRA_VAR, N, which)
105
106#define GEMV_UNUSED(N, which) GEMV_UNROLL3(GEMV_UNUSED_VAR, N, which)
107
108#define GEMV_INIT_MMA(iter, N) \
109 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
110 __builtin_mma_xxsetaccz(&e##iter); \
111 }
112
113#if EIGEN_COMP_LLVM
114#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \
115 GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_COL(iter2), GEMV_LOADPACKET_COL((iter2) + 1));
116#else
117#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \
118 const LhsScalar& src##iter1 = lhs(i + ((iter1 * 32) / sizeof(LhsScalar)), j); \
119 b##iter1 = *reinterpret_cast<__vector_pair*>(const_cast<LhsScalar*>(&src##iter1));
120#endif
121
122#define GEMV_LOAD1A_COL_MMA(iter, N) \
123 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
124 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
125 g##iter = GEMV_LOADPACKET_COL(iter); \
126 EIGEN_UNUSED_VARIABLE(b##iter); \
127 } else { \
128 GEMV_LOADPAIR_COL_MMA(iter, iter << 1) \
129 EIGEN_UNUSED_VARIABLE(g##iter); \
130 } \
131 } else { \
132 EIGEN_UNUSED_VARIABLE(b##iter); \
133 EIGEN_UNUSED_VARIABLE(g##iter); \
134 }
135
136#define GEMV_WORK1A_COL_MMA(iter, N) \
137 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
138 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
139 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter, a0, g##iter); \
140 } else { \
141 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter, b##iter, a0); \
142 } \
143 }
144
145#define GEMV_LOAD1B_COL_MMA(iter1, iter2, iter3, N) \
146 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
147 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
148 GEMV_LOADPAIR_COL_MMA(iter2, iter2) \
149 EIGEN_UNUSED_VARIABLE(b##iter3); \
150 } else { \
151 GEMV_LOADPAIR_COL_MMA(iter2, iter2 << 1) \
152 GEMV_LOADPAIR_COL_MMA(iter3, iter3 << 1) \
153 } \
154 } else { \
155 EIGEN_UNUSED_VARIABLE(b##iter2); \
156 EIGEN_UNUSED_VARIABLE(b##iter3); \
157 } \
158 EIGEN_UNUSED_VARIABLE(g##iter2); \
159 EIGEN_UNUSED_VARIABLE(g##iter3);
160
161#define GEMV_WORK1B_COL_MMA(iter1, iter2, iter3, N) \
162 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
163 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
164 LhsPacket h[2]; \
165 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(h), &b##iter2); \
166 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter2, a0, h[0]); \
167 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter3, a0, h[1]); \
168 } else { \
169 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter2, b##iter2, a0); \
170 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&e##iter3, b##iter3, a0); \
171 } \
172 }
173
174#if EIGEN_COMP_LLVM
175#define GEMV_LOAD_COL_MMA(N) \
176 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > 1) { \
177 GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) \
178 } else { \
179 GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \
180 }
181
182#define GEMV_WORK_COL_MMA(N) \
183 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > 1) { \
184 GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) \
185 } else { \
186 GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \
187 }
188#else
189#define GEMV_LOAD_COL_MMA(N) GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N)
190
191#define GEMV_WORK_COL_MMA(N) GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N)
192#endif
193
194#define GEMV_DISASSEMBLE_MMA(iter, N) \
195 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
196 __builtin_mma_disassemble_acc(&result##iter.packet, &e##iter); \
197 EIGEN_IF_CONSTEXPR (!GEMV_IS_FLOAT) { \
198 result##iter.packet[0][1] = result##iter.packet[1][0]; \
199 result##iter.packet[2][1] = result##iter.packet[3][0]; \
200 } \
201 }
202
203#define GEMV_LOADPAIR2_COL_MMA(iter1, iter2) \
204 b##iter1 = *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize));
205
206#define GEMV_LOAD2_COL_MMA(iter1, iter2, iter3, N) \
207 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
208 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
209 GEMV_LOADPAIR2_COL_MMA(iter2, iter2); \
210 EIGEN_UNUSED_VARIABLE(b##iter3); \
211 } else { \
212 GEMV_LOADPAIR2_COL_MMA(iter2, iter2 << 1); \
213 GEMV_LOADPAIR2_COL_MMA(iter3, iter3 << 1); \
214 } \
215 } else { \
216 EIGEN_UNUSED_VARIABLE(b##iter2); \
217 EIGEN_UNUSED_VARIABLE(b##iter3); \
218 }
219
220#if EIGEN_COMP_LLVM
221#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \
222 ResPacket f##iter2[2]; \
223 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(f##iter2), &b##iter2); \
224 f##iter2[0] = pmadd(result##iter2.packet[0], palpha, f##iter2[0]); \
225 f##iter2[1] = pmadd(result##iter3.packet[(iter2 == iter3) ? 2 : 0], palpha, f##iter2[1]); \
226 GEMV_BUILDPAIR_MMA(b##iter2, f##iter2[0], f##iter2[1]);
227#else
228#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \
229 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
230 __asm__("xvmaddasp %0,%x1,%x3\n\txvmaddasp %L0,%x2,%x3" \
231 : "+&d"(b##iter2) \
232 : "wa"(result##iter3.packet[0]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \
233 } else { \
234 __asm__("xvmaddadp %0,%x1,%x3\n\txvmaddadp %L0,%x2,%x3" \
235 : "+&d"(b##iter2) \
236 : "wa"(result##iter2.packet[2]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \
237 }
238#endif
239
240#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \
241 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
242 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
243 GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); \
244 } else { \
245 GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \
246 GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \
247 } \
248 }
249
250#define GEMV_STOREPAIR2_COL_MMA(iter1, iter2) \
251 *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)) = b##iter1;
252
253#define GEMV_STORE_COL_MMA(iter, N) \
254 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
255 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
256 storeMaddData<ResPacket, ResScalar>(res + i + (iter * ResPacketSize), palpha, result##iter.packet[0]); \
257 } else { \
258 GEMV_LOADPAIR2_COL_MMA(iter, iter << 1) \
259 GEMV_WORKPAIR2_COL_MMA(iter, iter, iter << 1) \
260 GEMV_STOREPAIR2_COL_MMA(iter, iter << 1) \
261 } \
262 }
263
264#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \
265 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter1) { \
266 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
267 GEMV_STOREPAIR2_COL_MMA(iter2, iter2); \
268 } else { \
269 GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \
270 GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \
271 } \
272 }
273
274#define GEMV_PROCESS_COL_ONE_MMA(N) \
275 GEMV_UNROLL(GEMV_INIT_MMA, N) \
276 Index j = j2; \
277 __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \
278 do { \
279 LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \
280 RhsPacket a0 = pset1<RhsPacket>(rhs2(j, 0)); \
281 GEMV_UNROLL(GEMV_PREFETCH, N) \
282 GEMV_LOAD_COL_MMA(N) \
283 GEMV_WORK_COL_MMA(N) \
284 } while (++j < jend); \
285 GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \
286 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= 1) { \
287 GEMV_UNROLL(GEMV_STORE_COL_MMA, N) \
288 } else { \
289 GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) \
290 GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1)) \
291 } \
292 i += (ResPacketSize * N);
293#endif
294
295#define GEMV_INIT(iter, N) \
296 EIGEN_IF_CONSTEXPR (N > iter) { \
297 c##iter = pset1<ResPacket>(ResScalar(0)); \
298 } else { \
299 EIGEN_UNUSED_VARIABLE(c##iter); \
300 }
301
302#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
303#define GEMV_PREFETCH(iter, N) \
304 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \
305 lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \
306 }
307#else
308#define GEMV_PREFETCH(iter, N)
309#endif
310
311#define GEMV_WORK_COL(iter, N) \
312 EIGEN_IF_CONSTEXPR (N > iter) { \
313 c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); \
314 }
315
316#define GEMV_STORE_COL(iter, N) \
317 EIGEN_IF_CONSTEXPR (N > iter) { \
318 pstoreu(res + i + (iter * ResPacketSize), \
319 pmadd(c##iter, palpha, ploadu<ResPacket>(res + i + (iter * ResPacketSize)))); \
320 }
321
323#define GEMV_PROCESS_COL_ONE(N) \
324 GEMV_UNROLL(GEMV_INIT, N) \
325 Index j = j2; \
326 do { \
327 RhsPacket a0 = pset1<RhsPacket>(rhs2(j, 0)); \
328 GEMV_UNROLL(GEMV_PREFETCH, N) \
329 GEMV_UNROLL(GEMV_WORK_COL, N) \
330 } while (++j < jend); \
331 GEMV_UNROLL(GEMV_STORE_COL, N) \
332 i += (ResPacketSize * N);
333
334#ifdef USE_GEMV_MMA
335#define GEMV_PROCESS_COL(N) GEMV_PROCESS_COL_ONE_MMA(N)
336#else
337#define GEMV_PROCESS_COL(N) GEMV_PROCESS_COL_ONE(N)
338#endif
339
341#ifdef USE_GEMV_MMA
342template <typename LhsPacket, typename RhsPacket, bool accumulate>
343EIGEN_ALWAYS_INLINE void pger_vecMMA_acc(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) {
344 if (accumulate) {
345 __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
346 } else {
347 __builtin_mma_xvf32ger(acc, (__vector unsigned char)a, (__vector unsigned char)b);
348 }
349}
350
352template <typename LhsPacket, typename RhsPacket, bool accumulate>
353EIGEN_ALWAYS_INLINE void pger_vecMMA_acc(__vector_quad* acc, __vector_pair& a, const LhsPacket& b) {
354 if (accumulate) {
355 __builtin_mma_xvf64gerpp(acc, a, (__vector unsigned char)b);
356 } else {
357 __builtin_mma_xvf64ger(acc, a, (__vector unsigned char)b);
358 }
359}
360#endif
361
362template <typename LhsScalar, typename LhsMapper, typename RhsScalar, typename RhsMapper, typename ResScalar>
363EIGEN_STRONG_INLINE void gemv_col(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols, const LhsMapper& alhs,
364 const RhsMapper& rhs, ResScalar* res, mapper_index_t<LhsMapper> resIncr,
365 ResScalar alpha) {
366 using Index = mapper_index_t<LhsMapper>;
367 using Traits = gemv_traits<LhsScalar, RhsScalar>;
368
369 using LhsPacket = typename Traits::LhsPacket;
370 using RhsPacket = typename Traits::RhsPacket;
371 using ResPacket = typename Traits::ResPacket;
372
373 EIGEN_UNUSED_VARIABLE(resIncr);
374 eigen_internal_assert(resIncr == 1);
375
376 // The following copy tells the compiler that lhs's attributes are not modified outside this function
377 // This helps GCC to generate proper code.
378 LhsMapper lhs(alhs);
379 RhsMapper rhs2(rhs);
380
381 conj_helper<LhsScalar, RhsScalar, false, false> cj;
382 conj_helper<LhsPacket, RhsPacket, false, false> pcj;
383
384 const Index lhsStride = lhs.stride();
385 // LhsAlignment stays Unaligned; enabling aligned reads would require
386 // propagating the Mapper's Alignment through the run() template.
387 enum {
388 LhsAlignment = Unaligned,
389 ResPacketSize = Traits::ResPacketSize,
390 LhsPacketSize = Traits::LhsPacketSize,
391 RhsPacketSize = Traits::RhsPacketSize,
392 };
393
394#ifndef GCC_ONE_VECTORPAIR_BUG
395 const Index n8 = rows - 8 * ResPacketSize + 1;
396 const Index n4 = rows - 4 * ResPacketSize + 1;
397 const Index n2 = rows - 2 * ResPacketSize + 1;
398#endif
399 const Index n1 = rows - 1 * ResPacketSize + 1;
400#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
401 const Index prefetch_dist = 64 * LhsPacketSize;
402#endif
403
404 // TODO: improve the following heuristic:
405 const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(LhsScalar) < 16000 ? 16 : 8);
406 ResPacket palpha = pset1<ResPacket>(alpha);
407
408 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
409 Index jend = numext::mini(j2 + block_cols, cols);
410 Index i = 0;
411 ResPacket c0, c1, c2, c3, c4, c5, c6, c7;
412#ifdef USE_GEMV_MMA
413 __vector_quad e0, e1, e2, e3, e4, e5, e6, e7;
414 PacketBlock<ResPacket, 4> result0, result1, result2, result3, result4, result5, result6, result7;
415 GEMV_UNUSED(8, e)
416 GEMV_UNUSED(8, result)
417 GEMV_UNUSED_EXTRA(1, c)
418#endif
419#ifndef GCC_ONE_VECTORPAIR_BUG
420 while (i < n8) {
421 GEMV_PROCESS_COL(8)
422 }
423 if (i < n4) {
424 GEMV_PROCESS_COL(4)
425 }
426 if (i < n2) {
427 GEMV_PROCESS_COL(2)
428 }
429 if (i < n1)
430#else
431 while (i < n1)
432#endif
433 {
434 GEMV_PROCESS_COL_ONE(1)
435 }
436 for (; i < rows; ++i) {
437 ResScalar d0(0);
438 Index j = j2;
439 do {
440 d0 += cj.pmul(lhs(i, j), rhs2(j, 0));
441 } while (++j < jend);
442 res[i] += alpha * d0;
443 }
444 }
445}
446
447template <bool extraRows>
448EIGEN_ALWAYS_INLINE void outputVecCol(const Packet4f& acc, float* result, const Packet4f& pAlpha, Index extra_rows) {
449 Packet4f d0 = ploadu<Packet4f>(result);
450 d0 = pmadd(acc, pAlpha, d0);
451 EIGEN_IF_CONSTEXPR (extraRows) {
452 pstoreu_partial(result, d0, extra_rows);
453 } else {
454 pstoreu(result, d0);
455 }
456}
457
458template <Index num_acc, bool extraRows, Index size>
459EIGEN_ALWAYS_INLINE void outputVecColResults(Packet4f (&acc)[num_acc][size], float* result, const Packet4f& pAlpha,
460 Index extra_rows) {
461 constexpr Index real_acc = (num_acc - (extraRows ? 1 : 0));
462 for (Index k = 0; k < real_acc; k++) {
463 outputVecCol<false>(acc[k][0], result + k * 4, pAlpha, extra_rows);
464 }
465 EIGEN_IF_CONSTEXPR (extraRows) {
466 outputVecCol<true>(acc[real_acc][0], result + real_acc * 4, pAlpha, extra_rows);
467 }
468}
469
470static Packet16uc p16uc_MERGE16_32_V1 = {0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17};
471static Packet16uc p16uc_MERGE16_32_V2 = {2, 3, 18, 19, 2, 3, 18, 19, 2, 3, 18, 19, 2, 3, 18, 19};
472
473template <Index num_acc, typename LhsMapper, bool zero>
474EIGEN_ALWAYS_INLINE void loadVecLoopVSX(Index k, LhsMapper& lhs, Packet4f (&a0)[num_acc][2]) {
475 const auto row = convert_index<mapper_index_t<LhsMapper>>(k * 4);
476 Packet8bf c0 = lhs.template loadPacket<Packet8bf>(row, 0);
477 Packet8bf b1;
478 EIGEN_IF_CONSTEXPR (!zero) {
479 b1 = lhs.template loadPacket<Packet8bf>(row, 1);
480
481 a0[k + 0][1] = oneConvertBF16Hi(b1.m_val);
482 }
483 a0[k + 0][0] = oneConvertBF16Hi(c0.m_val);
484
485 if (num_acc > (k + 1)) {
486 a0[k + 1][0] = oneConvertBF16Lo(c0.m_val);
487 EIGEN_IF_CONSTEXPR (!zero) {
488 a0[k + 1][1] = oneConvertBF16Lo(b1.m_val);
489 }
490 }
491}
492
493template <Index num_acc, bool zero>
494EIGEN_ALWAYS_INLINE void multVecVSX(Packet4f (&acc)[num_acc][2], Packet4f (&a0)[num_acc][2], Packet4f (&b0)[2]) {
495 for (Index k = 0; k < num_acc; k++) {
496 for (Index i = 0; i < (zero ? 1 : 2); i++) {
497 acc[k][i] = pmadd(b0[i], a0[k][i], acc[k][i]);
498 }
499 }
500}
501
502template <typename RhsMapper, bool linear>
503struct loadColData_impl {
504 // linear == false
505 template <typename Index>
506 static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) {
507 const Index n = unpacket_traits<Packet8bf>::size;
508 EIGEN_ALIGN16 bfloat16 to[n];
509 LOAD_STORE_UNROLL_16
510 for (Index i = 0; i < n; i++) {
511 to[i] = rhs(j + i, 0);
512 }
513 return pload<Packet8bf>(to);
514 }
515};
516
517template <typename RhsMapper>
518struct loadColData_impl<RhsMapper, true> {
519 // linear == true
520 template <typename Index>
521 static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) {
522 return rhs.template loadPacket<Packet8bf>(j + 0, 0);
523 }
524};
525
526template <typename RhsMapper, bool linear, typename Index>
527EIGEN_ALWAYS_INLINE Packet8bf loadColData(RhsMapper& rhs, Index j) {
528 return loadColData_impl<RhsMapper, linear>::run(rhs, j);
529}
530
531template <Index num_acc, typename LhsMapper, typename RhsMapper, bool zero, bool linear>
532EIGEN_ALWAYS_INLINE void vecColLoopVSX(Index j, LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2]) {
533 Packet4f a0[num_acc][2], b0[2];
534 const auto col = convert_index<mapper_index_t<LhsMapper>>(j);
535 Packet8bf b2 = loadColData<RhsMapper, linear>(rhs, col);
536
537 b0[0] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V1);
538 EIGEN_IF_CONSTEXPR (!zero) {
539 b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2);
540 }
541
542 using LhsSubMapper = typename LhsMapper::SubMapper;
543
544 LhsSubMapper lhs2 = lhs.getSubMapper(0, col);
545 for (Index k = 0; k < num_acc; k += 2) {
546 loadVecLoopVSX<num_acc, LhsSubMapper, zero>(k, lhs2, a0);
547 }
548
549 multVecVSX<num_acc, zero>(acc, a0, b0);
550}
551
552template <Index num_acc>
553EIGEN_ALWAYS_INLINE void addResultsVSX(Packet4f (&acc)[num_acc][2]) {
554 for (Index i = 0; i < num_acc; i++) {
555 acc[i][0] = acc[i][0] + acc[i][1];
556 }
557}
558
559// Uses 2X the accumulators or 4X the number of VSX registers
560#define MAX_BFLOAT16_VEC_ACC_VSX 8
561
562template <const Index num_acc, typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
563void colVSXVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f& pAlpha,
564 float* result) {
565 constexpr Index step = (num_acc * 4);
566 const Index extra_rows = (extraRows) ? (rows & 3) : 0;
567 constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC_VSX);
568
569 do {
570 Packet4f acc[num_acc][2];
571
572 zeroAccumulators<num_acc, 2>(acc);
573
574 using LhsSubMapper = typename LhsMapper::SubMapper;
575
576 LhsSubMapper lhs2 = lhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(row), 0);
577 for (Index j = 0; j + 2 <= cend; j += 2) {
578 vecColLoopVSX<num_acc, LhsSubMapper, RhsMapper, false, linear>(j, lhs2, rhs, acc);
579 }
580 if (cend & 1) {
581 vecColLoopVSX<num_acc, LhsSubMapper, RhsMapper, true, linear>(cend - 1, lhs2, rhs, acc);
582 }
583
584 addResultsVSX<num_acc>(acc);
585
586 outputVecColResults<num_acc, extraRows, 2>(acc, result, pAlpha, extra_rows);
587
588 result += step;
589 } while (multiIters && (step <= rows - (row += step)));
590}
591
592template <const Index num_acc, typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
593EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
594 const Packet4f& pAlpha, float* result) {
595 EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) {
596 colVSXVecColLoopBody<num_acc + (extraRows ? 1 : 0), LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs,
597 rhs, pAlpha, result);
598 }
599}
600
601template <typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
602EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
603 const Packet4f& pAlpha, float* result) {
604 switch ((rows - row) >> 2) {
605 case 7:
606 colVSXVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
607 break;
608 case 6:
609 colVSXVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
610 break;
611 case 5:
612 colVSXVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
613 break;
614 case 4:
615 colVSXVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
616 break;
617 case 3:
618 colVSXVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
619 break;
620 case 2:
621 colVSXVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
622 break;
623 case 1:
624 colVSXVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
625 break;
626 default:
627 EIGEN_IF_CONSTEXPR (extraRows) {
628 colVSXVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
629 }
630 break;
631 }
632}
633
634template <typename LhsMapper, typename RhsMapper, bool linear>
635EIGEN_ALWAYS_INLINE void calcVSXVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
636 const Packet4f& pAlpha, float* result) {
637 Index row = 0;
638 if (rows >= (MAX_BFLOAT16_VEC_ACC_VSX * 4)) {
639 colVSXVecColLoopBody<MAX_BFLOAT16_VEC_ACC_VSX, LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs,
640 pAlpha, result);
641 result += row;
642 }
643 if (rows & 3) {
644 colVSXVecColLoopBodyExtra<LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
645 } else {
646 colVSXVecColLoopBodyExtra<LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
647 }
648}
649
650template <const Index size, bool inc, Index delta>
651EIGEN_ALWAYS_INLINE void storeBF16fromResult(bfloat16* dst, const Packet8bf& data, Index resInc, Index extra) {
652 if (inc) {
653 EIGEN_IF_CONSTEXPR (size < 8) {
654 pscatter_partial(dst + delta * resInc, data, resInc, extra);
655 } else {
656 pscatter(dst + delta * resInc, data, resInc);
657 }
658 } else {
659 EIGEN_IF_CONSTEXPR (size < 8) {
660 pstoreu_partial(dst + delta, data, extra);
661 } else {
662 pstoreu(dst + delta, data);
663 }
664 }
665}
666
667template <const Index size, bool inc = false>
668EIGEN_ALWAYS_INLINE void convertPointerF32toBF16VSX(Index& i, float* result, Index rows, bfloat16*& dst,
669 Index resInc = 1) {
670 constexpr Index extra = ((size < 8) ? 8 : size);
671 while (i + size <= rows) {
672 PacketBlock<Packet8bf, (size + 7) / 8> r32;
673 r32.packet[0] = convertF32toBF16VSX(result + i + 0);
674 EIGEN_IF_CONSTEXPR (size >= 16) {
675 r32.packet[1] = convertF32toBF16VSX(result + i + 8);
676 }
677 EIGEN_IF_CONSTEXPR (size >= 32) {
678 r32.packet[2] = convertF32toBF16VSX(result + i + 16);
679 r32.packet[3] = convertF32toBF16VSX(result + i + 24);
680 }
681 storeBF16fromResult<size, inc, 0>(dst, r32.packet[0], resInc, rows & 7);
682 EIGEN_IF_CONSTEXPR (size >= 16) {
683 storeBF16fromResult<size, inc, 8>(dst, r32.packet[1], resInc);
684 }
685 EIGEN_IF_CONSTEXPR (size >= 32) {
686 storeBF16fromResult<size, inc, 16>(dst, r32.packet[2], resInc);
687 storeBF16fromResult<size, inc, 24>(dst, r32.packet[3], resInc);
688 }
689 i += extra;
690 dst += extra * resInc;
691 EIGEN_IF_CONSTEXPR (size != 32) break;
692 }
693}
694
695template <bool inc = false>
696EIGEN_ALWAYS_INLINE void convertArrayPointerF32toBF16VSX(float* result, Index rows, bfloat16* dst, Index resInc = 1) {
697 Index i = 0;
698 convertPointerF32toBF16VSX<32, inc>(i, result, rows, dst, resInc);
699 convertPointerF32toBF16VSX<16, inc>(i, result, rows, dst, resInc);
700 convertPointerF32toBF16VSX<8, inc>(i, result, rows, dst, resInc);
701 convertPointerF32toBF16VSX<1, inc>(i, result, rows, dst, resInc);
702}
703
704template <typename RhsMapper, typename LhsMapper, typename = void>
705struct UseStride : std::false_type {
706 static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
707 const Packet4f& pAlpha, float* result) {
708 using RhsSubMapper = typename RhsMapper::SubMapper;
709
710 RhsSubMapper rhs2 = rhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(j2), 0);
711 calcVSXVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
712 }
713};
714
715template <typename RhsMapper, typename LhsMapper>
716struct UseStride<RhsMapper, LhsMapper,
717 std::enable_if_t<std::is_member_function_pointer<decltype(&RhsMapper::stride)>::value>>
718 : std::true_type {
719 static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
720 const Packet4f& pAlpha, float* result) {
721 using RhsSubMapper = typename RhsMapper::SubMapper;
722
723 RhsSubMapper rhs2 = rhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(j2), 0);
724 if (rhs.stride() == 1) {
725 calcVSXVecColLoops<LhsMapper, RhsSubMapper, true>(jend - j2, rows, lhs, rhs2, pAlpha, result);
726 } else {
727 calcVSXVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
728 }
729 }
730};
731
732template <typename LhsMapper, typename RhsMapper>
733void gemv_bfloat16_col(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, bfloat16* res,
734 Index resIncr, bfloat16 alpha) {
735 EIGEN_UNUSED_VARIABLE(resIncr);
736 eigen_internal_assert(resIncr == 1);
737
738 // The following copy tells the compiler that lhs's attributes are not modified outside this function
739 // This helps GCC to generate proper code.
740 LhsMapper lhs(alhs);
741 RhsMapper rhs2(rhs);
742
743 const Index lhsStride = lhs.stride();
744
745 // TODO: improve the following heuristic:
746 const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8);
747 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
748 Packet4f pAlpha = pset1<Packet4f>(falpha);
749
750 ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
751
752 convertArrayPointerBF16toF32(result, 1, rows, res);
753
754 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
755 Index jend = numext::mini(j2 + block_cols, cols);
756
757 using LhsSubMapper = typename LhsMapper::SubMapper;
758
759 LhsSubMapper lhs2 = lhs.getSubMapper(0, convert_index<mapper_index_t<LhsMapper>>(j2));
760 UseStride<RhsMapper, LhsSubMapper>::run(j2, jend, rows, lhs2, rhs2, pAlpha, result);
761 }
762
763 convertArrayPointerF32toBF16VSX(result, rows, res);
764}
765
766template <Index num_acc, Index size>
767EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float* result, const Packet4f& pAlpha) {
768 constexpr Index extra = num_acc & 3;
769
770 for (Index k = 0; k < num_acc; k += 4) {
771 Packet4f d0 = ploadu<Packet4f>(result + k);
772 d0 = pmadd(acc[k + 0][0], pAlpha, d0);
773
774 if (num_acc > (k + 3)) {
775 pstoreu(result + k, d0);
776 } else {
777 if (extra == 3) {
778 pstoreu_partial(result + k, d0, extra);
779 } else {
780 memcpy((void*)(result + k), (void*)(&d0), sizeof(float) * extra);
781 }
782 }
783 }
784}
785
786template <Index num_acc>
787EIGEN_ALWAYS_INLINE void preduxVecResults2VSX(Packet4f (&acc)[num_acc][2], Index k) {
788 if (num_acc > (k + 1)) {
789 acc[k][1] = vec_mergel(acc[k + 0][0], acc[k + 1][0]);
790 acc[k][0] = vec_mergeh(acc[k + 0][0], acc[k + 1][0]);
791 acc[k][0] = acc[k][0] + acc[k][1];
792 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8);
793 } else {
794 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8);
795#ifdef _BIG_ENDIAN
796 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12);
797#else
798 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4);
799#endif
800 }
801}
802
803template <Index num_acc>
804EIGEN_ALWAYS_INLINE void preduxVecResultsVSX(Packet4f (&acc)[num_acc][2]) {
805 for (Index k = 0; k < num_acc; k += 4) {
806 preduxVecResults2VSX<num_acc>(acc, k + 0);
807 if (num_acc > (k + 2)) {
808 preduxVecResults2VSX<num_acc>(acc, k + 2);
809#ifdef EIGEN_VECTORIZE_VSX
810 acc[k + 0][0] = reinterpret_cast<Packet4f>(
811 vec_mergeh(reinterpret_cast<Packet2ul>(acc[k + 0][0]), reinterpret_cast<Packet2ul>(acc[k + 2][0])));
812#else
813 acc[k + 0][0] = reinterpret_cast<Packet4f>(vec_perm(acc[k + 0][0], acc[k + 2][0], p16uc_TRANSPOSE64_HI));
814#endif
815 }
816 }
817}
818
819#ifndef _ARCH_PWR9
820EIGEN_ALWAYS_INLINE Packet8us loadPacketPartialZero(const Packet8us& data, Index extra_cols) {
821 Packet16uc shift = pset1<Packet16uc>(8 * 2 * (8 - extra_cols));
822#ifdef _BIG_ENDIAN
823 return reinterpret_cast<Packet8us>(vec_slo(vec_sro(reinterpret_cast<Packet16uc>(data), shift), shift));
824#else
825 return reinterpret_cast<Packet8us>(vec_sro(vec_slo(reinterpret_cast<Packet16uc>(data), shift), shift));
826#endif
827}
828#endif
829
830template <Index num_acc, typename LhsMapper, typename RhsMapper, bool extra>
831EIGEN_ALWAYS_INLINE void multVSXVecLoop(Packet4f (&acc)[num_acc][2], const LhsMapper& lhs, RhsMapper& rhs, Index j,
832 Index extra_cols) {
833 Packet4f a0[num_acc][2], b0[2];
834 Packet8bf a1, b1;
835 using MapperIndex = mapper_index_t<LhsMapper>;
836 const MapperIndex col = convert_index<MapperIndex>(j);
837 const MapperIndex extraCols = convert_index<MapperIndex>(extra_cols);
838
839 if (extra) {
840 b1 = rhs.template loadPacketPartial<Packet8bf>(col, extraCols);
841#ifndef _ARCH_PWR9
842 b1 = loadPacketPartialZero(b1.m_val, extra_cols);
843#endif
844 } else {
845 b1 = rhs.template loadPacket<Packet8bf>(col);
846 }
847 b0[0] = oneConvertBF16Hi(b1.m_val);
848 b0[1] = oneConvertBF16Lo(b1.m_val);
849
850 const LhsMapper lhs2 = lhs.getSubMapper(0, col);
851 for (int k = 0; k < num_acc; k++) {
852 if (extra) {
853 a1 = lhs2.template loadPacketPartial<Packet8bf>(k, 0, extraCols);
854#ifndef _ARCH_PWR9
855 a1 = loadPacketPartialZero(a1.m_val, extra_cols);
856#endif
857 } else {
858 a1 = lhs2.template loadPacket<Packet8bf>(k, 0);
859 }
860 a0[k][0] = oneConvertBF16Hi(a1.m_val);
861 a0[k][1] = oneConvertBF16Lo(a1.m_val);
862 }
863
864 multVecVSX<num_acc, false>(acc, a0, b0);
865}
866
867template <Index num_acc, typename LhsMapper, typename RhsMapper>
868EIGEN_ALWAYS_INLINE void vecVSXLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2],
869 Index extra_cols) {
870 Index j = 0;
871 for (; j + 8 <= cols; j += 8) {
872 multVSXVecLoop<num_acc, LhsMapper, RhsMapper, false>(acc, lhs, rhs, j, extra_cols);
873 }
874
875 if (extra_cols) {
876 multVSXVecLoop<num_acc, LhsMapper, RhsMapper, true>(acc, lhs, rhs, j, extra_cols);
877 }
878}
879
880template <const Index num_acc, typename LhsMapper, typename RhsMapper>
881void colVSXVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f& pAlpha,
882 float* result) {
883 constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC_VSX);
884 const Index extra_cols = (cols & 7);
885
886 do {
887 Packet4f acc[num_acc][2];
888
889 zeroAccumulators<num_acc, 2>(acc);
890
891 const LhsMapper lhs2 = lhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(row), 0);
892 vecVSXLoop<num_acc, LhsMapper, RhsMapper>(cols, lhs2, rhs, acc, extra_cols);
893
894 addResultsVSX<num_acc>(acc);
895
896 preduxVecResultsVSX<num_acc>(acc);
897
898 outputVecResults<num_acc, 2>(acc, result, pAlpha);
899
900 result += num_acc;
901 } while (multiIters && (num_acc <= rows - (row += num_acc)));
902}
903
904template <const Index num_acc, typename LhsMapper, typename RhsMapper>
905EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
906 const Packet4f& pAlpha, float* result) {
907 EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) {
908 colVSXVecLoopBody<num_acc, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
909 }
910}
911
912template <typename LhsMapper, typename RhsMapper>
913EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
914 const Packet4f& pAlpha, float* result) {
915 switch (rows - row) {
916 case 7:
917 colVSXVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
918 break;
919 case 6:
920 colVSXVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
921 break;
922 case 5:
923 colVSXVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
924 break;
925 case 4:
926 colVSXVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
927 break;
928 case 3:
929 colVSXVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
930 break;
931 case 2:
932 colVSXVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
933 break;
934 case 1:
935 colVSXVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
936 break;
937 }
938}
939
940template <typename LhsMapper, typename RhsMapper>
941EIGEN_ALWAYS_INLINE void calcVSXVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f& pAlpha,
942 float* result) {
943 Index row = 0;
944 if (rows >= MAX_BFLOAT16_VEC_ACC_VSX) {
945 colVSXVecLoopBody<MAX_BFLOAT16_VEC_ACC_VSX, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
946 result += row;
947 }
948 colVSXVecLoopBodyExtra<LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
949}
950
951template <typename LhsMapper, typename RhsMapper>
952EIGEN_STRONG_INLINE void gemv_bfloat16_row(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs,
953 bfloat16* res, Index resIncr, bfloat16 alpha) {
954 using LinearMapper = typename RhsMapper::LinearMapper;
955
956 // The following copy tells the compiler that lhs's attributes are not modified outside this function
957 // This helps GCC to generate proper code.
958 LhsMapper lhs(alhs);
959 LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
960
961 eigen_internal_assert(rhs.stride() == 1);
962
963 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
964 const Packet4f pAlpha = pset1<Packet4f>(falpha);
965
966 ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
967 if (resIncr == 1) {
968 convertArrayPointerBF16toF32(result, 1, rows, res);
969 } else {
970 convertArrayPointerBF16toF32<true>(result, 1, rows, res, resIncr);
971 }
972 calcVSXVecLoops<LhsMapper, LinearMapper>(cols, rows, lhs, rhs2, pAlpha, result);
973 if (resIncr == 1) {
974 convertArrayPointerF32toBF16VSX(result, rows, res);
975 } else {
976 convertArrayPointerF32toBF16VSX<true>(result, rows, res, resIncr);
977 }
978}
979
980#undef MAX_BFLOAT16_VEC_ACC_VSX
981
982const Packet16uc p16uc_COMPLEX32_XORFLIP = {0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33,
983 0xcc, 0xdd, 0xee, 0xff, 0x88, 0x99, 0xaa, 0xbb};
984const Packet16uc p16uc_COMPLEX64_XORFLIP = {0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
985 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77};
986
987#ifdef _BIG_ENDIAN
988const Packet16uc p16uc_COMPLEX32_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
989 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00};
990const Packet16uc p16uc_COMPLEX64_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
991 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
992const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
993 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
994const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
995 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
996const Packet16uc p16uc_COMPLEX32_NEGATE = {0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
997 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00};
998const Packet16uc p16uc_COMPLEX64_NEGATE = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
999 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
1000#else
1001const Packet16uc p16uc_COMPLEX32_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
1002 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80};
1003const Packet16uc p16uc_COMPLEX64_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1004 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80};
1005const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = {0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
1006 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00};
1007const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
1008 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
1009const Packet16uc p16uc_COMPLEX32_NEGATE = {0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80,
1010 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80};
1011const Packet16uc p16uc_COMPLEX64_NEGATE = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80,
1012 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80};
1013#endif
1014
1015#ifdef _BIG_ENDIAN
1016#define COMPLEX_DELTA 0
1017#else
1018#define COMPLEX_DELTA 2
1019#endif
1020
1022EIGEN_ALWAYS_INLINE Packet2cf pconj2(const Packet2cf& a) {
1023 return Packet2cf(pxor(a.v, reinterpret_cast<Packet4f>(p16uc_COMPLEX32_CONJ_XOR)));
1024}
1025
1026EIGEN_ALWAYS_INLINE Packet1cd pconj2(const Packet1cd& a) {
1027 return Packet1cd(pxor(a.v, reinterpret_cast<Packet2d>(p16uc_COMPLEX64_CONJ_XOR)));
1028}
1029
1031EIGEN_ALWAYS_INLINE Packet2cf pconjinv(const Packet2cf& a) {
1032#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1033 return Packet2cf(Packet4f(vec_neg(Packet2d(a.v))));
1034#else
1035 return Packet2cf(pxor(a.v, reinterpret_cast<Packet4f>(p16uc_COMPLEX32_CONJ_XOR2)));
1036#endif
1037}
1038
1039EIGEN_ALWAYS_INLINE Packet1cd pconjinv(const Packet1cd& a) {
1040 return Packet1cd(pxor(a.v, reinterpret_cast<Packet2d>(p16uc_COMPLEX64_CONJ_XOR2)));
1041}
1042
1043#if defined(_ARCH_PWR8) && (!EIGEN_COMP_LLVM || __clang_major__ >= 12)
1044#define PERMXOR_GOOD // Clang had a bug with vec_permxor and endianness prior to version 12
1045#endif
1046
1048EIGEN_ALWAYS_INLINE Packet2cf pcplxflipconj(const Packet2cf& a) {
1049#ifdef PERMXOR_GOOD
1050 return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR, p16uc_COMPLEX32_XORFLIP)));
1051#else
1052 return pcplxflip(pconj2(a));
1053#endif
1054}
1055
1056EIGEN_ALWAYS_INLINE Packet1cd pcplxflipconj(const Packet1cd& a) {
1057#ifdef PERMXOR_GOOD
1058 return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR, p16uc_COMPLEX64_XORFLIP)));
1059#else
1060 return pcplxflip(pconj2(a));
1061#endif
1062}
1063
1065EIGEN_ALWAYS_INLINE Packet2cf pcplxconjflip(const Packet2cf& a) {
1066#ifdef PERMXOR_GOOD
1067 return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR2, p16uc_COMPLEX32_XORFLIP)));
1068#else
1069 return pconj2(pcplxflip(a));
1070#endif
1071}
1072
1073EIGEN_ALWAYS_INLINE Packet1cd pcplxconjflip(const Packet1cd& a) {
1074#ifdef PERMXOR_GOOD
1075 return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR2, p16uc_COMPLEX64_XORFLIP)));
1076#else
1077 return pconj2(pcplxflip(a));
1078#endif
1079}
1080
1082EIGEN_ALWAYS_INLINE Packet2cf pnegate2(const Packet2cf& a) {
1083#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1084 return Packet2cf(vec_neg(a.v));
1085#else
1086 return Packet2cf(pxor(a.v, reinterpret_cast<Packet4f>(p16uc_COMPLEX32_NEGATE)));
1087#endif
1088}
1089
1090EIGEN_ALWAYS_INLINE Packet1cd pnegate2(const Packet1cd& a) {
1091#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1092 return Packet1cd(vec_neg(a.v));
1093#else
1094 return Packet1cd(pxor(a.v, reinterpret_cast<Packet2d>(p16uc_COMPLEX64_NEGATE)));
1095#endif
1096}
1097
1099EIGEN_ALWAYS_INLINE Packet2cf pcplxflipnegate(const Packet2cf& a) {
1100#ifdef PERMXOR_GOOD
1101 return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_NEGATE, p16uc_COMPLEX32_XORFLIP)));
1102#else
1103 return pcplxflip(pnegate2(a));
1104#endif
1105}
1106
1107EIGEN_ALWAYS_INLINE Packet1cd pcplxflipnegate(const Packet1cd& a) {
1108#ifdef PERMXOR_GOOD
1109 return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_NEGATE, p16uc_COMPLEX64_XORFLIP)));
1110#else
1111 return pcplxflip(pnegate2(a));
1112#endif
1113}
1114
1116EIGEN_ALWAYS_INLINE Packet2cf pcplxflip2(const Packet2cf& a) {
1117 return Packet2cf(Packet4f(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX32_XORFLIP)));
1118}
1119
1120EIGEN_ALWAYS_INLINE Packet1cd pcplxflip2(const Packet1cd& a) {
1121#ifdef EIGEN_VECTORIZE_VSX
1122 return Packet1cd(__builtin_vsx_xxpermdi(a.v, a.v, 2));
1123#else
1124 return Packet1cd(Packet2d(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX64_XORFLIP)));
1125#endif
1126}
1127
1129EIGEN_ALWAYS_INLINE Packet4f pload_complex_half(std::complex<float>* src) {
1130 Packet4f t;
1131#ifdef EIGEN_VECTORIZE_VSX
1132 // Load float64/two float32 (doubleword alignment)
1133 __asm__("lxsdx %x0,%y1" : "=wa"(t) : "Z"(*src));
1134#else
1135 *reinterpret_cast<std::complex<float>*>(reinterpret_cast<float*>(&t) + COMPLEX_DELTA) = *src;
1136#endif
1137 return t;
1138}
1139
1141template <typename RhsScalar>
1142EIGEN_ALWAYS_INLINE void pload_realimag(RhsScalar* src, Packet4f& r, Packet4f& i) {
1143#ifdef _ARCH_PWR9
1144 __asm__("lxvwsx %x0,%y1" : "=wa"(r) : "Z"(*(reinterpret_cast<float*>(src) + 0)));
1145 __asm__("lxvwsx %x0,%y1" : "=wa"(i) : "Z"(*(reinterpret_cast<float*>(src) + 1)));
1146#else
1147 Packet4f t = pload_complex_half(src);
1148 r = vec_splat(t, COMPLEX_DELTA + 0);
1149 i = vec_splat(t, COMPLEX_DELTA + 1);
1150#endif
1151}
1152
1153template <typename RhsScalar>
1154EIGEN_ALWAYS_INLINE void pload_realimag(RhsScalar* src, Packet2d& r, Packet2d& i) {
1155#ifdef EIGEN_VECTORIZE_VSX
1156 __asm__("lxvdsx %x0,%y1" : "=wa"(r) : "Z"(*(reinterpret_cast<double*>(src) + 0)));
1157 __asm__("lxvdsx %x0,%y1" : "=wa"(i) : "Z"(*(reinterpret_cast<double*>(src) + 1)));
1158#else
1159 Packet2d t = ploadu<Packet2d>(reinterpret_cast<double*>(src));
1160 r = vec_splat(t, 0);
1161 i = vec_splat(t, 1);
1162#endif
1163}
1164
1165#ifndef EIGEN_VECTORIZE_POWER8_VECTOR
1166const Packet16uc p16uc_MERGEE = {0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
1167 0x08, 0x09, 0x0A, 0x0B, 0x18, 0x19, 0x1A, 0x1B};
1168
1169const Packet16uc p16uc_MERGEO = {0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17,
1170 0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F};
1171#endif
1172
1174template <typename RhsScalar>
1175EIGEN_ALWAYS_INLINE void pload_realimag_row(RhsScalar* src, Packet4f& r, Packet4f& i) {
1176 Packet4f t = ploadu<Packet4f>(reinterpret_cast<float*>(src));
1177#ifdef EIGEN_VECTORIZE_POWER8_VECTOR
1178 r = vec_mergee(t, t);
1179 i = vec_mergeo(t, t);
1180#else
1181 r = vec_perm(t, t, p16uc_MERGEE);
1182 i = vec_perm(t, t, p16uc_MERGEO);
1183#endif
1184}
1185
1186template <typename RhsScalar>
1187EIGEN_ALWAYS_INLINE void pload_realimag_row(RhsScalar* src, Packet2d& r, Packet2d& i) {
1188 return pload_realimag(src, r, i);
1189}
1190
1192EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine(std::complex<float>* src) {
1193#ifdef EIGEN_VECTORIZE_VSX
1194 Packet4f ret;
1195 __asm__("lxvdsx %x0,%y1" : "=wa"(ret) : "Z"(*(reinterpret_cast<double*>(src) + 0)));
1196 return ret;
1197#else
1198 return Packet4f(ploaddup<Packet2d>(reinterpret_cast<double*>(src)));
1199#endif
1200}
1201
1202EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine(std::complex<double>* src) { return ploadu<Packet1cd>(src).v; }
1203
1205EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine_row(std::complex<float>* src) { return ploadu<Packet2cf>(src).v; }
1206
1207EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine_row(std::complex<double>* src) { return ploadu<Packet1cd>(src).v; }
1208
1210template <typename ResPacket>
1211EIGEN_ALWAYS_INLINE Packet4f pload_complex(std::complex<float>* src) {
1212 EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) {
1213 return pload_complex_half(src);
1214 } else {
1215 return ploadu<Packet4f>(reinterpret_cast<float*>(src));
1216 }
1217}
1218
1219template <typename ResPacket>
1220EIGEN_ALWAYS_INLINE Packet2d pload_complex(std::complex<double>* src) {
1221 return ploadu<Packet2d>(reinterpret_cast<double*>(src));
1222}
1223
1225template <typename ResPacket>
1226EIGEN_ALWAYS_INLINE Packet4f pload_complex(Packet2cf* src) {
1227 return src->v;
1228}
1229
1230template <typename ResPacket>
1231EIGEN_ALWAYS_INLINE Packet2d pload_complex(Packet1cd* src) {
1232 return src->v;
1233}
1234
1236EIGEN_ALWAYS_INLINE Packet4f pload_complex_full(std::complex<float>* src) {
1237 return Packet4f(ploaddup<Packet2d>(reinterpret_cast<double*>(src)));
1238}
1239
1240EIGEN_ALWAYS_INLINE Packet2d pload_complex_full(std::complex<double>* src) { return ploadu<Packet1cd>(src).v; }
1241
1243EIGEN_ALWAYS_INLINE Packet4f pload_complex_full_row(std::complex<float>* src) { return ploadu<Packet2cf>(src).v; }
1244
1245EIGEN_ALWAYS_INLINE Packet2d pload_complex_full_row(std::complex<double>* src) { return pload_complex_full(src); }
1246
1248EIGEN_ALWAYS_INLINE Packet4f pload_real(float* src) { return pset1<Packet4f>(*src); }
1249
1250EIGEN_ALWAYS_INLINE Packet2d pload_real(double* src) { return pset1<Packet2d>(*src); }
1251
1252EIGEN_ALWAYS_INLINE Packet4f pload_real(const Packet4f& src) { return src; }
1253
1254EIGEN_ALWAYS_INLINE Packet2d pload_real(const Packet2d& src) { return src; }
1255
1257EIGEN_ALWAYS_INLINE Packet4f pload_real_full(float* src) { return ploaddup<Packet4f>(src); }
1258
1259EIGEN_ALWAYS_INLINE Packet2d pload_real_full(double* src) { return pload_real(src); }
1260
1261EIGEN_ALWAYS_INLINE Packet4f pload_real_full(std::complex<float>* src) {
1262 return pload_complex_full(src); // Just for compilation
1263}
1264
1265EIGEN_ALWAYS_INLINE Packet2d pload_real_full(std::complex<double>* src) {
1266 return pload_complex_full(src); // Just for compilation
1267}
1268
1270template <typename ResPacket>
1271EIGEN_ALWAYS_INLINE Packet4f pload_real_row(float* src) {
1272 EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) {
1273 return pload_real_full(src);
1274 } else {
1275 return ploadu<Packet4f>(src);
1276 }
1277}
1278
1279template <typename ResPacket>
1280EIGEN_ALWAYS_INLINE Packet2d pload_real_row(double* src) {
1281 return pload_real(src);
1282}
1283
1284// Never executed: predux_complex only forms these calls when ResPacket is a
1285// scalar, where GEMV_IS_COMPLEX_COMPLEX is false. They exist because under C++14
1286// EIGEN_IF_CONSTEXPR is a plain if, so that arm still has to compile, and its
1287// scalar operand arrives as a const reference.
1288EIGEN_ALWAYS_INLINE Packet2cf padd(const Packet2cf& a, const std::complex<float>& b) {
1289 EIGEN_UNUSED_VARIABLE(b);
1290 return a;
1291}
1292
1293EIGEN_ALWAYS_INLINE Packet1cd padd(const Packet1cd& a, const std::complex<double>& b) {
1294 EIGEN_UNUSED_VARIABLE(b);
1295 return a;
1296}
1297
1299template <typename Scalar, typename ResScalar>
1300EIGEN_ALWAYS_INLINE Scalar pset1_realimag(ResScalar& alpha, int which, int conj) {
1301 return (which) ? ((conj) ? -alpha.real() : alpha.real()) : ((conj) ? -alpha.imag() : alpha.imag());
1302}
1303
1305template <typename Scalar, typename ResScalar, typename ResPacket, int which>
1306EIGEN_ALWAYS_INLINE Packet2cf pset1_complex(std::complex<float>& alpha) {
1307 Packet2cf ret;
1308 ret.v[COMPLEX_DELTA + 0] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x01), (which & 0x04));
1309 ret.v[COMPLEX_DELTA + 1] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x02), (which & 0x08));
1310 ret.v[2 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 0];
1311 ret.v[3 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 1];
1312 return ret;
1313}
1314
1315template <typename Scalar, typename ResScalar, typename ResPacket, int which>
1316EIGEN_ALWAYS_INLINE Packet1cd pset1_complex(std::complex<double>& alpha) {
1317 Packet1cd ret;
1318 ret.v[0] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x01), (which & 0x04));
1319 ret.v[1] = pset1_realimag<Scalar, ResScalar>(alpha, (which & 0x02), (which & 0x08));
1320 return ret;
1321}
1322
1324template <typename Packet>
1325EIGEN_ALWAYS_INLINE Packet pset_zero() {
1326 return pset1<Packet>(__UNPACK_TYPE__(Packet)(0));
1327}
1328
1329template <>
1330EIGEN_ALWAYS_INLINE Packet2cf pset_zero<Packet2cf>() {
1331 return Packet2cf(pset1<Packet4f>(float(0)));
1332}
1333
1334template <>
1335EIGEN_ALWAYS_INLINE Packet1cd pset_zero<Packet1cd>() {
1336 return Packet1cd(pset1<Packet2d>(double(0)));
1337}
1338
1340template <typename Packet, typename LhsPacket, typename RhsPacket>
1341EIGEN_ALWAYS_INLINE Packet pset_init(const Packet& c1) {
1342 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
1343 EIGEN_UNUSED_VARIABLE(c1);
1344 return pset_zero<Packet>();
1345 } else {
1346 return c1; // Intentionally left uninitialized
1347 }
1348}
1349
1350template <typename PResPacket, typename ResPacket, typename ResScalar, typename Scalar>
1351struct alpha_store {
1352 alpha_store(ResScalar& alpha) {
1353 separate.r = pset1_complex<Scalar, ResScalar, ResPacket, 0x3>(alpha);
1354 separate.i = pset1_complex<Scalar, ResScalar, ResPacket, 0x0>(alpha);
1355 }
1356 struct ri {
1357 PResPacket r;
1358 PResPacket i;
1359 } separate;
1360};
1361
1363template <typename ScalarPacket, typename AlphaData>
1364EIGEN_ALWAYS_INLINE ScalarPacket pmadd_complex(const ScalarPacket& c0, const ScalarPacket& c2, const ScalarPacket& c4,
1365 AlphaData& b0) {
1366 return pmadd(c2, b0.separate.i.v, pmadd(c0, b0.separate.r.v, c4));
1367}
1368
1370template <typename Scalar, typename ScalarPacket, typename PResPacket, typename ResPacket, typename ResScalar,
1371 typename AlphaData>
1372EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(const PResPacket& c0, AlphaData& b0, ResScalar* res) {
1373 PResPacket c2 = pcplxflipconj(c0);
1374 EIGEN_IF_CONSTEXPR (GEMV_IS_SCALAR) {
1375 ScalarPacket c4 = ploadu<ScalarPacket>(reinterpret_cast<Scalar*>(res));
1376 ScalarPacket c3 = pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c4, b0);
1377 pstoreu(reinterpret_cast<Scalar*>(res), c3);
1378 } else {
1379 ScalarPacket c4 = pload_complex<ResPacket>(res);
1380 PResPacket c3 = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c4, b0));
1381 pstoreu(res, c3);
1382 }
1383}
1384
1385template <typename ScalarPacket, typename PResPacket, typename ResPacket, typename ResScalar, typename AlphaData,
1386 Index ResPacketSize, Index iter2>
1387EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(const PResPacket& c0, const PResPacket& c1, AlphaData& b0,
1388 ResScalar* res) {
1389 PResPacket c2 = pcplxflipconj(c0);
1390 PResPacket c3 = pcplxflipconj(c1);
1391#if !defined(_ARCH_PWR10)
1392 ScalarPacket c4 = pload_complex<ResPacket>(res + (iter2 * ResPacketSize));
1393 ScalarPacket c5 = pload_complex<ResPacket>(res + ((iter2 + 1) * ResPacketSize));
1394 PResPacket c6 = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c4, b0));
1395 PResPacket c7 = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c1.v, c3.v, c5, b0));
1396 pstoreu(res + (iter2 * ResPacketSize), c6);
1397 pstoreu(res + ((iter2 + 1) * ResPacketSize), c7);
1398#else
1399 __vector_pair a = *reinterpret_cast<__vector_pair*>(res + (iter2 * ResPacketSize));
1400#if EIGEN_COMP_LLVM
1401 PResPacket c6[2];
1402 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(c6), &a);
1403 c6[0] = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c0.v, c2.v, c6[0].v, b0));
1404 c6[1] = PResPacket(pmadd_complex<ScalarPacket, AlphaData>(c1.v, c3.v, c6[1].v, b0));
1405 GEMV_BUILDPAIR_MMA(a, c6[0].v, c6[1].v);
1406#else
1407 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) {
1408 __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v));
1409 __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v));
1410 } else {
1411 __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v));
1412 __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v));
1413 }
1414#endif
1415 *reinterpret_cast<__vector_pair*>(res + (iter2 * ResPacketSize)) = a;
1416#endif
1417}
1418
1420template <typename Scalar, typename LhsScalar, typename LhsMapper, typename LhsPacket>
1421EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, mapper_index_t<LhsMapper> i, mapper_index_t<LhsMapper> j) {
1422 EIGEN_IF_CONSTEXPR (sizeof(Scalar) == sizeof(LhsScalar)) {
1423 const LhsScalar& src = lhs(i + 0, j);
1424 return LhsPacket(pload_real_full(const_cast<LhsScalar*>(&src)));
1425 }
1426 return lhs.template load<LhsPacket, Unaligned>(i + 0, j);
1427}
1428
1430template <typename ComplexPacket, typename RealPacket, bool ConjugateLhs, bool ConjugateRhs, bool Negate>
1431EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(const RealPacket& a, const RealPacket& b, const RealPacket& c) {
1432 EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) {
1433 return vec_madd(a, pconj2(ComplexPacket(b)).v, c);
1434 } else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) {
1435 return vec_nmsub(a, b, c);
1436 } else {
1437 return vec_madd(a, b, c);
1438 }
1439}
1440
1442template <typename ComplexPacket, typename RealPacket, bool Conjugate>
1443EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_real(const RealPacket& a, const RealPacket& b, const RealPacket& c) {
1444 EIGEN_IF_CONSTEXPR (Conjugate) {
1445 return vec_madd(a, pconj2(ComplexPacket(b)).v, c);
1446 } else {
1447 return vec_madd(a, b, c);
1448 }
1449}
1450
1451template <typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket, bool ConjugateLhs,
1452 bool ConjugateRhs, int StorageOrder>
1453EIGEN_ALWAYS_INLINE void gemv_mult_generic(const LhsPacket& a0, RhsScalar* b, PResPacket& c0) {
1454 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs> pcj;
1455 RhsPacket b0;
1456 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1457 b0 = pset1<RhsPacket>(*b);
1458 } else {
1459 b0 = ploadu<RhsPacket>(b);
1460 }
1461 c0 = pcj.pmadd(a0, b0, c0);
1462}
1463
1465template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket,
1466 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder>
1467EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0, ResPacket& c1) {
1468 ScalarPacket br, bi;
1469 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1470 pload_realimag<RhsScalar>(b, br, bi);
1471 } else {
1472 pload_realimag_row<RhsScalar>(b, br, bi);
1473 }
1474 EIGEN_IF_CONSTEXPR (ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0);
1475 LhsPacket a1 = pcplxflipconj(a0);
1476 ScalarPacket cr = pmadd_complex_complex<LhsPacket, ScalarPacket, ConjugateLhs, ConjugateRhs, false>(a0.v, br, c0.v);
1477 ScalarPacket ci = pmadd_complex_complex<LhsPacket, ScalarPacket, ConjugateLhs, ConjugateRhs, true>(a1.v, bi, c1.v);
1478 c1 = ResPacket(ci);
1479 c0 = PResPacket(cr);
1480}
1481
1483template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket,
1484 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder>
1485EIGEN_ALWAYS_INLINE void gemv_mult_real_complex(const LhsPacket& a0, RhsScalar* b, PResPacket& c0) {
1486 ScalarPacket b0;
1487 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1488 b0 = pload_complex_full(b);
1489 } else {
1490 b0 = pload_complex_full_row(b);
1491 }
1492 ScalarPacket cri = pmadd_complex_real<PResPacket, ScalarPacket, ConjugateRhs>(a0, b0, c0.v);
1493 c0 = PResPacket(cri);
1494}
1495
1497template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket,
1498 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder>
1499EIGEN_ALWAYS_INLINE void gemv_mult_complex_real(LhsPacket& a0, RhsScalar* b, PResPacket& c0) {
1500 ScalarPacket a1 = pload_complex<ResPacket>(&a0);
1501 ScalarPacket b0;
1502 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1503 b0 = pload_real(b);
1504 } else {
1505 b0 = pload_real_row<ResPacket>(b);
1506 }
1507 ScalarPacket cri = pmadd_complex_real<PResPacket, ScalarPacket, ConjugateLhs>(a1, b0, c0.v);
1508 c0 = PResPacket(cri);
1509}
1510
1511#define GEMV_MULT_COMPLEX_COMPLEX(LhsType, RhsType, ResType) \
1512 template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket, \
1513 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1514 EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, ResType& c1) { \
1515 gemv_mult_complex_complex<ScalarPacket, LhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
1516 ConjugateRhs, StorageOrder>(a0, b, c0, c1); \
1517 }
1518
1519GEMV_MULT_COMPLEX_COMPLEX(Packet2cf, std::complex<float>, Packet2cf)
1520GEMV_MULT_COMPLEX_COMPLEX(Packet1cd, std::complex<double>, Packet1cd)
1521
1522#define GEMV_MULT_REAL_COMPLEX(LhsType, RhsType, ResType) \
1523 template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket, \
1524 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1525 EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, RhsType&) { \
1526 gemv_mult_real_complex<ScalarPacket, LhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
1527 ConjugateRhs, StorageOrder>(a0, b, c0); \
1528 }
1529
1530GEMV_MULT_REAL_COMPLEX(float, std::complex<float>, Packet2cf)
1531GEMV_MULT_REAL_COMPLEX(double, std::complex<double>, Packet1cd)
1532GEMV_MULT_REAL_COMPLEX(Packet4f, std::complex<float>, Packet2cf)
1533GEMV_MULT_REAL_COMPLEX(Packet2d, std::complex<double>, Packet1cd)
1534
1535#define GEMV_MULT_COMPLEX_REAL(LhsType, RhsType, ResType1, ResType2) \
1536 template <typename ScalarPacket, typename LhsPacket, typename RhsScalar, typename RhsPacket, typename PResPacket, \
1537 typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1538 EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType1& c0, ResType2&) { \
1539 gemv_mult_complex_real<ScalarPacket, LhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
1540 ConjugateRhs, StorageOrder>(a0, b, c0); \
1541 }
1542
1543GEMV_MULT_COMPLEX_REAL(Packet2cf, float, Packet2cf, std::complex<float>)
1544GEMV_MULT_COMPLEX_REAL(Packet1cd, double, Packet1cd, std::complex<double>)
1545GEMV_MULT_COMPLEX_REAL(std::complex<float>, float, Packet2cf, std::complex<float>)
1546GEMV_MULT_COMPLEX_REAL(std::complex<double>, double, Packet1cd, std::complex<double>)
1547
1548#ifdef USE_GEMV_MMA
1550template <typename T>
1551EIGEN_ALWAYS_INLINE T convertReal(T a) {
1552 return a;
1553}
1554
1555EIGEN_ALWAYS_INLINE Packet4f convertReal(const Packet2cf& a) { return a.v; }
1556
1557EIGEN_ALWAYS_INLINE Packet2d convertReal(const Packet1cd& a) { return a.v; }
1558
1560template <typename T>
1561EIGEN_ALWAYS_INLINE T convertComplex(T a) {
1562 return a;
1563}
1564
1565EIGEN_ALWAYS_INLINE Packet2cf convertComplex(const Packet4f& a) { return Packet2cf(a); }
1566
1567EIGEN_ALWAYS_INLINE Packet1cd convertComplex(const Packet2d& a) { return Packet1cd(a); }
1568
1570template <typename ScalarPacket, typename LhsPacket, typename SLhsPacket, typename ResPacket>
1571EIGEN_ALWAYS_INLINE void pload_complex_MMA(SLhsPacket& a) {
1572 a = SLhsPacket(pload_complex<ResPacket>(&a));
1573}
1574
1575template <typename ScalarPacket, typename LhsPacket, typename SLhsPacket, typename ResPacket>
1576EIGEN_ALWAYS_INLINE void pload_complex_MMA(__vector_pair&) {
1577 // Pass thru
1578}
1579
1581template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
1582EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) {
1583 EIGEN_IF_CONSTEXPR (NegativeAccumulate) {
1584 __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
1585 } else {
1586 __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b);
1587 }
1588}
1589
1591template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
1592EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, __vector_pair& a, const Packet2d& b) {
1593 EIGEN_IF_CONSTEXPR (NegativeAccumulate) {
1594 __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b);
1595 } else {
1596 __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
1597 }
1598}
1599
1600template <typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
1601EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad*, __vector_pair&, const Packet4f&) {
1602 // Just for compilation
1603}
1604
1606template <typename RealPacket, typename LhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool Negate>
1607EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(const LhsPacket& a, const RealPacket& b, __vector_quad* c) {
1608 EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) {
1609 RealPacket b2 = pconj2(convertComplex(b)).v;
1610 return pger_vecMMA<RealPacket, RealPacket, false>(c, b2, a.v);
1611 } else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) {
1612 return pger_vecMMA<RealPacket, RealPacket, true>(c, b, a.v);
1613 } else {
1614 return pger_vecMMA<RealPacket, RealPacket, false>(c, b, a.v);
1615 }
1616}
1617
1618template <typename RealPacket, typename LhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool Negate>
1619EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(__vector_pair& a, const RealPacket& b, __vector_quad* c) {
1620 EIGEN_IF_CONSTEXPR (ConjugateLhs && ConjugateRhs) {
1621 RealPacket b2 = pconj2(convertComplex(b)).v;
1622 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b2);
1623 } else EIGEN_IF_CONSTEXPR (Negate && !ConjugateLhs && ConjugateRhs) {
1624 return pger_vecMMA<RealPacket, __vector_pair, true>(c, a, b);
1625 } else {
1626 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b);
1627 }
1628}
1629
1631template <typename RealPacket, typename LhsPacket, bool Conjugate, int StorageOrder>
1632EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(const LhsPacket& a, const RealPacket& b, __vector_quad* c) {
1633 RealPacket a2 = convertReal(a);
1634 EIGEN_IF_CONSTEXPR (Conjugate) {
1635 RealPacket b2 = pconj2(convertComplex(b)).v;
1636 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1637 return pger_vecMMA<RealPacket, RealPacket, false>(c, b2, a2);
1638 } else {
1639 return pger_vecMMA<RealPacket, RealPacket, false>(c, a2, b2);
1640 }
1641 } else {
1642 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1643 return pger_vecMMA<RealPacket, RealPacket, false>(c, b, a2);
1644 } else {
1645 return pger_vecMMA<RealPacket, RealPacket, false>(c, a2, b);
1646 }
1647 }
1648}
1649
1651template <typename RealPacket, typename LhsPacket, bool Conjugate, int StorageOrder>
1652EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(__vector_pair& a, const RealPacket& b, __vector_quad* c) {
1653 EIGEN_IF_CONSTEXPR (Conjugate) {
1654 RealPacket b2 = pconj2(convertComplex(b)).v;
1655 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b2);
1656 } else {
1657 return pger_vecMMA<RealPacket, __vector_pair, false>(c, a, b);
1658 }
1659}
1660
1662template <typename ScalarPacket, typename LhsPacket, typename SLhsPacket, typename RhsScalar, typename ResPacket,
1663 bool ConjugateLhs, bool ConjugateRhs, int StorageOrder>
1664EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) {
1665 ScalarPacket b0;
1666 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1667 b0 = pload_realimag_combine(b);
1668 } else {
1669 b0 = pload_realimag_combine_row(b);
1670 }
1671 pmadd_complex_complex_MMA<ScalarPacket, LhsPacket, ConjugateLhs, ConjugateRhs, false>(a0, b0, c0);
1672}
1673
1675template <typename ScalarPacket, typename LhsPacket, typename SLhsPacket, typename RhsScalar, typename ResPacket,
1676 bool ConjugateLhs, bool ConjugateRhs, int StorageOrder>
1677EIGEN_ALWAYS_INLINE void gemv_mult_complex_real_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) {
1678 pload_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, ResPacket>(a0);
1679 ScalarPacket b0;
1680 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1681 b0 = pload_real(b);
1682 } else {
1683 b0 = pload_real_row<ResPacket>(b);
1684 }
1685 pmadd_complex_real_MMA<ScalarPacket, LhsPacket, ConjugateLhs, ColMajor>(a0, b0, c0);
1686}
1687
1689template <typename ScalarPacket, typename LhsPacket, typename SLhsPacket, typename RhsScalar, typename ResPacket,
1690 bool ConjugateLhs, bool ConjugateRhs, int StorageOrder>
1691EIGEN_ALWAYS_INLINE void gemv_mult_real_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) {
1692 ScalarPacket b0;
1693 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1694 b0 = pload_complex_full(b);
1695 } else {
1696 b0 = pload_complex_full_row(b);
1697 }
1698 pmadd_complex_real_MMA<ScalarPacket, LhsPacket, ConjugateRhs,
1699 (sizeof(RhsScalar) == sizeof(std::complex<float>)) ? StorageOrder : ColMajor>(a0, b0, c0);
1700}
1701
1702#define GEMV_MULT_COMPLEX_COMPLEX_MMA(LhsType, RhsType) \
1703 template <typename ScalarPacket, typename LhsScalar, typename LhsPacket, typename SLhsPacket, typename RhsScalar, \
1704 typename RhsPacket, typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1705 EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \
1706 gemv_mult_complex_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, \
1707 ConjugateRhs, StorageOrder>(a0, b, c0); \
1708 }
1709
1710GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet2cf, std::complex<float>)
1711GEMV_MULT_COMPLEX_COMPLEX_MMA(__vector_pair, std::complex<float>)
1712GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet1cd, std::complex<double>)
1713
1714
1715template <typename ScalarPacket, typename LhsScalar, typename LhsPacket, typename SLhsPacket, typename RhsScalar,
1716 typename RhsPacket, typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder>
1717EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(__vector_pair& a0, std::complex<double>* b, __vector_quad* c0) {
1718 EIGEN_IF_CONSTEXPR (sizeof(LhsScalar) == 16) {
1719 gemv_mult_complex_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs,
1720 StorageOrder>(a0, b, c0);
1721 } else {
1722 gemv_mult_real_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs,
1723 StorageOrder>(a0, b, c0);
1724 }
1725}
1726
1727#define GEMV_MULT_REAL_COMPLEX_MMA(LhsType, RhsType) \
1728 template <typename ScalarPacket, typename LhsScalar, typename LhsPacket, typename SLhsPacket, typename RhsScalar, \
1729 typename RhsPacket, typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1730 EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \
1731 gemv_mult_real_complex_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs, \
1732 StorageOrder>(a0, b, c0); \
1733 }
1734
1735GEMV_MULT_REAL_COMPLEX_MMA(Packet4f, std::complex<float>)
1736GEMV_MULT_REAL_COMPLEX_MMA(Packet2d, std::complex<double>)
1737
1738#define GEMV_MULT_COMPLEX_REAL_MMA(LhsType, RhsType) \
1739 template <typename ScalarPacket, typename LhsScalar, typename LhsPacket, typename SLhsPacket, typename RhsScalar, \
1740 typename RhsPacket, typename ResPacket, bool ConjugateLhs, bool ConjugateRhs, int StorageOrder> \
1741 EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \
1742 gemv_mult_complex_real_MMA<ScalarPacket, LhsPacket, SLhsPacket, RhsScalar, ResPacket, ConjugateLhs, ConjugateRhs, \
1743 StorageOrder>(a0, b, c0); \
1744 }
1745
1746GEMV_MULT_COMPLEX_REAL_MMA(Packet2cf, float)
1747GEMV_MULT_COMPLEX_REAL_MMA(Packet1cd, double)
1748GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair, float)
1749GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair, double)
1750
1751
1752template <typename Scalar, typename ScalarPacket, typename LhsPacket, typename RhsPacket, bool ConjugateLhs,
1753 bool ConjugateRhs>
1754EIGEN_ALWAYS_INLINE void disassembleResults2(__vector_quad* c0, PacketBlock<ScalarPacket, 4>& result0) {
1755 __builtin_mma_disassemble_acc(&result0.packet, c0);
1756 EIGEN_IF_CONSTEXPR (sizeof(LhsPacket) == 16) {
1757 EIGEN_IF_CONSTEXPR (sizeof(RhsPacket) == 16) {
1758 ScalarPacket tmp0, tmp2;
1759 tmp2 = vec_mergeh(result0.packet[2], result0.packet[3]);
1760 tmp0 = vec_mergeh(result0.packet[0], result0.packet[1]);
1761 result0.packet[3] = vec_mergel(result0.packet[3], result0.packet[2]);
1762 result0.packet[1] = vec_mergel(result0.packet[1], result0.packet[0]);
1763 result0.packet[2] = tmp2;
1764 result0.packet[0] = tmp0;
1765
1766 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
1767 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
1768 result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v;
1769 } else EIGEN_IF_CONSTEXPR (ConjugateRhs) {
1770 result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v;
1771 result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v;
1772 } else {
1773 result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v;
1774 result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v;
1775 }
1776 result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]);
1777 result0.packet[2] = vec_add(result0.packet[2], result0.packet[3]);
1778 } else {
1779 result0.packet[0][1] = result0.packet[1][1];
1780 result0.packet[2][1] = result0.packet[3][1];
1781 }
1782 }
1783}
1784
1785template <typename Scalar, typename ScalarPacket, typename LhsPacket, typename RhsPacket, bool ConjugateLhs,
1786 bool ConjugateRhs>
1787EIGEN_ALWAYS_INLINE void disassembleResults4(__vector_quad* c0, PacketBlock<ScalarPacket, 4>& result0) {
1788 __builtin_mma_disassemble_acc(&result0.packet, c0);
1789 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
1790 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
1791 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
1792 result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v;
1793 } else {
1794 EIGEN_IF_CONSTEXPR (ConjugateRhs) {
1795 result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v;
1796 } else {
1797 result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v;
1798 }
1799 }
1800 result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]);
1801 } else EIGEN_IF_CONSTEXPR (sizeof(LhsPacket) == sizeof(std::complex<float>)) {
1802 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
1803 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
1804 }
1805 } else {
1806 result0.packet[0] = vec_mergee(result0.packet[0], result0.packet[1]);
1807 }
1808}
1809
1810template <typename Scalar, typename ScalarPacket, int ResPacketSize, typename LhsPacket, typename RhsPacket,
1811 bool ConjugateLhs, bool ConjugateRhs>
1812EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock<ScalarPacket, 4>& result0) {
1813 EIGEN_IF_CONSTEXPR (!GEMV_IS_COMPLEX_FLOAT) {
1814 disassembleResults2<Scalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(c0, result0);
1815 } else {
1816 disassembleResults4<Scalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(c0, result0);
1817 }
1818}
1819#endif
1820
1821#define GEMV_GETN_COMPLEX(N) (((N) * ResPacketSize) >> 1)
1822
1823#define GEMV_LOADPACKET_COL_COMPLEX(iter) \
1824 loadLhsPacket<Scalar, LhsScalar, LhsMapper, PLhsPacket>(lhs, i + ((iter) * ResPacketSize), j)
1825
1826#define GEMV_LOADPACKET_COL_COMPLEX_DATA(iter) convertReal(GEMV_LOADPACKET_COL_COMPLEX(iter))
1827
1828#ifdef USE_GEMV_MMA
1829#define GEMV_INIT_COL_COMPLEX_MMA(iter, N) \
1830 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1831 __builtin_mma_xxsetaccz(&e0##iter); \
1832 }
1833
1834#if EIGEN_COMP_LLVM
1835#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \
1836 GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), \
1837 GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1)); \
1838 EIGEN_UNUSED_VARIABLE(f##iter1);
1839#else
1840#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \
1841 EIGEN_IF_CONSTEXPR (sizeof(LhsPacket) == 16) { \
1842 const LhsScalar& src = lhs(i + ((32 * iter1) / sizeof(LhsScalar)), j); \
1843 a##iter1 = *reinterpret_cast<__vector_pair*>(const_cast<LhsScalar*>(&src)); \
1844 EIGEN_UNUSED_VARIABLE(f##iter1); \
1845 } else { \
1846 f##iter1 = lhs.template load<PLhsPacket, Unaligned>(i + ((iter2) * ResPacketSize), j); \
1847 GEMV_BUILDPAIR_MMA(a##iter1, vec_splat(convertReal(f##iter1), 0), vec_splat(convertReal(f##iter1), 1)); \
1848 }
1849#endif
1850
1851#define GEMV_LOAD1_COL_COMPLEX_MMA(iter, N) \
1852 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1853 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1854 f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \
1855 EIGEN_UNUSED_VARIABLE(a##iter); \
1856 } else { \
1857 GEMV_LOADPAIR_COL_COMPLEX_MMA(iter, iter << 1) \
1858 } \
1859 } else { \
1860 EIGEN_UNUSED_VARIABLE(a##iter); \
1861 EIGEN_UNUSED_VARIABLE(f##iter); \
1862 }
1863
1864#define GEMV_WORK1_COL_COMPLEX_MMA(iter, N) \
1865 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1866 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1867 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
1868 ConjugateLhs, ConjugateRhs, ColMajor>(f##iter, b, &e0##iter); \
1869 } else { \
1870 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
1871 ConjugateLhs, ConjugateRhs, ColMajor>(a##iter, b, &e0##iter); \
1872 } \
1873 }
1874
1875#define GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter1, iter2) \
1876 GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1));
1877
1878#define GEMV_LOAD2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \
1879 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \
1880 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1881 GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2); \
1882 EIGEN_UNUSED_VARIABLE(a##iter3); \
1883 } else { \
1884 GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2 << 1); \
1885 GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter3, iter3 << 1); \
1886 } \
1887 } else { \
1888 EIGEN_UNUSED_VARIABLE(a##iter2); \
1889 EIGEN_UNUSED_VARIABLE(a##iter3); \
1890 } \
1891 EIGEN_UNUSED_VARIABLE(f##iter2); \
1892 EIGEN_UNUSED_VARIABLE(f##iter3);
1893
1894#define GEMV_WORK2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \
1895 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \
1896 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1897 PLhsPacket g[2]; \
1898 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(g), &a##iter2); \
1899 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
1900 ConjugateLhs, ConjugateRhs, ColMajor>(g[0], b, &e0##iter2); \
1901 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
1902 ConjugateLhs, ConjugateRhs, ColMajor>(g[1], b, &e0##iter3); \
1903 } else { \
1904 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
1905 ConjugateLhs, ConjugateRhs, ColMajor>(a##iter2, b, &e0##iter2); \
1906 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
1907 ConjugateLhs, ConjugateRhs, ColMajor>(a##iter3, b, &e0##iter3); \
1908 } \
1909 }
1910
1911#if EIGEN_COMP_LLVM
1912#define GEMV_LOAD_COL_COMPLEX_MMA(N) \
1913 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > 1) { \
1914 GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) \
1915 } else { \
1916 GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \
1917 }
1918
1919#define GEMV_WORK_COL_COMPLEX_MMA(N) \
1920 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > 1) { \
1921 GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) \
1922 } else { \
1923 GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \
1924 }
1925#else
1926#define GEMV_LOAD_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N)
1927
1928#define GEMV_WORK_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N)
1929#endif
1930
1931#define GEMV_DISASSEMBLE_COMPLEX_MMA(iter) \
1932 disassembleResults<Scalar, ScalarPacket, ResPacketSize, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>( \
1933 &e0##iter, result0##iter);
1934
1935#define GEMV_STORE_COL_COMPLEX_MMA(iter, N) \
1936 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
1937 GEMV_DISASSEMBLE_COMPLEX_MMA(iter); \
1938 c0##iter = PResPacket(result0##iter.packet[0]); \
1939 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1940 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
1941 c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \
1942 } else { \
1943 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
1944 c0##iter, alpha_data, res + i + ((iter << 1) * ResPacketSize)); \
1945 c0##iter = PResPacket(result0##iter.packet[2]); \
1946 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
1947 c0##iter, alpha_data, res + i + (((iter << 1) + 1) * ResPacketSize)); \
1948 } \
1949 }
1950
1951#define GEMV_STORE2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \
1952 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter1) { \
1953 GEMV_DISASSEMBLE_COMPLEX_MMA(iter2); \
1954 GEMV_DISASSEMBLE_COMPLEX_MMA(iter3); \
1955 c0##iter2 = PResPacket(result0##iter2.packet[0]); \
1956 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
1957 c0##iter3 = PResPacket(result0##iter3.packet[0]); \
1958 pstoreu_pmadd_complex<ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData, ResPacketSize, iter2>( \
1959 c0##iter2, c0##iter3, alpha_data, res + i); \
1960 } else { \
1961 c0##iter3 = PResPacket(result0##iter2.packet[2]); \
1962 pstoreu_pmadd_complex<ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData, ResPacketSize, iter2 << 1>( \
1963 c0##iter2, c0##iter3, alpha_data, res + i); \
1964 c0##iter2 = PResPacket(result0##iter3.packet[0]); \
1965 c0##iter3 = PResPacket(result0##iter3.packet[2]); \
1966 pstoreu_pmadd_complex<ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData, ResPacketSize, iter3 << 1>( \
1967 c0##iter2, c0##iter3, alpha_data, res + i); \
1968 } \
1969 }
1970
1971#define GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \
1972 GEMV_UNROLL(GEMV_INIT_COL_COMPLEX_MMA, N) \
1973 Index j = j2; \
1974 do { \
1975 const RhsScalar& b1 = rhs2(j, 0); \
1976 RhsScalar* b = const_cast<RhsScalar*>(&b1); \
1977 GEMV_UNROLL(GEMV_PREFETCH, N) \
1978 GEMV_LOAD_COL_COMPLEX_MMA(N) \
1979 GEMV_WORK_COL_COMPLEX_MMA(N) \
1980 } while (++j < jend); \
1981 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) <= 2) { \
1982 GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) \
1983 } else { \
1984 GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1)) \
1985 } \
1986 i += (ResPacketSize * N);
1987#endif
1988
1989#define GEMV_INIT_COMPLEX(iter, N) \
1990 EIGEN_IF_CONSTEXPR (N > iter) { \
1991 c0##iter = pset_zero<PResPacket>(); \
1992 c1##iter = pset_init<ResPacket, LhsPacket, RhsPacket>(c1##iter); \
1993 } else { \
1994 EIGEN_UNUSED_VARIABLE(c0##iter); \
1995 EIGEN_UNUSED_VARIABLE(c1##iter); \
1996 }
1997
1998#define GEMV_WORK_COL_COMPLEX(iter, N) \
1999 EIGEN_IF_CONSTEXPR (N > iter) { \
2000 f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \
2001 gemv_mult_complex<ScalarPacket, PLhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
2002 ConjugateRhs, ColMajor>(f##iter, b, c0##iter, c1##iter); \
2003 } else { \
2004 EIGEN_UNUSED_VARIABLE(f##iter); \
2005 }
2006
2007#define GEMV_STORE_COL_COMPLEX(iter, N) \
2008 EIGEN_IF_CONSTEXPR (N > iter) { \
2009 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { \
2010 c0##iter = padd(c0##iter, c1##iter); \
2011 } \
2012 pstoreu_pmadd_complex<Scalar, ScalarPacket, PResPacket, ResPacket, ResScalar, AlphaData>( \
2013 c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \
2014 }
2015
2017#define GEMV_PROCESS_COL_COMPLEX_ONE(N) \
2018 GEMV_UNROLL(GEMV_INIT_COMPLEX, N) \
2019 Index j = j2; \
2020 do { \
2021 const RhsScalar& b1 = rhs2(j, 0); \
2022 RhsScalar* b = const_cast<RhsScalar*>(&b1); \
2023 GEMV_UNROLL(GEMV_PREFETCH, N) \
2024 GEMV_UNROLL(GEMV_WORK_COL_COMPLEX, N) \
2025 } while (++j < jend); \
2026 GEMV_UNROLL(GEMV_STORE_COL_COMPLEX, N) \
2027 i += (ResPacketSize * N);
2028
2029#if defined(USE_GEMV_MMA) && (EIGEN_COMP_LLVM || defined(USE_SLOWER_GEMV_MMA))
2030#define USE_GEMV_COL_COMPLEX_MMA
2031#endif
2032
2033#ifdef USE_GEMV_COL_COMPLEX_MMA
2034#define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N)
2035#else
2036#if defined(USE_GEMV_MMA) && (__GNUC__ > 10)
2037#define GEMV_PROCESS_COL_COMPLEX(N) \
2038 EIGEN_IF_CONSTEXPR (sizeof(Scalar) != sizeof(LhsPacket)) { \
2039 GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \
2040 } else { \
2041 GEMV_PROCESS_COL_COMPLEX_ONE(N) \
2042 }
2043#else
2044#define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE(N)
2045#endif
2046#endif
2047
2048template <typename Scalar, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, bool LhsIsReal,
2049 typename RhsScalar, typename RhsMapper, bool ConjugateRhs, bool RhsIsReal, typename ResScalar>
2050EIGEN_STRONG_INLINE void gemv_complex_col(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols,
2051 const LhsMapper& alhs, const RhsMapper& rhs, ResScalar* res,
2052 mapper_index_t<LhsMapper> resIncr, ResScalar alpha) {
2053 using Index = mapper_index_t<LhsMapper>;
2054 // This kernel expands real packets itself rather than using the generic mixed-packet loaders.
2055 using Traits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketFull, false>;
2056
2057 using LhsPacket = typename Traits::LhsPacket;
2058 using RhsPacket = typename Traits::RhsPacket;
2059 using ResPacket = typename Traits::ResPacket;
2060
2061 using ScalarPacket = typename packet_traits<Scalar>::type;
2062 using PLhsPacket = typename packet_traits<LhsScalar>::type;
2063 using PResPacket = typename packet_traits<ResScalar>::type;
2064 using PTraits = gemv_traits<ResPacket, ResPacket>;
2065
2066 EIGEN_UNUSED_VARIABLE(resIncr);
2067 eigen_internal_assert(resIncr == 1);
2068
2069 // The following copy tells the compiler that lhs's attributes are not modified outside this function
2070 // This helps GCC to generate proper code.
2071 LhsMapper lhs(alhs);
2072 RhsMapper rhs2(rhs);
2073
2074 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
2075
2076 const Index lhsStride = lhs.stride();
2077 // LhsAlignment stays Unaligned; enabling aligned reads would require
2078 // propagating the Mapper's Alignment through the run() template.
2079 enum {
2080 LhsAlignment = Unaligned,
2081 ResPacketSize = PTraits::ResPacketSize,
2082 LhsPacketSize = PTraits::LhsPacketSize,
2083 RhsPacketSize = PTraits::RhsPacketSize,
2084 };
2085#ifdef EIGEN_POWER_USE_GEMV_PREFETCH
2086 const Index prefetch_dist = 64 * LhsPacketSize;
2087#endif
2088
2089#ifndef GCC_ONE_VECTORPAIR_BUG
2090 const Index n8 = rows - 8 * ResPacketSize + 1;
2091 const Index n4 = rows - 4 * ResPacketSize + 1;
2092 const Index n2 = rows - 2 * ResPacketSize + 1;
2093#endif
2094 const Index n1 = rows - 1 * ResPacketSize + 1;
2095
2096 // TODO: improve the following heuristic:
2097 const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(LhsScalar) < 16000 ? 16 : 8);
2098
2099 using AlphaData = alpha_store<PResPacket, ResPacket, ResScalar, Scalar>;
2100 AlphaData alpha_data(alpha);
2101
2102 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
2103 Index jend = numext::mini(j2 + block_cols, cols);
2104 Index i = 0;
2105 PResPacket c00, c01, c02, c03, c04, c05, c06, c07;
2106 ResPacket c10, c11, c12, c13, c14, c15, c16, c17;
2107 PLhsPacket f0, f1, f2, f3, f4, f5, f6, f7;
2108#ifdef USE_GEMV_MMA
2109 __vector_quad e00, e01, e02, e03, e04, e05, e06, e07;
2110 __vector_pair a0, a1, a2, a3, a4, a5, a6, a7;
2111 PacketBlock<ScalarPacket, 4> result00, result01, result02, result03, result04, result05, result06, result07;
2112 GEMV_UNUSED(8, e0)
2113 GEMV_UNUSED(8, result0)
2114 GEMV_UNUSED(8, a)
2115 GEMV_UNUSED(8, f)
2116#if !defined(GCC_ONE_VECTORPAIR_BUG) && defined(USE_GEMV_COL_COMPLEX_MMA)
2117 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT)
2118#endif
2119#endif
2120#ifndef GCC_ONE_VECTORPAIR_BUG
2121 {
2122 while (i < n8) {
2123 GEMV_PROCESS_COL_COMPLEX(8)
2124 }
2125 }
2126 while (i < n4) {
2127 GEMV_PROCESS_COL_COMPLEX(4)
2128 }
2129 if (i < n2) {
2130 GEMV_PROCESS_COL_COMPLEX(2)
2131 }
2132 if (i < n1)
2133#else
2134 while (i < n1)
2135#endif
2136 {
2137 GEMV_PROCESS_COL_COMPLEX_ONE(1)
2138 }
2139 for (; i < rows; ++i) {
2140 ResScalar d0(0);
2141 Index j = j2;
2142 do {
2143 d0 += cj.pmul(lhs(i, j), rhs2(j, 0));
2144 } while (++j < jend);
2145 res[i] += alpha * d0;
2146 }
2147 }
2148}
2149
2150template <typename Scalar, int N>
2151struct ScalarBlock {
2152 Scalar scalar[N];
2153};
2154
2155#ifdef USE_GEMV_MMA
2156static Packet16uc p16uc_ELEMENT_3 = {0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f,
2157 0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f};
2158
2160template <typename ResScalar, typename ResPacket>
2161EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_real(__vector_quad* acc0, __vector_quad* acc1) {
2162 PacketBlock<ResPacket, 4> result0, result1;
2163 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2164 __builtin_mma_disassemble_acc(&result1.packet, acc1);
2165 result0.packet[0] = vec_mergeh(result0.packet[0], result1.packet[0]);
2166 result0.packet[1] = vec_mergeo(result0.packet[1], result1.packet[1]);
2167 result0.packet[2] = vec_mergel(result0.packet[2], result1.packet[2]);
2168 result0.packet[3] = vec_perm(result0.packet[3], result1.packet[3], p16uc_ELEMENT_3);
2169 result0.packet[0] =
2170 vec_add(vec_add(result0.packet[0], result0.packet[2]), vec_add(result0.packet[1], result0.packet[3]));
2171 return *reinterpret_cast<ScalarBlock<ResScalar, 2>*>(&result0.packet[0]);
2172}
2173
2174template <>
2175EIGEN_ALWAYS_INLINE ScalarBlock<double, 2> predux_real<double, Packet2d>(__vector_quad* acc0, __vector_quad* acc1) {
2176 PacketBlock<Packet2d, 4> result0, result1;
2177 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2178 __builtin_mma_disassemble_acc(&result1.packet, acc1);
2179 result0.packet[0] =
2180 vec_add(vec_mergeh(result0.packet[0], result1.packet[0]), vec_mergel(result0.packet[1], result1.packet[1]));
2181 return *reinterpret_cast<ScalarBlock<double, 2>*>(&result0.packet[0]);
2182}
2183
2185template <typename LhsPacket, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs>
2186EIGEN_ALWAYS_INLINE ScalarBlock<std::complex<float>, 2> addComplexResults(PacketBlock<Packet4f, 4>& result0,
2187 const PacketBlock<Packet4f, 4>& result1) {
2188 ScalarBlock<std::complex<float>, 2> cc0;
2189 result0.packet[0] = reinterpret_cast<Packet4f>(
2190 vec_mergeh(reinterpret_cast<Packet2d>(result0.packet[0]), reinterpret_cast<Packet2d>(result1.packet[0])));
2191 result0.packet[2] = reinterpret_cast<Packet4f>(
2192 vec_mergel(reinterpret_cast<Packet2d>(result0.packet[2]), reinterpret_cast<Packet2d>(result1.packet[2])));
2193 result0.packet[0] = vec_add(result0.packet[0], result0.packet[2]);
2194 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
2195 result0.packet[1] = reinterpret_cast<Packet4f>(
2196 vec_mergeh(reinterpret_cast<Packet2d>(result0.packet[1]), reinterpret_cast<Packet2d>(result1.packet[1])));
2197 result0.packet[3] = reinterpret_cast<Packet4f>(
2198 vec_mergel(reinterpret_cast<Packet2d>(result0.packet[3]), reinterpret_cast<Packet2d>(result1.packet[3])));
2199 result0.packet[1] = vec_add(result0.packet[1], result0.packet[3]);
2200 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
2201 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
2202 result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v;
2203 } else EIGEN_IF_CONSTEXPR (ConjugateRhs) {
2204 result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v;
2205 } else {
2206 result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v;
2207 }
2208 result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]);
2209 } else {
2210 EIGEN_IF_CONSTEXPR (ConjugateLhs && (sizeof(LhsPacket) == sizeof(std::complex<float>))) {
2211 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
2212 }
2213 }
2214 cc0.scalar[0].real(result0.packet[0][0]);
2215 cc0.scalar[0].imag(result0.packet[0][1]);
2216 cc0.scalar[1].real(result0.packet[0][2]);
2217 cc0.scalar[1].imag(result0.packet[0][3]);
2218 return cc0;
2219}
2220
2221template <typename LhsPacket, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs>
2222EIGEN_ALWAYS_INLINE ScalarBlock<std::complex<double>, 2> addComplexResults(PacketBlock<Packet2d, 4>&,
2223 const PacketBlock<Packet2d, 4>&) {
2224 ScalarBlock<std::complex<double>, 2> cc0;
2225 EIGEN_UNUSED_VARIABLE(cc0);
2226 return cc0; // Just for compilation
2227}
2228
2230template <typename ResScalar, typename ResPacket, typename LhsPacket, typename RhsPacket, bool ConjugateLhs,
2231 bool ConjugateRhs>
2232EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(__vector_quad* acc0, __vector_quad* acc1) {
2233 PacketBlock<ResPacket, 4> result0, result1;
2234 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2235 __builtin_mma_disassemble_acc(&result1.packet, acc1);
2236 return addComplexResults<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(result0, result1);
2237}
2238
2239template <typename ResScalar, typename ResPacket>
2240EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_real(__vector_quad* acc0) {
2241 PacketBlock<ResPacket, 4> result0;
2242 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2243 result0.packet[0] =
2244 vec_add(vec_mergeh(result0.packet[0], result0.packet[2]), vec_mergel(result0.packet[1], result0.packet[3]));
2245 return *reinterpret_cast<ScalarBlock<ResScalar, 2>*>(&result0.packet[0]);
2246}
2247
2248template <typename ResScalar, typename ResPacket, typename LhsPacket, typename RhsPacket, bool ConjugateLhs,
2249 bool ConjugateRhs>
2250EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(__vector_quad* acc0) {
2251 ScalarBlock<ResScalar, 2> cc0;
2252 PacketBlock<ResPacket, 4> result0;
2253 __builtin_mma_disassemble_acc(&result0.packet, acc0);
2254 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
2255 EIGEN_IF_CONSTEXPR (ConjugateLhs) {
2256 result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v;
2257 result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v;
2258 } else EIGEN_IF_CONSTEXPR (ConjugateRhs) {
2259 result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v;
2260 result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v;
2261 } else {
2262 result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v;
2263 result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v;
2264 }
2265 result0.packet[0] = vec_add(result0.packet[0], __builtin_vsx_xxpermdi(result0.packet[1], result0.packet[1], 2));
2266 result0.packet[2] = vec_add(result0.packet[2], __builtin_vsx_xxpermdi(result0.packet[3], result0.packet[3], 2));
2267 } else {
2268 result0.packet[0] = __builtin_vsx_xxpermdi(result0.packet[0], result0.packet[1], 1);
2269 result0.packet[2] = __builtin_vsx_xxpermdi(result0.packet[2], result0.packet[3], 1);
2270 }
2271 cc0.scalar[0].real(result0.packet[0][0]);
2272 cc0.scalar[0].imag(result0.packet[0][1]);
2273 cc0.scalar[1].real(result0.packet[2][0]);
2274 cc0.scalar[1].imag(result0.packet[2][1]);
2275 return cc0;
2276}
2277#endif
2278
2279template <typename ResScalar, typename ResPacket>
2280EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_real(const ResPacket& a, const ResPacket& b) {
2281 ScalarBlock<ResScalar, 2> cc0;
2282 cc0.scalar[0] = predux(a);
2283 cc0.scalar[1] = predux(b);
2284 return cc0;
2285}
2286
2287template <typename ResScalar, typename ResPacket>
2288EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(const ResPacket& a, const ResPacket& b) {
2289 return predux_real<ResScalar, ResPacket>(a, b);
2290}
2291
2292#define GEMV_UNROLL_ROW(func, N) func(0, N) func(1, N) func(2, N) func(3, N) func(4, N) func(5, N) func(6, N) func(7, N)
2293
2294#define GEMV_UNROLL_ROW_HALF(func, N) func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N)
2295
2296#define GEMV_LOADPACKET_ROW(iter) lhs.template load<LhsPacket, Unaligned>(i + (iter), j)
2297
2298#ifdef USE_GEMV_MMA
2299#define GEMV_UNROLL3_ROW(func, N, which) \
2300 func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) func(4, N, which) func(5, N, which) \
2301 func(6, N, which) func(7, N, which)
2302
2303#define GEMV_UNUSED_ROW(N, which) GEMV_UNROLL3_ROW(GEMV_UNUSED_VAR, N, which)
2304
2305#define GEMV_INIT_ROW(iter, N) \
2306 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
2307 __builtin_mma_xxsetaccz(&c##iter); \
2308 }
2309
2310#define GEMV_LOADPAIR_ROW(iter1, iter2) \
2311 GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_ROW(iter2), GEMV_LOADPACKET_ROW((iter2) + 1));
2312
2313#define GEMV_WORK_ROW(iter, N) \
2314 EIGEN_IF_CONSTEXPR (GEMV_GETN(N) > iter) { \
2315 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
2316 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&c##iter, a0, GEMV_LOADPACKET_ROW(iter)); \
2317 } else { \
2318 __vector_pair b##iter; \
2319 GEMV_LOADPAIR_ROW(iter, iter << 1) \
2320 pger_vecMMA_acc<LhsPacket, RhsPacket, true>(&c##iter, b##iter, a0); \
2321 } \
2322 }
2323
2324#define GEMV_PREDUX2(iter1, iter2, iter3, N) \
2325 EIGEN_IF_CONSTEXPR (N > iter1) { \
2326 EIGEN_IF_CONSTEXPR (GEMV_IS_FLOAT) { \
2327 cc##iter1 = predux_real<ResScalar, ResPacket>(&c##iter2, &c##iter3); \
2328 } else { \
2329 cc##iter1 = predux_real<ResScalar, ResPacket>(&c##iter1); \
2330 } \
2331 } else { \
2332 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2333 }
2334#else
2335#define GEMV_INIT_ROW(iter, N) \
2336 EIGEN_IF_CONSTEXPR (N > iter) { \
2337 c##iter = pset1<ResPacket>(ResScalar(0)); \
2338 } else { \
2339 EIGEN_UNUSED_VARIABLE(c##iter); \
2340 }
2341
2342#define GEMV_WORK_ROW(iter, N) \
2343 EIGEN_IF_CONSTEXPR (N > iter) { \
2344 c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); \
2345 }
2346
2347#define GEMV_PREDUX2(iter1, iter2, iter3, N) \
2348 EIGEN_IF_CONSTEXPR (N > iter1) { \
2349 cc##iter1 = predux_real<ResScalar, ResPacket>(c##iter2, c##iter3); \
2350 } else { \
2351 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2352 }
2353#endif
2354
2355#define GEMV_MULT(iter1, iter2, iter3, N) \
2356 EIGEN_IF_CONSTEXPR (N > iter1) { \
2357 cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), a0); \
2358 cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), a0); \
2359 }
2360
2361#define GEMV_STORE_ROW(iter1, iter2, iter3, N) \
2362 EIGEN_IF_CONSTEXPR (N > iter1) { \
2363 storeMaddData<ResScalar>(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \
2364 storeMaddData<ResScalar>(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \
2365 }
2366
2368#define GEMV_PROCESS_ROW(N) \
2369 for (; i < n##N; i += N) { \
2370 GEMV_UNROLL_ROW(GEMV_INIT_ROW, N) \
2371 Index j = 0; \
2372 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \
2373 RhsPacket a0 = rhs2.template load<RhsPacket, Unaligned>(j); \
2374 GEMV_UNROLL_ROW(GEMV_WORK_ROW, N) \
2375 } \
2376 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX2, (N >> 1)) \
2377 for (; j < cols; ++j) { \
2378 RhsScalar a0 = rhs2(j); \
2379 GEMV_UNROLL_ROW_HALF(GEMV_MULT, (N >> 1)) \
2380 } \
2381 GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW, (N >> 1)) \
2382 }
2383
2384template <typename LhsScalar, typename LhsMapper, typename RhsScalar, typename RhsMapper, typename ResScalar>
2385EIGEN_STRONG_INLINE void gemv_row(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols, const LhsMapper& alhs,
2386 const RhsMapper& rhs, ResScalar* res, mapper_index_t<LhsMapper> resIncr,
2387 ResScalar alpha) {
2388 using Index = mapper_index_t<LhsMapper>;
2389 using Traits = gemv_traits<LhsScalar, RhsScalar>;
2390
2391 using LhsPacket = typename Traits::LhsPacket;
2392 using RhsPacket = typename Traits::RhsPacket;
2393 using ResPacket = typename Traits::ResPacket;
2394
2395 // The following copy tells the compiler that lhs's attributes are not modified outside this function
2396 // This helps GCC to generate proper code.
2397 LhsMapper lhs(alhs);
2398 typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
2399
2400 eigen_internal_assert(rhs.stride() == 1);
2401 conj_helper<LhsScalar, RhsScalar, false, false> cj;
2402 conj_helper<LhsPacket, RhsPacket, false, false> pcj;
2403
2404 // TODO: fine tune the following heuristic. The rationale is that if the matrix is very large,
2405 // processing 8 rows at once might be counter productive wrt cache.
2406#ifndef GCC_ONE_VECTORPAIR_BUG
2407 const Index n8 = lhs.stride() * sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7);
2408 const Index n4 = rows - 3;
2409 const Index n2 = rows - 1;
2410#endif
2411
2412 // LhsAlignment stays Unaligned; enabling aligned reads would require
2413 // propagating the Mapper's Alignment through the run() template.
2414 enum {
2415 LhsAlignment = Unaligned,
2416 ResPacketSize = Traits::ResPacketSize,
2417 LhsPacketSize = Traits::LhsPacketSize,
2418 RhsPacketSize = Traits::RhsPacketSize,
2419 };
2420
2421 Index i = 0;
2422#ifdef USE_GEMV_MMA
2423 __vector_quad c0, c1, c2, c3, c4, c5, c6, c7;
2424 GEMV_UNUSED_ROW(8, c)
2425#else
2426 ResPacket c0, c1, c2, c3, c4, c5, c6, c7;
2427#endif
2428#ifndef GCC_ONE_VECTORPAIR_BUG
2429 ScalarBlock<ResScalar, 2> cc0, cc1, cc2, cc3;
2430 GEMV_PROCESS_ROW(8)
2431 GEMV_PROCESS_ROW(4)
2432 GEMV_PROCESS_ROW(2)
2433#endif
2434 for (; i < rows; ++i) {
2435 ResPacket d0 = pset1<ResPacket>(ResScalar(0));
2436 Index j = 0;
2437 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) {
2438 RhsPacket b0 = rhs2.template load<RhsPacket, Unaligned>(j);
2439
2440 d0 = pcj.pmadd(lhs.template load<LhsPacket, LhsAlignment>(i + 0, j), b0, d0);
2441 }
2442 ResScalar dd0 = predux(d0);
2443 for (; j < cols; ++j) {
2444 dd0 += cj.pmul(lhs(i, j), rhs2(j));
2445 }
2446 res[i * resIncr] += alpha * dd0;
2447 }
2448}
2449
2450#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(Scalar) \
2451 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2452 struct general_matrix_vector_product<Index, Scalar, LhsMapper, ColMajor, ConjugateLhs, Scalar, RhsMapper, \
2453 ConjugateRhs, Version> { \
2454 using ResScalar = typename ScalarBinaryOpTraits<Scalar, Scalar>::ReturnType; \
2455 \
2456 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2457 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2458 ResScalar alpha) { \
2459 if (numext::is_exactly_zero(alpha)) return; \
2460 gemv_col<Scalar, LhsMapper, Scalar, RhsMapper, ResScalar>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2461 } \
2462 };
2463
2464#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(Scalar) \
2465 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2466 struct general_matrix_vector_product<Index, Scalar, LhsMapper, RowMajor, ConjugateLhs, Scalar, RhsMapper, \
2467 ConjugateRhs, Version> { \
2468 using ResScalar = typename ScalarBinaryOpTraits<Scalar, Scalar>::ReturnType; \
2469 \
2470 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2471 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2472 ResScalar alpha) { \
2473 if (numext::is_exactly_zero(alpha)) return; \
2474 gemv_row<Scalar, LhsMapper, Scalar, RhsMapper, ResScalar>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2475 } \
2476 };
2477
2478EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(float)
2479EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(double)
2480EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(float)
2481EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(double)
2482
2483#ifdef USE_GEMV_MMA
2484#define gemv_bf16_col gemvMMA_bfloat16_col
2485#define gemv_bf16_row gemvMMA_bfloat16_row
2486#else
2487#define gemv_bf16_col gemv_bfloat16_col
2488#define gemv_bf16_row gemv_bfloat16_row
2489#endif
2490
2491#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16() \
2492 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2493 struct general_matrix_vector_product<Index, bfloat16, LhsMapper, ColMajor, ConjugateLhs, bfloat16, RhsMapper, \
2494 ConjugateRhs, Version> { \
2495 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2496 const RhsMapper& rhs, bfloat16* res, Index resIncr, \
2497 bfloat16 alpha) { \
2498 if (numext::is_exactly_zero(alpha)) return; \
2499 gemv_bf16_col<LhsMapper, RhsMapper>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2500 } \
2501 };
2502
2503#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() \
2504 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2505 struct general_matrix_vector_product<Index, bfloat16, LhsMapper, RowMajor, ConjugateLhs, bfloat16, RhsMapper, \
2506 ConjugateRhs, Version> { \
2507 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2508 const RhsMapper& rhs, bfloat16* res, Index resIncr, \
2509 bfloat16 alpha) { \
2510 if (numext::is_exactly_zero(alpha)) return; \
2511 gemv_bf16_row<LhsMapper, RhsMapper>(rows, cols, lhs, rhs, res, resIncr, alpha); \
2512 } \
2513 };
2514
2515EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16()
2516EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16()
2517
2518template <typename ResScalar, typename PResPacket, typename ResPacket, typename LhsPacket, typename RhsPacket>
2519EIGEN_ALWAYS_INLINE ScalarBlock<ResScalar, 2> predux_complex(PResPacket& a0, PResPacket& b0, const ResPacket& a1,
2520 const ResPacket& b1) {
2521 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) {
2522 a0 = padd(a0, a1);
2523 b0 = padd(b0, b1);
2524 }
2525 return predux_complex<ResScalar, PResPacket>(a0, b0);
2526}
2527
2528#define GEMV_LOADPACKET_ROW_COMPLEX(iter) loadLhsPacket<Scalar, LhsScalar, LhsMapper, PLhsPacket>(lhs, i + (iter), j)
2529
2530#define GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter) convertReal(GEMV_LOADPACKET_ROW_COMPLEX(iter))
2531
2532#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(which, N) \
2533 j = 0; \
2534 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \
2535 const RhsScalar& b1 = rhs2(j); \
2536 RhsScalar* b = const_cast<RhsScalar*>(&b1); \
2537 GEMV_UNROLL_ROW(which, N) \
2538 }
2539
2540#define GEMV_PROCESS_END_ROW_COMPLEX(N) \
2541 for (; j < cols; ++j) { \
2542 RhsScalar b0 = rhs2(j); \
2543 GEMV_UNROLL_ROW_HALF(GEMV_MULT_COMPLEX, (N >> 1)) \
2544 } \
2545 GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW_COMPLEX, (N >> 1))
2546
2547#ifdef USE_GEMV_MMA
2548#define GEMV_INIT_ROW_COMPLEX_MMA(iter, N) \
2549 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
2550 __builtin_mma_xxsetaccz(&e0##iter); \
2551 }
2552
2553#define GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter1, iter2) \
2554 GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter2), GEMV_LOADPACKET_ROW_COMPLEX_DATA((iter2) + 1));
2555
2556#define GEMV_WORK_ROW_COMPLEX_MMA(iter, N) \
2557 EIGEN_IF_CONSTEXPR (GEMV_GETN_COMPLEX(N) > iter) { \
2558 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
2559 PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \
2560 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, PLhsPacket, RhsScalar, RhsPacket, ResPacket, \
2561 ConjugateLhs, ConjugateRhs, RowMajor>(a##iter, b, &e0##iter); \
2562 } else { \
2563 __vector_pair a##iter; \
2564 GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter, iter << 1) \
2565 gemv_mult_complex_MMA<ScalarPacket, LhsScalar, PLhsPacket, __vector_pair, RhsScalar, RhsPacket, ResPacket, \
2566 ConjugateLhs, ConjugateRhs, RowMajor>(a##iter, b, &e0##iter); \
2567 } \
2568 }
2569
2570#define GEMV_PREDUX4_COMPLEX_MMA(iter1, iter2, iter3, N) \
2571 EIGEN_IF_CONSTEXPR (N > iter1) { \
2572 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_FLOAT) { \
2573 cc##iter1 = predux_complex<ResScalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>( \
2574 &e0##iter2, &e0##iter3); \
2575 } else { \
2576 cc##iter1 = \
2577 predux_complex<ResScalar, ScalarPacket, LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs>(&e0##iter1); \
2578 } \
2579 } else { \
2580 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2581 }
2582
2583#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \
2584 GEMV_UNROLL_ROW(GEMV_INIT_ROW_COMPLEX_MMA, N) \
2585 GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX_MMA, N)
2586
2587#define GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N) \
2588 for (; i < n##N; i += N) { \
2589 GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \
2590 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_MMA, (N >> 1)) \
2591 GEMV_PROCESS_END_ROW_COMPLEX(N); \
2592 }
2593#endif
2594
2595#define GEMV_WORK_ROW_COMPLEX(iter, N) \
2596 EIGEN_IF_CONSTEXPR (N > iter) { \
2597 PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \
2598 gemv_mult_complex<ScalarPacket, PLhsPacket, RhsScalar, RhsPacket, PResPacket, ResPacket, ConjugateLhs, \
2599 ConjugateRhs, RowMajor>(a##iter, b, c0##iter, c1##iter); \
2600 }
2601
2602#define GEMV_PREDUX4_COMPLEX(iter1, iter2, iter3, N) \
2603 EIGEN_IF_CONSTEXPR (N > iter1) { \
2604 cc##iter1 = predux_complex<ResScalar, PResPacket, ResPacket, LhsPacket, RhsPacket>(c0##iter2, c0##iter3, \
2605 c1##iter2, c1##iter3); \
2606 } else { \
2607 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2608 }
2609
2610#define GEMV_MULT_COMPLEX(iter1, iter2, iter3, N) \
2611 EIGEN_IF_CONSTEXPR (N > iter1) { \
2612 cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), b0); \
2613 cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), b0); \
2614 }
2615
2616#define GEMV_STORE_ROW_COMPLEX(iter1, iter2, iter3, N) \
2617 EIGEN_IF_CONSTEXPR (N > iter1) { \
2618 storeMaddData<ResScalar>(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \
2619 storeMaddData<ResScalar>(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \
2620 }
2621
2622#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \
2623 GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX, N) \
2624 GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX, N)
2625
2628#define GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \
2629 for (; i < n##N; i += N) { \
2630 GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \
2631 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX, (N >> 1)) \
2632 GEMV_PROCESS_END_ROW_COMPLEX(N); \
2633 }
2634
2635#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \
2636 EIGEN_IF_CONSTEXPR (GEMV_IS_COMPLEX_COMPLEX) { \
2637 c0##iter = padd(c0##iter, c1##iter); \
2638 } \
2639 dd0 = predux(c0##iter);
2640
2641#if EIGEN_COMP_LLVM
2642#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N)
2643
2644#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N)
2645
2646#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter)
2647#else
2648// gcc seems to be reading and writing registers unnecessarily to memory.
2649// Use the old way for complex double until it is fixed.
2650
2651#define GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter) lhs.template load<LhsPacket, LhsAlignment>(i + (iter), j)
2652
2653#define GEMV_INIT_COMPLEX_OLD(iter, N) \
2654 EIGEN_UNUSED_VARIABLE(c0##iter); \
2655 EIGEN_IF_CONSTEXPR (N > iter) { \
2656 c1##iter = pset_zero<ResPacket>(); \
2657 } else { \
2658 EIGEN_UNUSED_VARIABLE(c1##iter); \
2659 }
2660
2661#define GEMV_WORK_ROW_COMPLEX_OLD(iter, N) \
2662 EIGEN_IF_CONSTEXPR (N > iter) { \
2663 LhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter); \
2664 c1##iter = pcj.pmadd(a##iter, b0, c1##iter); \
2665 }
2666
2667#define GEMV_PREDUX4_COMPLEX_OLD(iter1, iter2, iter3, N) \
2668 EIGEN_IF_CONSTEXPR (N > iter1) { \
2669 cc##iter1.scalar[0] = predux(c1##iter2); \
2670 cc##iter1.scalar[1] = predux(c1##iter3); \
2671 } else { \
2672 EIGEN_UNUSED_VARIABLE(cc##iter1); \
2673 }
2674
2675#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \
2676 GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX_OLD, N) \
2677 j = 0; \
2678 for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \
2679 RhsPacket b0 = rhs2.template load<RhsPacket, Unaligned>(j); \
2680 GEMV_UNROLL_ROW(GEMV_WORK_ROW_COMPLEX_OLD, N) \
2681 }
2682
2683#define GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \
2684 for (; i < n##N; i += N) { \
2685 GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \
2686 GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_OLD, (N >> 1)) \
2687 GEMV_PROCESS_END_ROW_COMPLEX(N) \
2688 }
2689
2690#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) dd0 = predux(c1##iter);
2691
2692#if (__GNUC__ > 10)
2693#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW 1
2694#else
2695#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW (sizeof(Scalar) == sizeof(float)) || GEMV_IS_COMPLEX_COMPLEX
2696#endif
2697
2698#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) \
2699 if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \
2700 GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \
2701 } else { \
2702 GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \
2703 }
2704
2705#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) \
2706 if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \
2707 GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \
2708 } else { \
2709 GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \
2710 }
2711
2712#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) \
2713 if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \
2714 GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \
2715 } else { \
2716 GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) \
2717 }
2718#endif
2719
2720#ifdef USE_GEMV_MMA
2721#define GEMV_PROCESS_ROW_COMPLEX(N) GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N)
2722#else
2723#define GEMV_PROCESS_ROW_COMPLEX(N) GEMV_PROCESS_ROW_COMPLEX_ONE(N)
2724#endif
2725
2726template <typename Scalar, typename LhsScalar, typename LhsMapper, bool ConjugateLhs, bool LhsIsReal,
2727 typename RhsScalar, typename RhsMapper, bool ConjugateRhs, bool RhsIsReal, typename ResScalar>
2728EIGEN_STRONG_INLINE void gemv_complex_row(mapper_index_t<LhsMapper> rows, mapper_index_t<LhsMapper> cols,
2729 const LhsMapper& alhs, const RhsMapper& rhs, ResScalar* res,
2730 mapper_index_t<LhsMapper> resIncr, ResScalar alpha) {
2731 using Index = mapper_index_t<LhsMapper>;
2732 // This kernel expands real packets itself rather than using the generic mixed-packet loaders.
2733 using Traits = gemv_traits<LhsScalar, RhsScalar, GEBPPacketFull, false>;
2734
2735 using LhsPacket = typename Traits::LhsPacket;
2736 using RhsPacket = typename Traits::RhsPacket;
2737 using ResPacket = typename Traits::ResPacket;
2738
2739 using ScalarPacket = typename packet_traits<Scalar>::type;
2740 using PLhsPacket = typename packet_traits<LhsScalar>::type;
2741 using PResPacket = typename packet_traits<ResScalar>::type;
2742 using PTraits = gemv_traits<ResPacket, ResPacket>;
2743
2744 // The following copy tells the compiler that lhs's attributes are not modified outside this function
2745 // This helps GCC to generate proper code.
2746 LhsMapper lhs(alhs);
2747 typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
2748
2749 eigen_internal_assert(rhs.stride() == 1);
2750 conj_helper<LhsScalar, RhsScalar, ConjugateLhs, ConjugateRhs> cj;
2751#if !EIGEN_COMP_LLVM
2752 conj_helper<LhsPacket, RhsPacket, ConjugateLhs, ConjugateRhs> pcj;
2753#endif
2754
2755 // TODO: fine tune the following heuristic. The rationale is that if the matrix is very large,
2756 // processing 8 rows at once might be counter productive wrt cache.
2757#ifndef GCC_ONE_VECTORPAIR_BUG
2758 const Index n8 = lhs.stride() * sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7);
2759 const Index n4 = rows - 3;
2760 const Index n2 = rows - 1;
2761#endif
2762
2763 // LhsAlignment stays Unaligned; enabling aligned reads would require
2764 // propagating the Mapper's Alignment through the run() template.
2765 enum {
2766 LhsAlignment = Unaligned,
2767 ResPacketSize = PTraits::ResPacketSize,
2768 LhsPacketSize = PTraits::LhsPacketSize,
2769 RhsPacketSize = PTraits::RhsPacketSize,
2770 };
2771
2772 Index i = 0, j;
2773 PResPacket c00, c01, c02, c03, c04, c05, c06, c07;
2774 ResPacket c10, c11, c12, c13, c14, c15, c16, c17;
2775#ifdef USE_GEMV_MMA
2776 __vector_quad e00, e01, e02, e03, e04, e05, e06, e07;
2777 GEMV_UNUSED_ROW(8, e0)
2778 GEMV_UNUSED_EXTRA(1, c0)
2779 GEMV_UNUSED_EXTRA(1, c1)
2780#endif
2781 ResScalar dd0;
2782#ifndef GCC_ONE_VECTORPAIR_BUG
2783 ScalarBlock<ResScalar, 2> cc0, cc1, cc2, cc3;
2784#ifdef USE_GEMV_MMA
2785 EIGEN_IF_CONSTEXPR (!GEMV_IS_COMPLEX_COMPLEX)
2786#endif
2787 {
2788 GEMV_PROCESS_ROW_COMPLEX(8)
2789 }
2790 GEMV_PROCESS_ROW_COMPLEX(4)
2791 GEMV_PROCESS_ROW_COMPLEX(2)
2792#endif
2793 for (; i < rows; ++i) {
2794 GEMV_PROCESS_ROW_COMPLEX_SINGLE(1)
2795 GEMV_PROCESS_ROW_COMPLEX_PREDUX(0)
2796 for (; j < cols; ++j) {
2797 dd0 += cj.pmul(lhs(i, j), rhs2(j));
2798 }
2799 res[i * resIncr] += alpha * dd0;
2800 }
2801}
2802
2803#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(Scalar, LhsScalar, RhsScalar) \
2804 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2805 struct general_matrix_vector_product<Index, LhsScalar, LhsMapper, ColMajor, ConjugateLhs, RhsScalar, RhsMapper, \
2806 ConjugateRhs, Version> { \
2807 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType; \
2808 \
2809 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2810 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2811 ResScalar alpha) { \
2812 if (numext::is_exactly_zero(alpha)) return; \
2813 gemv_complex_col<Scalar, LhsScalar, LhsMapper, ConjugateLhs, sizeof(Scalar) == sizeof(LhsScalar), RhsScalar, \
2814 RhsMapper, ConjugateRhs, sizeof(Scalar) == sizeof(RhsScalar), ResScalar>(rows, cols, lhs, rhs, \
2815 res, resIncr, alpha); \
2816 } \
2817 };
2818
2819#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(Scalar, LhsScalar, RhsScalar) \
2820 template <typename Index, typename LhsMapper, bool ConjugateLhs, typename RhsMapper, bool ConjugateRhs, int Version> \
2821 struct general_matrix_vector_product<Index, LhsScalar, LhsMapper, RowMajor, ConjugateLhs, RhsScalar, RhsMapper, \
2822 ConjugateRhs, Version> { \
2823 using ResScalar = typename ScalarBinaryOpTraits<LhsScalar, RhsScalar>::ReturnType; \
2824 \
2825 EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \
2826 const RhsMapper& rhs, ResScalar* res, Index resIncr, \
2827 ResScalar alpha) { \
2828 if (numext::is_exactly_zero(alpha)) return; \
2829 gemv_complex_row<Scalar, LhsScalar, LhsMapper, ConjugateLhs, sizeof(Scalar) == sizeof(LhsScalar), RhsScalar, \
2830 RhsMapper, ConjugateRhs, sizeof(Scalar) == sizeof(RhsScalar), ResScalar>(rows, cols, lhs, rhs, \
2831 res, resIncr, alpha); \
2832 } \
2833 };
2834
2835EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, float, std::complex<float>)
2836EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, std::complex<float>, float)
2837EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, std::complex<float>, std::complex<float>)
2838EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, double, std::complex<double>)
2839EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, std::complex<double>, double)
2840EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, std::complex<double>, std::complex<double>)
2841EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, float, std::complex<float>)
2842EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, std::complex<float>, float)
2843EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, std::complex<float>, std::complex<float>)
2844EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, double, std::complex<double>)
2845EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, std::complex<double>, double)
2846EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, std::complex<double>, std::complex<double>)
2847
2848#endif // EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H
@ Unaligned
Definition Constants.h:236
@ ColMajor
Definition Constants.h:319