Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
MatrixProduct.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2020 Everton Constantino (everton.constantino@ibm.com)
5// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com)
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_MATRIX_PRODUCT_ALTIVEC_H
13#define EIGEN_MATRIX_PRODUCT_ALTIVEC_H
14
15#ifndef EIGEN_ALTIVEC_USE_CUSTOM_PACK
16#define EIGEN_ALTIVEC_USE_CUSTOM_PACK 1
17#endif
18
19#if !defined(EIGEN_ALTIVEC_DISABLE_MMA)
20#define EIGEN_ALTIVEC_DISABLE_MMA 0
21#endif
22
23// Check for MMA builtin support.
24#if !EIGEN_ALTIVEC_DISABLE_MMA && defined(__has_builtin)
25#if __has_builtin(__builtin_mma_assemble_acc)
26#define EIGEN_ALTIVEC_MMA_SUPPORT
27#endif
28#endif
29
30// Check if and how we should actually use MMA if supported.
31#if defined(EIGEN_ALTIVEC_MMA_SUPPORT)
32
33#if !defined(EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH)
34#define EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH 0
35#endif
36
37// Check if we want to enable dynamic dispatch. Not supported by LLVM.
38#if EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH && !EIGEN_COMP_LLVM
39#define EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH 1
40// Otherwise, use MMA by default if available.
41#elif defined(__MMA__)
42#define EIGEN_ALTIVEC_MMA_ONLY 1
43#endif
44
45#endif // EIGEN_ALTIVEC_MMA_SUPPORT
46
47#include "MatrixProductCommon.h"
48
49#if defined(EIGEN_ALTIVEC_MMA_ONLY) || defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
50#include "MatrixProductMMA.h"
51#endif
52
53// IWYU pragma: private
54#include "../../InternalHeaderCheck.h"
55
56namespace Eigen {
57
58namespace internal {
59
60/**************************
61 * Constants and typedefs *
62 **************************/
63template <typename Scalar>
64struct quad_traits {
65 using vectortype = typename packet_traits<Scalar>::type;
66 using type = PacketBlock<vectortype, 4>;
67 using rhstype = vectortype;
68 enum { vectorsize = packet_traits<Scalar>::size, size = 4, rows = 4 };
69};
70
71template <>
72struct quad_traits<double> {
73 using vectortype = Packet2d;
74 using type = PacketBlock<vectortype, 4>;
75 using rhstype = PacketBlock<Packet2d, 2>;
76 enum { vectorsize = packet_traits<double>::size, size = 2, rows = 4 };
77};
78
79template <>
80struct quad_traits<bfloat16> {
81 using vectortype = Packet8bf;
82 using type = PacketBlock<vectortype, 4>;
83 using rhstype = vectortype;
84 enum { vectorsize = packet_traits<bfloat16>::size, size = 8, rows = 4 };
85};
86
87// MatrixProduct decomposes real/imaginary vectors into a real vector and an imaginary vector, this turned out
88// to be faster than Eigen's usual approach of having real/imaginary pairs on a single vector. These constants then
89// are responsible to extract from convert between Eigen's and MatrixProduct approach.
90
91const static Packet16uc p16uc_GETREAL32 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};
92
93const static Packet16uc p16uc_GETIMAG32 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
94
95const static Packet16uc p16uc_GETREAL32b = {0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27};
96
97const static Packet16uc p16uc_GETIMAG32b = {4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31};
98
99/*********************************************
100 * Single precision real and complex packing *
101 * *******************************************/
102
117template <typename Scalar, int StorageOrder>
118EIGEN_ALWAYS_INLINE std::complex<Scalar> getAdjointVal(
119 Index i, Index j, const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder>& dt) {
120 std::complex<Scalar> v;
121 if (i < j) {
122 v.real(dt(j, i).real());
123 v.imag(-dt(j, i).imag());
124 } else if (i > j) {
125 v.real(dt(i, j).real());
126 v.imag(dt(i, j).imag());
127 } else {
128 v.real(dt(i, j).real());
129 v.imag((Scalar)0.0);
130 }
131 return v;
132}
133
134template <typename Scalar, int StorageOrder, int N>
135EIGEN_STRONG_INLINE void symm_pack_complex_rhs_helper(std::complex<Scalar>* blockB, const std::complex<Scalar>* _rhs,
136 Index rhsStride, Index rows, Index cols, Index k2) {
137 const Index depth = k2 + rows;
138 const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder> rhs(_rhs, rhsStride);
139 const Index vectorSize = N * quad_traits<Scalar>::vectorsize;
140 const Index vectorDelta = vectorSize * rows;
141 Scalar* blockBf = reinterpret_cast<Scalar*>(blockB);
142
143 Index rir = 0, rii, j = 0;
144 for (; j + vectorSize <= cols; j += vectorSize) {
145 rii = rir + vectorDelta;
146
147 for (Index i = k2; i < depth; i++) {
148 for (Index k = 0; k < vectorSize; k++) {
149 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(i, j + k, rhs);
150
151 blockBf[rir + k] = v.real();
152 blockBf[rii + k] = v.imag();
153 }
154 rir += vectorSize;
155 rii += vectorSize;
156 }
157
158 rir += vectorDelta;
159 }
160
161 for (; j < cols; j++) {
162 rii = rir + rows;
163
164 for (Index i = k2; i < depth; i++) {
165 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(i, j, rhs);
166
167 blockBf[rir] = v.real();
168 blockBf[rii] = v.imag();
169
170 rir += 1;
171 rii += 1;
172 }
173
174 rir += rows;
175 }
176}
177
178template <typename Scalar, int StorageOrder>
179EIGEN_STRONG_INLINE void symm_pack_complex_lhs_helper(std::complex<Scalar>* blockA, const std::complex<Scalar>* _lhs,
180 Index lhsStride, Index cols, Index rows) {
181 const Index depth = cols;
182 const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder> lhs(_lhs, lhsStride);
183 const Index vectorSize = quad_traits<Scalar>::vectorsize;
184 const Index vectorDelta = vectorSize * depth;
185 Scalar* blockAf = reinterpret_cast<Scalar*>(blockA);
186
187 Index rir = 0, rii, j = 0;
188 for (; j + vectorSize <= rows; j += vectorSize) {
189 rii = rir + vectorDelta;
190
191 for (Index i = 0; i < depth; i++) {
192 for (Index k = 0; k < vectorSize; k++) {
193 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(j + k, i, lhs);
194
195 blockAf[rir + k] = v.real();
196 blockAf[rii + k] = v.imag();
197 }
198 rir += vectorSize;
199 rii += vectorSize;
200 }
201
202 rir += vectorDelta;
203 }
204
205 if (j < rows) {
206 rii = rir + ((rows - j) * depth);
207
208 for (Index i = 0; i < depth; i++) {
209 Index k = j;
210 for (; k < rows; k++) {
211 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(k, i, lhs);
212
213 blockAf[rir] = v.real();
214 blockAf[rii] = v.imag();
215
216 rir += 1;
217 rii += 1;
218 }
219 }
220 }
221}
222
223template <typename Scalar, int StorageOrder, int N>
224EIGEN_STRONG_INLINE void symm_pack_rhs_helper(Scalar* blockB, const Scalar* _rhs, Index rhsStride, Index rows,
225 Index cols, Index k2) {
226 const Index depth = k2 + rows;
227 const_blas_data_mapper<Scalar, Index, StorageOrder> rhs(_rhs, rhsStride);
228 const Index vectorSize = quad_traits<Scalar>::vectorsize;
229
230 Index ri = 0, j = 0;
231 for (; j + N * vectorSize <= cols; j += N * vectorSize) {
232 Index i = k2;
233 for (; i < depth; i++) {
234 for (Index k = 0; k < N * vectorSize; k++) {
235 if (i <= j + k)
236 blockB[ri + k] = rhs(j + k, i);
237 else
238 blockB[ri + k] = rhs(i, j + k);
239 }
240 ri += N * vectorSize;
241 }
242 }
243
244 for (; j < cols; j++) {
245 for (Index i = k2; i < depth; i++) {
246 if (j <= i)
247 blockB[ri] = rhs(i, j);
248 else
249 blockB[ri] = rhs(j, i);
250 ri += 1;
251 }
252 }
253}
254
255template <typename Scalar, int StorageOrder>
256EIGEN_STRONG_INLINE void symm_pack_lhs_helper(Scalar* blockA, const Scalar* _lhs, Index lhsStride, Index cols,
257 Index rows) {
258 const Index depth = cols;
259 const_blas_data_mapper<Scalar, Index, StorageOrder> lhs(_lhs, lhsStride);
260 const Index vectorSize = quad_traits<Scalar>::vectorsize;
261
262 Index ri = 0, j = 0;
263 for (; j + vectorSize <= rows; j += vectorSize) {
264 Index i = 0;
265
266 for (; i < depth; i++) {
267 for (Index k = 0; k < vectorSize; k++) {
268 if (i <= j + k)
269 blockA[ri + k] = lhs(j + k, i);
270 else
271 blockA[ri + k] = lhs(i, j + k);
272 }
273 ri += vectorSize;
274 }
275 }
276
277 if (j < rows) {
278 for (Index i = 0; i < depth; i++) {
279 Index k = j;
280 for (; k < rows; k++) {
281 if (i <= k)
282 blockA[ri] = lhs(k, i);
283 else
284 blockA[ri] = lhs(i, k);
285 ri += 1;
286 }
287 }
288 }
289}
290
291template <typename Index, int nr, int StorageOrder>
292struct symm_pack_rhs<std::complex<float>, Index, nr, StorageOrder> {
293 void operator()(std::complex<float>* blockB, const std::complex<float>* _rhs, Index rhsStride, Index rows, Index cols,
294 Index k2) {
295 symm_pack_complex_rhs_helper<float, StorageOrder, 1>(blockB, _rhs, rhsStride, rows, cols, k2);
296 }
297};
298
299template <typename Index, int Pack1, int Pack2_dummy, int StorageOrder>
300struct symm_pack_lhs<std::complex<float>, Index, Pack1, Pack2_dummy, StorageOrder> {
301 void operator()(std::complex<float>* blockA, const std::complex<float>* _lhs, Index lhsStride, Index cols,
302 Index rows) {
303 symm_pack_complex_lhs_helper<float, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
304 }
305};
306
307// *********** symm_pack std::complex<float64> ***********
308
309template <typename Index, int nr, int StorageOrder>
310struct symm_pack_rhs<std::complex<double>, Index, nr, StorageOrder> {
311 void operator()(std::complex<double>* blockB, const std::complex<double>* _rhs, Index rhsStride, Index rows,
312 Index cols, Index k2) {
313 symm_pack_complex_rhs_helper<double, StorageOrder, 2>(blockB, _rhs, rhsStride, rows, cols, k2);
314 }
315};
316
317template <typename Index, int Pack1, int Pack2_dummy, int StorageOrder>
318struct symm_pack_lhs<std::complex<double>, Index, Pack1, Pack2_dummy, StorageOrder> {
319 void operator()(std::complex<double>* blockA, const std::complex<double>* _lhs, Index lhsStride, Index cols,
320 Index rows) {
321 symm_pack_complex_lhs_helper<double, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
322 }
323};
324
325// *********** symm_pack float32 ***********
326template <typename Index, int nr, int StorageOrder>
327struct symm_pack_rhs<float, Index, nr, StorageOrder> {
328 void operator()(float* blockB, const float* _rhs, Index rhsStride, Index rows, Index cols, Index k2) {
329 symm_pack_rhs_helper<float, StorageOrder, 1>(blockB, _rhs, rhsStride, rows, cols, k2);
330 }
331};
332
333template <typename Index, int Pack1, int Pack2_dummy, int StorageOrder>
334struct symm_pack_lhs<float, Index, Pack1, Pack2_dummy, StorageOrder> {
335 void operator()(float* blockA, const float* _lhs, Index lhsStride, Index cols, Index rows) {
336 symm_pack_lhs_helper<float, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
337 }
338};
339
340// *********** symm_pack float64 ***********
341template <typename Index, int nr, int StorageOrder>
342struct symm_pack_rhs<double, Index, nr, StorageOrder> {
343 void operator()(double* blockB, const double* _rhs, Index rhsStride, Index rows, Index cols, Index k2) {
344 symm_pack_rhs_helper<double, StorageOrder, 2>(blockB, _rhs, rhsStride, rows, cols, k2);
345 }
346};
347
348template <typename Index, int Pack1, int Pack2_dummy, int StorageOrder>
349struct symm_pack_lhs<double, Index, Pack1, Pack2_dummy, StorageOrder> {
350 void operator()(double* blockA, const double* _lhs, Index lhsStride, Index cols, Index rows) {
351 symm_pack_lhs_helper<double, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
352 }
353};
354
365
366template <typename Scalar, typename Packet, int N>
367EIGEN_ALWAYS_INLINE void storeBlock(Scalar* to, PacketBlock<Packet, N>& block) {
368 const Index size = 16 / sizeof(Scalar);
369 pstore<Scalar>(to + (0 * size), block.packet[0]);
370 pstore<Scalar>(to + (1 * size), block.packet[1]);
371 EIGEN_IF_CONSTEXPR (N > 2) {
372 pstore<Scalar>(to + (2 * size), block.packet[2]);
373 }
374 EIGEN_IF_CONSTEXPR (N > 3) {
375 pstore<Scalar>(to + (3 * size), block.packet[3]);
376 }
377}
378
379// General template for lhs & rhs complex packing.
380template <typename Scalar, typename DataMapper, typename Packet, typename PacketC, int StorageOrder, bool Conjugate,
381 bool PanelMode, bool UseLhs>
382struct dhs_cpack {
383 using Index = mapper_index_t<DataMapper>;
384 template <bool transpose>
385 EIGEN_ALWAYS_INLINE void dhs_cblock(PacketBlock<PacketC, 8>& cblock, PacketBlock<Packet, 4>& block,
386 const Packet16uc& permute) {
387 if (transpose) {
388 block.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, permute);
389 block.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, permute);
390 block.packet[2] = vec_perm(cblock.packet[4].v, cblock.packet[5].v, permute);
391 block.packet[3] = vec_perm(cblock.packet[6].v, cblock.packet[7].v, permute);
392
393 Packet4f t0, t1, t2, t3;
394#ifdef EIGEN_VECTORIZE_VSX
395 t0 = reinterpret_cast<Packet>(
396 vec_mergeh(reinterpret_cast<Packet2ul>(block.packet[0]), reinterpret_cast<Packet2ul>(block.packet[1])));
397 t1 = reinterpret_cast<Packet>(
398 vec_mergel(reinterpret_cast<Packet2ul>(block.packet[0]), reinterpret_cast<Packet2ul>(block.packet[1])));
399 t2 = reinterpret_cast<Packet>(
400 vec_mergeh(reinterpret_cast<Packet2ul>(block.packet[2]), reinterpret_cast<Packet2ul>(block.packet[3])));
401 t3 = reinterpret_cast<Packet>(
402 vec_mergel(reinterpret_cast<Packet2ul>(block.packet[2]), reinterpret_cast<Packet2ul>(block.packet[3])));
403#else
404 t0 = reinterpret_cast<Packet>(vec_perm(block.packet[0], block.packet[1], p16uc_TRANSPOSE64_HI));
405 t1 = reinterpret_cast<Packet>(vec_perm(block.packet[0], block.packet[1], p16uc_TRANSPOSE64_LO));
406 t2 = reinterpret_cast<Packet>(vec_perm(block.packet[2], block.packet[3], p16uc_TRANSPOSE64_HI));
407 t3 = reinterpret_cast<Packet>(vec_perm(block.packet[2], block.packet[3], p16uc_TRANSPOSE64_LO));
408#endif
409
410 block.packet[0] = t0;
411 block.packet[1] = t1;
412 block.packet[2] = t2;
413 block.packet[3] = t3;
414 } else {
415 block.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, permute);
416 block.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, permute);
417 block.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, permute);
418 block.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, permute);
419 }
420 }
421
422 EIGEN_ALWAYS_INLINE void dhs_ccopy(Scalar* blockAt, const DataMapper& lhs2, Index& i, Index& rir, Index& rii,
423 Index depth, const Index vectorSize) {
424 PacketBlock<Packet, 4> blockr, blocki;
425 PacketBlock<PacketC, 8> cblock;
426
427 for (; i + vectorSize <= depth; i += vectorSize) {
428 EIGEN_IF_CONSTEXPR (UseLhs) {
429 bload<DataMapper, PacketC, 2, StorageOrder, true, 4>(cblock, lhs2, 0, i);
430 } else {
431 bload<DataMapper, PacketC, 2, StorageOrder, true, 4>(cblock, lhs2, i, 0);
432 }
433
434 EIGEN_IF_CONSTEXPR (((StorageOrder == RowMajor) && UseLhs) || (((StorageOrder == ColMajor) && !UseLhs))) {
435 dhs_cblock<true>(cblock, blockr, p16uc_GETREAL32b);
436 dhs_cblock<true>(cblock, blocki, p16uc_GETIMAG32b);
437 } else {
438 dhs_cblock<false>(cblock, blockr, p16uc_GETREAL32);
439 dhs_cblock<false>(cblock, blocki, p16uc_GETIMAG32);
440 }
441
442 EIGEN_IF_CONSTEXPR (Conjugate) {
443 blocki.packet[0] = -blocki.packet[0];
444 blocki.packet[1] = -blocki.packet[1];
445 blocki.packet[2] = -blocki.packet[2];
446 blocki.packet[3] = -blocki.packet[3];
447 }
448
449 storeBlock<Scalar, Packet, 4>(blockAt + rir, blockr);
450 storeBlock<Scalar, Packet, 4>(blockAt + rii, blocki);
451
452 rir += 4 * vectorSize;
453 rii += 4 * vectorSize;
454 }
455 }
456
457 EIGEN_STRONG_INLINE void operator()(std::complex<Scalar>* blockA, const DataMapper& lhs, Index depth, Index rows,
458 Index stride, Index offset) {
459 const Index vectorSize = quad_traits<Scalar>::vectorsize;
460 const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth);
461 Index rir = ((PanelMode) ? (vectorSize * offset) : 0), rii;
462 Scalar* blockAt = reinterpret_cast<Scalar*>(blockA);
463 Index j = 0;
464
465 for (; j + vectorSize <= rows; j += vectorSize) {
466 const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j);
467 Index i = 0;
468
469 rii = rir + vectorDelta;
470
471 dhs_ccopy(blockAt, lhs2, i, rir, rii, depth, vectorSize);
472
473 for (; i < depth; i++) {
474 PacketBlock<Packet, 1> blockr, blocki;
475 PacketBlock<PacketC, 2> cblock;
476
477 EIGEN_IF_CONSTEXPR (((StorageOrder == ColMajor) && UseLhs) || (((StorageOrder == RowMajor) && !UseLhs))) {
478 EIGEN_IF_CONSTEXPR (UseLhs) {
479 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i);
480 cblock.packet[1] = lhs2.template loadPacket<PacketC>(2, i);
481 } else {
482 cblock.packet[0] = lhs2.template loadPacket<PacketC>(i, 0);
483 cblock.packet[1] = lhs2.template loadPacket<PacketC>(i, 2);
484 }
485 } else {
486 EIGEN_IF_CONSTEXPR (UseLhs) {
487 cblock.packet[0] = pload2(lhs2(0, i), lhs2(1, i));
488 cblock.packet[1] = pload2(lhs2(2, i), lhs2(3, i));
489 } else {
490 cblock.packet[0] = pload2(lhs2(i, 0), lhs2(i, 1));
491 cblock.packet[1] = pload2(lhs2(i, 2), lhs2(i, 3));
492 }
493 }
494
495 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL32);
496 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG32);
497
498 EIGEN_IF_CONSTEXPR (Conjugate) {
499 blocki.packet[0] = -blocki.packet[0];
500 }
501
502 pstore<Scalar>(blockAt + rir, blockr.packet[0]);
503 pstore<Scalar>(blockAt + rii, blocki.packet[0]);
504
505 rir += vectorSize;
506 rii += vectorSize;
507 }
508
509 rir += ((PanelMode) ? (vectorSize * (2 * stride - depth)) : vectorDelta);
510 }
511
512 EIGEN_IF_CONSTEXPR (!UseLhs) {
513 EIGEN_IF_CONSTEXPR (PanelMode) rir -= (offset * (vectorSize - 1));
514
515 for (; j < rows; j++) {
516 const DataMapper lhs2 = lhs.getSubMapper(0, j);
517 rii = rir + ((PanelMode) ? stride : depth);
518
519 for (Index i = 0; i < depth; i++) {
520 blockAt[rir] = lhs2(i, 0).real();
521
522 EIGEN_IF_CONSTEXPR (Conjugate)
523 blockAt[rii] = -lhs2(i, 0).imag();
524 else
525 blockAt[rii] = lhs2(i, 0).imag();
526
527 rir += 1;
528 rii += 1;
529 }
530
531 rir += ((PanelMode) ? (2 * stride - depth) : depth);
532 }
533 } else {
534 if (j < rows) {
535 EIGEN_IF_CONSTEXPR (PanelMode) rir += (offset * (rows - j - vectorSize));
536 rii = rir + (((PanelMode) ? stride : depth) * (rows - j));
537
538 for (Index i = 0; i < depth; i++) {
539 Index k = j;
540 for (; k < rows; k++) {
541 blockAt[rir] = lhs(k, i).real();
542
543 EIGEN_IF_CONSTEXPR (Conjugate)
544 blockAt[rii] = -lhs(k, i).imag();
545 else
546 blockAt[rii] = lhs(k, i).imag();
547
548 rir += 1;
549 rii += 1;
550 }
551 }
552 }
553 }
554 }
555};
556
557// General template for lhs & rhs packing.
558template <typename Scalar, typename DataMapper, typename Packet, int StorageOrder, bool PanelMode, bool UseLhs>
559struct dhs_pack {
560 using Index = mapper_index_t<DataMapper>;
561 template <Index n>
562 EIGEN_ALWAYS_INLINE void dhs_copy(Scalar* blockA, const DataMapper& lhs2, Index& i, Index& ri, Index depth,
563 const Index vectorSize) {
564 PacketBlock<Packet, 4> block[n];
565
566 for (; i + n * vectorSize <= depth; i += n * vectorSize) {
567 for (Index k = 0; k < n; k++) {
568 EIGEN_IF_CONSTEXPR (UseLhs) {
569 bload<DataMapper, Packet, 4, StorageOrder, false, 4>(block[k], lhs2, 0, i + k * vectorSize);
570 } else {
571 bload<DataMapper, Packet, 4, StorageOrder, false, 4>(block[k], lhs2, i + k * vectorSize, 0);
572 }
573 }
574
575 EIGEN_IF_CONSTEXPR (((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) {
576 for (Index k = 0; k < n; k++) {
577 ptranspose(block[k]);
578 }
579 }
580
581 for (Index k = 0; k < n; k++) {
582 storeBlock<Scalar, Packet, 4>(blockA + ri + k * 4 * vectorSize, block[k]);
583 }
584
585 ri += n * 4 * vectorSize;
586 }
587 }
588
589 EIGEN_STRONG_INLINE void operator()(Scalar* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride,
590 Index offset) {
591 const Index vectorSize = quad_traits<Scalar>::vectorsize;
592 Index ri = 0, j = 0;
593
594 for (; j + vectorSize <= rows; j += vectorSize) {
595 const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j);
596 Index i = 0;
597
598 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset;
599
600 dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize);
601 dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize);
602 dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize);
603
604 for (; i < depth; i++) {
605 EIGEN_IF_CONSTEXPR (((StorageOrder == RowMajor) && UseLhs) || ((StorageOrder == ColMajor) && !UseLhs)) {
606 EIGEN_IF_CONSTEXPR (UseLhs) {
607 blockA[ri + 0] = lhs2(0, i);
608 blockA[ri + 1] = lhs2(1, i);
609 blockA[ri + 2] = lhs2(2, i);
610 blockA[ri + 3] = lhs2(3, i);
611 } else {
612 blockA[ri + 0] = lhs2(i, 0);
613 blockA[ri + 1] = lhs2(i, 1);
614 blockA[ri + 2] = lhs2(i, 2);
615 blockA[ri + 3] = lhs2(i, 3);
616 }
617 } else {
618 Packet lhsV;
619 EIGEN_IF_CONSTEXPR (UseLhs) {
620 lhsV = lhs2.template loadPacket<Packet>(0, i);
621 } else {
622 lhsV = lhs2.template loadPacket<Packet>(i, 0);
623 }
624 pstore<Scalar>(blockA + ri, lhsV);
625 }
626
627 ri += vectorSize;
628 }
629
630 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth);
631 }
632
633 EIGEN_IF_CONSTEXPR (!UseLhs) {
634 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset;
635
636 for (; j < rows; j++) {
637 const DataMapper lhs2 = lhs.getSubMapper(0, j);
638 for (Index i = 0; i < depth; i++) {
639 blockA[ri] = lhs2(i, 0);
640 ri += 1;
641 }
642
643 EIGEN_IF_CONSTEXPR (PanelMode) ri += stride - depth;
644 }
645 } else {
646 if (j < rows) {
647 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j);
648
649 for (Index i = 0; i < depth; i++) {
650 Index k = j;
651 for (; k < rows; k++) {
652 blockA[ri] = lhs(k, i);
653 ri += 1;
654 }
655 }
656 }
657 }
658 }
659};
660
661// General template for lhs packing, float64 specialization.
662template <typename DataMapper, int StorageOrder, bool PanelMode>
663struct dhs_pack<double, DataMapper, Packet2d, StorageOrder, PanelMode, true> {
664 using Index = mapper_index_t<DataMapper>;
665 template <Index n>
666 EIGEN_ALWAYS_INLINE void dhs_copy(double* blockA, const DataMapper& lhs2, Index& i, Index& ri, Index depth,
667 const Index vectorSize) {
668 PacketBlock<Packet2d, 2> block[n];
669
670 for (; i + n * vectorSize <= depth; i += n * vectorSize) {
671 for (Index k = 0; k < n; k++) {
672 EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) {
673 block[k].packet[0] = lhs2.template loadPacket<Packet2d>(0, i + k * vectorSize);
674 block[k].packet[1] = lhs2.template loadPacket<Packet2d>(1, i + k * vectorSize);
675 } else {
676 block[k].packet[0] = lhs2.template loadPacket<Packet2d>(0, i + k * vectorSize + 0);
677 block[k].packet[1] = lhs2.template loadPacket<Packet2d>(0, i + k * vectorSize + 1);
678 }
679 }
680
681 EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) {
682 for (Index k = 0; k < n; k++) {
683 ptranspose(block[k]);
684 }
685 }
686
687 for (Index k = 0; k < n; k++) {
688 storeBlock<double, Packet2d, 2>(blockA + ri + k * 2 * vectorSize, block[k]);
689 }
690
691 ri += n * 2 * vectorSize;
692 }
693 }
694
695 EIGEN_STRONG_INLINE void operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride,
696 Index offset) {
697 const Index vectorSize = quad_traits<double>::vectorsize;
698 Index ri = 0, j = 0;
699
700 for (; j + vectorSize <= rows; j += vectorSize) {
701 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
702 Index i = 0;
703
704 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset;
705
706 dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize);
707 dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize);
708 dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize);
709
710 for (; i < depth; i++) {
711 EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) {
712 blockA[ri + 0] = lhs2(0, i);
713 blockA[ri + 1] = lhs2(1, i);
714 } else {
715 Packet2d lhsV = lhs2.template loadPacket<Packet2d>(0, i);
716 pstore<double>(blockA + ri, lhsV);
717 }
718
719 ri += vectorSize;
720 }
721
722 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth);
723 }
724
725 if (j < rows) {
726 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j);
727
728 for (Index i = 0; i < depth; i++) {
729 Index k = j;
730 for (; k < rows; k++) {
731 blockA[ri] = lhs(k, i);
732 ri += 1;
733 }
734 }
735 }
736 }
737};
738
739// General template for rhs packing, float64 specialization.
740template <typename DataMapper, int StorageOrder, bool PanelMode>
741struct dhs_pack<double, DataMapper, Packet2d, StorageOrder, PanelMode, false> {
742 using Index = mapper_index_t<DataMapper>;
743 template <Index n>
744 EIGEN_ALWAYS_INLINE void dhs_copy(double* blockB, const DataMapper& rhs2, Index& i, Index& ri, Index depth,
745 const Index vectorSize) {
746 PacketBlock<Packet2d, 2> block1[n], block2[n];
747 PacketBlock<Packet2d, 4> block3[n];
748
749 for (; i + n * vectorSize <= depth; i += n * vectorSize) {
750 for (Index k = 0; k < n; k++) {
751 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
752 block1[k].packet[0] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 0);
753 block1[k].packet[1] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 1);
754 block2[k].packet[0] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 2);
755 block2[k].packet[1] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 3);
756 } else {
757 block3[k].packet[0] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 0, 0); //[a1 a2]
758 block3[k].packet[1] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 0, 2); //[a3 a4]
759 block3[k].packet[2] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 1, 0); //[b1 b2]
760 block3[k].packet[3] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 1, 2); //[b3 b4]
761 }
762 }
763
764 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
765 for (Index k = 0; k < n; k++) {
766 ptranspose(block1[k]);
767 ptranspose(block2[k]);
768 }
769 }
770
771 for (Index k = 0; k < n; k++) {
772 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
773 pstore<double>(blockB + ri + k * 4 * vectorSize, block1[k].packet[0]);
774 pstore<double>(blockB + ri + k * 4 * vectorSize + 2, block2[k].packet[0]);
775 pstore<double>(blockB + ri + k * 4 * vectorSize + 4, block1[k].packet[1]);
776 pstore<double>(blockB + ri + k * 4 * vectorSize + 6, block2[k].packet[1]);
777 } else {
778 storeBlock<double, Packet2d, 4>(blockB + ri + k * 4 * vectorSize, block3[k]);
779 }
780 }
781
782 ri += n * 4 * vectorSize;
783 }
784 }
785
786 EIGEN_STRONG_INLINE void operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride,
787 Index offset) {
788 const Index vectorSize = quad_traits<double>::vectorsize;
789 Index ri = 0, j = 0;
790
791 for (; j + 2 * vectorSize <= cols; j += 2 * vectorSize) {
792 const DataMapper rhs2 = rhs.getSubMapper(0, j);
793 Index i = 0;
794
795 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (2 * vectorSize);
796
797 dhs_copy<4>(blockB, rhs2, i, ri, depth, vectorSize);
798 dhs_copy<2>(blockB, rhs2, i, ri, depth, vectorSize);
799 dhs_copy<1>(blockB, rhs2, i, ri, depth, vectorSize);
800
801 for (; i < depth; i++) {
802 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
803 blockB[ri + 0] = rhs2(i, 0);
804 blockB[ri + 1] = rhs2(i, 1);
805
806 ri += vectorSize;
807
808 blockB[ri + 0] = rhs2(i, 2);
809 blockB[ri + 1] = rhs2(i, 3);
810 } else {
811 Packet2d rhsV = rhs2.template loadPacket<Packet2d>(i, 0);
812 pstore<double>(blockB + ri, rhsV);
813
814 ri += vectorSize;
815
816 rhsV = rhs2.template loadPacket<Packet2d>(i, 2);
817 pstore<double>(blockB + ri, rhsV);
818 }
819 ri += vectorSize;
820 }
821
822 EIGEN_IF_CONSTEXPR (PanelMode) ri += (2 * vectorSize) * (stride - offset - depth);
823 }
824
825 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset;
826
827 for (; j < cols; j++) {
828 const DataMapper rhs2 = rhs.getSubMapper(0, j);
829 for (Index i = 0; i < depth; i++) {
830 blockB[ri] = rhs2(i, 0);
831 ri += 1;
832 }
833
834 EIGEN_IF_CONSTEXPR (PanelMode) ri += stride - depth;
835 }
836 }
837};
838
839// General template for lhs packing, bfloat16 specialization.
840template <typename DataMapper, int StorageOrder, bool PanelMode>
841struct dhs_pack<bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true> {
842 using Index = mapper_index_t<DataMapper>;
843 EIGEN_STRONG_INLINE void operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride,
844 Index offset) {
845 const Index vectorSize = quad_traits<bfloat16>::vectorsize;
846 Index ri = 0, j = 0;
847
848 for (; j + 2 * vectorSize <= rows; j += 2 * vectorSize) {
849 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
850 Index i = 0;
851
852 EIGEN_IF_CONSTEXPR (PanelMode) ri += 2 * vectorSize * offset;
853
854 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
855 for (; i + 2 <= depth; i += 2) {
856 PacketBlock<Packet8bf, 4> block;
857
858 block.packet[0] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
859 block.packet[1] = lhs2.template loadPacket<Packet8bf>(1 * vectorSize, i + 0);
860 block.packet[2] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 1);
861 block.packet[3] = lhs2.template loadPacket<Packet8bf>(1 * vectorSize, i + 1);
862
863 Packet8bf t0, t1;
864 t0 = vec_mergeh(block.packet[0].m_val, block.packet[2].m_val);
865 t1 = vec_mergel(block.packet[0].m_val, block.packet[2].m_val);
866 block.packet[2] = vec_mergeh(block.packet[1].m_val, block.packet[3].m_val);
867 block.packet[3] = vec_mergel(block.packet[1].m_val, block.packet[3].m_val);
868 block.packet[0] = t0;
869 block.packet[1] = t1;
870
871 storeBlock<bfloat16, Packet8bf, 4>(blockA + ri, block);
872
873 ri += 2 * 2 * vectorSize;
874 }
875 if (depth & 1) {
876 PacketBlock<Packet8bf, 2> block;
877
878 block.packet[0] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
879 block.packet[1] = lhs2.template loadPacket<Packet8bf>(1 * vectorSize, i + 0);
880
881 storeBlock<bfloat16, Packet8bf, 2>(blockA + ri, block);
882
883 ri += 2 * vectorSize;
884 }
885 } else {
886 for (; i + vectorSize <= depth; i += vectorSize) {
887 PacketBlock<Packet8bf, 8> block1, block2;
888
889 bload<DataMapper, Packet8bf, 8, StorageOrder, false, 8>(block1, lhs2, 0 * vectorSize, i);
890 bload<DataMapper, Packet8bf, 8, StorageOrder, false, 8>(block2, lhs2, 1 * vectorSize, i);
891
892 Packet4ui v1[8], v2[8];
893
894 v1[0] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[0].m_val),
895 reinterpret_cast<Packet4ui>(block1.packet[1].m_val));
896 v1[1] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[0].m_val),
897 reinterpret_cast<Packet4ui>(block1.packet[1].m_val));
898 v1[2] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[2].m_val),
899 reinterpret_cast<Packet4ui>(block1.packet[3].m_val));
900 v1[3] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[2].m_val),
901 reinterpret_cast<Packet4ui>(block1.packet[3].m_val));
902 v1[4] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[4].m_val),
903 reinterpret_cast<Packet4ui>(block1.packet[5].m_val));
904 v1[5] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[4].m_val),
905 reinterpret_cast<Packet4ui>(block1.packet[5].m_val));
906 v1[6] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[6].m_val),
907 reinterpret_cast<Packet4ui>(block1.packet[7].m_val));
908 v1[7] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[6].m_val),
909 reinterpret_cast<Packet4ui>(block1.packet[7].m_val));
910 v2[0] = vec_mergeh(reinterpret_cast<Packet4ui>(block2.packet[0].m_val),
911 reinterpret_cast<Packet4ui>(block2.packet[1].m_val));
912 v2[1] = vec_mergel(reinterpret_cast<Packet4ui>(block2.packet[0].m_val),
913 reinterpret_cast<Packet4ui>(block2.packet[1].m_val));
914 v2[2] = vec_mergeh(reinterpret_cast<Packet4ui>(block2.packet[2].m_val),
915 reinterpret_cast<Packet4ui>(block2.packet[3].m_val));
916 v2[3] = vec_mergel(reinterpret_cast<Packet4ui>(block2.packet[2].m_val),
917 reinterpret_cast<Packet4ui>(block2.packet[3].m_val));
918 v2[4] = vec_mergeh(reinterpret_cast<Packet4ui>(block2.packet[4].m_val),
919 reinterpret_cast<Packet4ui>(block2.packet[5].m_val));
920 v2[5] = vec_mergel(reinterpret_cast<Packet4ui>(block2.packet[4].m_val),
921 reinterpret_cast<Packet4ui>(block2.packet[5].m_val));
922 v2[6] = vec_mergeh(reinterpret_cast<Packet4ui>(block2.packet[6].m_val),
923 reinterpret_cast<Packet4ui>(block2.packet[7].m_val));
924 v2[7] = vec_mergel(reinterpret_cast<Packet4ui>(block2.packet[6].m_val),
925 reinterpret_cast<Packet4ui>(block2.packet[7].m_val));
926
927#ifdef EIGEN_VECTORIZE_VSX
928 block1.packet[0] = reinterpret_cast<Packet8us>(
929 vec_mergeh(reinterpret_cast<Packet2ul>(v1[0]), reinterpret_cast<Packet2ul>(v1[2])));
930 block1.packet[2] = reinterpret_cast<Packet8us>(
931 vec_mergel(reinterpret_cast<Packet2ul>(v1[0]), reinterpret_cast<Packet2ul>(v1[2])));
932 block1.packet[4] = reinterpret_cast<Packet8us>(
933 vec_mergeh(reinterpret_cast<Packet2ul>(v1[1]), reinterpret_cast<Packet2ul>(v1[3])));
934 block1.packet[6] = reinterpret_cast<Packet8us>(
935 vec_mergel(reinterpret_cast<Packet2ul>(v1[1]), reinterpret_cast<Packet2ul>(v1[3])));
936 block1.packet[1] = reinterpret_cast<Packet8us>(
937 vec_mergeh(reinterpret_cast<Packet2ul>(v1[4]), reinterpret_cast<Packet2ul>(v1[6])));
938 block1.packet[3] = reinterpret_cast<Packet8us>(
939 vec_mergel(reinterpret_cast<Packet2ul>(v1[4]), reinterpret_cast<Packet2ul>(v1[6])));
940 block1.packet[5] = reinterpret_cast<Packet8us>(
941 vec_mergeh(reinterpret_cast<Packet2ul>(v1[5]), reinterpret_cast<Packet2ul>(v1[7])));
942 block1.packet[7] = reinterpret_cast<Packet8us>(
943 vec_mergel(reinterpret_cast<Packet2ul>(v1[5]), reinterpret_cast<Packet2ul>(v1[7])));
944 block2.packet[0] = reinterpret_cast<Packet8us>(
945 vec_mergeh(reinterpret_cast<Packet2ul>(v2[0]), reinterpret_cast<Packet2ul>(v2[2])));
946 block2.packet[2] = reinterpret_cast<Packet8us>(
947 vec_mergel(reinterpret_cast<Packet2ul>(v2[0]), reinterpret_cast<Packet2ul>(v2[2])));
948 block2.packet[4] = reinterpret_cast<Packet8us>(
949 vec_mergeh(reinterpret_cast<Packet2ul>(v2[1]), reinterpret_cast<Packet2ul>(v2[3])));
950 block2.packet[6] = reinterpret_cast<Packet8us>(
951 vec_mergel(reinterpret_cast<Packet2ul>(v2[1]), reinterpret_cast<Packet2ul>(v2[3])));
952 block2.packet[1] = reinterpret_cast<Packet8us>(
953 vec_mergeh(reinterpret_cast<Packet2ul>(v2[4]), reinterpret_cast<Packet2ul>(v2[6])));
954 block2.packet[3] = reinterpret_cast<Packet8us>(
955 vec_mergel(reinterpret_cast<Packet2ul>(v2[4]), reinterpret_cast<Packet2ul>(v2[6])));
956 block2.packet[5] = reinterpret_cast<Packet8us>(
957 vec_mergeh(reinterpret_cast<Packet2ul>(v2[5]), reinterpret_cast<Packet2ul>(v2[7])));
958 block2.packet[7] = reinterpret_cast<Packet8us>(
959 vec_mergel(reinterpret_cast<Packet2ul>(v2[5]), reinterpret_cast<Packet2ul>(v2[7])));
960#else
961 block1.packet[0] = reinterpret_cast<Packet8us>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_HI));
962 block1.packet[2] = reinterpret_cast<Packet8us>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_LO));
963 block1.packet[4] = reinterpret_cast<Packet8us>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_HI));
964 block1.packet[6] = reinterpret_cast<Packet8us>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_LO));
965 block1.packet[1] = reinterpret_cast<Packet8us>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_HI));
966 block1.packet[3] = reinterpret_cast<Packet8us>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_LO));
967 block1.packet[5] = reinterpret_cast<Packet8us>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_HI));
968 block1.packet[7] = reinterpret_cast<Packet8us>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_LO));
969 block2.packet[0] = reinterpret_cast<Packet8us>(vec_perm(v2[0], v2[2], p16uc_TRANSPOSE64_HI));
970 block2.packet[2] = reinterpret_cast<Packet8us>(vec_perm(v2[0], v2[2], p16uc_TRANSPOSE64_LO));
971 block2.packet[4] = reinterpret_cast<Packet8us>(vec_perm(v2[1], v2[3], p16uc_TRANSPOSE64_HI));
972 block2.packet[6] = reinterpret_cast<Packet8us>(vec_perm(v2[1], v2[3], p16uc_TRANSPOSE64_LO));
973 block2.packet[1] = reinterpret_cast<Packet8us>(vec_perm(v2[4], v2[6], p16uc_TRANSPOSE64_HI));
974 block2.packet[3] = reinterpret_cast<Packet8us>(vec_perm(v2[4], v2[6], p16uc_TRANSPOSE64_LO));
975 block2.packet[5] = reinterpret_cast<Packet8us>(vec_perm(v2[5], v2[7], p16uc_TRANSPOSE64_HI));
976 block2.packet[7] = reinterpret_cast<Packet8us>(vec_perm(v2[5], v2[7], p16uc_TRANSPOSE64_LO));
977#endif
978
979 for (Index M = 0; M < 8; M += 2) {
980 pstore<bfloat16>(blockA + ri + (0 * vectorSize) + (2 * vectorSize * M), block1.packet[M + 0]);
981 pstore<bfloat16>(blockA + ri + (1 * vectorSize) + (2 * vectorSize * M), block1.packet[M + 1]);
982 pstore<bfloat16>(blockA + ri + (2 * vectorSize) + (2 * vectorSize * M), block2.packet[M + 0]);
983 pstore<bfloat16>(blockA + ri + (3 * vectorSize) + (2 * vectorSize * M), block2.packet[M + 1]);
984 }
985
986 ri += 2 * vectorSize * vectorSize;
987 }
988 for (; i + 2 <= depth; i += 2) {
989 for (Index M = 0; M < 2 * vectorSize; M++) {
990 blockA[ri + (M * 2) + 0] = lhs2(M, i + 0);
991 blockA[ri + (M * 2) + 1] = lhs2(M, i + 1);
992 }
993
994 ri += 2 * 2 * vectorSize;
995 }
996 if (depth & 1) {
997 for (Index M = 0; M < 2 * vectorSize; M++) {
998 blockA[ri + M] = lhs2(M, i);
999 }
1000 ri += 2 * vectorSize;
1001 }
1002 }
1003
1004 EIGEN_IF_CONSTEXPR (PanelMode) ri += 2 * vectorSize * (stride - offset - depth);
1005 }
1006 for (; j + vectorSize <= rows; j += vectorSize) {
1007 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
1008 Index i = 0;
1009
1010 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset;
1011
1012 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1013 for (; i + 2 <= depth; i += 2) {
1014 PacketBlock<Packet8bf, 2> block;
1015
1016 block.packet[0] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
1017 block.packet[1] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 1);
1018
1019 Packet8bf t0;
1020 t0 = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1021 block.packet[1] = vec_mergel(block.packet[0].m_val, block.packet[1].m_val);
1022 block.packet[0] = t0;
1023
1024 storeBlock<bfloat16, Packet8bf, 2>(blockA + ri, block);
1025
1026 ri += 2 * vectorSize;
1027 }
1028 if (depth & 1) {
1029 Packet8bf lhsV = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
1030 pstore<bfloat16>(blockA + ri, lhsV);
1031
1032 ri += vectorSize;
1033 }
1034 } else {
1035 for (; i + vectorSize <= depth; i += vectorSize) {
1036 PacketBlock<Packet8bf, 8> block1;
1037
1038 bload<DataMapper, Packet8bf, 8, StorageOrder, false, 8>(block1, lhs2, 0 * vectorSize, i);
1039
1040 Packet4ui v1[8];
1041
1042 // This is transposing and interleaving data
1043 v1[0] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[0].m_val),
1044 reinterpret_cast<Packet4ui>(block1.packet[1].m_val));
1045 v1[1] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[0].m_val),
1046 reinterpret_cast<Packet4ui>(block1.packet[1].m_val));
1047 v1[2] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[2].m_val),
1048 reinterpret_cast<Packet4ui>(block1.packet[3].m_val));
1049 v1[3] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[2].m_val),
1050 reinterpret_cast<Packet4ui>(block1.packet[3].m_val));
1051 v1[4] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[4].m_val),
1052 reinterpret_cast<Packet4ui>(block1.packet[5].m_val));
1053 v1[5] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[4].m_val),
1054 reinterpret_cast<Packet4ui>(block1.packet[5].m_val));
1055 v1[6] = vec_mergeh(reinterpret_cast<Packet4ui>(block1.packet[6].m_val),
1056 reinterpret_cast<Packet4ui>(block1.packet[7].m_val));
1057 v1[7] = vec_mergel(reinterpret_cast<Packet4ui>(block1.packet[6].m_val),
1058 reinterpret_cast<Packet4ui>(block1.packet[7].m_val));
1059
1060#ifdef EIGEN_VECTORIZE_VSX
1061 block1.packet[0] = reinterpret_cast<Packet8us>(
1062 vec_mergeh(reinterpret_cast<Packet2ul>(v1[0]), reinterpret_cast<Packet2ul>(v1[2])));
1063 block1.packet[2] = reinterpret_cast<Packet8us>(
1064 vec_mergel(reinterpret_cast<Packet2ul>(v1[0]), reinterpret_cast<Packet2ul>(v1[2])));
1065 block1.packet[4] = reinterpret_cast<Packet8us>(
1066 vec_mergeh(reinterpret_cast<Packet2ul>(v1[1]), reinterpret_cast<Packet2ul>(v1[3])));
1067 block1.packet[6] = reinterpret_cast<Packet8us>(
1068 vec_mergel(reinterpret_cast<Packet2ul>(v1[1]), reinterpret_cast<Packet2ul>(v1[3])));
1069 block1.packet[1] = reinterpret_cast<Packet8us>(
1070 vec_mergeh(reinterpret_cast<Packet2ul>(v1[4]), reinterpret_cast<Packet2ul>(v1[6])));
1071 block1.packet[3] = reinterpret_cast<Packet8us>(
1072 vec_mergel(reinterpret_cast<Packet2ul>(v1[4]), reinterpret_cast<Packet2ul>(v1[6])));
1073 block1.packet[5] = reinterpret_cast<Packet8us>(
1074 vec_mergeh(reinterpret_cast<Packet2ul>(v1[5]), reinterpret_cast<Packet2ul>(v1[7])));
1075 block1.packet[7] = reinterpret_cast<Packet8us>(
1076 vec_mergel(reinterpret_cast<Packet2ul>(v1[5]), reinterpret_cast<Packet2ul>(v1[7])));
1077#else
1078 block1.packet[0] = reinterpret_cast<Packet8us>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_HI));
1079 block1.packet[2] = reinterpret_cast<Packet8us>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_LO));
1080 block1.packet[4] = reinterpret_cast<Packet8us>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_HI));
1081 block1.packet[6] = reinterpret_cast<Packet8us>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_LO));
1082 block1.packet[1] = reinterpret_cast<Packet8us>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_HI));
1083 block1.packet[3] = reinterpret_cast<Packet8us>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_LO));
1084 block1.packet[5] = reinterpret_cast<Packet8us>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_HI));
1085 block1.packet[7] = reinterpret_cast<Packet8us>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_LO));
1086#endif
1087
1088 for (Index M = 0; M < 8; M++) {
1089 pstore<bfloat16>(blockA + ri + (vectorSize * M), block1.packet[M]);
1090 }
1091
1092 ri += vectorSize * vectorSize;
1093 }
1094 for (; i + 2 <= depth; i += 2) {
1095 for (Index M = 0; M < vectorSize; M++) {
1096 blockA[ri + (M * 2) + 0] = lhs2(M, i + 0);
1097 blockA[ri + (M * 2) + 1] = lhs2(M, i + 1);
1098 }
1099
1100 ri += 2 * vectorSize;
1101 }
1102 if (depth & 1) {
1103 for (Index M = 0; M < vectorSize; M++) {
1104 blockA[ri + M] = lhs2(M, i);
1105 }
1106
1107 ri += vectorSize;
1108 }
1109 }
1110
1111 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth);
1112 }
1113 if (j + 4 <= rows) {
1114 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
1115 Index i = 0;
1116
1117 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * offset;
1118
1119 for (; i + 2 <= depth; i += 2) {
1120 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1121 PacketBlock<Packet8bf, 2> block;
1122
1123 block.packet[0] = lhs2.template loadPacketPartial<Packet8bf>(0, i + 0, 4);
1124 block.packet[1] = lhs2.template loadPacketPartial<Packet8bf>(0, i + 1, 4);
1125
1126 block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1127
1128 pstore<bfloat16>(blockA + ri, block.packet[0]);
1129 } else {
1130 blockA[ri + 0] = lhs2(0, i + 0);
1131 blockA[ri + 1] = lhs2(0, i + 1);
1132 blockA[ri + 2] = lhs2(1, i + 0);
1133 blockA[ri + 3] = lhs2(1, i + 1);
1134 blockA[ri + 4] = lhs2(2, i + 0);
1135 blockA[ri + 5] = lhs2(2, i + 1);
1136 blockA[ri + 6] = lhs2(3, i + 0);
1137 blockA[ri + 7] = lhs2(3, i + 1);
1138 }
1139
1140 ri += 2 * 4;
1141 }
1142 if (depth & 1) {
1143 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1144 Packet8bf lhsV = lhs2.template loadPacketPartial<Packet8bf>(0, i + 0, 4);
1145
1146 pstore_partial<bfloat16>(blockA + ri, lhsV, 4);
1147 } else {
1148 blockA[ri + 0] = lhs2(0, i);
1149 blockA[ri + 1] = lhs2(1, i);
1150 blockA[ri + 2] = lhs2(2, i);
1151 blockA[ri + 3] = lhs2(3, i);
1152 }
1153
1154 ri += 4;
1155 }
1156
1157 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * (stride - offset - depth);
1158 j += 4;
1159 }
1160
1161 if (j < rows) {
1162 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j);
1163
1164 Index i = 0;
1165 for (; i + 2 <= depth; i += 2) {
1166 Index k = j;
1167 for (; k < rows; k++) {
1168 blockA[ri + 0] = lhs(k, i + 0);
1169 blockA[ri + 1] = lhs(k, i + 1);
1170 ri += 2;
1171 }
1172 }
1173 if (depth & 1) {
1174 for (; j < rows; j++) {
1175 blockA[ri] = lhs(j, i);
1176 ri += 1;
1177 }
1178 }
1179 }
1180 }
1181};
1182
1183// General template for rhs packing, bfloat16 specialization.
1184template <typename DataMapper, int StorageOrder, bool PanelMode>
1185struct dhs_pack<bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, false> {
1186 using Index = mapper_index_t<DataMapper>;
1187 EIGEN_STRONG_INLINE void operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride,
1188 Index offset) {
1189 const Index vectorSize = quad_traits<bfloat16>::vectorsize;
1190 Index ri = 0, j = 0;
1191
1192 for (; j + 4 <= cols; j += 4) {
1193 const DataMapper rhs2 = rhs.getSubMapper(0, j);
1194 Index i = 0;
1195
1196 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * offset;
1197
1198 for (; i + vectorSize <= depth; i += vectorSize) {
1199 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1200 PacketBlock<Packet8bf, 4> block;
1201
1202 bload<DataMapper, Packet8bf, 4, StorageOrder, false, 4>(block, rhs2, i, 0);
1203
1204 Packet4ui t0, t1, t2, t3;
1205
1206 t0 = vec_mergeh(reinterpret_cast<Packet4ui>(block.packet[0].m_val),
1207 reinterpret_cast<Packet4ui>(block.packet[1].m_val));
1208 t1 = vec_mergel(reinterpret_cast<Packet4ui>(block.packet[0].m_val),
1209 reinterpret_cast<Packet4ui>(block.packet[1].m_val));
1210 t2 = vec_mergeh(reinterpret_cast<Packet4ui>(block.packet[2].m_val),
1211 reinterpret_cast<Packet4ui>(block.packet[3].m_val));
1212 t3 = vec_mergel(reinterpret_cast<Packet4ui>(block.packet[2].m_val),
1213 reinterpret_cast<Packet4ui>(block.packet[3].m_val));
1214
1215#ifdef EIGEN_VECTORIZE_VSX
1216 block.packet[0] =
1217 reinterpret_cast<Packet8us>(vec_mergeh(reinterpret_cast<Packet2ul>(t0), reinterpret_cast<Packet2ul>(t2)));
1218 block.packet[1] =
1219 reinterpret_cast<Packet8us>(vec_mergel(reinterpret_cast<Packet2ul>(t0), reinterpret_cast<Packet2ul>(t2)));
1220 block.packet[2] =
1221 reinterpret_cast<Packet8us>(vec_mergeh(reinterpret_cast<Packet2ul>(t1), reinterpret_cast<Packet2ul>(t3)));
1222 block.packet[3] =
1223 reinterpret_cast<Packet8us>(vec_mergel(reinterpret_cast<Packet2ul>(t1), reinterpret_cast<Packet2ul>(t3)));
1224#else
1225 block.packet[0] = reinterpret_cast<Packet8us>(vec_perm(t0, t2, p16uc_TRANSPOSE64_HI));
1226 block.packet[1] = reinterpret_cast<Packet8us>(vec_perm(t0, t2, p16uc_TRANSPOSE64_LO));
1227 block.packet[2] = reinterpret_cast<Packet8us>(vec_perm(t1, t3, p16uc_TRANSPOSE64_HI));
1228 block.packet[3] = reinterpret_cast<Packet8us>(vec_perm(t1, t3, p16uc_TRANSPOSE64_LO));
1229#endif
1230
1231 storeBlock<bfloat16, Packet8bf, 4>(blockB + ri, block);
1232 } else {
1233 PacketBlock<Packet8bf, 8> block;
1234
1235 for (int M = 0; M < 8; M++) {
1236 block.packet[M] = rhs2.template loadPacketPartial<Packet8bf>(i + M, 0, 4);
1237 }
1238
1239 block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1240 block.packet[1] = vec_mergeh(block.packet[2].m_val, block.packet[3].m_val);
1241 block.packet[2] = vec_mergeh(block.packet[4].m_val, block.packet[5].m_val);
1242 block.packet[3] = vec_mergeh(block.packet[6].m_val, block.packet[7].m_val);
1243
1244 const Index size = 16 / sizeof(bfloat16);
1245
1246 for (int M = 0; M < 4; M++) {
1247 pstore<bfloat16>(blockB + ri + (M * size), block.packet[M]);
1248 }
1249 }
1250
1251 ri += 4 * vectorSize;
1252 }
1253 for (; i + 2 <= depth; i += 2) {
1254 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1255 blockB[ri + 0] = rhs2(i + 0, 0);
1256 blockB[ri + 1] = rhs2(i + 1, 0);
1257 blockB[ri + 2] = rhs2(i + 0, 1);
1258 blockB[ri + 3] = rhs2(i + 1, 1);
1259 blockB[ri + 4] = rhs2(i + 0, 2);
1260 blockB[ri + 5] = rhs2(i + 1, 2);
1261 blockB[ri + 6] = rhs2(i + 0, 3);
1262 blockB[ri + 7] = rhs2(i + 1, 3);
1263 } else {
1264 PacketBlock<Packet8bf, 2> block;
1265
1266 for (int M = 0; M < 2; M++) {
1267 block.packet[M] = rhs2.template loadPacketPartial<Packet8bf>(i + M, 0, 4);
1268 }
1269
1270 block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1271
1272 pstore<bfloat16>(blockB + ri, block.packet[0]);
1273 }
1274
1275 ri += 4 * 2;
1276 }
1277 if (depth & 1) {
1278 blockB[ri + 0] = rhs2(i, 0);
1279 blockB[ri + 1] = rhs2(i, 1);
1280 blockB[ri + 2] = rhs2(i, 2);
1281 blockB[ri + 3] = rhs2(i, 3);
1282
1283 ri += 4;
1284 }
1285
1286 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * (stride - offset - depth);
1287 }
1288
1289 if (j < cols) {
1290 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (cols - j);
1291
1292 Index i = 0;
1293 for (; i + 2 <= depth; i += 2) {
1294 Index k = j;
1295 for (; k < cols; k++) {
1296 blockB[ri + 0] = rhs(i + 0, k);
1297 blockB[ri + 1] = rhs(i + 1, k);
1298 ri += 2;
1299 }
1300 }
1301 if (depth & 1) {
1302 for (; j < cols; j++) {
1303 blockB[ri] = rhs(i, j);
1304 ri += 1;
1305 }
1306 }
1307 }
1308 }
1309};
1310
1311// General template for lhs complex packing, float64 specialization.
1312template <typename DataMapper, typename Packet, typename PacketC, int StorageOrder, bool Conjugate, bool PanelMode>
1313struct dhs_cpack<double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, true> {
1314 using Index = mapper_index_t<DataMapper>;
1315 EIGEN_ALWAYS_INLINE void dhs_ccopy(double* blockAt, const DataMapper& lhs2, Index& i, Index& rir, Index& rii,
1316 Index depth, const Index vectorSize) {
1317 PacketBlock<Packet, 2> blockr, blocki;
1318 PacketBlock<PacketC, 4> cblock;
1319
1320 for (; i + vectorSize <= depth; i += vectorSize) {
1321 EIGEN_IF_CONSTEXPR (StorageOrder == ColMajor) {
1322 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i + 0); //[a1 a1i]
1323 cblock.packet[1] = lhs2.template loadPacket<PacketC>(0, i + 1); //[b1 b1i]
1324
1325 cblock.packet[2] = lhs2.template loadPacket<PacketC>(1, i + 0); //[a2 a2i]
1326 cblock.packet[3] = lhs2.template loadPacket<PacketC>(1, i + 1); //[b2 b2i]
1327
1328 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[2].v); //[a1 a2]
1329 blockr.packet[1] = vec_mergeh(cblock.packet[1].v, cblock.packet[3].v); //[b1 b2]
1330
1331 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[2].v);
1332 blocki.packet[1] = vec_mergel(cblock.packet[1].v, cblock.packet[3].v);
1333 } else {
1334 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i); //[a1 a1i]
1335 cblock.packet[1] = lhs2.template loadPacket<PacketC>(1, i); //[a2 a2i]
1336
1337 cblock.packet[2] = lhs2.template loadPacket<PacketC>(0, i + 1); //[b1 b1i]
1338 cblock.packet[3] = lhs2.template loadPacket<PacketC>(1, i + 1); //[b2 b2i]
1339
1340 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v); //[a1 a2]
1341 blockr.packet[1] = vec_mergeh(cblock.packet[2].v, cblock.packet[3].v); //[b1 b2]
1342
1343 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v);
1344 blocki.packet[1] = vec_mergel(cblock.packet[2].v, cblock.packet[3].v);
1345 }
1346
1347 EIGEN_IF_CONSTEXPR (Conjugate) {
1348 blocki.packet[0] = -blocki.packet[0];
1349 blocki.packet[1] = -blocki.packet[1];
1350 }
1351
1352 storeBlock<double, Packet, 2>(blockAt + rir, blockr);
1353 storeBlock<double, Packet, 2>(blockAt + rii, blocki);
1354
1355 rir += 2 * vectorSize;
1356 rii += 2 * vectorSize;
1357 }
1358 }
1359
1360 EIGEN_STRONG_INLINE void operator()(std::complex<double>* blockA, const DataMapper& lhs, Index depth, Index rows,
1361 Index stride, Index offset) {
1362 const Index vectorSize = quad_traits<double>::vectorsize;
1363 const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth);
1364 Index rir = ((PanelMode) ? (vectorSize * offset) : 0), rii;
1365 double* blockAt = reinterpret_cast<double*>(blockA);
1366 Index j = 0;
1367
1368 for (; j + vectorSize <= rows; j += vectorSize) {
1369 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
1370 Index i = 0;
1371
1372 rii = rir + vectorDelta;
1373
1374 dhs_ccopy(blockAt, lhs2, i, rir, rii, depth, vectorSize);
1375
1376 for (; i < depth; i++) {
1377 PacketBlock<Packet, 1> blockr, blocki;
1378 PacketBlock<PacketC, 2> cblock;
1379
1380 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i);
1381 cblock.packet[1] = lhs2.template loadPacket<PacketC>(1, i);
1382
1383 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v);
1384 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v);
1385
1386 EIGEN_IF_CONSTEXPR (Conjugate) {
1387 blocki.packet[0] = -blocki.packet[0];
1388 }
1389
1390 pstore<double>(blockAt + rir, blockr.packet[0]);
1391 pstore<double>(blockAt + rii, blocki.packet[0]);
1392
1393 rir += vectorSize;
1394 rii += vectorSize;
1395 }
1396
1397 rir += ((PanelMode) ? (vectorSize * (2 * stride - depth)) : vectorDelta);
1398 }
1399
1400 if (j < rows) {
1401 EIGEN_IF_CONSTEXPR (PanelMode) rir += (offset * (rows - j - vectorSize));
1402 rii = rir + (((PanelMode) ? stride : depth) * (rows - j));
1403
1404 for (Index i = 0; i < depth; i++) {
1405 Index k = j;
1406 for (; k < rows; k++) {
1407 blockAt[rir] = lhs(k, i).real();
1408
1409 EIGEN_IF_CONSTEXPR (Conjugate)
1410 blockAt[rii] = -lhs(k, i).imag();
1411 else
1412 blockAt[rii] = lhs(k, i).imag();
1413
1414 rir += 1;
1415 rii += 1;
1416 }
1417 }
1418 }
1419 }
1420};
1421
1422// General template for rhs complex packing, float64 specialization.
1423template <typename DataMapper, typename Packet, typename PacketC, int StorageOrder, bool Conjugate, bool PanelMode>
1424struct dhs_cpack<double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, false> {
1425 using Index = mapper_index_t<DataMapper>;
1426 EIGEN_ALWAYS_INLINE void dhs_ccopy(double* blockBt, const DataMapper& rhs2, Index& i, Index& rir, Index& rii,
1427 Index depth, const Index vectorSize) {
1428 for (; i < depth; i++) {
1429 PacketBlock<PacketC, 4> cblock;
1430 PacketBlock<Packet, 2> blockr, blocki;
1431
1432 bload<DataMapper, PacketC, 2, ColMajor, false, 4>(cblock, rhs2, i, 0);
1433
1434 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v);
1435 blockr.packet[1] = vec_mergeh(cblock.packet[2].v, cblock.packet[3].v);
1436
1437 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v);
1438 blocki.packet[1] = vec_mergel(cblock.packet[2].v, cblock.packet[3].v);
1439
1440 EIGEN_IF_CONSTEXPR (Conjugate) {
1441 blocki.packet[0] = -blocki.packet[0];
1442 blocki.packet[1] = -blocki.packet[1];
1443 }
1444
1445 storeBlock<double, Packet, 2>(blockBt + rir, blockr);
1446 storeBlock<double, Packet, 2>(blockBt + rii, blocki);
1447
1448 rir += 2 * vectorSize;
1449 rii += 2 * vectorSize;
1450 }
1451 }
1452
1453 EIGEN_STRONG_INLINE void operator()(std::complex<double>* blockB, const DataMapper& rhs, Index depth, Index cols,
1454 Index stride, Index offset) {
1455 const Index vectorSize = quad_traits<double>::vectorsize;
1456 const Index vectorDelta = 2 * vectorSize * ((PanelMode) ? stride : depth);
1457 Index rir = ((PanelMode) ? (2 * vectorSize * offset) : 0), rii;
1458 double* blockBt = reinterpret_cast<double*>(blockB);
1459 Index j = 0;
1460
1461 for (; j + 2 * vectorSize <= cols; j += 2 * vectorSize) {
1462 const DataMapper rhs2 = rhs.getSubMapper(0, j);
1463 Index i = 0;
1464
1465 rii = rir + vectorDelta;
1466
1467 dhs_ccopy(blockBt, rhs2, i, rir, rii, depth, vectorSize);
1468
1469 rir += ((PanelMode) ? (2 * vectorSize * (2 * stride - depth)) : vectorDelta);
1470 }
1471
1472 EIGEN_IF_CONSTEXPR (PanelMode) rir -= (offset * (2 * vectorSize - 1));
1473
1474 for (; j < cols; j++) {
1475 const DataMapper rhs2 = rhs.getSubMapper(0, j);
1476 rii = rir + ((PanelMode) ? stride : depth);
1477
1478 for (Index i = 0; i < depth; i++) {
1479 blockBt[rir] = rhs2(i, 0).real();
1480
1481 EIGEN_IF_CONSTEXPR (Conjugate)
1482 blockBt[rii] = -rhs2(i, 0).imag();
1483 else
1484 blockBt[rii] = rhs2(i, 0).imag();
1485
1486 rir += 1;
1487 rii += 1;
1488 }
1489
1490 rir += ((PanelMode) ? (2 * stride - depth) : depth);
1491 }
1492 }
1493};
1494
1495/**************
1496 * GEMM utils *
1497 **************/
1498
1499// 512-bits rank1-update of acc. It can either positive or negative accumulate (useful for complex gemm).
1500template <typename Packet, bool NegativeAccumulate, int N>
1501EIGEN_ALWAYS_INLINE void pger_common(PacketBlock<Packet, N>* acc, const Packet& lhsV, const Packet* rhsV) {
1502 EIGEN_IF_CONSTEXPR (NegativeAccumulate) {
1503 for (int M = 0; M < N; M++) {
1504 acc->packet[M] = vec_nmsub(lhsV, rhsV[M], acc->packet[M]);
1505 }
1506 } else {
1507 for (int M = 0; M < N; M++) {
1508 acc->packet[M] = vec_madd(lhsV, rhsV[M], acc->packet[M]);
1509 }
1510 }
1511}
1512
1513template <int N, typename Scalar, typename Packet, bool NegativeAccumulate, Index remaining_rows = 0>
1514EIGEN_ALWAYS_INLINE void pger(PacketBlock<Packet, N>* acc, const Scalar* lhs, const Packet* rhsV) {
1515 Packet lhsV;
1516 EIGEN_IF_CONSTEXPR (remaining_rows > 0) {
1517 lhsV = ploadu_partial<Packet>(lhs, remaining_rows);
1518 } else {
1519 lhsV = ploadLhs<Packet>(lhs);
1520 }
1521
1522 pger_common<Packet, NegativeAccumulate, N>(acc, lhsV, rhsV);
1523}
1524
1525// 512-bits rank1-update of complex acc. It takes decoupled accumulators as entries. It also takes care of mixed types
1526// real * complex and complex * real.
1527template <int N, typename Packet, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
1528EIGEN_ALWAYS_INLINE void pgerc_common(PacketBlock<Packet, N>* accReal, PacketBlock<Packet, N>* accImag,
1529 const Packet& lhsV, Packet& lhsVi, const Packet* rhsV, const Packet* rhsVi) {
1530 pger_common<Packet, false, N>(accReal, lhsV, rhsV);
1531 EIGEN_IF_CONSTEXPR (LhsIsReal) {
1532 pger_common<Packet, ConjugateRhs, N>(accImag, lhsV, rhsVi);
1533 EIGEN_UNUSED_VARIABLE(lhsVi);
1534 } else {
1535 EIGEN_IF_CONSTEXPR (!RhsIsReal) {
1536 pger_common<Packet, ConjugateLhs == ConjugateRhs, N>(accReal, lhsVi, rhsVi);
1537 pger_common<Packet, ConjugateRhs, N>(accImag, lhsV, rhsVi);
1538 } else {
1539 EIGEN_UNUSED_VARIABLE(rhsVi);
1540 }
1541 pger_common<Packet, ConjugateLhs, N>(accImag, lhsVi, rhsV);
1542 }
1543}
1544
1545template <int N, typename Scalar, typename Packet, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal,
1546 Index remaining_rows = 0>
1547EIGEN_ALWAYS_INLINE void pgerc(PacketBlock<Packet, N>* accReal, PacketBlock<Packet, N>* accImag, const Scalar* lhs_ptr,
1548 const Scalar* lhs_ptr_imag, const Packet* rhsV, const Packet* rhsVi) {
1549 Packet lhsV;
1550 EIGEN_IF_CONSTEXPR (remaining_rows > 0) {
1551 lhsV = ploadu_partial<Packet>(lhs_ptr, remaining_rows);
1552 } else {
1553 lhsV = ploadLhs<Packet>(lhs_ptr);
1554 }
1555 Packet lhsVi;
1556 EIGEN_IF_CONSTEXPR (!LhsIsReal) {
1557 EIGEN_IF_CONSTEXPR (remaining_rows > 0) {
1558 lhsVi = ploadu_partial<Packet>(lhs_ptr_imag, remaining_rows);
1559 } else {
1560 lhsVi = ploadLhs<Packet>(lhs_ptr_imag);
1561 }
1562 } else {
1563 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1564 }
1565
1566 pgerc_common<N, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi);
1567}
1568
1569template <typename Packet>
1570EIGEN_ALWAYS_INLINE Packet ploadLhs(const __UNPACK_TYPE__(Packet) * lhs) {
1571 return ploadu<Packet>(lhs);
1572}
1573
1574// Zero the accumulator on PacketBlock.
1575template <typename Packet, int N>
1576EIGEN_ALWAYS_INLINE void bsetzero(PacketBlock<Packet, N>& acc) {
1577 for (int M = 0; M < N; M++) {
1578 acc.packet[M] = pset1<Packet>((__UNPACK_TYPE__(Packet))0);
1579 }
1580}
1581
1582template <typename Packet, int N>
1583EIGEN_ALWAYS_INLINE void bscalec_common(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ,
1584 const Packet& pAlpha) {
1585 for (int M = 0; M < N; M++) {
1586 acc.packet[M] = vec_mul(accZ.packet[M], pAlpha);
1587 }
1588}
1589
1590template <typename Packet, int N>
1591EIGEN_ALWAYS_INLINE void band(PacketBlock<Packet, N>& acc, const Packet& pMask) {
1592 for (int M = 0; M < N; M++) {
1593 acc.packet[M] = pand<Packet>(acc.packet[M], pMask);
1594 }
1595}
1596
1597// Complex version of PacketBlock scaling.
1598template <typename Packet, int N, bool mask>
1599EIGEN_ALWAYS_INLINE void bscalec(PacketBlock<Packet, N>& aReal, PacketBlock<Packet, N>& aImag, const Packet& bReal,
1600 const Packet& bImag, PacketBlock<Packet, N>& cReal, PacketBlock<Packet, N>& cImag,
1601 const Packet& pMask) {
1602 EIGEN_IF_CONSTEXPR (mask && (sizeof(__UNPACK_TYPE__(Packet)) == sizeof(float))) {
1603 band<Packet, N>(aReal, pMask);
1604 band<Packet, N>(aImag, pMask);
1605 } else {
1606 EIGEN_UNUSED_VARIABLE(pMask);
1607 }
1608
1609 bscalec_common<Packet, N>(cReal, aReal, bReal);
1610
1611 bscalec_common<Packet, N>(cImag, aImag, bReal);
1612
1613 pger_common<Packet, true, N>(&cReal, bImag, aImag.packet);
1614
1615 pger_common<Packet, false, N>(&cImag, bImag, aReal.packet);
1616}
1617
1618// Load a PacketBlock, the N parameters make tuning gemm easier so we can add more accumulators as needed.
1619//
1620// full = operate (load) on the entire PacketBlock or only half
1621template <typename DataMapper, typename Packet, const Index accCols, int StorageOrder, bool Complex, int N, bool full>
1622EIGEN_ALWAYS_INLINE void bload(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc, const DataMapper& res, Index row,
1623 Index col) {
1624 using MapperIndex = mapper_index_t<DataMapper>;
1625 const MapperIndex mapperRow = convert_index<MapperIndex>(row);
1626 const MapperIndex mapperCol = convert_index<MapperIndex>(col);
1627 EIGEN_IF_CONSTEXPR (StorageOrder == RowMajor) {
1628 for (int M = 0; M < N; M++) {
1629 acc.packet[M] = res.template loadPacket<Packet>(mapperRow + M, mapperCol);
1630 }
1631 EIGEN_IF_CONSTEXPR (Complex) {
1632 for (int M = 0; M < N; M++) {
1633 acc.packet[M + N] = res.template loadPacket<Packet>(mapperRow + M, mapperCol + MapperIndex(accCols));
1634 }
1635 }
1636 } else {
1637 for (int M = 0; M < N; M++) {
1638 acc.packet[M] = res.template loadPacket<Packet>(mapperRow, mapperCol + M);
1639 }
1640 EIGEN_IF_CONSTEXPR (Complex && full) {
1641 for (int M = 0; M < N; M++) {
1642 acc.packet[M + N] = res.template loadPacket<Packet>(mapperRow + MapperIndex(accCols), mapperCol + M);
1643 }
1644 }
1645 }
1646}
1647
1648template <typename DataMapper, typename Packet, int N>
1649EIGEN_ALWAYS_INLINE void bstore(PacketBlock<Packet, N>& acc, const DataMapper& res, Index row) {
1650 const auto mapperRow = convert_index<mapper_index_t<DataMapper>>(row);
1651 for (int M = 0; M < N; M++) {
1652 res.template storePacket<Packet>(mapperRow, M, acc.packet[M]);
1653 }
1654}
1655
1656// When Complex && full, the first N packets are full loads and `elements` counts only the second half at row + accCols.
1657template <typename DataMapper, typename Packet, const Index accCols, bool Complex, Index N, bool full>
1658EIGEN_ALWAYS_INLINE void bload_partial(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc, const DataMapper& res, Index row,
1659 Index elements) {
1660 using MapperIndex = mapper_index_t<DataMapper>;
1661 const MapperIndex mapperRow = convert_index<MapperIndex>(row);
1662 const MapperIndex mapperElements = convert_index<MapperIndex>(elements);
1663 EIGEN_IF_CONSTEXPR (Complex && full) {
1664 for (int M = 0; M < N; M++) {
1665 acc.packet[M] = res.template loadPacket<Packet>(mapperRow, M);
1666 acc.packet[M + N] = res.template loadPacketPartial<Packet>(mapperRow + MapperIndex(accCols), M, mapperElements);
1667 }
1668 } else {
1669 for (int M = 0; M < N; M++) {
1670 acc.packet[M] = res.template loadPacketPartial<Packet>(mapperRow, M, mapperElements);
1671 }
1672 }
1673}
1674
1675template <typename DataMapper, typename Packet, Index N>
1676EIGEN_ALWAYS_INLINE void bstore_partial(PacketBlock<Packet, N>& acc, const DataMapper& res, Index row, Index elements) {
1677 const auto mapperRow = convert_index<mapper_index_t<DataMapper>>(row);
1678 const auto mapperElements = convert_index<mapper_index_t<DataMapper>>(elements);
1679 for (int M = 0; M < N; M++) {
1680 res.template storePacketPartial<Packet>(mapperRow, M, acc.packet[M], mapperElements);
1681 }
1682}
1683
1684#ifdef _ARCH_PWR10
1685#define USE_P10_AND_PVIPR2_0 (EIGEN_COMP_LLVM || (__GNUC__ >= 11))
1686#else
1687#define USE_P10_AND_PVIPR2_0 0
1688#endif
1689
1690#if !USE_P10_AND_PVIPR2_0
1691const static Packet4i mask4[4] = {{0, 0, 0, 0}, {-1, 0, 0, 0}, {-1, -1, 0, 0}, {-1, -1, -1, 0}};
1692#endif
1693
1694template <typename Packet>
1695EIGEN_ALWAYS_INLINE Packet bmask(const Index remaining_rows) {
1696#if USE_P10_AND_PVIPR2_0
1697#ifdef _BIG_ENDIAN
1698 return Packet(vec_reve(vec_genwm((1 << remaining_rows) - 1)));
1699#else
1700 return Packet(vec_genwm((1 << remaining_rows) - 1));
1701#endif
1702#else
1703 return Packet(mask4[remaining_rows]);
1704#endif
1705}
1706
1707template <>
1708EIGEN_ALWAYS_INLINE Packet2d bmask<Packet2d>(const Index remaining_rows) {
1709#if USE_P10_AND_PVIPR2_0
1710 Packet2d mask2 = Packet2d(vec_gendm(remaining_rows));
1711#ifdef _BIG_ENDIAN
1712 return preverse(mask2);
1713#else
1714 return mask2;
1715#endif
1716#else
1717 Packet2l ret = {-remaining_rows, 0};
1718 return Packet2d(ret);
1719#endif
1720}
1721
1722template <typename Packet, int N>
1723EIGEN_ALWAYS_INLINE void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ, const Packet& pAlpha) {
1724 for (int M = 0; M < N; M++) {
1725 acc.packet[M] = pmadd<Packet>(pAlpha, accZ.packet[M], acc.packet[M]);
1726 }
1727}
1728
1729// Scale the PacketBlock vectors by alpha.
1730template <typename Packet, int N, bool mask>
1731EIGEN_ALWAYS_INLINE void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ, const Packet& pAlpha,
1732 const Packet& pMask) {
1733 EIGEN_IF_CONSTEXPR (mask) {
1734 band<Packet, N>(accZ, pMask);
1735 } else {
1736 EIGEN_UNUSED_VARIABLE(pMask);
1737 }
1738
1739 bscale<Packet, N>(acc, accZ, pAlpha);
1740}
1741
1742template <typename Packet, int N, bool real>
1743EIGEN_ALWAYS_INLINE void pbroadcastN(const __UNPACK_TYPE__(Packet) * ap0, const __UNPACK_TYPE__(Packet) * ap1,
1744 const __UNPACK_TYPE__(Packet) * ap2, Packet& a0, Packet& a1, Packet& a2,
1745 Packet& a3) {
1746 a0 = pset1<Packet>(ap0[0]);
1747 EIGEN_IF_CONSTEXPR (N == 4) {
1748 a1 = pset1<Packet>(ap0[1]);
1749 a2 = pset1<Packet>(ap0[2]);
1750 a3 = pset1<Packet>(ap0[3]);
1751 EIGEN_UNUSED_VARIABLE(ap1);
1752 EIGEN_UNUSED_VARIABLE(ap2);
1753 } else {
1754 EIGEN_IF_CONSTEXPR (N > 1) {
1755 a1 = pset1<Packet>(ap1[0]);
1756 } else {
1757 EIGEN_UNUSED_VARIABLE(a1);
1758 EIGEN_UNUSED_VARIABLE(ap1);
1759 }
1760 EIGEN_IF_CONSTEXPR (N > 2) {
1761 a2 = pset1<Packet>(ap2[0]);
1762 } else {
1763 EIGEN_UNUSED_VARIABLE(a2);
1764 EIGEN_UNUSED_VARIABLE(ap2);
1765 }
1766 }
1767}
1768
1769template <>
1770EIGEN_ALWAYS_INLINE void pbroadcastN<Packet4f, 4, true>(const float* ap0, const float*, const float*, Packet4f& a0,
1771 Packet4f& a1, Packet4f& a2, Packet4f& a3) {
1772 pbroadcast4<Packet4f>(ap0, a0, a1, a2, a3);
1773}
1774
1775template <>
1776EIGEN_ALWAYS_INLINE void pbroadcastN<Packet4f, 4, false>(const float* ap0, const float* ap1, const float* ap2,
1777 Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
1778 pbroadcastN<Packet4f, 4, true>(ap0, ap1, ap2, a0, a1, a2, a3);
1779}
1780
1781template <>
1782EIGEN_ALWAYS_INLINE void pbroadcastN<Packet2d, 4, false>(const double* ap0, const double*, const double*, Packet2d& a0,
1783 Packet2d& a1, Packet2d& a2, Packet2d& a3) {
1784 a1 = pload<Packet2d>(ap0);
1785 a3 = pload<Packet2d>(ap0 + 2);
1786 a0 = vec_splat(a1, 0);
1787 a1 = vec_splat(a1, 1);
1788 a2 = vec_splat(a3, 0);
1789 a3 = vec_splat(a3, 1);
1790}
1791
1792// Grab two decouples real/imaginary PacketBlocks and return two coupled (real/imaginary pairs) PacketBlocks.
1793template <typename Packet, typename Packetc, int N, bool full>
1794EIGEN_ALWAYS_INLINE void bcouple_common(PacketBlock<Packet, N>& taccReal, PacketBlock<Packet, N>& taccImag,
1795 PacketBlock<Packetc, N>& acc1, PacketBlock<Packetc, N>& acc2) {
1796 for (int M = 0; M < N; M++) {
1797 acc1.packet[M].v = vec_mergeh(taccReal.packet[M], taccImag.packet[M]);
1798 }
1799
1800 EIGEN_IF_CONSTEXPR (full) {
1801 for (int M = 0; M < N; M++) {
1802 acc2.packet[M].v = vec_mergel(taccReal.packet[M], taccImag.packet[M]);
1803 }
1804 }
1805}
1806
1807template <typename Packet, typename Packetc, int N, bool full>
1808EIGEN_ALWAYS_INLINE void bcouple(PacketBlock<Packet, N>& taccReal, PacketBlock<Packet, N>& taccImag,
1809 PacketBlock<Packetc, N * 2>& tRes, PacketBlock<Packetc, N>& acc1,
1810 PacketBlock<Packetc, N>& acc2) {
1811 bcouple_common<Packet, Packetc, N, full>(taccReal, taccImag, acc1, acc2);
1812
1813 for (int M = 0; M < N; M++) {
1814 acc1.packet[M] = padd<Packetc>(tRes.packet[M], acc1.packet[M]);
1815 }
1816
1817 EIGEN_IF_CONSTEXPR (full) {
1818 for (int M = 0; M < N; M++) {
1819 acc2.packet[M] = padd<Packetc>(tRes.packet[M + N], acc2.packet[M]);
1820 }
1821 }
1822}
1823
1824// PEEL loop factor.
1825#define PEEL 7
1826#define PEEL_ROW 7
1827
1828#define MICRO_UNROLL(func) func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
1829
1830#define MICRO_NORMAL_ROWS accRows == quad_traits<Scalar>::rows || accRows == 1
1831
1832#define MICRO_NEW_ROWS ((MICRO_NORMAL_ROWS) ? accRows : 1)
1833
1834#define MICRO_RHS(ptr, N) rhs_##ptr##N
1835
1836#define MICRO_ZERO_PEEL(peel) \
1837 EIGEN_IF_CONSTEXPR ((PEEL_ROW > peel) && (peel != 0)) { \
1838 bsetzero<Packet, accRows>(accZero##peel); \
1839 } else { \
1840 EIGEN_UNUSED_VARIABLE(accZero##peel); \
1841 }
1842
1843#define MICRO_ADD(ptr, N) \
1844 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \
1845 MICRO_RHS(ptr, 0) += (accRows * N); \
1846 } else { \
1847 MICRO_RHS(ptr, 0) += N; \
1848 MICRO_RHS(ptr, 1) += N; \
1849 EIGEN_IF_CONSTEXPR (accRows == 3) { \
1850 MICRO_RHS(ptr, 2) += N; \
1851 } \
1852 }
1853
1854#define MICRO_ADD_ROWS(N) MICRO_ADD(ptr, N)
1855
1856#define MICRO_BROADCAST1(peel, ptr, rhsV, real) \
1857 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \
1858 pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0) + (accRows * peel), MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 0), \
1859 rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \
1860 } else { \
1861 pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0) + peel, MICRO_RHS(ptr, 1) + peel, MICRO_RHS(ptr, 2) + peel, \
1862 rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \
1863 }
1864
1865#define MICRO_BROADCAST(peel) MICRO_BROADCAST1(peel, ptr, rhsV, true)
1866
1867#define MICRO_BROADCAST_EXTRA1(ptr, rhsV, real) \
1868 pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 1), MICRO_RHS(ptr, 2), rhsV[0], rhsV[1], \
1869 rhsV[2], rhsV[3]);
1870
1871#define MICRO_BROADCAST_EXTRA \
1872 Packet rhsV[4]; \
1873 MICRO_BROADCAST_EXTRA1(ptr, rhsV, true) \
1874 MICRO_ADD_ROWS(1)
1875
1876#define MICRO_SRC2(ptr, N, M) \
1877 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \
1878 EIGEN_UNUSED_VARIABLE(strideB); \
1879 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 1)); \
1880 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 2)); \
1881 } else { \
1882 MICRO_RHS(ptr, 1) = rhs_base + N + M; \
1883 EIGEN_IF_CONSTEXPR (accRows == 3) { \
1884 MICRO_RHS(ptr, 2) = rhs_base + N * 2 + M; \
1885 } else { \
1886 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 2)); \
1887 } \
1888 }
1889
1890#define MICRO_SRC2_PTR MICRO_SRC2(ptr, strideB, 0)
1891
1892#define MICRO_ZERO_PEEL_ROW MICRO_UNROLL(MICRO_ZERO_PEEL)
1893
1894#define MICRO_WORK_PEEL(peel) \
1895 EIGEN_IF_CONSTEXPR (PEEL_ROW > peel) { \
1896 MICRO_BROADCAST(peel) \
1897 pger<accRows, Scalar, Packet, false>(&accZero##peel, lhs_ptr + (remaining_rows * peel), rhsV##peel); \
1898 } else { \
1899 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
1900 }
1901
1902#define MICRO_WORK_PEEL_ROW \
1903 Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4], rhsV4[4], rhsV5[4], rhsV6[4], rhsV7[4]; \
1904 MICRO_UNROLL(MICRO_WORK_PEEL) \
1905 lhs_ptr += (remaining_rows * PEEL_ROW); \
1906 MICRO_ADD_ROWS(PEEL_ROW)
1907
1908#define MICRO_ADD_PEEL(peel, sum) \
1909 EIGEN_IF_CONSTEXPR (PEEL_ROW > peel) { \
1910 for (Index i = 0; i < accRows; i++) { \
1911 accZero##sum.packet[i] += accZero##peel.packet[i]; \
1912 } \
1913 }
1914
1915#define MICRO_ADD_PEEL_ROW \
1916 MICRO_ADD_PEEL(4, 0) \
1917 MICRO_ADD_PEEL(5, 1) \
1918 MICRO_ADD_PEEL(6, 2) MICRO_ADD_PEEL(7, 3) MICRO_ADD_PEEL(2, 0) MICRO_ADD_PEEL(3, 1) MICRO_ADD_PEEL(1, 0)
1919
1920#define MICRO_PREFETCHN1(ptr, N) \
1921 EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 0)); \
1922 EIGEN_IF_CONSTEXPR (N == 2 || N == 3) { \
1923 EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 1)); \
1924 EIGEN_IF_CONSTEXPR (N == 3) { \
1925 EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 2)); \
1926 } \
1927 }
1928
1929#define MICRO_PREFETCHN(N) MICRO_PREFETCHN1(ptr, N)
1930
1931#define MICRO_COMPLEX_PREFETCHN(N) \
1932 MICRO_PREFETCHN1(ptr_real, N); \
1933 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
1934 MICRO_PREFETCHN1(ptr_imag, N); \
1935 }
1936
1937template <typename Scalar, typename Packet, const Index accRows, const Index remaining_rows, const Index load_rows = 0>
1938EIGEN_ALWAYS_INLINE void MICRO_EXTRA_ROW(const Scalar*& lhs_ptr, const Scalar*& rhs_ptr0, const Scalar*& rhs_ptr1,
1939 const Scalar*& rhs_ptr2, PacketBlock<Packet, accRows>& accZero) {
1940 MICRO_BROADCAST_EXTRA
1941 pger<accRows, Scalar, Packet, false, load_rows>(&accZero, lhs_ptr, rhsV);
1942 lhs_ptr += remaining_rows;
1943}
1944
1945template <typename Scalar, typename Packet, typename DataMapper, const Index accRows, const Index accCols,
1946 const Index remaining_rows>
1947EIGEN_ALWAYS_INLINE void gemm_unrolled_row_iteration(const DataMapper& res, const Scalar* lhs_base,
1948 const Scalar* rhs_base, Index depth, Index strideA, Index offsetA,
1949 Index strideB, Index row, Index rows, const Packet& pAlpha,
1950 const Packet& pMask) {
1951 const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = nullptr, *rhs_ptr2 = nullptr;
1952 const Scalar* lhs_ptr = lhs_base + row * strideA + remaining_rows * offsetA;
1953 PacketBlock<Packet, accRows> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7, acc;
1954
1955 MICRO_SRC2_PTR
1956 bsetzero<Packet, accRows>(accZero0);
1957
1958 const Index peel_depth = depth - (accCols - remaining_rows);
1959 Index k = 0;
1960 if (peel_depth >= PEEL_ROW) {
1961 MICRO_ZERO_PEEL_ROW
1962 do {
1963 MICRO_PREFETCHN(accRows)
1964 EIGEN_POWER_PREFETCH(lhs_ptr);
1965 MICRO_WORK_PEEL_ROW
1966 } while ((k += PEEL_ROW) + PEEL_ROW <= peel_depth);
1967 MICRO_ADD_PEEL_ROW
1968 }
1969 for (; k < peel_depth; k++) {
1970 MICRO_EXTRA_ROW<Scalar, Packet, accRows, remaining_rows>(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2, accZero0);
1971 }
1972 for (; k < depth; k++) {
1973 MICRO_EXTRA_ROW<Scalar, Packet, accRows, remaining_rows, remaining_rows>(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2,
1974 accZero0);
1975 }
1976
1977 EIGEN_UNUSED_VARIABLE(rows);
1978 EIGEN_UNUSED_VARIABLE(pMask);
1979 bload_partial<DataMapper, Packet, 0, false, accRows>(acc, res, row, remaining_rows);
1980 bscale<Packet, accRows>(acc, accZero0, pAlpha);
1981 bstore_partial<DataMapper, Packet, accRows>(acc, res, row, remaining_rows);
1982}
1983
1984#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \
1985 switch (value) { \
1986 default: \
1987 MICRO_EXTRA_UNROLL(1) \
1988 break; \
1989 case 2: \
1990 EIGEN_IF_CONSTEXPR (is_col || (sizeof(Scalar) == sizeof(float))) { \
1991 MICRO_EXTRA_UNROLL(2) \
1992 } \
1993 break; \
1994 case 3: \
1995 EIGEN_IF_CONSTEXPR (is_col || (sizeof(Scalar) == sizeof(float))) { \
1996 MICRO_EXTRA_UNROLL(3) \
1997 } \
1998 break; \
1999 }
2000
2001#define MICRO_EXTRA_ROWS(N) \
2002 gemm_unrolled_row_iteration<Scalar, Packet, DataMapper, accRows, accCols, N>( \
2003 res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlpha, pMask);
2004
2005template <typename Scalar, typename Packet, typename DataMapper, const Index accRows, const Index accCols>
2006EIGEN_ALWAYS_INLINE void gemm_extra_row(const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base,
2007 Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows,
2008 Index remaining_rows, const Packet& pAlpha, const Packet& pMask) {
2009 MICRO_EXTRA(MICRO_EXTRA_ROWS, remaining_rows, false)
2010}
2011
2012#define MICRO_UNROLL_WORK(func, func2, peel) \
2013 MICRO_UNROLL(func2); \
2014 func(0, peel) func(1, peel) func(2, peel) func(3, peel) func(4, peel) func(5, peel) func(6, peel) func(7, peel)
2015
2016#define MICRO_WORK_ONE(iter, peel) \
2017 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2018 pger_common<Packet, false, accRows>(&accZero##iter, lhsV##iter, rhsV##peel); \
2019 }
2020
2021#define MICRO_TYPE_PEEL4(func, func2, peel) \
2022 EIGEN_IF_CONSTEXPR (PEEL > peel) { \
2023 Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
2024 MICRO_BROADCAST(peel) \
2025 MICRO_UNROLL_WORK(func, func2, peel) \
2026 } else { \
2027 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
2028 }
2029
2030#define MICRO_UNROLL_TYPE_PEEL(M, func, func1, func2) \
2031 Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M]; \
2032 func(func1, func2, 0) func(func1, func2, 1) func(func1, func2, 2) func(func1, func2, 3) func(func1, func2, 4) \
2033 func(func1, func2, 5) func(func1, func2, 6) func(func1, func2, 7)
2034
2035#define MICRO_UNROLL_TYPE_ONE(M, func, func1, func2) \
2036 Packet rhsV0[M]; \
2037 func(func1, func2, 0)
2038
2039#define MICRO_UNROLL_TYPE(MICRO_TYPE, size) \
2040 MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE) \
2041 MICRO_ADD_ROWS(size)
2042
2043#define MICRO_UNROLL_TYPE_PARTIAL(MICRO_TYPE, size) \
2044 MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_PARTIAL_ONE) \
2045 MICRO_ADD_ROWS(size)
2046
2047#define MICRO_ONE_PEEL4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_PEEL, PEEL)
2048
2049#define MICRO_ONE4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1)
2050
2051#define MICRO_ONE4_PARTIAL MICRO_UNROLL_TYPE_PARTIAL(MICRO_UNROLL_TYPE_ONE, 1)
2052
2053#define MICRO_DST_PTR_ONE(iter) \
2054 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2055 bsetzero<Packet, accRows>(accZero##iter); \
2056 } else { \
2057 EIGEN_UNUSED_VARIABLE(accZero##iter); \
2058 }
2059
2060#define MICRO_DST_PTR MICRO_UNROLL(MICRO_DST_PTR_ONE)
2061
2062#define MICRO_SRC_PTR MICRO_UNROLL(MICRO_SRC_PTR_ONE)
2063
2064#define MICRO_PREFETCH MICRO_UNROLL(MICRO_PREFETCH_ONE)
2065
2066#define MICRO_STORE_ONE(iter) \
2067 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2068 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_PARTIAL(iter)) { \
2069 bload<DataMapper, Packet, 0, ColMajor, false, accRows>(acc, res, row + iter * accCols, 0); \
2070 bscale<Packet, accRows>(acc, accZero##iter, pAlpha); \
2071 bstore<DataMapper, Packet, accRows>(acc, res, row + iter * accCols); \
2072 } else { \
2073 bload_partial<DataMapper, Packet, 0, false, accRows>(acc, res, row + iter * accCols, accCols2); \
2074 bscale<Packet, accRows>(acc, accZero##iter, pAlpha); \
2075 bstore_partial<DataMapper, Packet, accRows>(acc, res, row + iter * accCols, accCols2); \
2076 } \
2077 }
2078
2079#define MICRO_STORE MICRO_UNROLL(MICRO_STORE_ONE)
2080
2081template <int unroll_factor, typename Scalar, typename Packet, typename DataMapper, const Index accRows,
2082 const Index accCols, bool full>
2083EIGEN_ALWAYS_INLINE void gemm_unrolled_iteration(const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base,
2084 Index depth, Index strideA, Index offsetA, Index strideB, Index& row,
2085 const Packet& pAlpha, Index accCols2) {
2086 const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 = nullptr, *rhs_ptr2 = nullptr;
2087 const Scalar *lhs_ptr0 = nullptr, *lhs_ptr1 = nullptr, *lhs_ptr2 = nullptr, *lhs_ptr3 = nullptr, *lhs_ptr4 = nullptr,
2088 *lhs_ptr5 = nullptr, *lhs_ptr6 = nullptr, *lhs_ptr7 = nullptr;
2089 PacketBlock<Packet, accRows> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
2090 PacketBlock<Packet, accRows> acc;
2091
2092 MICRO_SRC2_PTR
2093 MICRO_SRC_PTR
2094 MICRO_DST_PTR
2095
2096 const Index peel_depth = full ? depth : (depth - (accCols - accCols2));
2097 Index k = 0;
2098 for (; k + PEEL <= peel_depth; k += PEEL) {
2099 MICRO_PREFETCHN(accRows)
2100 MICRO_PREFETCH
2101 MICRO_ONE_PEEL4
2102 }
2103 for (; k < peel_depth; k++) {
2104 MICRO_ONE4
2105 }
2106 EIGEN_IF_CONSTEXPR (!full) {
2107 for (; k < depth; k++) {
2108 MICRO_ONE4_PARTIAL
2109 }
2110 }
2111 MICRO_STORE
2112
2113 MICRO_UPDATE
2114}
2115
2116#define MICRO_UNROLL_ITER2(N, M) \
2117 gemm_unrolled_iteration<N + ((M) ? 1 : 0), Scalar, Packet, DataMapper, accRows, accCols, !M>( \
2118 res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlpha, M ? remaining_rows : accCols); \
2119 if (M) return;
2120
2121template <typename Scalar, typename Packet, typename DataMapper, const Index accRows, const Index accCols>
2122EIGEN_ALWAYS_INLINE void gemm_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index depth,
2123 Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows,
2124 Index remaining_rows, const Packet& pAlpha, const Packet& pMask) {
2125 const DataMapper res3 = res.getSubMapper(0, convert_index<mapper_index_t<DataMapper>>(col));
2126
2127 const Scalar* rhs_base = blockB + col * strideB + MICRO_NEW_ROWS * offsetB;
2128 const Scalar* lhs_base = blockA + accCols * offsetA;
2129 Index row = 0;
2130
2131#define MAX_UNROLL 7
2132 while (row + MAX_UNROLL * accCols <= rows) {
2133 MICRO_UNROLL_ITER2(MAX_UNROLL, 0);
2134 }
2135 switch ((rows - row) / accCols) {
2136#if MAX_UNROLL > 7
2137 case 7:
2138 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 7)
2139 break;
2140#endif
2141#if MAX_UNROLL > 6
2142 case 6:
2143 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 6)
2144 break;
2145#endif
2146#if MAX_UNROLL > 5
2147 case 5:
2148 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 5)
2149 break;
2150#endif
2151#if MAX_UNROLL > 4
2152 case 4:
2153 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 4)
2154 break;
2155#endif
2156#if MAX_UNROLL > 3
2157 case 3:
2158 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 3)
2159 break;
2160#endif
2161#if MAX_UNROLL > 2
2162 case 2:
2163 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 2)
2164 break;
2165#endif
2166#if MAX_UNROLL > 1
2167 case 1:
2168 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 1)
2169 break;
2170#endif
2171 default:
2172 break;
2173 }
2174#undef MAX_UNROLL
2175
2176 if (remaining_rows > 0) {
2177 gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>(res3, blockA, rhs_base, depth, strideA, offsetA,
2178 strideB, row, rows, remaining_rows, pAlpha, pMask);
2179 }
2180}
2181
2182#define MICRO_EXTRA_COLS(N) \
2183 gemm_cols<Scalar, Packet, DataMapper, N, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, \
2184 col, rows, remaining_rows, pAlpha, pMask);
2185
2186template <typename Scalar, typename Packet, typename DataMapper, const Index accCols>
2187EIGEN_ALWAYS_INLINE void gemm_extra_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index depth,
2188 Index strideA, Index offsetA, Index strideB, Index offsetB, Index col,
2189 Index rows, Index cols, Index remaining_rows, const Packet& pAlpha,
2190 const Packet& pMask) {
2191 MICRO_EXTRA(MICRO_EXTRA_COLS, cols - col, true)
2192}
2193
2194/****************
2195 * GEMM kernels *
2196 * **************/
2197template <typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows,
2198 const Index accCols>
2199EIGEN_STRONG_INLINE void gemm(const DataMapper& res, const Scalar* blockA, const Scalar* blockB, Index rows,
2200 Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA,
2201 Index offsetB) {
2202 const Index remaining_rows = rows % accCols;
2203
2204 if (strideA == -1) strideA = depth;
2205 if (strideB == -1) strideB = depth;
2206
2207 const Packet pAlpha = pset1<Packet>(alpha);
2208 const Packet pMask = bmask<Packet>(remaining_rows);
2209
2210 Index col = 0;
2211 for (; col + accRows <= cols; col += accRows) {
2212 gemm_cols<Scalar, Packet, DataMapper, accRows, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB,
2213 offsetB, col, rows, remaining_rows, pAlpha, pMask);
2214 }
2215
2216 if (col != cols) {
2217 gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB,
2218 col, rows, cols, remaining_rows, pAlpha, pMask);
2219 }
2220}
2221
2222#define accColsC (accCols / 2)
2223#define advanceRows ((LhsIsReal) ? 1 : 2)
2224#define advanceCols ((RhsIsReal) ? 1 : 2)
2225
2226// PEEL_COMPLEX loop factor.
2227#define PEEL_COMPLEX 3
2228#define PEEL_COMPLEX_ROW 3
2229
2230#define MICRO_COMPLEX_UNROLL(func) func(0) func(1) func(2) func(3)
2231
2232#define MICRO_COMPLEX_ZERO_PEEL(peel) \
2233 EIGEN_IF_CONSTEXPR ((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \
2234 bsetzero<Packet, accRows>(accReal##peel); \
2235 bsetzero<Packet, accRows>(accImag##peel); \
2236 } else { \
2237 EIGEN_UNUSED_VARIABLE(accReal##peel); \
2238 EIGEN_UNUSED_VARIABLE(accImag##peel); \
2239 }
2240
2241#define MICRO_COMPLEX_ADD_ROWS(N, used) \
2242 MICRO_ADD(ptr_real, N) \
2243 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2244 MICRO_ADD(ptr_imag, N) \
2245 } else if (used) { \
2246 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \
2247 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \
2248 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \
2249 }
2250
2251#define MICRO_COMPLEX_BROADCAST(peel) \
2252 MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \
2253 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2254 MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) \
2255 } else { \
2256 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
2257 }
2258
2259#define MICRO_COMPLEX_BROADCAST_EXTRA \
2260 Packet rhsV[4], rhsVi[4]; \
2261 MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \
2262 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2263 MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) \
2264 } else { \
2265 EIGEN_UNUSED_VARIABLE(rhsVi); \
2266 } \
2267 MICRO_COMPLEX_ADD_ROWS(1, true)
2268
2269#define MICRO_COMPLEX_SRC2_PTR \
2270 MICRO_SRC2(ptr_real, strideB* advanceCols, 0) \
2271 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2272 MICRO_RHS(ptr_imag, 0) = rhs_base + MICRO_NEW_ROWS * strideB; \
2273 MICRO_SRC2(ptr_imag, strideB* advanceCols, strideB) \
2274 } else { \
2275 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \
2276 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \
2277 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \
2278 }
2279
2280#define MICRO_COMPLEX_ZERO_PEEL_ROW MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_ZERO_PEEL)
2281
2282#define MICRO_COMPLEX_WORK_PEEL(peel) \
2283 EIGEN_IF_CONSTEXPR (PEEL_COMPLEX_ROW > peel) { \
2284 MICRO_COMPLEX_BROADCAST(peel) \
2285 pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
2286 &accReal##peel, &accImag##peel, lhs_ptr_real + (remaining_rows * peel), \
2287 lhs_ptr_imag + (remaining_rows * peel), rhsV##peel, rhsVi##peel); \
2288 } else { \
2289 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
2290 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
2291 }
2292
2293#define MICRO_COMPLEX_ADD_COLS(size) \
2294 lhs_ptr_real += (remaining_rows * size); \
2295 EIGEN_IF_CONSTEXPR (!LhsIsReal) \
2296 lhs_ptr_imag += (remaining_rows * size); \
2297 else \
2298 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
2299
2300#define MICRO_COMPLEX_WORK_PEEL_ROW \
2301 Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4]; \
2302 Packet rhsVi0[4], rhsVi1[4], rhsVi2[4], rhsVi3[4]; \
2303 MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_WORK_PEEL) \
2304 MICRO_COMPLEX_ADD_COLS(PEEL_COMPLEX_ROW) \
2305 MICRO_COMPLEX_ADD_ROWS(PEEL_COMPLEX_ROW, false)
2306
2307#define MICRO_COMPLEX_ADD_PEEL(peel, sum) \
2308 EIGEN_IF_CONSTEXPR (PEEL_COMPLEX_ROW > peel) { \
2309 for (Index i = 0; i < accRows; i++) { \
2310 accReal##sum.packet[i] += accReal##peel.packet[i]; \
2311 accImag##sum.packet[i] += accImag##peel.packet[i]; \
2312 } \
2313 }
2314
2315#define MICRO_COMPLEX_ADD_PEEL_ROW \
2316 MICRO_COMPLEX_ADD_PEEL(2, 0) MICRO_COMPLEX_ADD_PEEL(3, 1) MICRO_COMPLEX_ADD_PEEL(1, 0)
2317
2318template <typename Scalar, typename Packet, const Index accRows, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal,
2319 bool RhsIsReal, const Index remaining_rows, const Index load_rows = 0>
2320EIGEN_ALWAYS_INLINE void MICRO_COMPLEX_EXTRA_ROW(const Scalar*& lhs_ptr_real, const Scalar*& lhs_ptr_imag,
2321 const Scalar*& rhs_ptr_real0, const Scalar*& rhs_ptr_real1,
2322 const Scalar*& rhs_ptr_real2, const Scalar*& rhs_ptr_imag0,
2323 const Scalar*& rhs_ptr_imag1, const Scalar*& rhs_ptr_imag2,
2324 PacketBlock<Packet, accRows>& accReal,
2325 PacketBlock<Packet, accRows>& accImag) {
2326 MICRO_COMPLEX_BROADCAST_EXTRA
2327 pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, load_rows>(
2328 &accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi);
2329 MICRO_COMPLEX_ADD_COLS(1)
2330}
2331
2332template <typename Scalar, typename Packet, typename Packetc, typename DataMapper, const Index accRows,
2333 const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal,
2334 const Index remaining_rows>
2335EIGEN_ALWAYS_INLINE void gemm_unrolled_complex_row_iteration(const DataMapper& res, const Scalar* lhs_base,
2336 const Scalar* rhs_base, Index depth, Index strideA,
2337 Index offsetA, Index strideB, Index row, Index rows,
2338 const Packet& pAlphaReal, const Packet& pAlphaImag,
2339 const Packet& pMask) {
2340 const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = nullptr, *rhs_ptr_real2 = nullptr;
2341 const Scalar *rhs_ptr_imag0 = nullptr, *rhs_ptr_imag1 = nullptr, *rhs_ptr_imag2 = nullptr;
2342 const Scalar* lhs_ptr_real = lhs_base + advanceRows * row * strideA + remaining_rows * offsetA;
2343 const Scalar* lhs_ptr_imag = nullptr;
2344 EIGEN_IF_CONSTEXPR (!LhsIsReal)
2345 lhs_ptr_imag = lhs_ptr_real + remaining_rows * strideA;
2346 else
2347 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
2348 PacketBlock<Packet, accRows> accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;
2349 PacketBlock<Packet, accRows> taccReal, taccImag;
2350 PacketBlock<Packetc, accRows> acc0, acc1;
2351 PacketBlock<Packetc, accRows * 2> tRes;
2352
2353 MICRO_COMPLEX_SRC2_PTR
2354
2355 bsetzero<Packet, accRows>(accReal0);
2356 bsetzero<Packet, accRows>(accImag0);
2357
2358 const Index peel_depth = depth - (accCols - remaining_rows);
2359 Index k = 0;
2360 if (peel_depth >= PEEL_COMPLEX_ROW) {
2361 MICRO_COMPLEX_ZERO_PEEL_ROW
2362 do {
2363 MICRO_COMPLEX_PREFETCHN(accRows)
2364 EIGEN_POWER_PREFETCH(lhs_ptr_real);
2365 EIGEN_IF_CONSTEXPR (!LhsIsReal) {
2366 EIGEN_POWER_PREFETCH(lhs_ptr_imag);
2367 }
2368 MICRO_COMPLEX_WORK_PEEL_ROW
2369 } while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= peel_depth);
2370 MICRO_COMPLEX_ADD_PEEL_ROW
2371 }
2372 for (; k < peel_depth; k++) {
2373 MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, remaining_rows>(
2374 lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2, rhs_ptr_imag0, rhs_ptr_imag1,
2375 rhs_ptr_imag2, accReal0, accImag0);
2376 }
2377 for (; k < depth; k++) {
2378 MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, remaining_rows,
2379 remaining_rows>(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2,
2380 rhs_ptr_imag0, rhs_ptr_imag1, rhs_ptr_imag2, accReal0, accImag0);
2381 }
2382
2383 EIGEN_UNUSED_VARIABLE(rows);
2384 constexpr bool full = (remaining_rows > accColsC);
2385 constexpr bool odd = (sizeof(Scalar) == sizeof(float)) && (remaining_rows & 1);
2386 EIGEN_IF_CONSTEXPR (odd) {
2387 bload_partial<DataMapper, Packetc, accColsC, true, accRows, full>(tRes, res, row, 1);
2388 } else {
2389 bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row, 0);
2390 }
2391 bscalec<Packet, accRows, true>(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask);
2392 bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1);
2393 EIGEN_IF_CONSTEXPR (odd && !full) {
2394 bstore_partial<DataMapper, Packetc, accRows>(acc0, res, row + 0, 1);
2395 } else {
2396 bstore<DataMapper, Packetc, accRows>(acc0, res, row + 0);
2397 }
2398 EIGEN_IF_CONSTEXPR (full) {
2399 EIGEN_IF_CONSTEXPR (odd) {
2400 bstore_partial<DataMapper, Packetc, accRows>(acc1, res, row + accColsC, 1);
2401 } else {
2402 bstore<DataMapper, Packetc, accRows>(acc1, res, row + accColsC);
2403 }
2404 }
2405}
2406
2407#define MICRO_COMPLEX_EXTRA_ROWS(N) \
2408 gemm_unrolled_complex_row_iteration<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, \
2409 ConjugateRhs, LhsIsReal, RhsIsReal, N>( \
2410 res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlphaReal, pAlphaImag, pMask);
2411
2412template <typename Scalar, typename Packet, typename Packetc, typename DataMapper, const Index accRows,
2413 const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
2414EIGEN_ALWAYS_INLINE void gemm_complex_extra_row(const DataMapper& res, const Scalar* lhs_base, const Scalar* rhs_base,
2415 Index depth, Index strideA, Index offsetA, Index strideB, Index row,
2416 Index rows, Index remaining_rows, const Packet& pAlphaReal,
2417 const Packet& pAlphaImag, const Packet& pMask) {
2418 MICRO_EXTRA(MICRO_COMPLEX_EXTRA_ROWS, remaining_rows, false)
2419}
2420
2421#define MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \
2422 MICRO_COMPLEX_UNROLL(func2); \
2423 func(0, peel) func(1, peel) func(2, peel) func(3, peel)
2424
2425#define MICRO_COMPLEX_WORK_ONE4(iter, peel) \
2426 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2427 pgerc_common<accRows, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
2428 &accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \
2429 }
2430
2431#define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel) \
2432 EIGEN_IF_CONSTEXPR (PEEL_COMPLEX > peel) { \
2433 Packet lhsV0, lhsV1, lhsV2, lhsV3; \
2434 Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \
2435 MICRO_COMPLEX_BROADCAST(peel) \
2436 MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \
2437 } else { \
2438 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
2439 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
2440 }
2441
2442#define MICRO_COMPLEX_UNROLL_TYPE_PEEL(M, func, func1, func2) \
2443 Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M]; \
2444 Packet rhsVi0[M], rhsVi1[M], rhsVi2[M], rhsVi3[M]; \
2445 func(func1, func2, 0) func(func1, func2, 1) func(func1, func2, 2) func(func1, func2, 3)
2446
2447#define MICRO_COMPLEX_UNROLL_TYPE_ONE(M, func, func1, func2) \
2448 Packet rhsV0[M], rhsVi0[M]; \
2449 func(func1, func2, 0)
2450
2451#define MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_TYPE, size) \
2452 MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE) \
2453 MICRO_COMPLEX_ADD_ROWS(size, false)
2454
2455#define MICRO_COMPLEX_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_TYPE, size) \
2456 MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_PARTIAL_ONE) \
2457 MICRO_COMPLEX_ADD_ROWS(size, false)
2458
2459#define MICRO_COMPLEX_ONE_PEEL4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_PEEL, PEEL_COMPLEX)
2460
2461#define MICRO_COMPLEX_ONE4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1)
2462
2463#define MICRO_COMPLEX_ONE4_PARTIAL MICRO_COMPLEX_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1)
2464
2465#define MICRO_COMPLEX_DST_PTR_ONE(iter) \
2466 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2467 bsetzero<Packet, accRows>(accReal##iter); \
2468 bsetzero<Packet, accRows>(accImag##iter); \
2469 } else { \
2470 EIGEN_UNUSED_VARIABLE(accReal##iter); \
2471 EIGEN_UNUSED_VARIABLE(accImag##iter); \
2472 }
2473
2474#define MICRO_COMPLEX_DST_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_DST_PTR_ONE)
2475
2476#define MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)
2477
2478#define MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)
2479
2480#define MICRO_COMPLEX_STORE_ONE(iter) \
2481 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2482 constexpr bool full = ((MICRO_NORMAL(iter)) || (accCols2 > accColsC)); \
2483 constexpr bool odd = !(MICRO_NORMAL(iter)) && (sizeof(Scalar) == sizeof(float)) && (accCols2 & 1); \
2484 EIGEN_IF_CONSTEXPR (odd) { \
2485 bload_partial<DataMapper, Packetc, accColsC, true, accRows, full>(tRes, res, row + iter * accCols, 1); \
2486 } else { \
2487 bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row + iter * accCols, 0); \
2488 } \
2489 bscalec<Packet, accRows, !(MICRO_NORMAL(iter))>(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, \
2490 taccImag, pMask); \
2491 bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1); \
2492 EIGEN_IF_CONSTEXPR (odd && !full) { \
2493 bstore_partial<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0, 1); \
2494 } else { \
2495 bstore<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0); \
2496 } \
2497 EIGEN_IF_CONSTEXPR (full) { \
2498 EIGEN_IF_CONSTEXPR (odd) { \
2499 bstore_partial<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC, 1); \
2500 } else { \
2501 bstore<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC); \
2502 } \
2503 } \
2504 }
2505
2506#define MICRO_COMPLEX_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE)
2507
2508template <int unroll_factor, typename Scalar, typename Packet, typename Packetc, typename DataMapper,
2509 const Index accRows, const Index accCols, const Index accCols2, bool ConjugateLhs, bool ConjugateRhs,
2510 bool LhsIsReal, bool RhsIsReal>
2511EIGEN_ALWAYS_INLINE void gemm_complex_unrolled_iteration(const DataMapper& res, const Scalar* lhs_base,
2512 const Scalar* rhs_base, Index depth, Index strideA,
2513 Index offsetA, Index strideB, Index& row,
2514 const Packet& pAlphaReal, const Packet& pAlphaImag,
2515 const Packet& pMask) {
2516 const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 = nullptr, *rhs_ptr_real2 = nullptr;
2517 const Scalar *rhs_ptr_imag0 = nullptr, *rhs_ptr_imag1 = nullptr, *rhs_ptr_imag2 = nullptr;
2518 const Index imag_delta = accCols * strideA;
2519 const Index imag_delta2 = accCols2 * strideA;
2520 const Scalar *lhs_ptr_real0 = nullptr, *lhs_ptr_real1 = nullptr;
2521 const Scalar *lhs_ptr_real2 = nullptr, *lhs_ptr_real3 = nullptr;
2522 PacketBlock<Packet, accRows> accReal0, accImag0, accReal1, accImag1;
2523 PacketBlock<Packet, accRows> accReal2, accImag2, accReal3, accImag3;
2524 PacketBlock<Packet, accRows> taccReal, taccImag;
2525 PacketBlock<Packetc, accRows> acc0, acc1;
2526 PacketBlock<Packetc, accRows * 2> tRes;
2527
2528 MICRO_COMPLEX_SRC2_PTR
2529 MICRO_COMPLEX_SRC_PTR
2530 MICRO_COMPLEX_DST_PTR
2531
2532 const Index peel_depth = depth - (accCols - accCols2);
2533 Index k = 0;
2534 for (; k + PEEL_COMPLEX <= peel_depth; k += PEEL_COMPLEX) {
2535 MICRO_COMPLEX_PREFETCHN(accRows)
2536 MICRO_COMPLEX_PREFETCH
2537 MICRO_COMPLEX_ONE_PEEL4
2538 }
2539 for (; k < peel_depth; k++) {
2540 MICRO_COMPLEX_ONE4
2541 }
2542 EIGEN_IF_CONSTEXPR (accCols != accCols2) {
2543 for (; k < depth; k++) {
2544 MICRO_COMPLEX_ONE4_PARTIAL
2545 }
2546 }
2547 MICRO_COMPLEX_STORE
2548
2549 MICRO_COMPLEX_UPDATE
2550}
2551
2552#define MICRO_COMPLEX_UNROLL_ITER2(N, M) \
2553 gemm_complex_unrolled_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, DataMapper, accRows, accCols, \
2554 M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
2555 res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlphaReal, pAlphaImag, pMask); \
2556 if (M) return;
2557
2558template <typename Scalar, typename Packet, typename Packetc, typename DataMapper, const Index accRows,
2559 const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
2560EIGEN_ALWAYS_INLINE void gemm_complex_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
2561 Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB,
2562 Index col, Index rows, Index remaining_rows, const Packet& pAlphaReal,
2563 const Packet& pAlphaImag, const Packet& pMask) {
2564 const DataMapper res3 = res.getSubMapper(0, convert_index<mapper_index_t<DataMapper>>(col));
2565
2566 const Scalar* rhs_base = blockB + advanceCols * col * strideB + MICRO_NEW_ROWS * offsetB;
2567 const Scalar* lhs_base = blockA + accCols * offsetA;
2568 Index row = 0;
2569
2570#define MAX_COMPLEX_UNROLL 4
2571 while (row + MAX_COMPLEX_UNROLL * accCols <= rows) {
2572 MICRO_COMPLEX_UNROLL_ITER2(MAX_COMPLEX_UNROLL, 0);
2573 }
2574 switch ((rows - row) / accCols) {
2575#if MAX_COMPLEX_UNROLL > 4
2576 case 4:
2577 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 4)
2578 break;
2579#endif
2580#if MAX_COMPLEX_UNROLL > 3
2581 case 3:
2582 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 3)
2583 break;
2584#endif
2585#if MAX_COMPLEX_UNROLL > 2
2586 case 2:
2587 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 2)
2588 break;
2589#endif
2590#if MAX_COMPLEX_UNROLL > 1
2591 case 1:
2592 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 1)
2593 break;
2594#endif
2595 default:
2596 break;
2597 }
2598#undef MAX_COMPLEX_UNROLL
2599
2600 if (remaining_rows > 0) {
2601 gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
2602 RhsIsReal>(res3, blockA, rhs_base, depth, strideA, offsetA, strideB, row, rows,
2603 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2604 }
2605}
2606
2607#define MICRO_COMPLEX_EXTRA_COLS(N) \
2608 gemm_complex_cols<Scalar, Packet, Packetc, DataMapper, N, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, \
2609 RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, \
2610 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2611
2612template <typename Scalar, typename Packet, typename Packetc, typename DataMapper, const Index accCols,
2613 bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
2614EIGEN_ALWAYS_INLINE void gemm_complex_extra_cols(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
2615 Index depth, Index strideA, Index offsetA, Index strideB,
2616 Index offsetB, Index col, Index rows, Index cols, Index remaining_rows,
2617 const Packet& pAlphaReal, const Packet& pAlphaImag,
2618 const Packet& pMask) {
2619 MICRO_EXTRA(MICRO_COMPLEX_EXTRA_COLS, cols - col, true)
2620}
2621
2622template <typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Packet, typename Packetc,
2623 typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs,
2624 bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
2625EIGEN_STRONG_INLINE void gemm_complex(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc,
2626 Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB,
2627 Index offsetA, Index offsetB) {
2628 const Index remaining_rows = rows % accCols;
2629
2630 if (strideA == -1) strideA = depth;
2631 if (strideB == -1) strideB = depth;
2632
2633 const Packet pAlphaReal = pset1<Packet>(alpha.real());
2634 const Packet pAlphaImag = pset1<Packet>(alpha.imag());
2635 const Packet pMask = bmask<Packet>(remaining_rows);
2636
2637 const Scalar* blockA = (Scalar*)blockAc;
2638 const Scalar* blockB = (Scalar*)blockBc;
2639
2640 Index col = 0;
2641 for (; col + accRows <= cols; col += accRows) {
2642 gemm_complex_cols<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
2643 RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows,
2644 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2645 }
2646
2647 if (col != cols) {
2648 gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
2649 RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols,
2650 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2651 }
2652}
2653
2654#undef accColsC
2655#undef advanceCols
2656#undef advanceRows
2657
2658EIGEN_ALWAYS_INLINE bool supportsMMA() {
2659#if defined(EIGEN_ALTIVEC_MMA_ONLY)
2660 return true;
2661#elif defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) && defined(__BUILTIN_CPU_SUPPORTS__)
2662 return __builtin_cpu_supports("arch_3_1") && __builtin_cpu_supports("mma");
2663#else
2664 return false; // No dynamic dispatch for LLVM or older GCC
2665#endif
2666}
2667
2668EIGEN_ALWAYS_INLINE Packet4f loadAndMultiplyF32(const Packet4f& acc, const Packet4f& pAlpha, float* result) {
2669 Packet4f result_block = ploadu<Packet4f>(result);
2670 return pmadd(acc, pAlpha, result_block);
2671}
2672
2673template <bool lhsExtraRows>
2674EIGEN_ALWAYS_INLINE void storeF32(float*& result, const Packet4f& result_block, Index rows, Index extra_rows) {
2675 EIGEN_IF_CONSTEXPR (lhsExtraRows) {
2676 pstoreu_partial(result, result_block, extra_rows);
2677 } else {
2678 pstoreu(result, result_block);
2679 }
2680 result += rows;
2681}
2682
2683template <bool rhsExtraCols, bool lhsExtraRows>
2684EIGEN_ALWAYS_INLINE void storeResults(Packet4f (&acc)[4], Index rows, const Packet4f& pAlpha, float* result,
2685 Index extra_cols, Index extra_rows) {
2686 Index x = 0;
2687 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
2688 do {
2689 Packet4f result_block = loadAndMultiplyF32(acc[x], pAlpha, result);
2690 storeF32<lhsExtraRows>(result, result_block, rows, extra_rows);
2691 } while (++x < extra_cols);
2692 } else {
2693 Packet4f result_block[4];
2694 float* result2 = result;
2695 do {
2696 result_block[x] = loadAndMultiplyF32(acc[x], pAlpha, result);
2697 result += rows;
2698 } while (++x < 4);
2699 x = 0;
2700 do {
2701 storeF32<lhsExtraRows>(result2, result_block[x], rows, extra_rows);
2702 } while (++x < 4);
2703 }
2704}
2705
2706EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Hi(const Packet8us& data) {
2707 Packet8us z = pset1<Packet8us>(0);
2708#ifdef _BIG_ENDIAN
2709 return reinterpret_cast<Packet4f>(vec_mergeh(data, z));
2710#else
2711 return reinterpret_cast<Packet4f>(vec_mergeh(z, data));
2712#endif
2713}
2714
2715EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Lo(const Packet8us& data) {
2716 Packet8us z = pset1<Packet8us>(0);
2717#ifdef _BIG_ENDIAN
2718 return reinterpret_cast<Packet4f>(vec_mergel(data, z));
2719#else
2720 return reinterpret_cast<Packet4f>(vec_mergel(z, data));
2721#endif
2722}
2723
2724template <Index N, Index M>
2725EIGEN_ALWAYS_INLINE void storeConvertTwoBF16(float* to, PacketBlock<Packet8bf, (N + 7) / 8>& block, Index extra = 0) {
2726 EIGEN_IF_CONSTEXPR (N < 4) {
2727 pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra);
2728 } else EIGEN_IF_CONSTEXPR (N >= (M * 8 + 4)) {
2729 pstoreu(to + 0, oneConvertBF16Hi(block.packet[M].m_val));
2730 EIGEN_IF_CONSTEXPR (N >= 8) {
2731 pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val));
2732 }
2733 }
2734}
2735
2736template <Index N>
2737EIGEN_ALWAYS_INLINE void storeConvertBlockBF16(float* to, PacketBlock<Packet8bf, (N + 7) / 8>& block, Index extra) {
2738 storeConvertTwoBF16<N, 0>(to + 0, block, extra);
2739 EIGEN_IF_CONSTEXPR (N >= 16) {
2740 storeConvertTwoBF16<N, 1>(to + 8, block);
2741 }
2742 EIGEN_IF_CONSTEXPR (N >= 32) {
2743 storeConvertTwoBF16<N, 2>(to + 16, block);
2744 storeConvertTwoBF16<N, 3>(to + 24, block);
2745 }
2746}
2747
2748template <bool non_unit_stride, Index delta>
2749EIGEN_ALWAYS_INLINE Packet8bf loadBF16fromResult(bfloat16* src, Index resInc) {
2750 EIGEN_IF_CONSTEXPR (non_unit_stride) {
2751 return pgather<bfloat16, Packet8bf>(src + delta * resInc, resInc);
2752 } else {
2753 return ploadu<Packet8bf>(src + delta);
2754 }
2755}
2756
2757static Packet16uc p16uc_MERGE16_32_1 = {0, 1, 16, 17, 2, 3, 18, 19, 0, 1, 16, 17, 2, 3, 18, 19};
2758static Packet16uc p16uc_MERGE16_32_2 = {4, 5, 20, 21, 6, 7, 22, 23, 4, 5, 20, 21, 6, 7, 22, 23};
2759static Packet16uc p16uc_MERGE16_32_3 = {8, 9, 24, 25, 10, 11, 26, 27, 8, 9, 24, 25, 10, 11, 26, 27};
2760static Packet16uc p16uc_MERGE16_32_4 = {12, 13, 28, 29, 14, 15, 30, 31, 12, 13, 28, 29, 14, 15, 30, 31};
2761
2762static Packet16uc p16uc_MERGE16_32_5 = {0, 1, 16, 17, 16, 17, 16, 17, 0, 1, 16, 17, 16, 17, 16, 17};
2763static Packet16uc p16uc_MERGE16_32_6 = {2, 3, 18, 19, 18, 19, 18, 19, 2, 3, 18, 19, 18, 19, 18, 19};
2764static Packet16uc p16uc_MERGE16_32_7 = {4, 5, 20, 21, 20, 21, 20, 21, 4, 5, 20, 21, 20, 21, 20, 21};
2765static Packet16uc p16uc_MERGE16_32_8 = {6, 7, 22, 23, 22, 23, 22, 23, 6, 7, 22, 23, 22, 23, 22, 23};
2766
2767EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Perm(const Packet8us& data, const Packet16uc& mask) {
2768 Packet8us z = pset1<Packet8us>(0);
2769#ifdef _BIG_ENDIAN
2770 return reinterpret_cast<Packet4f>(vec_perm(data, z, mask));
2771#else
2772 return reinterpret_cast<Packet4f>(vec_perm(z, data, mask));
2773#endif
2774}
2775
2776template <bool lhsExtraRows, bool odd, Index size>
2777EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32DupOne(float* result, Index rows, const bfloat16* src,
2778 Index extra_rows) {
2779 Packet4f dup[4 * 4];
2780 Packet8bf data[4];
2781
2782 for (Index i = 0; i < size; i++) {
2783 data[i] = ploadu<Packet8bf>(src + rows * i);
2784 }
2785
2786 for (Index i = 0, j = 0; i < size; i++, j += 4) {
2787 dup[j + 0] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_5 : p16uc_MERGE16_32_1);
2788 dup[j + 1] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_6 : p16uc_MERGE16_32_2);
2789 dup[j + 2] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_7 : p16uc_MERGE16_32_3);
2790 dup[j + 3] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_8 : p16uc_MERGE16_32_4);
2791 }
2792
2793 for (Index j = 0; j < 4 * size; j += 4) {
2794 EIGEN_IF_CONSTEXPR (lhsExtraRows) {
2795 Packet4f z = pset1<Packet4f>(float(0));
2796 Index i = 0;
2797 do {
2798 pstoreu(result + (j + i) * 4, dup[j + i]);
2799 } while (++i < extra_rows);
2800 do {
2801 pstoreu(result + (j + i) * 4, z);
2802 } while (++i < 4);
2803 } else {
2804 for (Index i = 0; i < 4; i++) {
2805 pstoreu(result + (j + i) * 4, dup[j + i]);
2806 }
2807 }
2808 }
2809}
2810
2811template <bool lhsExtraRows>
2812EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32Dup(float* result, Index cols, Index rows, const bfloat16* src,
2813 Index delta, Index extra_rows) {
2814 Index col = 0;
2815 src += delta * 2;
2816 for (; col + 4 * 2 <= cols; col += 4 * 2, result += 4 * 4 * 4, src += 4 * rows) {
2817 convertArrayPointerBF16toF32DupOne<lhsExtraRows, false, 4>(result, rows, src, extra_rows);
2818 }
2819 for (; col + 2 <= cols; col += 2, result += 4 * 4, src += rows) {
2820 convertArrayPointerBF16toF32DupOne<lhsExtraRows, false, 1>(result, rows, src, extra_rows);
2821 }
2822 if (cols & 1) {
2823 convertArrayPointerBF16toF32DupOne<lhsExtraRows, true, 1>(result, rows, src - delta, extra_rows);
2824 }
2825}
2826
2827template <const Index size, bool non_unit_stride>
2828EIGEN_ALWAYS_INLINE void convertPointerBF16toF32(Index& i, float* result, Index rows, bfloat16*& src, Index resInc) {
2829 while (i + size <= rows) {
2830 const Index count = size == 1 ? rows - i : size;
2831 PacketBlock<Packet8bf, (size + 7) / 8> r32;
2832 EIGEN_IF_CONSTEXPR (size < 8) {
2833 r32.packet[0] = pgather_partial<bfloat16, Packet8bf>(src, non_unit_stride ? resInc : 1, count);
2834 } else {
2835 r32.packet[0] = loadBF16fromResult<non_unit_stride, 0>(src, resInc);
2836 }
2837 EIGEN_IF_CONSTEXPR (size >= 16) {
2838 r32.packet[1] = loadBF16fromResult<non_unit_stride, 8>(src, resInc);
2839 }
2840 EIGEN_IF_CONSTEXPR (size >= 32) {
2841 r32.packet[2] = loadBF16fromResult<non_unit_stride, 16>(src, resInc);
2842 r32.packet[3] = loadBF16fromResult<non_unit_stride, 24>(src, resInc);
2843 }
2844 storeConvertBlockBF16<size>(result + i, r32, rows & 3);
2845 i += count;
2846 if (i < rows) src += count * resInc;
2847 EIGEN_IF_CONSTEXPR (size != 32) break;
2848 }
2849}
2850
2851template <bool non_unit_stride>
2852EIGEN_ALWAYS_INLINE void convertArrayPointerBF16toF32(float* result, Index cols, Index rows, bfloat16* src,
2853 Index resInc) {
2854 for (Index col = 0; col < cols; col++, result += rows) {
2855 Index i = 0;
2856 bfloat16* src2 = src;
2857 convertPointerBF16toF32<32, non_unit_stride>(i, result, rows, src2, resInc);
2858 convertPointerBF16toF32<16, non_unit_stride>(i, result, rows, src2, resInc);
2859 convertPointerBF16toF32<8, non_unit_stride>(i, result, rows, src2, resInc);
2860 convertPointerBF16toF32<4, non_unit_stride>(i, result, rows, src2, resInc);
2861 convertPointerBF16toF32<1, non_unit_stride>(i, result, rows, src2, resInc);
2862 if (col + 1 < cols) src += rows * resInc;
2863 }
2864}
2865
2866template <Index num_acc, Index size = 4>
2867EIGEN_ALWAYS_INLINE void zeroAccumulators(Packet4f (&acc)[num_acc][size]) {
2868 Packet4f z = pset1<Packet4f>(float(0));
2869
2870 for (Index k = 0; k < num_acc; k++) {
2871 for (Index j = 0; j < size; j++) {
2872 acc[k][j] = z;
2873 }
2874 }
2875}
2876
2877template <Index num_acc>
2878EIGEN_ALWAYS_INLINE void tranposeResults(Packet4f (&acc)[num_acc][4]) {
2879 for (Index i = 0; i < num_acc; i++) {
2880 Packet4ui t0, t1, t2, t3;
2881 t0 = vec_mergeh(reinterpret_cast<Packet4ui>(acc[i][0]), reinterpret_cast<Packet4ui>(acc[i][2]));
2882 t1 = vec_mergel(reinterpret_cast<Packet4ui>(acc[i][0]), reinterpret_cast<Packet4ui>(acc[i][2]));
2883 t2 = vec_mergeh(reinterpret_cast<Packet4ui>(acc[i][1]), reinterpret_cast<Packet4ui>(acc[i][3]));
2884 t3 = vec_mergel(reinterpret_cast<Packet4ui>(acc[i][1]), reinterpret_cast<Packet4ui>(acc[i][3]));
2885 acc[i][0] = reinterpret_cast<Packet4f>(vec_mergeh(t0, t2));
2886 acc[i][1] = reinterpret_cast<Packet4f>(vec_mergel(t0, t2));
2887 acc[i][2] = reinterpret_cast<Packet4f>(vec_mergeh(t1, t3));
2888 acc[i][3] = reinterpret_cast<Packet4f>(vec_mergel(t1, t3));
2889 }
2890}
2891
2892template <Index num_acc>
2893EIGEN_ALWAYS_INLINE void addResults(Packet4f (&acc)[num_acc][4]) {
2894 for (Index i = 0, j = 0; j < num_acc; i++, j += 2) {
2895 for (Index x = 0, y = 0; x < 2; x++, y += 2) {
2896 for (Index w = 0, z = 0; w < 2; w++, z += 2) {
2897 acc[i][y + w] = acc[j + x][z + 0] + acc[j + x][z + 1];
2898 }
2899 }
2900 }
2901}
2902
2903template <Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs>
2904EIGEN_ALWAYS_INLINE void outputResultsVSX(Packet4f (&acc)[num_acc][4], Index rows, const Packet4f& pAlpha,
2905 float* result, const Index extra_cols, Index extra_rows) {
2906 tranposeResults<num_acc>(acc);
2907 addResults<num_acc>(acc);
2908
2909 constexpr Index real_rhs = ((num_rhs / 2) - (rhsExtraCols ? 1 : 0));
2910 Index k = 0;
2911 for (Index i = 0; i < real_rhs; i++, result += 4 * rows, k++) {
2912 storeResults<false, lhsExtraRows>(acc[k], rows, pAlpha, result, extra_cols, extra_rows);
2913 }
2914 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
2915 storeResults<rhsExtraCols, lhsExtraRows>(acc[k], rows, pAlpha, result, extra_cols, extra_rows);
2916 }
2917}
2918
2919template <bool zero>
2920EIGEN_ALWAYS_INLINE void loadTwoRhsFloat32(const float* block, Index strideB, Index i, Packet4f& dhs0, Packet4f& dhs1) {
2921 dhs0 = ploadu<Packet4f>(block + strideB * i + 0);
2922 EIGEN_IF_CONSTEXPR (zero) {
2923 Packet4f dhs2 = pset1<Packet4f>(float(0));
2924 dhs1 = vec_mergel(dhs0, dhs2);
2925 dhs0 = vec_mergeh(dhs0, dhs2);
2926 } else {
2927 dhs1 = ploadu<Packet4f>(block + strideB * i + 4);
2928 }
2929}
2930
2931template <Index num_acc, bool zero, bool rhsExtraCols, Index num_rhs>
2932EIGEN_ALWAYS_INLINE void KLoop(const float* indexA, const float* indexB, Packet4f (&acc)[num_acc][4], Index strideB,
2933 Index k, Index offsetB, Index extra_cols) {
2934 constexpr Index num_lhs = 4;
2935 Packet4f lhs[num_lhs], rhs[num_rhs];
2936
2937 constexpr Index real_rhs = (num_rhs - (rhsExtraCols ? 2 : 0));
2938 for (Index i = 0; i < real_rhs; i += 2) {
2939 loadTwoRhsFloat32<zero>(indexB + k * 4, strideB, i, rhs[i + 0], rhs[i + 1]);
2940 }
2941 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
2942 loadTwoRhsFloat32<zero>(indexB + k * extra_cols - offsetB, strideB, real_rhs, rhs[real_rhs + 0], rhs[real_rhs + 1]);
2943 }
2944
2945 indexA += 2 * k * 4;
2946 for (Index j = 0; j < num_lhs; j++) {
2947 lhs[j] = ploadu<Packet4f>(indexA + j * 4);
2948 }
2949
2950 for (Index j = 0; j < num_rhs; j++) {
2951 for (Index i = 0; i < num_lhs; i++) {
2952 acc[j][i] = pmadd(rhs[j], lhs[i], acc[j][i]);
2953 }
2954 }
2955}
2956
2957template <const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
2958EIGEN_ALWAYS_INLINE void colVSXLoopBodyIter(Index depth, Index rows, const Packet4f& pAlpha, const float* indexA,
2959 const float* indexB, Index strideB, Index offsetB, float* result,
2960 const Index extra_cols, const Index extra_rows) {
2961 constexpr Index num_rhs = num_acc;
2962
2963 Packet4f acc[num_acc][4];
2964
2965 zeroAccumulators<num_acc>(acc);
2966
2967 Index k;
2968 for (k = 0; k + 2 <= depth; k += 2) {
2969 KLoop<num_acc, false, rhsExtraCols, num_rhs>(indexA, indexB, acc, strideB, k, offsetB, extra_cols);
2970 }
2971 if (depth & 1) {
2972 KLoop<num_acc, true, rhsExtraCols, num_rhs>(indexA, indexB, acc, strideB, k, offsetB, extra_cols);
2973 }
2974
2975 outputResultsVSX<num_acc, rhsExtraCols, lhsExtraRows, num_rhs>(acc, rows, pAlpha, result, extra_cols, extra_rows);
2976}
2977
2978// No more than 4 (uses 2X the accumulators or 8X the number of VSX registers)
2979#define MAX_BFLOAT16_ACC_VSX 4
2980
2981template <const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
2982void colVSXLoopBody(Index& col, Index depth, Index cols, Index rows, const Packet4f& pAlpha, const float* indexA,
2983 const float* indexB, Index strideB, Index offsetB, float* result) {
2984 constexpr Index step = (num_acc * 4); // each accumulator has 4 elements
2985 const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0;
2986 const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0;
2987 constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC_VSX);
2988
2989 do {
2990 colVSXLoopBodyIter<num_acc * 2, rhsExtraCols, lhsExtraRows>(depth, rows, pAlpha, indexA, indexB, strideB, offsetB,
2991 result, extra_cols, extra_rows);
2992
2993 indexB += strideB * (num_acc * 2);
2994 result += rows * step;
2995 } while (multiIters && (step <= cols - (col += step)));
2996}
2997
2998template <const Index num_acc, bool rhsExtraCols, bool lhsExtraRows>
2999EIGEN_ALWAYS_INLINE void colVSXLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f& pAlpha,
3000 const float* indexA, const float* blockB, Index strideB, Index offsetB,
3001 float* result) {
3002 EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_ACC_VSX > num_acc) {
3003 colVSXLoopBody<num_acc + (rhsExtraCols ? 1 : 0), rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA,
3004 blockB, strideB, offsetB, result);
3005 }
3006}
3007
3008template <bool rhsExtraCols, bool lhsExtraRows>
3009void colVSXLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const Packet4f& pAlpha, const float* indexA,
3010 const float* blockB, Index strideB, Index offsetB, float* result) {
3011 switch ((cols - col) >> 2) {
3012 case 3:
3013 colVSXLoopBodyExtraN<3, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
3014 offsetB, result);
3015 break;
3016 case 2:
3017 colVSXLoopBodyExtraN<2, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
3018 offsetB, result);
3019 break;
3020 case 1:
3021 colVSXLoopBodyExtraN<1, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
3022 offsetB, result);
3023 break;
3024 default:
3025 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
3026 colVSXLoopBody<1, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result);
3027 }
3028 break;
3029 }
3030}
3031
3032template <Index size, bool lhsExtraRows = false>
3033EIGEN_ALWAYS_INLINE void colVSXLoops(Index depth, Index cols, Index rows, const Packet4f& pAlpha,
3034 const bfloat16* indexA, const float* indexA2, const float* blockB2, Index strideA,
3035 Index strideB, Index offsetB, float* result2) {
3036 Index delta_rows = 2 * (lhsExtraRows ? (rows & 3) : size);
3037 for (Index row = 0; row < size; row += 4) {
3038 convertArrayPointerBF16toF32Dup<lhsExtraRows>(const_cast<float*>(indexA2), strideA, delta_rows, indexA, row,
3039 rows & 3);
3040
3041 const float* blockB = blockB2;
3042 float* result = result2 + row;
3043
3044 Index col = 0;
3045 if (cols >= (MAX_BFLOAT16_ACC_VSX * 4)) {
3046 colVSXLoopBody<MAX_BFLOAT16_ACC_VSX, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB,
3047 strideB, 0, result);
3048 blockB += (strideB >> 1) * col;
3049 result += rows * col;
3050 }
3051 if (cols & 3) {
3052 colVSXLoopBodyExtra<true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, offsetB,
3053 result);
3054 } else {
3055 colVSXLoopBodyExtra<false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, 0, result);
3056 }
3057 }
3058}
3059
3060template <Index size>
3061EIGEN_ALWAYS_INLINE void calcVSXColLoops(const bfloat16*& indexA, const float* indexA2, Index& row, Index depth,
3062 Index cols, Index rows, const Packet4f& pAlpha, const float* indexB,
3063 Index strideA, Index strideB, Index offsetA, Index offsetB, Index bigSuffix,
3064 float* result) {
3065 if ((size == 16) || (rows & size)) {
3066 indexA += size * offsetA;
3067 colVSXLoops<size>(depth, cols, rows, pAlpha, indexA, indexA2, indexB, strideA, strideB, offsetB, result + row);
3068 row += size;
3069 indexA += bigSuffix * size / 16;
3070 }
3071}
3072
3073template <const Index size, typename DataMapper, typename MapperIndex>
3074EIGEN_ALWAYS_INLINE void convertBF16toF32(Index& i, float* result, Index rows, const DataMapper& src) {
3075 constexpr Index extra = ((size < 4) ? 4 : size);
3076 while (i + size <= rows) {
3077 PacketBlock<Packet8bf, (size + 7) / 8> r32;
3078 const MapperIndex mapperRow = convert_index<MapperIndex>(i);
3079 r32.packet[0] = src.template loadPacket<Packet8bf>(mapperRow);
3080 EIGEN_IF_CONSTEXPR (size >= 16) {
3081 r32.packet[1] = src.template loadPacket<Packet8bf>(mapperRow + 8);
3082 }
3083 EIGEN_IF_CONSTEXPR (size >= 32) {
3084 r32.packet[2] = src.template loadPacket<Packet8bf>(mapperRow + 16);
3085 r32.packet[3] = src.template loadPacket<Packet8bf>(mapperRow + 24);
3086 }
3087 storeConvertBlockBF16<size>(result + i, r32, rows & 3);
3088 i += extra;
3089 EIGEN_IF_CONSTEXPR (size != 32) break;
3090 }
3091}
3092
3093template <typename DataMapper>
3094EIGEN_ALWAYS_INLINE void convertArrayBF16toF32(float* result, Index cols, Index rows, const DataMapper& src) {
3095 using LinearMapper = typename DataMapper::LinearMapper;
3096 using MapperIndex = mapper_index_t<DataMapper>;
3097 for (Index j = 0; j < cols; j++, result += rows) {
3098 const LinearMapper src2 = src.getLinearMapper(0, convert_index<MapperIndex>(j));
3099 Index i = 0;
3100 convertBF16toF32<32, LinearMapper, MapperIndex>(i, result, rows, src2);
3101 convertBF16toF32<16, LinearMapper, MapperIndex>(i, result, rows, src2);
3102 convertBF16toF32<8, LinearMapper, MapperIndex>(i, result, rows, src2);
3103 convertBF16toF32<4, LinearMapper, MapperIndex>(i, result, rows, src2);
3104 convertBF16toF32<1, LinearMapper, MapperIndex>(i, result, rows, src2);
3105 }
3106}
3107
3108EIGEN_ALWAYS_INLINE Packet8bf convertF32toBF16VSX(const float* res) {
3109 return F32ToBf16Both(ploadu<Packet4f>(res + 0), ploadu<Packet4f>(res + 4));
3110}
3111
3112template <typename DataMapper, const Index size>
3113EIGEN_ALWAYS_INLINE void convertArrayF32toBF16ColVSX(float* result, Index col, Index rows, const DataMapper& res) {
3114 const DataMapper res2 = res.getSubMapper(0, convert_index<mapper_index_t<DataMapper>>(col));
3115 Index row;
3116 float* result2 = result + col * rows;
3117 for (row = 0; row + 8 <= rows; row += 8, result2 += 8) {
3118 // get and save block
3119 PacketBlock<Packet8bf, size> block;
3120 for (Index j = 0; j < size; j++) {
3121 block.packet[j] = convertF32toBF16VSX(result2 + j * rows);
3122 }
3123 res2.template storePacketBlock<Packet8bf, size>(convert_index<mapper_index_t<DataMapper>>(row), 0, block);
3124 }
3125 // extra rows
3126 if (row < rows) {
3127 for (int j = 0; j < size; j++) {
3128 Packet8bf fp16 = convertF32toBF16VSX(result2 + j * rows);
3129 res2.template storePacketPartial<Packet8bf>(convert_index<mapper_index_t<DataMapper>>(row), j, fp16,
3130 convert_index<mapper_index_t<DataMapper>>(rows & 7));
3131 }
3132 }
3133}
3134
3135template <typename DataMapper>
3136EIGEN_ALWAYS_INLINE void convertArrayF32toBF16VSX(float* result, Index cols, Index rows, const DataMapper& res) {
3137 Index col;
3138 for (col = 0; col + 4 <= cols; col += 4) {
3139 convertArrayF32toBF16ColVSX<DataMapper, 4>(result, col, rows, res);
3140 }
3141 // extra cols
3142 switch (cols - col) {
3143 case 1:
3144 convertArrayF32toBF16ColVSX<DataMapper, 1>(result, col, rows, res);
3145 break;
3146 case 2:
3147 convertArrayF32toBF16ColVSX<DataMapper, 2>(result, col, rows, res);
3148 break;
3149 case 3:
3150 convertArrayF32toBF16ColVSX<DataMapper, 3>(result, col, rows, res);
3151 break;
3152 }
3153}
3154
3155template <typename DataMapper>
3156void gemmbfloat16(const DataMapper& res, const bfloat16* indexA, const bfloat16* indexB, Index rows, Index depth,
3157 Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3158 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
3159 const Packet4f pAlpha = pset1<Packet4f>(falpha);
3160
3161 if (strideA == -1) strideA = depth;
3162 if (strideB == -1) strideB = depth;
3163
3164 ei_declare_aligned_stack_constructed_variable(float, result, cols* rows, 0);
3165 ei_declare_aligned_stack_constructed_variable(float, indexB2, strideB* cols, 0);
3166 ei_declare_aligned_stack_constructed_variable(float, indexA2, ((strideA + 1) & -2) * 4 * 2, 0);
3167
3168 convertArrayBF16toF32<DataMapper>(result, cols, rows, res);
3169 convertArrayPointerBF16toF32(indexB2, cols, strideB, const_cast<bfloat16*>(indexB));
3170
3171 Index bigSuffix = 2 * 8 * (strideA - offsetA);
3172 float* indexBF32 = indexB2 + 4 * offsetB;
3173 offsetB *= 3;
3174 strideB *= 2;
3175
3176 Index row = 0;
3177 // LHS (8x16) block
3178 while (row + 16 <= rows) {
3179 calcVSXColLoops<16>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
3180 bigSuffix, result);
3181 }
3182 // LHS (8x8) block
3183 calcVSXColLoops<8>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
3184 bigSuffix, result);
3185 // LHS (8x4) block
3186 calcVSXColLoops<4>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
3187 bigSuffix, result);
3188 // extra rows
3189 if (rows & 3) {
3190 // This index is the beginning of remaining block.
3191 colVSXLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexA2, indexBF32, strideA, strideB, offsetB,
3192 result + row);
3193 }
3194
3195 // Convert back to bfloat16
3196 convertArrayF32toBF16VSX<DataMapper>(result, cols, rows, res);
3197}
3198
3199#undef MAX_BFLOAT16_ACC_VSX
3200
3201#include "MatrixVectorProduct.inc"
3202
3203/************************************
3204 * ppc64le template specializations *
3205 * **********************************/
3206template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3207struct gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> {
3208 void operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3209};
3210
3211template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3212void gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>::operator()(
3213 double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3214 dhs_pack<double, DataMapper, Packet2d, ColMajor, PanelMode, true> pack;
3215 pack(blockA, lhs, depth, rows, stride, offset);
3216}
3217
3218template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3219struct gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> {
3220 void operator()(double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3221};
3222
3223template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3224void gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>::operator()(
3225 double* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3226 dhs_pack<double, DataMapper, Packet2d, RowMajor, PanelMode, true> pack;
3227 pack(blockA, lhs, depth, rows, stride, offset);
3228}
3229
3230#if EIGEN_ALTIVEC_USE_CUSTOM_PACK
3231template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3232struct gemm_pack_rhs<double, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> {
3233 void operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3234};
3235
3236template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3237void gemm_pack_rhs<double, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3238 double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3239 dhs_pack<double, DataMapper, Packet2d, ColMajor, PanelMode, false> pack;
3240 pack(blockB, rhs, depth, cols, stride, offset);
3241}
3242
3243template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3244struct gemm_pack_rhs<double, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> {
3245 void operator()(double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3246};
3247
3248template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3249void gemm_pack_rhs<double, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3250 double* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3251 dhs_pack<double, DataMapper, Packet2d, RowMajor, PanelMode, false> pack;
3252 pack(blockB, rhs, depth, cols, stride, offset);
3253}
3254
3255template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3256struct gemm_pack_rhs<bfloat16, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> {
3257 void operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3258};
3259
3260template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3261void gemm_pack_rhs<bfloat16, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3262 bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3263 dhs_pack<bfloat16, DataMapper, Packet8bf, ColMajor, PanelMode, false> pack;
3264 pack(blockB, rhs, depth, cols, stride, offset);
3265}
3266
3267template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3268struct gemm_pack_rhs<bfloat16, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> {
3269 void operator()(bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3270};
3271
3272template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3273void gemm_pack_rhs<bfloat16, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3274 bfloat16* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3275 dhs_pack<bfloat16, DataMapper, Packet8bf, RowMajor, PanelMode, false> pack;
3276 pack(blockB, rhs, depth, cols, stride, offset);
3277}
3278#endif
3279
3280template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3281struct gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> {
3282 void operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3283};
3284
3285template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3286void gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>::operator()(
3287 bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3288 dhs_pack<bfloat16, DataMapper, Packet8bf, ColMajor, PanelMode, true> pack;
3289 pack(blockA, lhs, depth, rows, stride, offset);
3290}
3291
3292template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3293struct gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> {
3294 void operator()(bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3295};
3296
3297template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3298void gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>::operator()(
3299 bfloat16* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3300 dhs_pack<bfloat16, DataMapper, Packet8bf, RowMajor, PanelMode, true> pack;
3301 pack(blockA, lhs, depth, rows, stride, offset);
3302}
3303
3304template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3305struct gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> {
3306 void operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3307};
3308
3309template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3310void gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>::operator()(
3311 float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3312 dhs_pack<float, DataMapper, Packet4f, RowMajor, PanelMode, true> pack;
3313 pack(blockA, lhs, depth, rows, stride, offset);
3314}
3315
3316template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3317struct gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> {
3318 void operator()(float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3319};
3320
3321template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3322void gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>::operator()(
3323 float* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3324 dhs_pack<float, DataMapper, Packet4f, ColMajor, PanelMode, true> pack;
3325 pack(blockA, lhs, depth, rows, stride, offset);
3326}
3327
3328template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3329struct gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> {
3330 void operator()(std::complex<float>* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3331 Index offset = 0);
3332};
3333
3334template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3335void gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate,
3336 PanelMode>::operator()(std::complex<float>* blockA, const DataMapper& lhs, Index depth, Index rows,
3337 Index stride, Index offset) {
3338 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, RowMajor, Conjugate, PanelMode, true> pack;
3339 pack(blockA, lhs, depth, rows, stride, offset);
3340}
3341
3342template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3343struct gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> {
3344 void operator()(std::complex<float>* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3345 Index offset = 0);
3346};
3347
3348template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3349void gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate,
3350 PanelMode>::operator()(std::complex<float>* blockA, const DataMapper& lhs, Index depth, Index rows,
3351 Index stride, Index offset) {
3352 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, ColMajor, Conjugate, PanelMode, true> pack;
3353 pack(blockA, lhs, depth, rows, stride, offset);
3354}
3355
3356#if EIGEN_ALTIVEC_USE_CUSTOM_PACK
3357template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3358struct gemm_pack_rhs<float, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> {
3359 void operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3360};
3361
3362template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3363void gemm_pack_rhs<float, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3364 float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3365 dhs_pack<float, DataMapper, Packet4f, ColMajor, PanelMode, false> pack;
3366 pack(blockB, rhs, depth, cols, stride, offset);
3367}
3368
3369template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3370struct gemm_pack_rhs<float, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> {
3371 void operator()(float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3372};
3373
3374template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3375void gemm_pack_rhs<float, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3376 float* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3377 dhs_pack<float, DataMapper, Packet4f, RowMajor, PanelMode, false> pack;
3378 pack(blockB, rhs, depth, cols, stride, offset);
3379}
3380#endif
3381
3382template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3383struct gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> {
3384 void operator()(std::complex<float>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3385 Index offset = 0);
3386};
3387
3388template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3389void gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3390 std::complex<float>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3391 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, ColMajor, Conjugate, PanelMode, false> pack;
3392 pack(blockB, rhs, depth, cols, stride, offset);
3393}
3394
3395template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3396struct gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> {
3397 void operator()(std::complex<float>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3398 Index offset = 0);
3399};
3400
3401template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3402void gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3403 std::complex<float>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3404 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, RowMajor, Conjugate, PanelMode, false> pack;
3405 pack(blockB, rhs, depth, cols, stride, offset);
3406}
3407
3408template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3409struct gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode> {
3410 void operator()(std::complex<double>* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3411 Index offset = 0);
3412};
3413
3414template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3415void gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate,
3416 PanelMode>::operator()(std::complex<double>* blockA, const DataMapper& lhs, Index depth, Index rows,
3417 Index stride, Index offset) {
3418 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, RowMajor, Conjugate, PanelMode, true> pack;
3419 pack(blockA, lhs, depth, rows, stride, offset);
3420}
3421
3422template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3423struct gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode> {
3424 void operator()(std::complex<double>* blockA, const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3425 Index offset = 0);
3426};
3427
3428template <typename Index, typename DataMapper, int Pack1, int Pack2, typename Packet, bool Conjugate, bool PanelMode>
3429void gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate,
3430 PanelMode>::operator()(std::complex<double>* blockA, const DataMapper& lhs, Index depth, Index rows,
3431 Index stride, Index offset) {
3432 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, ColMajor, Conjugate, PanelMode, true> pack;
3433 pack(blockA, lhs, depth, rows, stride, offset);
3434}
3435
3436template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3437struct gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode> {
3438 void operator()(std::complex<double>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3439 Index offset = 0);
3440};
3441
3442template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3443void gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3444 std::complex<double>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3445 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, ColMajor, Conjugate, PanelMode, false> pack;
3446 pack(blockB, rhs, depth, cols, stride, offset);
3447}
3448
3449template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3450struct gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode> {
3451 void operator()(std::complex<double>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3452 Index offset = 0);
3453};
3454
3455template <typename Index, typename DataMapper, int nr, bool Conjugate, bool PanelMode>
3456void gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3457 std::complex<double>* blockB, const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3458 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, RowMajor, Conjugate, PanelMode, false> pack;
3459 pack(blockB, rhs, depth, cols, stride, offset);
3460}
3461
3462// ********* gebp specializations *********
3463template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3464struct gebp_kernel<float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3465 using Packet = typename quad_traits<float>::vectortype;
3466 using RhsPacket = typename quad_traits<float>::rhstype;
3467
3468 void operator()(const DataMapper& res, const float* blockA, const float* blockB, Index rows, Index depth, Index cols,
3469 float alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0, Index offsetB = 0);
3470};
3471
3472template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3473void gebp_kernel<float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3474 const DataMapper& res, const float* blockA, const float* blockB, Index rows, Index depth, Index cols, float alpha,
3475 Index strideA, Index strideB, Index offsetA, Index offsetB) {
3476 const Eigen::Index accRows = quad_traits<float>::rows;
3477 const Eigen::Index accCols = quad_traits<float>::size;
3478 // The kernels below take Eigen::Index, which need not be the Index this kernel is instantiated with: a Tensor
3479 // contraction instantiates gebp_kernel with the tensor's StorageIndex.
3480 static void (*gemm_function)(const DataMapper&, const float*, const float*, Eigen::Index, Eigen::Index, Eigen::Index,
3481 float, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) =
3482#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3483 (supportsMMA()) ? &Eigen::internal::gemmMMA<float, Packet, RhsPacket, DataMapper, accRows, accCols> :
3484#endif
3485 &Eigen::internal::gemm<float, Packet, RhsPacket, DataMapper, accRows, accCols>;
3486 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3487}
3488
3489template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3490struct gebp_kernel<std::complex<float>, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3491 using Packet = Packet4f;
3492 using Packetc = Packet2cf;
3493 using RhsPacket = Packet4f;
3494
3495 void operator()(const DataMapper& res, const std::complex<float>* blockA, const std::complex<float>* blockB,
3496 Index rows, Index depth, Index cols, std::complex<float> alpha, Index strideA = -1,
3497 Index strideB = -1, Index offsetA = 0, Index offsetB = 0);
3498};
3499
3500template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3501void gebp_kernel<std::complex<float>, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs,
3502 ConjugateRhs>::operator()(const DataMapper& res, const std::complex<float>* blockA,
3503 const std::complex<float>* blockB, Index rows, Index depth, Index cols,
3504 std::complex<float> alpha, Index strideA, Index strideB, Index offsetA,
3505 Index offsetB) {
3506 const Eigen::Index accRows = quad_traits<float>::rows;
3507 const Eigen::Index accCols = quad_traits<float>::size;
3508 static void (*gemm_function)(const DataMapper&, const std::complex<float>*, const std::complex<float>*, Eigen::Index,
3509 Eigen::Index, Eigen::Index, std::complex<float>, Eigen::Index, Eigen::Index,
3510 Eigen::Index, Eigen::Index) =
3511#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3512 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<std::complex<float>, std::complex<float>, std::complex<float>,
3513 float, Packet, Packetc, RhsPacket, DataMapper, accRows,
3514 accCols, ConjugateLhs, ConjugateRhs, false, false>
3515 :
3516#endif
3517 &Eigen::internal::gemm_complex<std::complex<float>, std::complex<float>, std::complex<float>,
3518 float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3519 ConjugateLhs, ConjugateRhs, false, false>;
3520 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3521}
3522
3523template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3524struct gebp_kernel<float, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3525 using Packet = Packet4f;
3526 using Packetc = Packet2cf;
3527 using RhsPacket = Packet4f;
3528
3529 void operator()(const DataMapper& res, const float* blockA, const std::complex<float>* blockB, Index rows,
3530 Index depth, Index cols, std::complex<float> alpha, Index strideA = -1, Index strideB = -1,
3531 Index offsetA = 0, Index offsetB = 0);
3532};
3533
3534template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3535void gebp_kernel<float, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3536 const DataMapper& res, const float* blockA, const std::complex<float>* blockB, Index rows, Index depth, Index cols,
3537 std::complex<float> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3538 const Eigen::Index accRows = quad_traits<float>::rows;
3539 const Eigen::Index accCols = quad_traits<float>::size;
3540 static void (*gemm_function)(const DataMapper&, const float*, const std::complex<float>*, Eigen::Index, Eigen::Index,
3541 Eigen::Index, std::complex<float>, Eigen::Index, Eigen::Index, Eigen::Index,
3542 Eigen::Index) =
3543#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3544 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<float, std::complex<float>, std::complex<float>, float,
3545 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3546 ConjugateLhs, ConjugateRhs, true, false>
3547 :
3548#endif
3549 &Eigen::internal::gemm_complex<float, std::complex<float>, std::complex<float>, float, Packet,
3550 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3551 ConjugateRhs, true, false>;
3552 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3553}
3554
3555template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3556struct gebp_kernel<std::complex<float>, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3557 using Packet = Packet4f;
3558 using Packetc = Packet2cf;
3559 using RhsPacket = Packet4f;
3560
3561 void operator()(const DataMapper& res, const std::complex<float>* blockA, const float* blockB, Index rows,
3562 Index depth, Index cols, std::complex<float> alpha, Index strideA = -1, Index strideB = -1,
3563 Index offsetA = 0, Index offsetB = 0);
3564};
3565
3566template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3567void gebp_kernel<std::complex<float>, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3568 const DataMapper& res, const std::complex<float>* blockA, const float* blockB, Index rows, Index depth, Index cols,
3569 std::complex<float> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3570 const Eigen::Index accRows = quad_traits<float>::rows;
3571 const Eigen::Index accCols = quad_traits<float>::size;
3572 static void (*gemm_function)(const DataMapper&, const std::complex<float>*, const float*, Eigen::Index, Eigen::Index,
3573 Eigen::Index, std::complex<float>, Eigen::Index, Eigen::Index, Eigen::Index,
3574 Eigen::Index) =
3575#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3576 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<std::complex<float>, float, std::complex<float>, float,
3577 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3578 ConjugateLhs, ConjugateRhs, false, true>
3579 :
3580#endif
3581 &Eigen::internal::gemm_complex<std::complex<float>, float, std::complex<float>, float, Packet,
3582 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3583 ConjugateRhs, false, true>;
3584 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3585}
3586
3587template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3588struct gebp_kernel<double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3589 using Packet = typename quad_traits<double>::vectortype;
3590 using RhsPacket = typename quad_traits<double>::rhstype;
3591
3592 void operator()(const DataMapper& res, const double* blockA, const double* blockB, Index rows, Index depth,
3593 Index cols, double alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0,
3594 Index offsetB = 0);
3595};
3596
3597template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3598void gebp_kernel<double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3599 const DataMapper& res, const double* blockA, const double* blockB, Index rows, Index depth, Index cols,
3600 double alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3601 const Eigen::Index accRows = quad_traits<double>::rows;
3602 const Eigen::Index accCols = quad_traits<double>::size;
3603 static void (*gemm_function)(const DataMapper&, const double*, const double*, Eigen::Index, Eigen::Index,
3604 Eigen::Index, double, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) =
3605#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3606 (supportsMMA()) ? &Eigen::internal::gemmMMA<double, Packet, RhsPacket, DataMapper, accRows, accCols> :
3607#endif
3608 &Eigen::internal::gemm<double, Packet, RhsPacket, DataMapper, accRows, accCols>;
3609 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3610}
3611
3612template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3613struct gebp_kernel<std::complex<double>, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3614 using Packet = quad_traits<double>::vectortype;
3615 using Packetc = Packet1cd;
3616 using RhsPacket = quad_traits<double>::rhstype;
3617
3618 void operator()(const DataMapper& res, const std::complex<double>* blockA, const std::complex<double>* blockB,
3619 Index rows, Index depth, Index cols, std::complex<double> alpha, Index strideA = -1,
3620 Index strideB = -1, Index offsetA = 0, Index offsetB = 0);
3621};
3622
3623template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3624void gebp_kernel<std::complex<double>, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs,
3625 ConjugateRhs>::operator()(const DataMapper& res, const std::complex<double>* blockA,
3626 const std::complex<double>* blockB, Index rows, Index depth, Index cols,
3627 std::complex<double> alpha, Index strideA, Index strideB, Index offsetA,
3628 Index offsetB) {
3629 const Eigen::Index accRows = quad_traits<double>::rows;
3630 const Eigen::Index accCols = quad_traits<double>::size;
3631 static void (*gemm_function)(const DataMapper&, const std::complex<double>*, const std::complex<double>*,
3632 Eigen::Index, Eigen::Index, Eigen::Index, std::complex<double>, Eigen::Index,
3633 Eigen::Index, Eigen::Index, Eigen::Index) =
3634#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3635 (supportsMMA())
3636 ? &Eigen::internal::gemm_complexMMA<std::complex<double>, std::complex<double>, std::complex<double>, double,
3637 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3638 ConjugateRhs, false, false>
3639 :
3640#endif
3641 &Eigen::internal::gemm_complex<std::complex<double>, std::complex<double>, std::complex<double>, double,
3642 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3643 ConjugateRhs, false, false>;
3644 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3645}
3646
3647template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3648struct gebp_kernel<std::complex<double>, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3649 using Packet = quad_traits<double>::vectortype;
3650 using Packetc = Packet1cd;
3651 using RhsPacket = quad_traits<double>::rhstype;
3652
3653 void operator()(const DataMapper& res, const std::complex<double>* blockA, const double* blockB, Index rows,
3654 Index depth, Index cols, std::complex<double> alpha, Index strideA = -1, Index strideB = -1,
3655 Index offsetA = 0, Index offsetB = 0);
3656};
3657
3658template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3659void gebp_kernel<std::complex<double>, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3660 const DataMapper& res, const std::complex<double>* blockA, const double* blockB, Index rows, Index depth,
3661 Index cols, std::complex<double> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3662 const Eigen::Index accRows = quad_traits<double>::rows;
3663 const Eigen::Index accCols = quad_traits<double>::size;
3664 static void (*gemm_function)(const DataMapper&, const std::complex<double>*, const double*, Eigen::Index,
3665 Eigen::Index, Eigen::Index, std::complex<double>, Eigen::Index, Eigen::Index,
3666 Eigen::Index, Eigen::Index) =
3667#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3668 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<std::complex<double>, double, std::complex<double>, double,
3669 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3670 ConjugateLhs, ConjugateRhs, false, true>
3671 :
3672#endif
3673 &Eigen::internal::gemm_complex<std::complex<double>, double, std::complex<double>, double, Packet,
3674 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3675 ConjugateRhs, false, true>;
3676 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3677}
3678
3679template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3680struct gebp_kernel<double, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3681 using Packet = quad_traits<double>::vectortype;
3682 using Packetc = Packet1cd;
3683 using RhsPacket = quad_traits<double>::rhstype;
3684
3685 void operator()(const DataMapper& res, const double* blockA, const std::complex<double>* blockB, Index rows,
3686 Index depth, Index cols, std::complex<double> alpha, Index strideA = -1, Index strideB = -1,
3687 Index offsetA = 0, Index offsetB = 0);
3688};
3689
3690template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3691void gebp_kernel<double, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3692 const DataMapper& res, const double* blockA, const std::complex<double>* blockB, Index rows, Index depth,
3693 Index cols, std::complex<double> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3694 const Eigen::Index accRows = quad_traits<double>::rows;
3695 const Eigen::Index accCols = quad_traits<double>::size;
3696 static void (*gemm_function)(const DataMapper&, const double*, const std::complex<double>*, Eigen::Index,
3697 Eigen::Index, Eigen::Index, std::complex<double>, Eigen::Index, Eigen::Index,
3698 Eigen::Index, Eigen::Index) =
3699#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3700 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<double, std::complex<double>, std::complex<double>, double,
3701 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3702 ConjugateLhs, ConjugateRhs, true, false>
3703 :
3704#endif
3705 &Eigen::internal::gemm_complex<double, std::complex<double>, std::complex<double>, double, Packet,
3706 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3707 ConjugateRhs, true, false>;
3708 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3709}
3710
3711template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3712struct gebp_kernel<bfloat16, bfloat16, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3713 using Packet = typename quad_traits<bfloat16>::vectortype;
3714 using RhsPacket = typename quad_traits<bfloat16>::rhstype;
3715
3716 void operator()(const DataMapper& res, const bfloat16* blockA, const bfloat16* blockB, Index rows, Index depth,
3717 Index cols, bfloat16 alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0,
3718 Index offsetB = 0);
3719};
3720
3721template <typename Index, typename DataMapper, int mr, int nr, bool ConjugateLhs, bool ConjugateRhs>
3722void gebp_kernel<bfloat16, bfloat16, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3723 const DataMapper& res, const bfloat16* blockA, const bfloat16* blockB, Index rows, Index depth, Index cols,
3724 bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3725 static void (*gemm_function)(const DataMapper&, const bfloat16*, const bfloat16*, Eigen::Index, Eigen::Index,
3726 Eigen::Index, bfloat16, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) =
3727#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3728 (supportsMMA()) ? &Eigen::internal::gemmMMAbfloat16<DataMapper> :
3729#endif
3730 &Eigen::internal::gemmbfloat16<DataMapper>;
3731 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3732}
3733} // end namespace internal
3734
3735} // end namespace Eigen
3736
3737#endif // EIGEN_MATRIX_PRODUCT_ALTIVEC_H
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321