12#ifndef EIGEN_MATRIX_PRODUCT_ALTIVEC_H
13#define EIGEN_MATRIX_PRODUCT_ALTIVEC_H
15#ifndef EIGEN_ALTIVEC_USE_CUSTOM_PACK
16#define EIGEN_ALTIVEC_USE_CUSTOM_PACK 1
19#if !defined(EIGEN_ALTIVEC_DISABLE_MMA)
20#define EIGEN_ALTIVEC_DISABLE_MMA 0
24#if !EIGEN_ALTIVEC_DISABLE_MMA && defined(__has_builtin)
25#if __has_builtin(__builtin_mma_assemble_acc)
26#define EIGEN_ALTIVEC_MMA_SUPPORT
31#if defined(EIGEN_ALTIVEC_MMA_SUPPORT)
33#if !defined(EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH)
34#define EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH 0
38#if EIGEN_ALTIVEC_ENABLE_MMA_DYNAMIC_DISPATCH && !EIGEN_COMP_LLVM
39#define EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH 1
42#define EIGEN_ALTIVEC_MMA_ONLY 1
47#include "MatrixProductCommon.h"
49#if defined(EIGEN_ALTIVEC_MMA_ONLY) || defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH)
50#include "MatrixProductMMA.h"
54#include "../../InternalHeaderCheck.h"
63template <
typename Scalar>
65 using vectortype =
typename packet_traits<Scalar>::type;
66 using type = PacketBlock<vectortype, 4>;
67 using rhstype = vectortype;
68 enum { vectorsize = packet_traits<Scalar>::size, size = 4, rows = 4 };
72struct quad_traits<double> {
73 using vectortype = Packet2d;
74 using type = PacketBlock<vectortype, 4>;
75 using rhstype = PacketBlock<Packet2d, 2>;
76 enum { vectorsize = packet_traits<double>::size, size = 2, rows = 4 };
80struct quad_traits<bfloat16> {
81 using vectortype = Packet8bf;
82 using type = PacketBlock<vectortype, 4>;
83 using rhstype = vectortype;
84 enum { vectorsize = packet_traits<bfloat16>::size, size = 8, rows = 4 };
91const static Packet16uc p16uc_GETREAL32 = {0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27};
93const static Packet16uc p16uc_GETIMAG32 = {4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31};
95const static Packet16uc p16uc_GETREAL32b = {0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27};
97const static Packet16uc p16uc_GETIMAG32b = {4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31};
117template <
typename Scalar,
int StorageOrder>
118EIGEN_ALWAYS_INLINE std::complex<Scalar> getAdjointVal(
119 Index i, Index j, const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder>& dt) {
120 std::complex<Scalar> v;
122 v.real(dt(j, i).real());
123 v.imag(-dt(j, i).imag());
125 v.real(dt(i, j).real());
126 v.imag(dt(i, j).imag());
128 v.real(dt(i, j).real());
134template <
typename Scalar,
int StorageOrder,
int N>
135EIGEN_STRONG_INLINE
void symm_pack_complex_rhs_helper(std::complex<Scalar>* blockB,
const std::complex<Scalar>* _rhs,
136 Index rhsStride, Index rows, Index cols, Index k2) {
137 const Index depth = k2 + rows;
138 const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder> rhs(_rhs, rhsStride);
139 const Index vectorSize = N * quad_traits<Scalar>::vectorsize;
140 const Index vectorDelta = vectorSize * rows;
141 Scalar* blockBf =
reinterpret_cast<Scalar*
>(blockB);
143 Index rir = 0, rii, j = 0;
144 for (; j + vectorSize <= cols; j += vectorSize) {
145 rii = rir + vectorDelta;
147 for (Index i = k2; i < depth; i++) {
148 for (Index k = 0; k < vectorSize; k++) {
149 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(i, j + k, rhs);
151 blockBf[rir + k] = v.real();
152 blockBf[rii + k] = v.imag();
161 for (; j < cols; j++) {
164 for (Index i = k2; i < depth; i++) {
165 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(i, j, rhs);
167 blockBf[rir] = v.real();
168 blockBf[rii] = v.imag();
178template <
typename Scalar,
int StorageOrder>
179EIGEN_STRONG_INLINE
void symm_pack_complex_lhs_helper(std::complex<Scalar>* blockA,
const std::complex<Scalar>* _lhs,
180 Index lhsStride, Index cols, Index rows) {
181 const Index depth = cols;
182 const_blas_data_mapper<std::complex<Scalar>, Index, StorageOrder> lhs(_lhs, lhsStride);
183 const Index vectorSize = quad_traits<Scalar>::vectorsize;
184 const Index vectorDelta = vectorSize * depth;
185 Scalar* blockAf =
reinterpret_cast<Scalar*
>(blockA);
187 Index rir = 0, rii, j = 0;
188 for (; j + vectorSize <= rows; j += vectorSize) {
189 rii = rir + vectorDelta;
191 for (Index i = 0; i < depth; i++) {
192 for (Index k = 0; k < vectorSize; k++) {
193 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(j + k, i, lhs);
195 blockAf[rir + k] = v.real();
196 blockAf[rii + k] = v.imag();
206 rii = rir + ((rows - j) * depth);
208 for (Index i = 0; i < depth; i++) {
210 for (; k < rows; k++) {
211 std::complex<Scalar> v = getAdjointVal<Scalar, StorageOrder>(k, i, lhs);
213 blockAf[rir] = v.real();
214 blockAf[rii] = v.imag();
223template <
typename Scalar,
int StorageOrder,
int N>
224EIGEN_STRONG_INLINE
void symm_pack_rhs_helper(Scalar* blockB,
const Scalar* _rhs, Index rhsStride, Index rows,
225 Index cols, Index k2) {
226 const Index depth = k2 + rows;
227 const_blas_data_mapper<Scalar, Index, StorageOrder> rhs(_rhs, rhsStride);
228 const Index vectorSize = quad_traits<Scalar>::vectorsize;
231 for (; j + N * vectorSize <= cols; j += N * vectorSize) {
233 for (; i < depth; i++) {
234 for (Index k = 0; k < N * vectorSize; k++) {
236 blockB[ri + k] = rhs(j + k, i);
238 blockB[ri + k] = rhs(i, j + k);
240 ri += N * vectorSize;
244 for (; j < cols; j++) {
245 for (Index i = k2; i < depth; i++) {
247 blockB[ri] = rhs(i, j);
249 blockB[ri] = rhs(j, i);
255template <
typename Scalar,
int StorageOrder>
256EIGEN_STRONG_INLINE
void symm_pack_lhs_helper(Scalar* blockA,
const Scalar* _lhs, Index lhsStride, Index cols,
258 const Index depth = cols;
259 const_blas_data_mapper<Scalar, Index, StorageOrder> lhs(_lhs, lhsStride);
260 const Index vectorSize = quad_traits<Scalar>::vectorsize;
263 for (; j + vectorSize <= rows; j += vectorSize) {
266 for (; i < depth; i++) {
267 for (Index k = 0; k < vectorSize; k++) {
269 blockA[ri + k] = lhs(j + k, i);
271 blockA[ri + k] = lhs(i, j + k);
278 for (Index i = 0; i < depth; i++) {
280 for (; k < rows; k++) {
282 blockA[ri] = lhs(k, i);
284 blockA[ri] = lhs(i, k);
291template <
typename Index,
int nr,
int StorageOrder>
292struct symm_pack_rhs<std::complex<float>, Index, nr, StorageOrder> {
293 void operator()(std::complex<float>* blockB,
const std::complex<float>* _rhs, Index rhsStride, Index rows, Index cols,
295 symm_pack_complex_rhs_helper<float, StorageOrder, 1>(blockB, _rhs, rhsStride, rows, cols, k2);
299template <
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
300struct symm_pack_lhs<std::complex<float>, Index, Pack1, Pack2_dummy, StorageOrder> {
301 void operator()(std::complex<float>* blockA,
const std::complex<float>* _lhs, Index lhsStride, Index cols,
303 symm_pack_complex_lhs_helper<float, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
309template <
typename Index,
int nr,
int StorageOrder>
310struct symm_pack_rhs<std::complex<double>, Index, nr, StorageOrder> {
311 void operator()(std::complex<double>* blockB,
const std::complex<double>* _rhs, Index rhsStride, Index rows,
312 Index cols, Index k2) {
313 symm_pack_complex_rhs_helper<double, StorageOrder, 2>(blockB, _rhs, rhsStride, rows, cols, k2);
317template <
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
318struct symm_pack_lhs<std::complex<double>, Index, Pack1, Pack2_dummy, StorageOrder> {
319 void operator()(std::complex<double>* blockA,
const std::complex<double>* _lhs, Index lhsStride, Index cols,
321 symm_pack_complex_lhs_helper<double, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
326template <
typename Index,
int nr,
int StorageOrder>
327struct symm_pack_rhs<float, Index, nr, StorageOrder> {
328 void operator()(
float* blockB,
const float* _rhs, Index rhsStride, Index rows, Index cols, Index k2) {
329 symm_pack_rhs_helper<float, StorageOrder, 1>(blockB, _rhs, rhsStride, rows, cols, k2);
333template <
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
334struct symm_pack_lhs<float, Index, Pack1, Pack2_dummy, StorageOrder> {
335 void operator()(
float* blockA,
const float* _lhs, Index lhsStride, Index cols, Index rows) {
336 symm_pack_lhs_helper<float, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
341template <
typename Index,
int nr,
int StorageOrder>
342struct symm_pack_rhs<double, Index, nr, StorageOrder> {
343 void operator()(
double* blockB,
const double* _rhs, Index rhsStride, Index rows, Index cols, Index k2) {
344 symm_pack_rhs_helper<double, StorageOrder, 2>(blockB, _rhs, rhsStride, rows, cols, k2);
348template <
typename Index,
int Pack1,
int Pack2_dummy,
int StorageOrder>
349struct symm_pack_lhs<double, Index, Pack1, Pack2_dummy, StorageOrder> {
350 void operator()(
double* blockA,
const double* _lhs, Index lhsStride, Index cols, Index rows) {
351 symm_pack_lhs_helper<double, StorageOrder>(blockA, _lhs, lhsStride, cols, rows);
366template <
typename Scalar,
typename Packet,
int N>
367EIGEN_ALWAYS_INLINE
void storeBlock(Scalar* to, PacketBlock<Packet, N>& block) {
368 const Index size = 16 /
sizeof(Scalar);
369 pstore<Scalar>(to + (0 * size), block.packet[0]);
370 pstore<Scalar>(to + (1 * size), block.packet[1]);
371 EIGEN_IF_CONSTEXPR (N > 2) {
372 pstore<Scalar>(to + (2 * size), block.packet[2]);
374 EIGEN_IF_CONSTEXPR (N > 3) {
375 pstore<Scalar>(to + (3 * size), block.packet[3]);
380template <
typename Scalar,
typename DataMapper,
typename Packet,
typename PacketC,
int StorageOrder,
bool Conjugate,
381 bool PanelMode,
bool UseLhs>
383 using Index = mapper_index_t<DataMapper>;
384 template <
bool transpose>
385 EIGEN_ALWAYS_INLINE
void dhs_cblock(PacketBlock<PacketC, 8>& cblock, PacketBlock<Packet, 4>& block,
386 const Packet16uc& permute) {
388 block.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, permute);
389 block.packet[1] = vec_perm(cblock.packet[2].v, cblock.packet[3].v, permute);
390 block.packet[2] = vec_perm(cblock.packet[4].v, cblock.packet[5].v, permute);
391 block.packet[3] = vec_perm(cblock.packet[6].v, cblock.packet[7].v, permute);
393 Packet4f t0, t1, t2, t3;
394#ifdef EIGEN_VECTORIZE_VSX
395 t0 =
reinterpret_cast<Packet
>(
396 vec_mergeh(
reinterpret_cast<Packet2ul
>(block.packet[0]),
reinterpret_cast<Packet2ul
>(block.packet[1])));
397 t1 =
reinterpret_cast<Packet
>(
398 vec_mergel(
reinterpret_cast<Packet2ul
>(block.packet[0]),
reinterpret_cast<Packet2ul
>(block.packet[1])));
399 t2 =
reinterpret_cast<Packet
>(
400 vec_mergeh(
reinterpret_cast<Packet2ul
>(block.packet[2]),
reinterpret_cast<Packet2ul
>(block.packet[3])));
401 t3 =
reinterpret_cast<Packet
>(
402 vec_mergel(
reinterpret_cast<Packet2ul
>(block.packet[2]),
reinterpret_cast<Packet2ul
>(block.packet[3])));
404 t0 =
reinterpret_cast<Packet
>(vec_perm(block.packet[0], block.packet[1], p16uc_TRANSPOSE64_HI));
405 t1 =
reinterpret_cast<Packet
>(vec_perm(block.packet[0], block.packet[1], p16uc_TRANSPOSE64_LO));
406 t2 =
reinterpret_cast<Packet
>(vec_perm(block.packet[2], block.packet[3], p16uc_TRANSPOSE64_HI));
407 t3 =
reinterpret_cast<Packet
>(vec_perm(block.packet[2], block.packet[3], p16uc_TRANSPOSE64_LO));
410 block.packet[0] = t0;
411 block.packet[1] = t1;
412 block.packet[2] = t2;
413 block.packet[3] = t3;
415 block.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[4].v, permute);
416 block.packet[1] = vec_perm(cblock.packet[1].v, cblock.packet[5].v, permute);
417 block.packet[2] = vec_perm(cblock.packet[2].v, cblock.packet[6].v, permute);
418 block.packet[3] = vec_perm(cblock.packet[3].v, cblock.packet[7].v, permute);
422 EIGEN_ALWAYS_INLINE
void dhs_ccopy(Scalar* blockAt,
const DataMapper& lhs2, Index& i, Index& rir, Index& rii,
423 Index depth,
const Index vectorSize) {
424 PacketBlock<Packet, 4> blockr, blocki;
425 PacketBlock<PacketC, 8> cblock;
427 for (; i + vectorSize <= depth; i += vectorSize) {
428 EIGEN_IF_CONSTEXPR (UseLhs) {
429 bload<DataMapper, PacketC, 2, StorageOrder, true, 4>(cblock, lhs2, 0, i);
431 bload<DataMapper, PacketC, 2, StorageOrder, true, 4>(cblock, lhs2, i, 0);
434 EIGEN_IF_CONSTEXPR (((StorageOrder ==
RowMajor) && UseLhs) || (((StorageOrder ==
ColMajor) && !UseLhs))) {
435 dhs_cblock<true>(cblock, blockr, p16uc_GETREAL32b);
436 dhs_cblock<true>(cblock, blocki, p16uc_GETIMAG32b);
438 dhs_cblock<false>(cblock, blockr, p16uc_GETREAL32);
439 dhs_cblock<false>(cblock, blocki, p16uc_GETIMAG32);
442 EIGEN_IF_CONSTEXPR (Conjugate) {
443 blocki.packet[0] = -blocki.packet[0];
444 blocki.packet[1] = -blocki.packet[1];
445 blocki.packet[2] = -blocki.packet[2];
446 blocki.packet[3] = -blocki.packet[3];
449 storeBlock<Scalar, Packet, 4>(blockAt + rir, blockr);
450 storeBlock<Scalar, Packet, 4>(blockAt + rii, blocki);
452 rir += 4 * vectorSize;
453 rii += 4 * vectorSize;
457 EIGEN_STRONG_INLINE
void operator()(std::complex<Scalar>* blockA,
const DataMapper& lhs, Index depth, Index rows,
458 Index stride, Index offset) {
459 const Index vectorSize = quad_traits<Scalar>::vectorsize;
460 const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth);
461 Index rir = ((PanelMode) ? (vectorSize * offset) : 0), rii;
462 Scalar* blockAt =
reinterpret_cast<Scalar*
>(blockA);
465 for (; j + vectorSize <= rows; j += vectorSize) {
466 const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j);
469 rii = rir + vectorDelta;
471 dhs_ccopy(blockAt, lhs2, i, rir, rii, depth, vectorSize);
473 for (; i < depth; i++) {
474 PacketBlock<Packet, 1> blockr, blocki;
475 PacketBlock<PacketC, 2> cblock;
477 EIGEN_IF_CONSTEXPR (((StorageOrder ==
ColMajor) && UseLhs) || (((StorageOrder ==
RowMajor) && !UseLhs))) {
478 EIGEN_IF_CONSTEXPR (UseLhs) {
479 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i);
480 cblock.packet[1] = lhs2.template loadPacket<PacketC>(2, i);
482 cblock.packet[0] = lhs2.template loadPacket<PacketC>(i, 0);
483 cblock.packet[1] = lhs2.template loadPacket<PacketC>(i, 2);
486 EIGEN_IF_CONSTEXPR (UseLhs) {
487 cblock.packet[0] = pload2(lhs2(0, i), lhs2(1, i));
488 cblock.packet[1] = pload2(lhs2(2, i), lhs2(3, i));
490 cblock.packet[0] = pload2(lhs2(i, 0), lhs2(i, 1));
491 cblock.packet[1] = pload2(lhs2(i, 2), lhs2(i, 3));
495 blockr.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETREAL32);
496 blocki.packet[0] = vec_perm(cblock.packet[0].v, cblock.packet[1].v, p16uc_GETIMAG32);
498 EIGEN_IF_CONSTEXPR (Conjugate) {
499 blocki.packet[0] = -blocki.packet[0];
502 pstore<Scalar>(blockAt + rir, blockr.packet[0]);
503 pstore<Scalar>(blockAt + rii, blocki.packet[0]);
509 rir += ((PanelMode) ? (vectorSize * (2 * stride - depth)) : vectorDelta);
512 EIGEN_IF_CONSTEXPR (!UseLhs) {
513 EIGEN_IF_CONSTEXPR (PanelMode) rir -= (offset * (vectorSize - 1));
515 for (; j < rows; j++) {
516 const DataMapper lhs2 = lhs.getSubMapper(0, j);
517 rii = rir + ((PanelMode) ? stride : depth);
519 for (Index i = 0; i < depth; i++) {
520 blockAt[rir] = lhs2(i, 0).real();
522 EIGEN_IF_CONSTEXPR (Conjugate)
523 blockAt[rii] = -lhs2(i, 0).imag();
525 blockAt[rii] = lhs2(i, 0).imag();
531 rir += ((PanelMode) ? (2 * stride - depth) : depth);
535 EIGEN_IF_CONSTEXPR (PanelMode) rir += (offset * (rows - j - vectorSize));
536 rii = rir + (((PanelMode) ? stride : depth) * (rows - j));
538 for (Index i = 0; i < depth; i++) {
540 for (; k < rows; k++) {
541 blockAt[rir] = lhs(k, i).real();
543 EIGEN_IF_CONSTEXPR (Conjugate)
544 blockAt[rii] = -lhs(k, i).imag();
546 blockAt[rii] = lhs(k, i).imag();
558template <
typename Scalar,
typename DataMapper,
typename Packet,
int StorageOrder,
bool PanelMode,
bool UseLhs>
560 using Index = mapper_index_t<DataMapper>;
562 EIGEN_ALWAYS_INLINE
void dhs_copy(Scalar* blockA,
const DataMapper& lhs2, Index& i, Index& ri, Index depth,
563 const Index vectorSize) {
564 PacketBlock<Packet, 4> block[n];
566 for (; i + n * vectorSize <= depth; i += n * vectorSize) {
567 for (Index k = 0; k < n; k++) {
568 EIGEN_IF_CONSTEXPR (UseLhs) {
569 bload<DataMapper, Packet, 4, StorageOrder, false, 4>(block[k], lhs2, 0, i + k * vectorSize);
571 bload<DataMapper, Packet, 4, StorageOrder, false, 4>(block[k], lhs2, i + k * vectorSize, 0);
575 EIGEN_IF_CONSTEXPR (((StorageOrder ==
RowMajor) && UseLhs) || ((StorageOrder ==
ColMajor) && !UseLhs)) {
576 for (Index k = 0; k < n; k++) {
577 ptranspose(block[k]);
581 for (Index k = 0; k < n; k++) {
582 storeBlock<Scalar, Packet, 4>(blockA + ri + k * 4 * vectorSize, block[k]);
585 ri += n * 4 * vectorSize;
589 EIGEN_STRONG_INLINE
void operator()(Scalar* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride,
591 const Index vectorSize = quad_traits<Scalar>::vectorsize;
594 for (; j + vectorSize <= rows; j += vectorSize) {
595 const DataMapper lhs2 = UseLhs ? lhs.getSubMapper(j, 0) : lhs.getSubMapper(0, j);
598 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset;
600 dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize);
601 dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize);
602 dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize);
604 for (; i < depth; i++) {
605 EIGEN_IF_CONSTEXPR (((StorageOrder ==
RowMajor) && UseLhs) || ((StorageOrder ==
ColMajor) && !UseLhs)) {
606 EIGEN_IF_CONSTEXPR (UseLhs) {
607 blockA[ri + 0] = lhs2(0, i);
608 blockA[ri + 1] = lhs2(1, i);
609 blockA[ri + 2] = lhs2(2, i);
610 blockA[ri + 3] = lhs2(3, i);
612 blockA[ri + 0] = lhs2(i, 0);
613 blockA[ri + 1] = lhs2(i, 1);
614 blockA[ri + 2] = lhs2(i, 2);
615 blockA[ri + 3] = lhs2(i, 3);
619 EIGEN_IF_CONSTEXPR (UseLhs) {
620 lhsV = lhs2.template loadPacket<Packet>(0, i);
622 lhsV = lhs2.template loadPacket<Packet>(i, 0);
624 pstore<Scalar>(blockA + ri, lhsV);
630 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth);
633 EIGEN_IF_CONSTEXPR (!UseLhs) {
634 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset;
636 for (; j < rows; j++) {
637 const DataMapper lhs2 = lhs.getSubMapper(0, j);
638 for (Index i = 0; i < depth; i++) {
639 blockA[ri] = lhs2(i, 0);
643 EIGEN_IF_CONSTEXPR (PanelMode) ri += stride - depth;
647 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j);
649 for (Index i = 0; i < depth; i++) {
651 for (; k < rows; k++) {
652 blockA[ri] = lhs(k, i);
662template <
typename DataMapper,
int StorageOrder,
bool PanelMode>
663struct dhs_pack<double, DataMapper, Packet2d, StorageOrder, PanelMode, true> {
664 using Index = mapper_index_t<DataMapper>;
666 EIGEN_ALWAYS_INLINE
void dhs_copy(
double* blockA,
const DataMapper& lhs2, Index& i, Index& ri, Index depth,
667 const Index vectorSize) {
668 PacketBlock<Packet2d, 2> block[n];
670 for (; i + n * vectorSize <= depth; i += n * vectorSize) {
671 for (Index k = 0; k < n; k++) {
672 EIGEN_IF_CONSTEXPR (StorageOrder ==
RowMajor) {
673 block[k].packet[0] = lhs2.template loadPacket<Packet2d>(0, i + k * vectorSize);
674 block[k].packet[1] = lhs2.template loadPacket<Packet2d>(1, i + k * vectorSize);
676 block[k].packet[0] = lhs2.template loadPacket<Packet2d>(0, i + k * vectorSize + 0);
677 block[k].packet[1] = lhs2.template loadPacket<Packet2d>(0, i + k * vectorSize + 1);
681 EIGEN_IF_CONSTEXPR (StorageOrder ==
RowMajor) {
682 for (Index k = 0; k < n; k++) {
683 ptranspose(block[k]);
687 for (Index k = 0; k < n; k++) {
688 storeBlock<double, Packet2d, 2>(blockA + ri + k * 2 * vectorSize, block[k]);
691 ri += n * 2 * vectorSize;
695 EIGEN_STRONG_INLINE
void operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride,
697 const Index vectorSize = quad_traits<double>::vectorsize;
700 for (; j + vectorSize <= rows; j += vectorSize) {
701 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
704 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset;
706 dhs_copy<4>(blockA, lhs2, i, ri, depth, vectorSize);
707 dhs_copy<2>(blockA, lhs2, i, ri, depth, vectorSize);
708 dhs_copy<1>(blockA, lhs2, i, ri, depth, vectorSize);
710 for (; i < depth; i++) {
711 EIGEN_IF_CONSTEXPR (StorageOrder ==
RowMajor) {
712 blockA[ri + 0] = lhs2(0, i);
713 blockA[ri + 1] = lhs2(1, i);
715 Packet2d lhsV = lhs2.template loadPacket<Packet2d>(0, i);
716 pstore<double>(blockA + ri, lhsV);
722 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth);
726 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j);
728 for (Index i = 0; i < depth; i++) {
730 for (; k < rows; k++) {
731 blockA[ri] = lhs(k, i);
740template <
typename DataMapper,
int StorageOrder,
bool PanelMode>
741struct dhs_pack<double, DataMapper, Packet2d, StorageOrder, PanelMode, false> {
742 using Index = mapper_index_t<DataMapper>;
744 EIGEN_ALWAYS_INLINE
void dhs_copy(
double* blockB,
const DataMapper& rhs2, Index& i, Index& ri, Index depth,
745 const Index vectorSize) {
746 PacketBlock<Packet2d, 2> block1[n], block2[n];
747 PacketBlock<Packet2d, 4> block3[n];
749 for (; i + n * vectorSize <= depth; i += n * vectorSize) {
750 for (Index k = 0; k < n; k++) {
751 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
752 block1[k].packet[0] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 0);
753 block1[k].packet[1] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 1);
754 block2[k].packet[0] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 2);
755 block2[k].packet[1] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize, 3);
757 block3[k].packet[0] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 0, 0);
758 block3[k].packet[1] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 0, 2);
759 block3[k].packet[2] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 1, 0);
760 block3[k].packet[3] = rhs2.template loadPacket<Packet2d>(i + k * vectorSize + 1, 2);
764 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
765 for (Index k = 0; k < n; k++) {
766 ptranspose(block1[k]);
767 ptranspose(block2[k]);
771 for (Index k = 0; k < n; k++) {
772 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
773 pstore<double>(blockB + ri + k * 4 * vectorSize, block1[k].packet[0]);
774 pstore<double>(blockB + ri + k * 4 * vectorSize + 2, block2[k].packet[0]);
775 pstore<double>(blockB + ri + k * 4 * vectorSize + 4, block1[k].packet[1]);
776 pstore<double>(blockB + ri + k * 4 * vectorSize + 6, block2[k].packet[1]);
778 storeBlock<double, Packet2d, 4>(blockB + ri + k * 4 * vectorSize, block3[k]);
782 ri += n * 4 * vectorSize;
786 EIGEN_STRONG_INLINE
void operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride,
788 const Index vectorSize = quad_traits<double>::vectorsize;
791 for (; j + 2 * vectorSize <= cols; j += 2 * vectorSize) {
792 const DataMapper rhs2 = rhs.getSubMapper(0, j);
795 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (2 * vectorSize);
797 dhs_copy<4>(blockB, rhs2, i, ri, depth, vectorSize);
798 dhs_copy<2>(blockB, rhs2, i, ri, depth, vectorSize);
799 dhs_copy<1>(blockB, rhs2, i, ri, depth, vectorSize);
801 for (; i < depth; i++) {
802 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
803 blockB[ri + 0] = rhs2(i, 0);
804 blockB[ri + 1] = rhs2(i, 1);
808 blockB[ri + 0] = rhs2(i, 2);
809 blockB[ri + 1] = rhs2(i, 3);
811 Packet2d rhsV = rhs2.template loadPacket<Packet2d>(i, 0);
812 pstore<double>(blockB + ri, rhsV);
816 rhsV = rhs2.template loadPacket<Packet2d>(i, 2);
817 pstore<double>(blockB + ri, rhsV);
822 EIGEN_IF_CONSTEXPR (PanelMode) ri += (2 * vectorSize) * (stride - offset - depth);
825 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset;
827 for (; j < cols; j++) {
828 const DataMapper rhs2 = rhs.getSubMapper(0, j);
829 for (Index i = 0; i < depth; i++) {
830 blockB[ri] = rhs2(i, 0);
834 EIGEN_IF_CONSTEXPR (PanelMode) ri += stride - depth;
840template <
typename DataMapper,
int StorageOrder,
bool PanelMode>
841struct dhs_pack<bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, true> {
842 using Index = mapper_index_t<DataMapper>;
843 EIGEN_STRONG_INLINE
void operator()(bfloat16* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride,
845 const Index vectorSize = quad_traits<bfloat16>::vectorsize;
848 for (; j + 2 * vectorSize <= rows; j += 2 * vectorSize) {
849 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
852 EIGEN_IF_CONSTEXPR (PanelMode) ri += 2 * vectorSize * offset;
854 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
855 for (; i + 2 <= depth; i += 2) {
856 PacketBlock<Packet8bf, 4> block;
858 block.packet[0] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
859 block.packet[1] = lhs2.template loadPacket<Packet8bf>(1 * vectorSize, i + 0);
860 block.packet[2] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 1);
861 block.packet[3] = lhs2.template loadPacket<Packet8bf>(1 * vectorSize, i + 1);
864 t0 = vec_mergeh(block.packet[0].m_val, block.packet[2].m_val);
865 t1 = vec_mergel(block.packet[0].m_val, block.packet[2].m_val);
866 block.packet[2] = vec_mergeh(block.packet[1].m_val, block.packet[3].m_val);
867 block.packet[3] = vec_mergel(block.packet[1].m_val, block.packet[3].m_val);
868 block.packet[0] = t0;
869 block.packet[1] = t1;
871 storeBlock<bfloat16, Packet8bf, 4>(blockA + ri, block);
873 ri += 2 * 2 * vectorSize;
876 PacketBlock<Packet8bf, 2> block;
878 block.packet[0] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
879 block.packet[1] = lhs2.template loadPacket<Packet8bf>(1 * vectorSize, i + 0);
881 storeBlock<bfloat16, Packet8bf, 2>(blockA + ri, block);
883 ri += 2 * vectorSize;
886 for (; i + vectorSize <= depth; i += vectorSize) {
887 PacketBlock<Packet8bf, 8> block1, block2;
889 bload<DataMapper, Packet8bf, 8, StorageOrder, false, 8>(block1, lhs2, 0 * vectorSize, i);
890 bload<DataMapper, Packet8bf, 8, StorageOrder, false, 8>(block2, lhs2, 1 * vectorSize, i);
892 Packet4ui v1[8], v2[8];
894 v1[0] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[0].m_val),
895 reinterpret_cast<Packet4ui
>(block1.packet[1].m_val));
896 v1[1] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[0].m_val),
897 reinterpret_cast<Packet4ui
>(block1.packet[1].m_val));
898 v1[2] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[2].m_val),
899 reinterpret_cast<Packet4ui
>(block1.packet[3].m_val));
900 v1[3] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[2].m_val),
901 reinterpret_cast<Packet4ui
>(block1.packet[3].m_val));
902 v1[4] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[4].m_val),
903 reinterpret_cast<Packet4ui
>(block1.packet[5].m_val));
904 v1[5] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[4].m_val),
905 reinterpret_cast<Packet4ui
>(block1.packet[5].m_val));
906 v1[6] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[6].m_val),
907 reinterpret_cast<Packet4ui
>(block1.packet[7].m_val));
908 v1[7] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[6].m_val),
909 reinterpret_cast<Packet4ui
>(block1.packet[7].m_val));
910 v2[0] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block2.packet[0].m_val),
911 reinterpret_cast<Packet4ui
>(block2.packet[1].m_val));
912 v2[1] = vec_mergel(
reinterpret_cast<Packet4ui
>(block2.packet[0].m_val),
913 reinterpret_cast<Packet4ui
>(block2.packet[1].m_val));
914 v2[2] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block2.packet[2].m_val),
915 reinterpret_cast<Packet4ui
>(block2.packet[3].m_val));
916 v2[3] = vec_mergel(
reinterpret_cast<Packet4ui
>(block2.packet[2].m_val),
917 reinterpret_cast<Packet4ui
>(block2.packet[3].m_val));
918 v2[4] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block2.packet[4].m_val),
919 reinterpret_cast<Packet4ui
>(block2.packet[5].m_val));
920 v2[5] = vec_mergel(
reinterpret_cast<Packet4ui
>(block2.packet[4].m_val),
921 reinterpret_cast<Packet4ui
>(block2.packet[5].m_val));
922 v2[6] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block2.packet[6].m_val),
923 reinterpret_cast<Packet4ui
>(block2.packet[7].m_val));
924 v2[7] = vec_mergel(
reinterpret_cast<Packet4ui
>(block2.packet[6].m_val),
925 reinterpret_cast<Packet4ui
>(block2.packet[7].m_val));
927#ifdef EIGEN_VECTORIZE_VSX
928 block1.packet[0] =
reinterpret_cast<Packet8us
>(
929 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[0]),
reinterpret_cast<Packet2ul
>(v1[2])));
930 block1.packet[2] =
reinterpret_cast<Packet8us
>(
931 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[0]),
reinterpret_cast<Packet2ul
>(v1[2])));
932 block1.packet[4] =
reinterpret_cast<Packet8us
>(
933 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[1]),
reinterpret_cast<Packet2ul
>(v1[3])));
934 block1.packet[6] =
reinterpret_cast<Packet8us
>(
935 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[1]),
reinterpret_cast<Packet2ul
>(v1[3])));
936 block1.packet[1] =
reinterpret_cast<Packet8us
>(
937 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[4]),
reinterpret_cast<Packet2ul
>(v1[6])));
938 block1.packet[3] =
reinterpret_cast<Packet8us
>(
939 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[4]),
reinterpret_cast<Packet2ul
>(v1[6])));
940 block1.packet[5] =
reinterpret_cast<Packet8us
>(
941 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[5]),
reinterpret_cast<Packet2ul
>(v1[7])));
942 block1.packet[7] =
reinterpret_cast<Packet8us
>(
943 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[5]),
reinterpret_cast<Packet2ul
>(v1[7])));
944 block2.packet[0] =
reinterpret_cast<Packet8us
>(
945 vec_mergeh(
reinterpret_cast<Packet2ul
>(v2[0]),
reinterpret_cast<Packet2ul
>(v2[2])));
946 block2.packet[2] =
reinterpret_cast<Packet8us
>(
947 vec_mergel(
reinterpret_cast<Packet2ul
>(v2[0]),
reinterpret_cast<Packet2ul
>(v2[2])));
948 block2.packet[4] =
reinterpret_cast<Packet8us
>(
949 vec_mergeh(
reinterpret_cast<Packet2ul
>(v2[1]),
reinterpret_cast<Packet2ul
>(v2[3])));
950 block2.packet[6] =
reinterpret_cast<Packet8us
>(
951 vec_mergel(
reinterpret_cast<Packet2ul
>(v2[1]),
reinterpret_cast<Packet2ul
>(v2[3])));
952 block2.packet[1] =
reinterpret_cast<Packet8us
>(
953 vec_mergeh(
reinterpret_cast<Packet2ul
>(v2[4]),
reinterpret_cast<Packet2ul
>(v2[6])));
954 block2.packet[3] =
reinterpret_cast<Packet8us
>(
955 vec_mergel(
reinterpret_cast<Packet2ul
>(v2[4]),
reinterpret_cast<Packet2ul
>(v2[6])));
956 block2.packet[5] =
reinterpret_cast<Packet8us
>(
957 vec_mergeh(
reinterpret_cast<Packet2ul
>(v2[5]),
reinterpret_cast<Packet2ul
>(v2[7])));
958 block2.packet[7] =
reinterpret_cast<Packet8us
>(
959 vec_mergel(
reinterpret_cast<Packet2ul
>(v2[5]),
reinterpret_cast<Packet2ul
>(v2[7])));
961 block1.packet[0] =
reinterpret_cast<Packet8us
>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_HI));
962 block1.packet[2] =
reinterpret_cast<Packet8us
>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_LO));
963 block1.packet[4] =
reinterpret_cast<Packet8us
>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_HI));
964 block1.packet[6] =
reinterpret_cast<Packet8us
>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_LO));
965 block1.packet[1] =
reinterpret_cast<Packet8us
>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_HI));
966 block1.packet[3] =
reinterpret_cast<Packet8us
>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_LO));
967 block1.packet[5] =
reinterpret_cast<Packet8us
>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_HI));
968 block1.packet[7] =
reinterpret_cast<Packet8us
>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_LO));
969 block2.packet[0] =
reinterpret_cast<Packet8us
>(vec_perm(v2[0], v2[2], p16uc_TRANSPOSE64_HI));
970 block2.packet[2] =
reinterpret_cast<Packet8us
>(vec_perm(v2[0], v2[2], p16uc_TRANSPOSE64_LO));
971 block2.packet[4] =
reinterpret_cast<Packet8us
>(vec_perm(v2[1], v2[3], p16uc_TRANSPOSE64_HI));
972 block2.packet[6] =
reinterpret_cast<Packet8us
>(vec_perm(v2[1], v2[3], p16uc_TRANSPOSE64_LO));
973 block2.packet[1] =
reinterpret_cast<Packet8us
>(vec_perm(v2[4], v2[6], p16uc_TRANSPOSE64_HI));
974 block2.packet[3] =
reinterpret_cast<Packet8us
>(vec_perm(v2[4], v2[6], p16uc_TRANSPOSE64_LO));
975 block2.packet[5] =
reinterpret_cast<Packet8us
>(vec_perm(v2[5], v2[7], p16uc_TRANSPOSE64_HI));
976 block2.packet[7] =
reinterpret_cast<Packet8us
>(vec_perm(v2[5], v2[7], p16uc_TRANSPOSE64_LO));
979 for (Index M = 0; M < 8; M += 2) {
980 pstore<bfloat16>(blockA + ri + (0 * vectorSize) + (2 * vectorSize * M), block1.packet[M + 0]);
981 pstore<bfloat16>(blockA + ri + (1 * vectorSize) + (2 * vectorSize * M), block1.packet[M + 1]);
982 pstore<bfloat16>(blockA + ri + (2 * vectorSize) + (2 * vectorSize * M), block2.packet[M + 0]);
983 pstore<bfloat16>(blockA + ri + (3 * vectorSize) + (2 * vectorSize * M), block2.packet[M + 1]);
986 ri += 2 * vectorSize * vectorSize;
988 for (; i + 2 <= depth; i += 2) {
989 for (Index M = 0; M < 2 * vectorSize; M++) {
990 blockA[ri + (M * 2) + 0] = lhs2(M, i + 0);
991 blockA[ri + (M * 2) + 1] = lhs2(M, i + 1);
994 ri += 2 * 2 * vectorSize;
997 for (Index M = 0; M < 2 * vectorSize; M++) {
998 blockA[ri + M] = lhs2(M, i);
1000 ri += 2 * vectorSize;
1004 EIGEN_IF_CONSTEXPR (PanelMode) ri += 2 * vectorSize * (stride - offset - depth);
1006 for (; j + vectorSize <= rows; j += vectorSize) {
1007 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
1010 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * offset;
1012 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
1013 for (; i + 2 <= depth; i += 2) {
1014 PacketBlock<Packet8bf, 2> block;
1016 block.packet[0] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
1017 block.packet[1] = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 1);
1020 t0 = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1021 block.packet[1] = vec_mergel(block.packet[0].m_val, block.packet[1].m_val);
1022 block.packet[0] = t0;
1024 storeBlock<bfloat16, Packet8bf, 2>(blockA + ri, block);
1026 ri += 2 * vectorSize;
1029 Packet8bf lhsV = lhs2.template loadPacket<Packet8bf>(0 * vectorSize, i + 0);
1030 pstore<bfloat16>(blockA + ri, lhsV);
1035 for (; i + vectorSize <= depth; i += vectorSize) {
1036 PacketBlock<Packet8bf, 8> block1;
1038 bload<DataMapper, Packet8bf, 8, StorageOrder, false, 8>(block1, lhs2, 0 * vectorSize, i);
1043 v1[0] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[0].m_val),
1044 reinterpret_cast<Packet4ui
>(block1.packet[1].m_val));
1045 v1[1] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[0].m_val),
1046 reinterpret_cast<Packet4ui
>(block1.packet[1].m_val));
1047 v1[2] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[2].m_val),
1048 reinterpret_cast<Packet4ui
>(block1.packet[3].m_val));
1049 v1[3] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[2].m_val),
1050 reinterpret_cast<Packet4ui
>(block1.packet[3].m_val));
1051 v1[4] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[4].m_val),
1052 reinterpret_cast<Packet4ui
>(block1.packet[5].m_val));
1053 v1[5] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[4].m_val),
1054 reinterpret_cast<Packet4ui
>(block1.packet[5].m_val));
1055 v1[6] = vec_mergeh(
reinterpret_cast<Packet4ui
>(block1.packet[6].m_val),
1056 reinterpret_cast<Packet4ui
>(block1.packet[7].m_val));
1057 v1[7] = vec_mergel(
reinterpret_cast<Packet4ui
>(block1.packet[6].m_val),
1058 reinterpret_cast<Packet4ui
>(block1.packet[7].m_val));
1060#ifdef EIGEN_VECTORIZE_VSX
1061 block1.packet[0] =
reinterpret_cast<Packet8us
>(
1062 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[0]),
reinterpret_cast<Packet2ul
>(v1[2])));
1063 block1.packet[2] =
reinterpret_cast<Packet8us
>(
1064 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[0]),
reinterpret_cast<Packet2ul
>(v1[2])));
1065 block1.packet[4] =
reinterpret_cast<Packet8us
>(
1066 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[1]),
reinterpret_cast<Packet2ul
>(v1[3])));
1067 block1.packet[6] =
reinterpret_cast<Packet8us
>(
1068 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[1]),
reinterpret_cast<Packet2ul
>(v1[3])));
1069 block1.packet[1] =
reinterpret_cast<Packet8us
>(
1070 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[4]),
reinterpret_cast<Packet2ul
>(v1[6])));
1071 block1.packet[3] =
reinterpret_cast<Packet8us
>(
1072 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[4]),
reinterpret_cast<Packet2ul
>(v1[6])));
1073 block1.packet[5] =
reinterpret_cast<Packet8us
>(
1074 vec_mergeh(
reinterpret_cast<Packet2ul
>(v1[5]),
reinterpret_cast<Packet2ul
>(v1[7])));
1075 block1.packet[7] =
reinterpret_cast<Packet8us
>(
1076 vec_mergel(
reinterpret_cast<Packet2ul
>(v1[5]),
reinterpret_cast<Packet2ul
>(v1[7])));
1078 block1.packet[0] =
reinterpret_cast<Packet8us
>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_HI));
1079 block1.packet[2] =
reinterpret_cast<Packet8us
>(vec_perm(v1[0], v1[2], p16uc_TRANSPOSE64_LO));
1080 block1.packet[4] =
reinterpret_cast<Packet8us
>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_HI));
1081 block1.packet[6] =
reinterpret_cast<Packet8us
>(vec_perm(v1[1], v1[3], p16uc_TRANSPOSE64_LO));
1082 block1.packet[1] =
reinterpret_cast<Packet8us
>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_HI));
1083 block1.packet[3] =
reinterpret_cast<Packet8us
>(vec_perm(v1[4], v1[6], p16uc_TRANSPOSE64_LO));
1084 block1.packet[5] =
reinterpret_cast<Packet8us
>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_HI));
1085 block1.packet[7] =
reinterpret_cast<Packet8us
>(vec_perm(v1[5], v1[7], p16uc_TRANSPOSE64_LO));
1088 for (Index M = 0; M < 8; M++) {
1089 pstore<bfloat16>(blockA + ri + (vectorSize * M), block1.packet[M]);
1092 ri += vectorSize * vectorSize;
1094 for (; i + 2 <= depth; i += 2) {
1095 for (Index M = 0; M < vectorSize; M++) {
1096 blockA[ri + (M * 2) + 0] = lhs2(M, i + 0);
1097 blockA[ri + (M * 2) + 1] = lhs2(M, i + 1);
1100 ri += 2 * vectorSize;
1103 for (Index M = 0; M < vectorSize; M++) {
1104 blockA[ri + M] = lhs2(M, i);
1111 EIGEN_IF_CONSTEXPR (PanelMode) ri += vectorSize * (stride - offset - depth);
1113 if (j + 4 <= rows) {
1114 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
1117 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * offset;
1119 for (; i + 2 <= depth; i += 2) {
1120 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
1121 PacketBlock<Packet8bf, 2> block;
1123 block.packet[0] = lhs2.template loadPacketPartial<Packet8bf>(0, i + 0, 4);
1124 block.packet[1] = lhs2.template loadPacketPartial<Packet8bf>(0, i + 1, 4);
1126 block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1128 pstore<bfloat16>(blockA + ri, block.packet[0]);
1130 blockA[ri + 0] = lhs2(0, i + 0);
1131 blockA[ri + 1] = lhs2(0, i + 1);
1132 blockA[ri + 2] = lhs2(1, i + 0);
1133 blockA[ri + 3] = lhs2(1, i + 1);
1134 blockA[ri + 4] = lhs2(2, i + 0);
1135 blockA[ri + 5] = lhs2(2, i + 1);
1136 blockA[ri + 6] = lhs2(3, i + 0);
1137 blockA[ri + 7] = lhs2(3, i + 1);
1143 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
1144 Packet8bf lhsV = lhs2.template loadPacketPartial<Packet8bf>(0, i + 0, 4);
1146 pstore_partial<bfloat16>(blockA + ri, lhsV, 4);
1148 blockA[ri + 0] = lhs2(0, i);
1149 blockA[ri + 1] = lhs2(1, i);
1150 blockA[ri + 2] = lhs2(2, i);
1151 blockA[ri + 3] = lhs2(3, i);
1157 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * (stride - offset - depth);
1162 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (rows - j);
1165 for (; i + 2 <= depth; i += 2) {
1167 for (; k < rows; k++) {
1168 blockA[ri + 0] = lhs(k, i + 0);
1169 blockA[ri + 1] = lhs(k, i + 1);
1174 for (; j < rows; j++) {
1175 blockA[ri] = lhs(j, i);
1184template <
typename DataMapper,
int StorageOrder,
bool PanelMode>
1185struct dhs_pack<bfloat16, DataMapper, Packet8bf, StorageOrder, PanelMode, false> {
1186 using Index = mapper_index_t<DataMapper>;
1187 EIGEN_STRONG_INLINE
void operator()(bfloat16* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride,
1189 const Index vectorSize = quad_traits<bfloat16>::vectorsize;
1190 Index ri = 0, j = 0;
1192 for (; j + 4 <= cols; j += 4) {
1193 const DataMapper rhs2 = rhs.getSubMapper(0, j);
1196 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * offset;
1198 for (; i + vectorSize <= depth; i += vectorSize) {
1199 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
1200 PacketBlock<Packet8bf, 4> block;
1202 bload<DataMapper, Packet8bf, 4, StorageOrder, false, 4>(block, rhs2, i, 0);
1204 Packet4ui t0, t1, t2, t3;
1206 t0 = vec_mergeh(
reinterpret_cast<Packet4ui
>(block.packet[0].m_val),
1207 reinterpret_cast<Packet4ui
>(block.packet[1].m_val));
1208 t1 = vec_mergel(
reinterpret_cast<Packet4ui
>(block.packet[0].m_val),
1209 reinterpret_cast<Packet4ui
>(block.packet[1].m_val));
1210 t2 = vec_mergeh(
reinterpret_cast<Packet4ui
>(block.packet[2].m_val),
1211 reinterpret_cast<Packet4ui
>(block.packet[3].m_val));
1212 t3 = vec_mergel(
reinterpret_cast<Packet4ui
>(block.packet[2].m_val),
1213 reinterpret_cast<Packet4ui
>(block.packet[3].m_val));
1215#ifdef EIGEN_VECTORIZE_VSX
1217 reinterpret_cast<Packet8us
>(vec_mergeh(
reinterpret_cast<Packet2ul
>(t0),
reinterpret_cast<Packet2ul
>(t2)));
1219 reinterpret_cast<Packet8us
>(vec_mergel(
reinterpret_cast<Packet2ul
>(t0),
reinterpret_cast<Packet2ul
>(t2)));
1221 reinterpret_cast<Packet8us
>(vec_mergeh(
reinterpret_cast<Packet2ul
>(t1),
reinterpret_cast<Packet2ul
>(t3)));
1223 reinterpret_cast<Packet8us
>(vec_mergel(
reinterpret_cast<Packet2ul
>(t1),
reinterpret_cast<Packet2ul
>(t3)));
1225 block.packet[0] =
reinterpret_cast<Packet8us
>(vec_perm(t0, t2, p16uc_TRANSPOSE64_HI));
1226 block.packet[1] =
reinterpret_cast<Packet8us
>(vec_perm(t0, t2, p16uc_TRANSPOSE64_LO));
1227 block.packet[2] =
reinterpret_cast<Packet8us
>(vec_perm(t1, t3, p16uc_TRANSPOSE64_HI));
1228 block.packet[3] =
reinterpret_cast<Packet8us
>(vec_perm(t1, t3, p16uc_TRANSPOSE64_LO));
1231 storeBlock<bfloat16, Packet8bf, 4>(blockB + ri, block);
1233 PacketBlock<Packet8bf, 8> block;
1235 for (
int M = 0; M < 8; M++) {
1236 block.packet[M] = rhs2.template loadPacketPartial<Packet8bf>(i + M, 0, 4);
1239 block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1240 block.packet[1] = vec_mergeh(block.packet[2].m_val, block.packet[3].m_val);
1241 block.packet[2] = vec_mergeh(block.packet[4].m_val, block.packet[5].m_val);
1242 block.packet[3] = vec_mergeh(block.packet[6].m_val, block.packet[7].m_val);
1244 const Index size = 16 /
sizeof(bfloat16);
1246 for (
int M = 0; M < 4; M++) {
1247 pstore<bfloat16>(blockB + ri + (M * size), block.packet[M]);
1251 ri += 4 * vectorSize;
1253 for (; i + 2 <= depth; i += 2) {
1254 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
1255 blockB[ri + 0] = rhs2(i + 0, 0);
1256 blockB[ri + 1] = rhs2(i + 1, 0);
1257 blockB[ri + 2] = rhs2(i + 0, 1);
1258 blockB[ri + 3] = rhs2(i + 1, 1);
1259 blockB[ri + 4] = rhs2(i + 0, 2);
1260 blockB[ri + 5] = rhs2(i + 1, 2);
1261 blockB[ri + 6] = rhs2(i + 0, 3);
1262 blockB[ri + 7] = rhs2(i + 1, 3);
1264 PacketBlock<Packet8bf, 2> block;
1266 for (
int M = 0; M < 2; M++) {
1267 block.packet[M] = rhs2.template loadPacketPartial<Packet8bf>(i + M, 0, 4);
1270 block.packet[0] = vec_mergeh(block.packet[0].m_val, block.packet[1].m_val);
1272 pstore<bfloat16>(blockB + ri, block.packet[0]);
1278 blockB[ri + 0] = rhs2(i, 0);
1279 blockB[ri + 1] = rhs2(i, 1);
1280 blockB[ri + 2] = rhs2(i, 2);
1281 blockB[ri + 3] = rhs2(i, 3);
1286 EIGEN_IF_CONSTEXPR (PanelMode) ri += 4 * (stride - offset - depth);
1290 EIGEN_IF_CONSTEXPR (PanelMode) ri += offset * (cols - j);
1293 for (; i + 2 <= depth; i += 2) {
1295 for (; k < cols; k++) {
1296 blockB[ri + 0] = rhs(i + 0, k);
1297 blockB[ri + 1] = rhs(i + 1, k);
1302 for (; j < cols; j++) {
1303 blockB[ri] = rhs(i, j);
1312template <
typename DataMapper,
typename Packet,
typename PacketC,
int StorageOrder,
bool Conjugate,
bool PanelMode>
1313struct dhs_cpack<double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, true> {
1314 using Index = mapper_index_t<DataMapper>;
1315 EIGEN_ALWAYS_INLINE
void dhs_ccopy(
double* blockAt,
const DataMapper& lhs2, Index& i, Index& rir, Index& rii,
1316 Index depth,
const Index vectorSize) {
1317 PacketBlock<Packet, 2> blockr, blocki;
1318 PacketBlock<PacketC, 4> cblock;
1320 for (; i + vectorSize <= depth; i += vectorSize) {
1321 EIGEN_IF_CONSTEXPR (StorageOrder ==
ColMajor) {
1322 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i + 0);
1323 cblock.packet[1] = lhs2.template loadPacket<PacketC>(0, i + 1);
1325 cblock.packet[2] = lhs2.template loadPacket<PacketC>(1, i + 0);
1326 cblock.packet[3] = lhs2.template loadPacket<PacketC>(1, i + 1);
1328 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[2].v);
1329 blockr.packet[1] = vec_mergeh(cblock.packet[1].v, cblock.packet[3].v);
1331 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[2].v);
1332 blocki.packet[1] = vec_mergel(cblock.packet[1].v, cblock.packet[3].v);
1334 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i);
1335 cblock.packet[1] = lhs2.template loadPacket<PacketC>(1, i);
1337 cblock.packet[2] = lhs2.template loadPacket<PacketC>(0, i + 1);
1338 cblock.packet[3] = lhs2.template loadPacket<PacketC>(1, i + 1);
1340 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v);
1341 blockr.packet[1] = vec_mergeh(cblock.packet[2].v, cblock.packet[3].v);
1343 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v);
1344 blocki.packet[1] = vec_mergel(cblock.packet[2].v, cblock.packet[3].v);
1347 EIGEN_IF_CONSTEXPR (Conjugate) {
1348 blocki.packet[0] = -blocki.packet[0];
1349 blocki.packet[1] = -blocki.packet[1];
1352 storeBlock<double, Packet, 2>(blockAt + rir, blockr);
1353 storeBlock<double, Packet, 2>(blockAt + rii, blocki);
1355 rir += 2 * vectorSize;
1356 rii += 2 * vectorSize;
1360 EIGEN_STRONG_INLINE
void operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows,
1361 Index stride, Index offset) {
1362 const Index vectorSize = quad_traits<double>::vectorsize;
1363 const Index vectorDelta = vectorSize * ((PanelMode) ? stride : depth);
1364 Index rir = ((PanelMode) ? (vectorSize * offset) : 0), rii;
1365 double* blockAt =
reinterpret_cast<double*
>(blockA);
1368 for (; j + vectorSize <= rows; j += vectorSize) {
1369 const DataMapper lhs2 = lhs.getSubMapper(j, 0);
1372 rii = rir + vectorDelta;
1374 dhs_ccopy(blockAt, lhs2, i, rir, rii, depth, vectorSize);
1376 for (; i < depth; i++) {
1377 PacketBlock<Packet, 1> blockr, blocki;
1378 PacketBlock<PacketC, 2> cblock;
1380 cblock.packet[0] = lhs2.template loadPacket<PacketC>(0, i);
1381 cblock.packet[1] = lhs2.template loadPacket<PacketC>(1, i);
1383 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v);
1384 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v);
1386 EIGEN_IF_CONSTEXPR (Conjugate) {
1387 blocki.packet[0] = -blocki.packet[0];
1390 pstore<double>(blockAt + rir, blockr.packet[0]);
1391 pstore<double>(blockAt + rii, blocki.packet[0]);
1397 rir += ((PanelMode) ? (vectorSize * (2 * stride - depth)) : vectorDelta);
1401 EIGEN_IF_CONSTEXPR (PanelMode) rir += (offset * (rows - j - vectorSize));
1402 rii = rir + (((PanelMode) ? stride : depth) * (rows - j));
1404 for (Index i = 0; i < depth; i++) {
1406 for (; k < rows; k++) {
1407 blockAt[rir] = lhs(k, i).real();
1409 EIGEN_IF_CONSTEXPR (Conjugate)
1410 blockAt[rii] = -lhs(k, i).imag();
1412 blockAt[rii] = lhs(k, i).imag();
1423template <
typename DataMapper,
typename Packet,
typename PacketC,
int StorageOrder,
bool Conjugate,
bool PanelMode>
1424struct dhs_cpack<double, DataMapper, Packet, PacketC, StorageOrder, Conjugate, PanelMode, false> {
1425 using Index = mapper_index_t<DataMapper>;
1426 EIGEN_ALWAYS_INLINE
void dhs_ccopy(
double* blockBt,
const DataMapper& rhs2, Index& i, Index& rir, Index& rii,
1427 Index depth,
const Index vectorSize) {
1428 for (; i < depth; i++) {
1429 PacketBlock<PacketC, 4> cblock;
1430 PacketBlock<Packet, 2> blockr, blocki;
1432 bload<DataMapper, PacketC, 2, ColMajor, false, 4>(cblock, rhs2, i, 0);
1434 blockr.packet[0] = vec_mergeh(cblock.packet[0].v, cblock.packet[1].v);
1435 blockr.packet[1] = vec_mergeh(cblock.packet[2].v, cblock.packet[3].v);
1437 blocki.packet[0] = vec_mergel(cblock.packet[0].v, cblock.packet[1].v);
1438 blocki.packet[1] = vec_mergel(cblock.packet[2].v, cblock.packet[3].v);
1440 EIGEN_IF_CONSTEXPR (Conjugate) {
1441 blocki.packet[0] = -blocki.packet[0];
1442 blocki.packet[1] = -blocki.packet[1];
1445 storeBlock<double, Packet, 2>(blockBt + rir, blockr);
1446 storeBlock<double, Packet, 2>(blockBt + rii, blocki);
1448 rir += 2 * vectorSize;
1449 rii += 2 * vectorSize;
1453 EIGEN_STRONG_INLINE
void operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols,
1454 Index stride, Index offset) {
1455 const Index vectorSize = quad_traits<double>::vectorsize;
1456 const Index vectorDelta = 2 * vectorSize * ((PanelMode) ? stride : depth);
1457 Index rir = ((PanelMode) ? (2 * vectorSize * offset) : 0), rii;
1458 double* blockBt =
reinterpret_cast<double*
>(blockB);
1461 for (; j + 2 * vectorSize <= cols; j += 2 * vectorSize) {
1462 const DataMapper rhs2 = rhs.getSubMapper(0, j);
1465 rii = rir + vectorDelta;
1467 dhs_ccopy(blockBt, rhs2, i, rir, rii, depth, vectorSize);
1469 rir += ((PanelMode) ? (2 * vectorSize * (2 * stride - depth)) : vectorDelta);
1472 EIGEN_IF_CONSTEXPR (PanelMode) rir -= (offset * (2 * vectorSize - 1));
1474 for (; j < cols; j++) {
1475 const DataMapper rhs2 = rhs.getSubMapper(0, j);
1476 rii = rir + ((PanelMode) ? stride : depth);
1478 for (Index i = 0; i < depth; i++) {
1479 blockBt[rir] = rhs2(i, 0).real();
1481 EIGEN_IF_CONSTEXPR (Conjugate)
1482 blockBt[rii] = -rhs2(i, 0).imag();
1484 blockBt[rii] = rhs2(i, 0).imag();
1490 rir += ((PanelMode) ? (2 * stride - depth) : depth);
1500template <
typename Packet,
bool NegativeAccumulate,
int N>
1501EIGEN_ALWAYS_INLINE
void pger_common(PacketBlock<Packet, N>* acc,
const Packet& lhsV,
const Packet* rhsV) {
1502 EIGEN_IF_CONSTEXPR (NegativeAccumulate) {
1503 for (
int M = 0; M < N; M++) {
1504 acc->packet[M] = vec_nmsub(lhsV, rhsV[M], acc->packet[M]);
1507 for (
int M = 0; M < N; M++) {
1508 acc->packet[M] = vec_madd(lhsV, rhsV[M], acc->packet[M]);
1513template <
int N,
typename Scalar,
typename Packet,
bool NegativeAccumulate, Index remaining_rows = 0>
1514EIGEN_ALWAYS_INLINE
void pger(PacketBlock<Packet, N>* acc,
const Scalar* lhs,
const Packet* rhsV) {
1516 EIGEN_IF_CONSTEXPR (remaining_rows > 0) {
1517 lhsV = ploadu_partial<Packet>(lhs, remaining_rows);
1519 lhsV = ploadLhs<Packet>(lhs);
1522 pger_common<Packet, NegativeAccumulate, N>(acc, lhsV, rhsV);
1527template <
int N,
typename Packet,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
1528EIGEN_ALWAYS_INLINE
void pgerc_common(PacketBlock<Packet, N>* accReal, PacketBlock<Packet, N>* accImag,
1529 const Packet& lhsV, Packet& lhsVi,
const Packet* rhsV,
const Packet* rhsVi) {
1530 pger_common<Packet, false, N>(accReal, lhsV, rhsV);
1531 EIGEN_IF_CONSTEXPR (LhsIsReal) {
1532 pger_common<Packet, ConjugateRhs, N>(accImag, lhsV, rhsVi);
1533 EIGEN_UNUSED_VARIABLE(lhsVi);
1535 EIGEN_IF_CONSTEXPR (!RhsIsReal) {
1536 pger_common<Packet, ConjugateLhs == ConjugateRhs, N>(accReal, lhsVi, rhsVi);
1537 pger_common<Packet, ConjugateRhs, N>(accImag, lhsV, rhsVi);
1539 EIGEN_UNUSED_VARIABLE(rhsVi);
1541 pger_common<Packet, ConjugateLhs, N>(accImag, lhsVi, rhsV);
1545template <
int N,
typename Scalar,
typename Packet,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal,
1546 Index remaining_rows = 0>
1547EIGEN_ALWAYS_INLINE
void pgerc(PacketBlock<Packet, N>* accReal, PacketBlock<Packet, N>* accImag,
const Scalar* lhs_ptr,
1548 const Scalar* lhs_ptr_imag,
const Packet* rhsV,
const Packet* rhsVi) {
1550 EIGEN_IF_CONSTEXPR (remaining_rows > 0) {
1551 lhsV = ploadu_partial<Packet>(lhs_ptr, remaining_rows);
1553 lhsV = ploadLhs<Packet>(lhs_ptr);
1556 EIGEN_IF_CONSTEXPR (!LhsIsReal) {
1557 EIGEN_IF_CONSTEXPR (remaining_rows > 0) {
1558 lhsVi = ploadu_partial<Packet>(lhs_ptr_imag, remaining_rows);
1560 lhsVi = ploadLhs<Packet>(lhs_ptr_imag);
1563 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
1566 pgerc_common<N, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(accReal, accImag, lhsV, lhsVi, rhsV, rhsVi);
1569template <
typename Packet>
1570EIGEN_ALWAYS_INLINE Packet ploadLhs(
const __UNPACK_TYPE__(Packet) * lhs) {
1571 return ploadu<Packet>(lhs);
1575template <
typename Packet,
int N>
1576EIGEN_ALWAYS_INLINE
void bsetzero(PacketBlock<Packet, N>& acc) {
1577 for (
int M = 0; M < N; M++) {
1578 acc.packet[M] = pset1<Packet>((__UNPACK_TYPE__(Packet))0);
1582template <
typename Packet,
int N>
1583EIGEN_ALWAYS_INLINE
void bscalec_common(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ,
1584 const Packet& pAlpha) {
1585 for (
int M = 0; M < N; M++) {
1586 acc.packet[M] = vec_mul(accZ.packet[M], pAlpha);
1590template <
typename Packet,
int N>
1591EIGEN_ALWAYS_INLINE
void band(PacketBlock<Packet, N>& acc,
const Packet& pMask) {
1592 for (
int M = 0; M < N; M++) {
1593 acc.packet[M] = pand<Packet>(acc.packet[M], pMask);
1598template <
typename Packet,
int N,
bool mask>
1599EIGEN_ALWAYS_INLINE
void bscalec(PacketBlock<Packet, N>& aReal, PacketBlock<Packet, N>& aImag,
const Packet& bReal,
1600 const Packet& bImag, PacketBlock<Packet, N>& cReal, PacketBlock<Packet, N>& cImag,
1601 const Packet& pMask) {
1602 EIGEN_IF_CONSTEXPR (mask && (
sizeof(__UNPACK_TYPE__(Packet)) ==
sizeof(
float))) {
1603 band<Packet, N>(aReal, pMask);
1604 band<Packet, N>(aImag, pMask);
1606 EIGEN_UNUSED_VARIABLE(pMask);
1609 bscalec_common<Packet, N>(cReal, aReal, bReal);
1611 bscalec_common<Packet, N>(cImag, aImag, bReal);
1613 pger_common<Packet, true, N>(&cReal, bImag, aImag.packet);
1615 pger_common<Packet, false, N>(&cImag, bImag, aReal.packet);
1621template <
typename DataMapper,
typename Packet, const Index accCols,
int StorageOrder,
bool Complex,
int N,
bool full>
1622EIGEN_ALWAYS_INLINE
void bload(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc,
const DataMapper& res, Index row,
1624 using MapperIndex = mapper_index_t<DataMapper>;
1625 const MapperIndex mapperRow = convert_index<MapperIndex>(row);
1626 const MapperIndex mapperCol = convert_index<MapperIndex>(col);
1627 EIGEN_IF_CONSTEXPR (StorageOrder ==
RowMajor) {
1628 for (
int M = 0; M < N; M++) {
1629 acc.packet[M] = res.template loadPacket<Packet>(mapperRow + M, mapperCol);
1631 EIGEN_IF_CONSTEXPR (Complex) {
1632 for (
int M = 0; M < N; M++) {
1633 acc.packet[M + N] = res.template loadPacket<Packet>(mapperRow + M, mapperCol + MapperIndex(accCols));
1637 for (
int M = 0; M < N; M++) {
1638 acc.packet[M] = res.template loadPacket<Packet>(mapperRow, mapperCol + M);
1640 EIGEN_IF_CONSTEXPR (Complex && full) {
1641 for (
int M = 0; M < N; M++) {
1642 acc.packet[M + N] = res.template loadPacket<Packet>(mapperRow + MapperIndex(accCols), mapperCol + M);
1648template <
typename DataMapper,
typename Packet,
int N>
1649EIGEN_ALWAYS_INLINE
void bstore(PacketBlock<Packet, N>& acc,
const DataMapper& res, Index row) {
1650 const auto mapperRow = convert_index<mapper_index_t<DataMapper>>(row);
1651 for (
int M = 0; M < N; M++) {
1652 res.template storePacket<Packet>(mapperRow, M, acc.packet[M]);
1657template <
typename DataMapper,
typename Packet, const Index accCols,
bool Complex, Index N,
bool full>
1658EIGEN_ALWAYS_INLINE
void bload_partial(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc,
const DataMapper& res, Index row,
1660 using MapperIndex = mapper_index_t<DataMapper>;
1661 const MapperIndex mapperRow = convert_index<MapperIndex>(row);
1662 const MapperIndex mapperElements = convert_index<MapperIndex>(elements);
1663 EIGEN_IF_CONSTEXPR (Complex && full) {
1664 for (
int M = 0; M < N; M++) {
1665 acc.packet[M] = res.template loadPacket<Packet>(mapperRow, M);
1666 acc.packet[M + N] = res.template loadPacketPartial<Packet>(mapperRow + MapperIndex(accCols), M, mapperElements);
1669 for (
int M = 0; M < N; M++) {
1670 acc.packet[M] = res.template loadPacketPartial<Packet>(mapperRow, M, mapperElements);
1675template <
typename DataMapper,
typename Packet, Index N>
1676EIGEN_ALWAYS_INLINE
void bstore_partial(PacketBlock<Packet, N>& acc,
const DataMapper& res, Index row, Index elements) {
1677 const auto mapperRow = convert_index<mapper_index_t<DataMapper>>(row);
1678 const auto mapperElements = convert_index<mapper_index_t<DataMapper>>(elements);
1679 for (
int M = 0; M < N; M++) {
1680 res.template storePacketPartial<Packet>(mapperRow, M, acc.packet[M], mapperElements);
1685#define USE_P10_AND_PVIPR2_0 (EIGEN_COMP_LLVM || (__GNUC__ >= 11))
1687#define USE_P10_AND_PVIPR2_0 0
1690#if !USE_P10_AND_PVIPR2_0
1691const static Packet4i mask4[4] = {{0, 0, 0, 0}, {-1, 0, 0, 0}, {-1, -1, 0, 0}, {-1, -1, -1, 0}};
1694template <
typename Packet>
1695EIGEN_ALWAYS_INLINE Packet bmask(
const Index remaining_rows) {
1696#if USE_P10_AND_PVIPR2_0
1698 return Packet(vec_reve(vec_genwm((1 << remaining_rows) - 1)));
1700 return Packet(vec_genwm((1 << remaining_rows) - 1));
1703 return Packet(mask4[remaining_rows]);
1708EIGEN_ALWAYS_INLINE Packet2d bmask<Packet2d>(
const Index remaining_rows) {
1709#if USE_P10_AND_PVIPR2_0
1710 Packet2d mask2 = Packet2d(vec_gendm(remaining_rows));
1712 return preverse(mask2);
1717 Packet2l ret = {-remaining_rows, 0};
1718 return Packet2d(ret);
1722template <
typename Packet,
int N>
1723EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ,
const Packet& pAlpha) {
1724 for (
int M = 0; M < N; M++) {
1725 acc.packet[M] = pmadd<Packet>(pAlpha, accZ.packet[M], acc.packet[M]);
1730template <
typename Packet,
int N,
bool mask>
1731EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ,
const Packet& pAlpha,
1732 const Packet& pMask) {
1733 EIGEN_IF_CONSTEXPR (mask) {
1734 band<Packet, N>(accZ, pMask);
1736 EIGEN_UNUSED_VARIABLE(pMask);
1739 bscale<Packet, N>(acc, accZ, pAlpha);
1742template <
typename Packet,
int N,
bool real>
1743EIGEN_ALWAYS_INLINE
void pbroadcastN(
const __UNPACK_TYPE__(Packet) * ap0,
const __UNPACK_TYPE__(Packet) * ap1,
1744 const __UNPACK_TYPE__(Packet) * ap2, Packet& a0, Packet& a1, Packet& a2,
1746 a0 = pset1<Packet>(ap0[0]);
1747 EIGEN_IF_CONSTEXPR (N == 4) {
1748 a1 = pset1<Packet>(ap0[1]);
1749 a2 = pset1<Packet>(ap0[2]);
1750 a3 = pset1<Packet>(ap0[3]);
1751 EIGEN_UNUSED_VARIABLE(ap1);
1752 EIGEN_UNUSED_VARIABLE(ap2);
1754 EIGEN_IF_CONSTEXPR (N > 1) {
1755 a1 = pset1<Packet>(ap1[0]);
1757 EIGEN_UNUSED_VARIABLE(a1);
1758 EIGEN_UNUSED_VARIABLE(ap1);
1760 EIGEN_IF_CONSTEXPR (N > 2) {
1761 a2 = pset1<Packet>(ap2[0]);
1763 EIGEN_UNUSED_VARIABLE(a2);
1764 EIGEN_UNUSED_VARIABLE(ap2);
1770EIGEN_ALWAYS_INLINE
void pbroadcastN<Packet4f, 4, true>(
const float* ap0,
const float*,
const float*, Packet4f& a0,
1771 Packet4f& a1, Packet4f& a2, Packet4f& a3) {
1772 pbroadcast4<Packet4f>(ap0, a0, a1, a2, a3);
1776EIGEN_ALWAYS_INLINE
void pbroadcastN<Packet4f, 4, false>(
const float* ap0,
const float* ap1,
const float* ap2,
1777 Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3) {
1778 pbroadcastN<Packet4f, 4, true>(ap0, ap1, ap2, a0, a1, a2, a3);
1782EIGEN_ALWAYS_INLINE
void pbroadcastN<Packet2d, 4, false>(
const double* ap0,
const double*,
const double*, Packet2d& a0,
1783 Packet2d& a1, Packet2d& a2, Packet2d& a3) {
1784 a1 = pload<Packet2d>(ap0);
1785 a3 = pload<Packet2d>(ap0 + 2);
1786 a0 = vec_splat(a1, 0);
1787 a1 = vec_splat(a1, 1);
1788 a2 = vec_splat(a3, 0);
1789 a3 = vec_splat(a3, 1);
1793template <
typename Packet,
typename Packetc,
int N,
bool full>
1794EIGEN_ALWAYS_INLINE
void bcouple_common(PacketBlock<Packet, N>& taccReal, PacketBlock<Packet, N>& taccImag,
1795 PacketBlock<Packetc, N>& acc1, PacketBlock<Packetc, N>& acc2) {
1796 for (
int M = 0; M < N; M++) {
1797 acc1.packet[M].v = vec_mergeh(taccReal.packet[M], taccImag.packet[M]);
1800 EIGEN_IF_CONSTEXPR (full) {
1801 for (
int M = 0; M < N; M++) {
1802 acc2.packet[M].v = vec_mergel(taccReal.packet[M], taccImag.packet[M]);
1807template <
typename Packet,
typename Packetc,
int N,
bool full>
1808EIGEN_ALWAYS_INLINE
void bcouple(PacketBlock<Packet, N>& taccReal, PacketBlock<Packet, N>& taccImag,
1809 PacketBlock<Packetc, N * 2>& tRes, PacketBlock<Packetc, N>& acc1,
1810 PacketBlock<Packetc, N>& acc2) {
1811 bcouple_common<Packet, Packetc, N, full>(taccReal, taccImag, acc1, acc2);
1813 for (
int M = 0; M < N; M++) {
1814 acc1.packet[M] = padd<Packetc>(tRes.packet[M], acc1.packet[M]);
1817 EIGEN_IF_CONSTEXPR (full) {
1818 for (
int M = 0; M < N; M++) {
1819 acc2.packet[M] = padd<Packetc>(tRes.packet[M + N], acc2.packet[M]);
1828#define MICRO_UNROLL(func) func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
1830#define MICRO_NORMAL_ROWS accRows == quad_traits<Scalar>::rows || accRows == 1
1832#define MICRO_NEW_ROWS ((MICRO_NORMAL_ROWS) ? accRows : 1)
1834#define MICRO_RHS(ptr, N) rhs_##ptr##N
1836#define MICRO_ZERO_PEEL(peel) \
1837 EIGEN_IF_CONSTEXPR ((PEEL_ROW > peel) && (peel != 0)) { \
1838 bsetzero<Packet, accRows>(accZero##peel); \
1840 EIGEN_UNUSED_VARIABLE(accZero##peel); \
1843#define MICRO_ADD(ptr, N) \
1844 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \
1845 MICRO_RHS(ptr, 0) += (accRows * N); \
1847 MICRO_RHS(ptr, 0) += N; \
1848 MICRO_RHS(ptr, 1) += N; \
1849 EIGEN_IF_CONSTEXPR (accRows == 3) { \
1850 MICRO_RHS(ptr, 2) += N; \
1854#define MICRO_ADD_ROWS(N) MICRO_ADD(ptr, N)
1856#define MICRO_BROADCAST1(peel, ptr, rhsV, real) \
1857 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \
1858 pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0) + (accRows * peel), MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 0), \
1859 rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \
1861 pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0) + peel, MICRO_RHS(ptr, 1) + peel, MICRO_RHS(ptr, 2) + peel, \
1862 rhsV##peel[0], rhsV##peel[1], rhsV##peel[2], rhsV##peel[3]); \
1865#define MICRO_BROADCAST(peel) MICRO_BROADCAST1(peel, ptr, rhsV, true)
1867#define MICRO_BROADCAST_EXTRA1(ptr, rhsV, real) \
1868 pbroadcastN<Packet, accRows, real>(MICRO_RHS(ptr, 0), MICRO_RHS(ptr, 1), MICRO_RHS(ptr, 2), rhsV[0], rhsV[1], \
1871#define MICRO_BROADCAST_EXTRA \
1873 MICRO_BROADCAST_EXTRA1(ptr, rhsV, true) \
1876#define MICRO_SRC2(ptr, N, M) \
1877 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_ROWS) { \
1878 EIGEN_UNUSED_VARIABLE(strideB); \
1879 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 1)); \
1880 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 2)); \
1882 MICRO_RHS(ptr, 1) = rhs_base + N + M; \
1883 EIGEN_IF_CONSTEXPR (accRows == 3) { \
1884 MICRO_RHS(ptr, 2) = rhs_base + N * 2 + M; \
1886 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr, 2)); \
1890#define MICRO_SRC2_PTR MICRO_SRC2(ptr, strideB, 0)
1892#define MICRO_ZERO_PEEL_ROW MICRO_UNROLL(MICRO_ZERO_PEEL)
1894#define MICRO_WORK_PEEL(peel) \
1895 EIGEN_IF_CONSTEXPR (PEEL_ROW > peel) { \
1896 MICRO_BROADCAST(peel) \
1897 pger<accRows, Scalar, Packet, false>(&accZero##peel, lhs_ptr + (remaining_rows * peel), rhsV##peel); \
1899 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
1902#define MICRO_WORK_PEEL_ROW \
1903 Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4], rhsV4[4], rhsV5[4], rhsV6[4], rhsV7[4]; \
1904 MICRO_UNROLL(MICRO_WORK_PEEL) \
1905 lhs_ptr += (remaining_rows * PEEL_ROW); \
1906 MICRO_ADD_ROWS(PEEL_ROW)
1908#define MICRO_ADD_PEEL(peel, sum) \
1909 EIGEN_IF_CONSTEXPR (PEEL_ROW > peel) { \
1910 for (Index i = 0; i < accRows; i++) { \
1911 accZero##sum.packet[i] += accZero##peel.packet[i]; \
1915#define MICRO_ADD_PEEL_ROW \
1916 MICRO_ADD_PEEL(4, 0) \
1917 MICRO_ADD_PEEL(5, 1) \
1918 MICRO_ADD_PEEL(6, 2) MICRO_ADD_PEEL(7, 3) MICRO_ADD_PEEL(2, 0) MICRO_ADD_PEEL(3, 1) MICRO_ADD_PEEL(1, 0)
1920#define MICRO_PREFETCHN1(ptr, N) \
1921 EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 0)); \
1922 EIGEN_IF_CONSTEXPR (N == 2 || N == 3) { \
1923 EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 1)); \
1924 EIGEN_IF_CONSTEXPR (N == 3) { \
1925 EIGEN_POWER_PREFETCH(MICRO_RHS(ptr, 2)); \
1929#define MICRO_PREFETCHN(N) MICRO_PREFETCHN1(ptr, N)
1931#define MICRO_COMPLEX_PREFETCHN(N) \
1932 MICRO_PREFETCHN1(ptr_real, N); \
1933 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
1934 MICRO_PREFETCHN1(ptr_imag, N); \
1937template <
typename Scalar,
typename Packet, const Index accRows, const Index remaining_rows, const Index load_rows = 0>
1938EIGEN_ALWAYS_INLINE
void MICRO_EXTRA_ROW(
const Scalar*& lhs_ptr,
const Scalar*& rhs_ptr0,
const Scalar*& rhs_ptr1,
1939 const Scalar*& rhs_ptr2, PacketBlock<Packet, accRows>& accZero) {
1940 MICRO_BROADCAST_EXTRA
1941 pger<accRows, Scalar, Packet, false, load_rows>(&accZero, lhs_ptr, rhsV);
1942 lhs_ptr += remaining_rows;
1945template <
typename Scalar,
typename Packet,
typename DataMapper,
const Index accRows,
const Index accCols,
1946 const Index remaining_rows>
1947EIGEN_ALWAYS_INLINE
void gemm_unrolled_row_iteration(
const DataMapper& res,
const Scalar* lhs_base,
1948 const Scalar* rhs_base, Index depth, Index strideA, Index offsetA,
1949 Index strideB, Index row, Index rows,
const Packet& pAlpha,
1950 const Packet& pMask) {
1951 const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 =
nullptr, *rhs_ptr2 =
nullptr;
1952 const Scalar* lhs_ptr = lhs_base + row * strideA + remaining_rows * offsetA;
1953 PacketBlock<Packet, accRows> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7, acc;
1956 bsetzero<Packet, accRows>(accZero0);
1958 const Index peel_depth = depth - (accCols - remaining_rows);
1960 if (peel_depth >= PEEL_ROW) {
1963 MICRO_PREFETCHN(accRows)
1964 EIGEN_POWER_PREFETCH(lhs_ptr);
1966 }
while ((k += PEEL_ROW) + PEEL_ROW <= peel_depth);
1969 for (; k < peel_depth; k++) {
1970 MICRO_EXTRA_ROW<Scalar, Packet, accRows, remaining_rows>(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2, accZero0);
1972 for (; k < depth; k++) {
1973 MICRO_EXTRA_ROW<Scalar, Packet, accRows, remaining_rows, remaining_rows>(lhs_ptr, rhs_ptr0, rhs_ptr1, rhs_ptr2,
1977 EIGEN_UNUSED_VARIABLE(rows);
1978 EIGEN_UNUSED_VARIABLE(pMask);
1979 bload_partial<DataMapper, Packet, 0, false, accRows>(acc, res, row, remaining_rows);
1980 bscale<Packet, accRows>(acc, accZero0, pAlpha);
1981 bstore_partial<DataMapper, Packet, accRows>(acc, res, row, remaining_rows);
1984#define MICRO_EXTRA(MICRO_EXTRA_UNROLL, value, is_col) \
1987 MICRO_EXTRA_UNROLL(1) \
1990 EIGEN_IF_CONSTEXPR (is_col || (sizeof(Scalar) == sizeof(float))) { \
1991 MICRO_EXTRA_UNROLL(2) \
1995 EIGEN_IF_CONSTEXPR (is_col || (sizeof(Scalar) == sizeof(float))) { \
1996 MICRO_EXTRA_UNROLL(3) \
2001#define MICRO_EXTRA_ROWS(N) \
2002 gemm_unrolled_row_iteration<Scalar, Packet, DataMapper, accRows, accCols, N>( \
2003 res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlpha, pMask);
2005template <
typename Scalar,
typename Packet,
typename DataMapper, const Index accRows, const Index accCols>
2006EIGEN_ALWAYS_INLINE
void gemm_extra_row(
const DataMapper& res,
const Scalar* lhs_base,
const Scalar* rhs_base,
2007 Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows,
2008 Index remaining_rows,
const Packet& pAlpha,
const Packet& pMask) {
2009 MICRO_EXTRA(MICRO_EXTRA_ROWS, remaining_rows,
false)
2012#define MICRO_UNROLL_WORK(func, func2, peel) \
2013 MICRO_UNROLL(func2); \
2014 func(0, peel) func(1, peel) func(2, peel) func(3, peel) func(4, peel) func(5, peel) func(6, peel) func(7, peel)
2016#define MICRO_WORK_ONE(iter, peel) \
2017 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2018 pger_common<Packet, false, accRows>(&accZero##iter, lhsV##iter, rhsV##peel); \
2021#define MICRO_TYPE_PEEL4(func, func2, peel) \
2022 EIGEN_IF_CONSTEXPR (PEEL > peel) { \
2023 Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
2024 MICRO_BROADCAST(peel) \
2025 MICRO_UNROLL_WORK(func, func2, peel) \
2027 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
2030#define MICRO_UNROLL_TYPE_PEEL(M, func, func1, func2) \
2031 Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M], rhsV4[M], rhsV5[M], rhsV6[M], rhsV7[M]; \
2032 func(func1, func2, 0) func(func1, func2, 1) func(func1, func2, 2) func(func1, func2, 3) func(func1, func2, 4) \
2033 func(func1, func2, 5) func(func1, func2, 6) func(func1, func2, 7)
2035#define MICRO_UNROLL_TYPE_ONE(M, func, func1, func2) \
2037 func(func1, func2, 0)
2039#define MICRO_UNROLL_TYPE(MICRO_TYPE, size) \
2040 MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_ONE) \
2041 MICRO_ADD_ROWS(size)
2043#define MICRO_UNROLL_TYPE_PARTIAL(MICRO_TYPE, size) \
2044 MICRO_TYPE(4, MICRO_TYPE_PEEL4, MICRO_WORK_ONE, MICRO_LOAD_PARTIAL_ONE) \
2045 MICRO_ADD_ROWS(size)
2047#define MICRO_ONE_PEEL4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_PEEL, PEEL)
2049#define MICRO_ONE4 MICRO_UNROLL_TYPE(MICRO_UNROLL_TYPE_ONE, 1)
2051#define MICRO_ONE4_PARTIAL MICRO_UNROLL_TYPE_PARTIAL(MICRO_UNROLL_TYPE_ONE, 1)
2053#define MICRO_DST_PTR_ONE(iter) \
2054 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2055 bsetzero<Packet, accRows>(accZero##iter); \
2057 EIGEN_UNUSED_VARIABLE(accZero##iter); \
2060#define MICRO_DST_PTR MICRO_UNROLL(MICRO_DST_PTR_ONE)
2062#define MICRO_SRC_PTR MICRO_UNROLL(MICRO_SRC_PTR_ONE)
2064#define MICRO_PREFETCH MICRO_UNROLL(MICRO_PREFETCH_ONE)
2066#define MICRO_STORE_ONE(iter) \
2067 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2068 EIGEN_IF_CONSTEXPR (MICRO_NORMAL_PARTIAL(iter)) { \
2069 bload<DataMapper, Packet, 0, ColMajor, false, accRows>(acc, res, row + iter * accCols, 0); \
2070 bscale<Packet, accRows>(acc, accZero##iter, pAlpha); \
2071 bstore<DataMapper, Packet, accRows>(acc, res, row + iter * accCols); \
2073 bload_partial<DataMapper, Packet, 0, false, accRows>(acc, res, row + iter * accCols, accCols2); \
2074 bscale<Packet, accRows>(acc, accZero##iter, pAlpha); \
2075 bstore_partial<DataMapper, Packet, accRows>(acc, res, row + iter * accCols, accCols2); \
2079#define MICRO_STORE MICRO_UNROLL(MICRO_STORE_ONE)
2081template <
int unroll_factor,
typename Scalar,
typename Packet,
typename DataMapper,
const Index accRows,
2082 const Index accCols,
bool full>
2083EIGEN_ALWAYS_INLINE
void gemm_unrolled_iteration(
const DataMapper& res,
const Scalar* lhs_base,
const Scalar* rhs_base,
2084 Index depth, Index strideA, Index offsetA, Index strideB, Index& row,
2085 const Packet& pAlpha, Index accCols2) {
2086 const Scalar *rhs_ptr0 = rhs_base, *rhs_ptr1 =
nullptr, *rhs_ptr2 =
nullptr;
2087 const Scalar *lhs_ptr0 =
nullptr, *lhs_ptr1 =
nullptr, *lhs_ptr2 =
nullptr, *lhs_ptr3 =
nullptr, *lhs_ptr4 =
nullptr,
2088 *lhs_ptr5 =
nullptr, *lhs_ptr6 =
nullptr, *lhs_ptr7 =
nullptr;
2089 PacketBlock<Packet, accRows> accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
2090 PacketBlock<Packet, accRows> acc;
2096 const Index peel_depth = full ? depth : (depth - (accCols - accCols2));
2098 for (; k + PEEL <= peel_depth; k += PEEL) {
2099 MICRO_PREFETCHN(accRows)
2103 for (; k < peel_depth; k++) {
2106 EIGEN_IF_CONSTEXPR (!full) {
2107 for (; k < depth; k++) {
2116#define MICRO_UNROLL_ITER2(N, M) \
2117 gemm_unrolled_iteration<N + ((M) ? 1 : 0), Scalar, Packet, DataMapper, accRows, accCols, !M>( \
2118 res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlpha, M ? remaining_rows : accCols); \
2121template <
typename Scalar,
typename Packet,
typename DataMapper, const Index accRows, const Index accCols>
2122EIGEN_ALWAYS_INLINE
void gemm_cols(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB, Index depth,
2123 Index strideA, Index offsetA, Index strideB, Index offsetB, Index col, Index rows,
2124 Index remaining_rows,
const Packet& pAlpha,
const Packet& pMask) {
2125 const DataMapper res3 = res.getSubMapper(0, convert_index<mapper_index_t<DataMapper>>(col));
2127 const Scalar* rhs_base = blockB + col * strideB + MICRO_NEW_ROWS * offsetB;
2128 const Scalar* lhs_base = blockA + accCols * offsetA;
2132 while (row + MAX_UNROLL * accCols <= rows) {
2133 MICRO_UNROLL_ITER2(MAX_UNROLL, 0);
2135 switch ((rows - row) / accCols) {
2138 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 7)
2143 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 6)
2148 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 5)
2153 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 4)
2158 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 3)
2163 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 2)
2168 MICRO_UNROLL_ITER(MICRO_UNROLL_ITER2, 1)
2176 if (remaining_rows > 0) {
2177 gemm_extra_row<Scalar, Packet, DataMapper, accRows, accCols>(res3, blockA, rhs_base, depth, strideA, offsetA,
2178 strideB, row, rows, remaining_rows, pAlpha, pMask);
2182#define MICRO_EXTRA_COLS(N) \
2183 gemm_cols<Scalar, Packet, DataMapper, N, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, \
2184 col, rows, remaining_rows, pAlpha, pMask);
2186template <
typename Scalar,
typename Packet,
typename DataMapper, const Index accCols>
2187EIGEN_ALWAYS_INLINE
void gemm_extra_cols(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB, Index depth,
2188 Index strideA, Index offsetA, Index strideB, Index offsetB, Index col,
2189 Index rows, Index cols, Index remaining_rows,
const Packet& pAlpha,
2190 const Packet& pMask) {
2191 MICRO_EXTRA(MICRO_EXTRA_COLS, cols - col,
true)
2197template <
typename Scalar,
typename Packet,
typename RhsPacket,
typename DataMapper,
const Index accRows,
2198 const Index accCols>
2199EIGEN_STRONG_INLINE
void gemm(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB, Index rows,
2200 Index depth, Index cols, Scalar alpha, Index strideA, Index strideB, Index offsetA,
2202 const Index remaining_rows = rows % accCols;
2204 if (strideA == -1) strideA = depth;
2205 if (strideB == -1) strideB = depth;
2207 const Packet pAlpha = pset1<Packet>(alpha);
2208 const Packet pMask = bmask<Packet>(remaining_rows);
2211 for (; col + accRows <= cols; col += accRows) {
2212 gemm_cols<Scalar, Packet, DataMapper, accRows, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB,
2213 offsetB, col, rows, remaining_rows, pAlpha, pMask);
2217 gemm_extra_cols<Scalar, Packet, DataMapper, accCols>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB,
2218 col, rows, cols, remaining_rows, pAlpha, pMask);
2222#define accColsC (accCols / 2)
2223#define advanceRows ((LhsIsReal) ? 1 : 2)
2224#define advanceCols ((RhsIsReal) ? 1 : 2)
2227#define PEEL_COMPLEX 3
2228#define PEEL_COMPLEX_ROW 3
2230#define MICRO_COMPLEX_UNROLL(func) func(0) func(1) func(2) func(3)
2232#define MICRO_COMPLEX_ZERO_PEEL(peel) \
2233 EIGEN_IF_CONSTEXPR ((PEEL_COMPLEX_ROW > peel) && (peel != 0)) { \
2234 bsetzero<Packet, accRows>(accReal##peel); \
2235 bsetzero<Packet, accRows>(accImag##peel); \
2237 EIGEN_UNUSED_VARIABLE(accReal##peel); \
2238 EIGEN_UNUSED_VARIABLE(accImag##peel); \
2241#define MICRO_COMPLEX_ADD_ROWS(N, used) \
2242 MICRO_ADD(ptr_real, N) \
2243 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2244 MICRO_ADD(ptr_imag, N) \
2245 } else if (used) { \
2246 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \
2247 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \
2248 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \
2251#define MICRO_COMPLEX_BROADCAST(peel) \
2252 MICRO_BROADCAST1(peel, ptr_real, rhsV, false) \
2253 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2254 MICRO_BROADCAST1(peel, ptr_imag, rhsVi, false) \
2256 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
2259#define MICRO_COMPLEX_BROADCAST_EXTRA \
2260 Packet rhsV[4], rhsVi[4]; \
2261 MICRO_BROADCAST_EXTRA1(ptr_real, rhsV, false) \
2262 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2263 MICRO_BROADCAST_EXTRA1(ptr_imag, rhsVi, false) \
2265 EIGEN_UNUSED_VARIABLE(rhsVi); \
2267 MICRO_COMPLEX_ADD_ROWS(1, true)
2269#define MICRO_COMPLEX_SRC2_PTR \
2270 MICRO_SRC2(ptr_real, strideB* advanceCols, 0) \
2271 EIGEN_IF_CONSTEXPR (!RhsIsReal) { \
2272 MICRO_RHS(ptr_imag, 0) = rhs_base + MICRO_NEW_ROWS * strideB; \
2273 MICRO_SRC2(ptr_imag, strideB* advanceCols, strideB) \
2275 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 0)); \
2276 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 1)); \
2277 EIGEN_UNUSED_VARIABLE(MICRO_RHS(ptr_imag, 2)); \
2280#define MICRO_COMPLEX_ZERO_PEEL_ROW MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_ZERO_PEEL)
2282#define MICRO_COMPLEX_WORK_PEEL(peel) \
2283 EIGEN_IF_CONSTEXPR (PEEL_COMPLEX_ROW > peel) { \
2284 MICRO_COMPLEX_BROADCAST(peel) \
2285 pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
2286 &accReal##peel, &accImag##peel, lhs_ptr_real + (remaining_rows * peel), \
2287 lhs_ptr_imag + (remaining_rows * peel), rhsV##peel, rhsVi##peel); \
2289 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
2290 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
2293#define MICRO_COMPLEX_ADD_COLS(size) \
2294 lhs_ptr_real += (remaining_rows * size); \
2295 EIGEN_IF_CONSTEXPR (!LhsIsReal) \
2296 lhs_ptr_imag += (remaining_rows * size); \
2298 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
2300#define MICRO_COMPLEX_WORK_PEEL_ROW \
2301 Packet rhsV0[4], rhsV1[4], rhsV2[4], rhsV3[4]; \
2302 Packet rhsVi0[4], rhsVi1[4], rhsVi2[4], rhsVi3[4]; \
2303 MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_WORK_PEEL) \
2304 MICRO_COMPLEX_ADD_COLS(PEEL_COMPLEX_ROW) \
2305 MICRO_COMPLEX_ADD_ROWS(PEEL_COMPLEX_ROW, false)
2307#define MICRO_COMPLEX_ADD_PEEL(peel, sum) \
2308 EIGEN_IF_CONSTEXPR (PEEL_COMPLEX_ROW > peel) { \
2309 for (Index i = 0; i < accRows; i++) { \
2310 accReal##sum.packet[i] += accReal##peel.packet[i]; \
2311 accImag##sum.packet[i] += accImag##peel.packet[i]; \
2315#define MICRO_COMPLEX_ADD_PEEL_ROW \
2316 MICRO_COMPLEX_ADD_PEEL(2, 0) MICRO_COMPLEX_ADD_PEEL(3, 1) MICRO_COMPLEX_ADD_PEEL(1, 0)
2318template <
typename Scalar,
typename Packet,
const Index accRows,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
2319 bool RhsIsReal,
const Index remaining_rows,
const Index load_rows = 0>
2320EIGEN_ALWAYS_INLINE
void MICRO_COMPLEX_EXTRA_ROW(
const Scalar*& lhs_ptr_real,
const Scalar*& lhs_ptr_imag,
2321 const Scalar*& rhs_ptr_real0,
const Scalar*& rhs_ptr_real1,
2322 const Scalar*& rhs_ptr_real2,
const Scalar*& rhs_ptr_imag0,
2323 const Scalar*& rhs_ptr_imag1,
const Scalar*& rhs_ptr_imag2,
2324 PacketBlock<Packet, accRows>& accReal,
2325 PacketBlock<Packet, accRows>& accImag) {
2326 MICRO_COMPLEX_BROADCAST_EXTRA
2327 pgerc<accRows, Scalar, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, load_rows>(
2328 &accReal, &accImag, lhs_ptr_real, lhs_ptr_imag, rhsV, rhsVi);
2329 MICRO_COMPLEX_ADD_COLS(1)
2332template <
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
const Index accRows,
2333 const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal,
2334 const Index remaining_rows>
2335EIGEN_ALWAYS_INLINE
void gemm_unrolled_complex_row_iteration(
const DataMapper& res,
const Scalar* lhs_base,
2336 const Scalar* rhs_base, Index depth, Index strideA,
2337 Index offsetA, Index strideB, Index row, Index rows,
2338 const Packet& pAlphaReal,
const Packet& pAlphaImag,
2339 const Packet& pMask) {
2340 const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 =
nullptr, *rhs_ptr_real2 =
nullptr;
2341 const Scalar *rhs_ptr_imag0 =
nullptr, *rhs_ptr_imag1 =
nullptr, *rhs_ptr_imag2 =
nullptr;
2342 const Scalar* lhs_ptr_real = lhs_base + advanceRows * row * strideA + remaining_rows * offsetA;
2343 const Scalar* lhs_ptr_imag =
nullptr;
2344 EIGEN_IF_CONSTEXPR (!LhsIsReal)
2345 lhs_ptr_imag = lhs_ptr_real + remaining_rows * strideA;
2347 EIGEN_UNUSED_VARIABLE(lhs_ptr_imag);
2348 PacketBlock<Packet, accRows> accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3;
2349 PacketBlock<Packet, accRows> taccReal, taccImag;
2350 PacketBlock<Packetc, accRows> acc0, acc1;
2351 PacketBlock<Packetc, accRows * 2> tRes;
2353 MICRO_COMPLEX_SRC2_PTR
2355 bsetzero<Packet, accRows>(accReal0);
2356 bsetzero<Packet, accRows>(accImag0);
2358 const Index peel_depth = depth - (accCols - remaining_rows);
2360 if (peel_depth >= PEEL_COMPLEX_ROW) {
2361 MICRO_COMPLEX_ZERO_PEEL_ROW
2363 MICRO_COMPLEX_PREFETCHN(accRows)
2364 EIGEN_POWER_PREFETCH(lhs_ptr_real);
2365 EIGEN_IF_CONSTEXPR (!LhsIsReal) {
2366 EIGEN_POWER_PREFETCH(lhs_ptr_imag);
2368 MICRO_COMPLEX_WORK_PEEL_ROW
2369 }
while ((k += PEEL_COMPLEX_ROW) + PEEL_COMPLEX_ROW <= peel_depth);
2370 MICRO_COMPLEX_ADD_PEEL_ROW
2372 for (; k < peel_depth; k++) {
2373 MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, remaining_rows>(
2374 lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2, rhs_ptr_imag0, rhs_ptr_imag1,
2375 rhs_ptr_imag2, accReal0, accImag0);
2377 for (; k < depth; k++) {
2378 MICRO_COMPLEX_EXTRA_ROW<Scalar, Packet, accRows, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal, remaining_rows,
2379 remaining_rows>(lhs_ptr_real, lhs_ptr_imag, rhs_ptr_real0, rhs_ptr_real1, rhs_ptr_real2,
2380 rhs_ptr_imag0, rhs_ptr_imag1, rhs_ptr_imag2, accReal0, accImag0);
2383 EIGEN_UNUSED_VARIABLE(rows);
2384 constexpr bool full = (remaining_rows > accColsC);
2385 constexpr bool odd = (
sizeof(Scalar) ==
sizeof(float)) && (remaining_rows & 1);
2386 EIGEN_IF_CONSTEXPR (odd) {
2387 bload_partial<DataMapper, Packetc, accColsC, true, accRows, full>(tRes, res, row, 1);
2389 bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row, 0);
2391 bscalec<Packet, accRows, true>(accReal0, accImag0, pAlphaReal, pAlphaImag, taccReal, taccImag, pMask);
2392 bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1);
2393 EIGEN_IF_CONSTEXPR (odd && !full) {
2394 bstore_partial<DataMapper, Packetc, accRows>(acc0, res, row + 0, 1);
2396 bstore<DataMapper, Packetc, accRows>(acc0, res, row + 0);
2398 EIGEN_IF_CONSTEXPR (full) {
2399 EIGEN_IF_CONSTEXPR (odd) {
2400 bstore_partial<DataMapper, Packetc, accRows>(acc1, res, row + accColsC, 1);
2402 bstore<DataMapper, Packetc, accRows>(acc1, res, row + accColsC);
2407#define MICRO_COMPLEX_EXTRA_ROWS(N) \
2408 gemm_unrolled_complex_row_iteration<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, \
2409 ConjugateRhs, LhsIsReal, RhsIsReal, N>( \
2410 res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, pAlphaReal, pAlphaImag, pMask);
2412template <
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
const Index accRows,
2413 const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2414EIGEN_ALWAYS_INLINE
void gemm_complex_extra_row(
const DataMapper& res,
const Scalar* lhs_base,
const Scalar* rhs_base,
2415 Index depth, Index strideA, Index offsetA, Index strideB, Index row,
2416 Index rows, Index remaining_rows,
const Packet& pAlphaReal,
2417 const Packet& pAlphaImag,
const Packet& pMask) {
2418 MICRO_EXTRA(MICRO_COMPLEX_EXTRA_ROWS, remaining_rows,
false)
2421#define MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \
2422 MICRO_COMPLEX_UNROLL(func2); \
2423 func(0, peel) func(1, peel) func(2, peel) func(3, peel)
2425#define MICRO_COMPLEX_WORK_ONE4(iter, peel) \
2426 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2427 pgerc_common<accRows, Packet, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
2428 &accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \
2431#define MICRO_COMPLEX_TYPE_PEEL4(func, func2, peel) \
2432 EIGEN_IF_CONSTEXPR (PEEL_COMPLEX > peel) { \
2433 Packet lhsV0, lhsV1, lhsV2, lhsV3; \
2434 Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3; \
2435 MICRO_COMPLEX_BROADCAST(peel) \
2436 MICRO_COMPLEX_UNROLL_WORK(func, func2, peel) \
2438 EIGEN_UNUSED_VARIABLE(rhsV##peel); \
2439 EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
2442#define MICRO_COMPLEX_UNROLL_TYPE_PEEL(M, func, func1, func2) \
2443 Packet rhsV0[M], rhsV1[M], rhsV2[M], rhsV3[M]; \
2444 Packet rhsVi0[M], rhsVi1[M], rhsVi2[M], rhsVi3[M]; \
2445 func(func1, func2, 0) func(func1, func2, 1) func(func1, func2, 2) func(func1, func2, 3)
2447#define MICRO_COMPLEX_UNROLL_TYPE_ONE(M, func, func1, func2) \
2448 Packet rhsV0[M], rhsVi0[M]; \
2449 func(func1, func2, 0)
2451#define MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_TYPE, size) \
2452 MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_ONE) \
2453 MICRO_COMPLEX_ADD_ROWS(size, false)
2455#define MICRO_COMPLEX_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_TYPE, size) \
2456 MICRO_COMPLEX_TYPE(4, MICRO_COMPLEX_TYPE_PEEL4, MICRO_COMPLEX_WORK_ONE4, MICRO_COMPLEX_LOAD_PARTIAL_ONE) \
2457 MICRO_COMPLEX_ADD_ROWS(size, false)
2459#define MICRO_COMPLEX_ONE_PEEL4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_PEEL, PEEL_COMPLEX)
2461#define MICRO_COMPLEX_ONE4 MICRO_COMPLEX_UNROLL_TYPE(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1)
2463#define MICRO_COMPLEX_ONE4_PARTIAL MICRO_COMPLEX_UNROLL_TYPE_PARTIAL(MICRO_COMPLEX_UNROLL_TYPE_ONE, 1)
2465#define MICRO_COMPLEX_DST_PTR_ONE(iter) \
2466 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2467 bsetzero<Packet, accRows>(accReal##iter); \
2468 bsetzero<Packet, accRows>(accImag##iter); \
2470 EIGEN_UNUSED_VARIABLE(accReal##iter); \
2471 EIGEN_UNUSED_VARIABLE(accImag##iter); \
2474#define MICRO_COMPLEX_DST_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_DST_PTR_ONE)
2476#define MICRO_COMPLEX_SRC_PTR MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_SRC_PTR_ONE)
2478#define MICRO_COMPLEX_PREFETCH MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_PREFETCH_ONE)
2480#define MICRO_COMPLEX_STORE_ONE(iter) \
2481 EIGEN_IF_CONSTEXPR (unroll_factor > iter) { \
2482 constexpr bool full = ((MICRO_NORMAL(iter)) || (accCols2 > accColsC)); \
2483 constexpr bool odd = !(MICRO_NORMAL(iter)) && (sizeof(Scalar) == sizeof(float)) && (accCols2 & 1); \
2484 EIGEN_IF_CONSTEXPR (odd) { \
2485 bload_partial<DataMapper, Packetc, accColsC, true, accRows, full>(tRes, res, row + iter * accCols, 1); \
2487 bload<DataMapper, Packetc, accColsC, ColMajor, true, accRows, full>(tRes, res, row + iter * accCols, 0); \
2489 bscalec<Packet, accRows, !(MICRO_NORMAL(iter))>(accReal##iter, accImag##iter, pAlphaReal, pAlphaImag, taccReal, \
2491 bcouple<Packet, Packetc, accRows, full>(taccReal, taccImag, tRes, acc0, acc1); \
2492 EIGEN_IF_CONSTEXPR (odd && !full) { \
2493 bstore_partial<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0, 1); \
2495 bstore<DataMapper, Packetc, accRows>(acc0, res, row + iter * accCols + 0); \
2497 EIGEN_IF_CONSTEXPR (full) { \
2498 EIGEN_IF_CONSTEXPR (odd) { \
2499 bstore_partial<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC, 1); \
2501 bstore<DataMapper, Packetc, accRows>(acc1, res, row + iter * accCols + accColsC); \
2506#define MICRO_COMPLEX_STORE MICRO_COMPLEX_UNROLL(MICRO_COMPLEX_STORE_ONE)
2508template <
int unroll_factor,
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
2509 const Index accRows,
const Index accCols,
const Index accCols2,
bool ConjugateLhs,
bool ConjugateRhs,
2510 bool LhsIsReal,
bool RhsIsReal>
2511EIGEN_ALWAYS_INLINE
void gemm_complex_unrolled_iteration(
const DataMapper& res,
const Scalar* lhs_base,
2512 const Scalar* rhs_base, Index depth, Index strideA,
2513 Index offsetA, Index strideB, Index& row,
2514 const Packet& pAlphaReal,
const Packet& pAlphaImag,
2515 const Packet& pMask) {
2516 const Scalar *rhs_ptr_real0 = rhs_base, *rhs_ptr_real1 =
nullptr, *rhs_ptr_real2 =
nullptr;
2517 const Scalar *rhs_ptr_imag0 =
nullptr, *rhs_ptr_imag1 =
nullptr, *rhs_ptr_imag2 =
nullptr;
2518 const Index imag_delta = accCols * strideA;
2519 const Index imag_delta2 = accCols2 * strideA;
2520 const Scalar *lhs_ptr_real0 =
nullptr, *lhs_ptr_real1 =
nullptr;
2521 const Scalar *lhs_ptr_real2 =
nullptr, *lhs_ptr_real3 =
nullptr;
2522 PacketBlock<Packet, accRows> accReal0, accImag0, accReal1, accImag1;
2523 PacketBlock<Packet, accRows> accReal2, accImag2, accReal3, accImag3;
2524 PacketBlock<Packet, accRows> taccReal, taccImag;
2525 PacketBlock<Packetc, accRows> acc0, acc1;
2526 PacketBlock<Packetc, accRows * 2> tRes;
2528 MICRO_COMPLEX_SRC2_PTR
2529 MICRO_COMPLEX_SRC_PTR
2530 MICRO_COMPLEX_DST_PTR
2532 const Index peel_depth = depth - (accCols - accCols2);
2534 for (; k + PEEL_COMPLEX <= peel_depth; k += PEEL_COMPLEX) {
2535 MICRO_COMPLEX_PREFETCHN(accRows)
2536 MICRO_COMPLEX_PREFETCH
2537 MICRO_COMPLEX_ONE_PEEL4
2539 for (; k < peel_depth; k++) {
2542 EIGEN_IF_CONSTEXPR (accCols != accCols2) {
2543 for (; k < depth; k++) {
2544 MICRO_COMPLEX_ONE4_PARTIAL
2549 MICRO_COMPLEX_UPDATE
2552#define MICRO_COMPLEX_UNROLL_ITER2(N, M) \
2553 gemm_complex_unrolled_iteration<N + (M ? 1 : 0), Scalar, Packet, Packetc, DataMapper, accRows, accCols, \
2554 M ? M : accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>( \
2555 res3, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, pAlphaReal, pAlphaImag, pMask); \
2558template <
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
const Index accRows,
2559 const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2560EIGEN_ALWAYS_INLINE
void gemm_complex_cols(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB,
2561 Index depth, Index strideA, Index offsetA, Index strideB, Index offsetB,
2562 Index col, Index rows, Index remaining_rows,
const Packet& pAlphaReal,
2563 const Packet& pAlphaImag,
const Packet& pMask) {
2564 const DataMapper res3 = res.getSubMapper(0, convert_index<mapper_index_t<DataMapper>>(col));
2566 const Scalar* rhs_base = blockB + advanceCols * col * strideB + MICRO_NEW_ROWS * offsetB;
2567 const Scalar* lhs_base = blockA + accCols * offsetA;
2570#define MAX_COMPLEX_UNROLL 4
2571 while (row + MAX_COMPLEX_UNROLL * accCols <= rows) {
2572 MICRO_COMPLEX_UNROLL_ITER2(MAX_COMPLEX_UNROLL, 0);
2574 switch ((rows - row) / accCols) {
2575#if MAX_COMPLEX_UNROLL > 4
2577 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 4)
2580#if MAX_COMPLEX_UNROLL > 3
2582 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 3)
2585#if MAX_COMPLEX_UNROLL > 2
2587 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 2)
2590#if MAX_COMPLEX_UNROLL > 1
2592 MICRO_COMPLEX_UNROLL_ITER(MICRO_COMPLEX_UNROLL_ITER2, 1)
2598#undef MAX_COMPLEX_UNROLL
2600 if (remaining_rows > 0) {
2601 gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
2602 RhsIsReal>(res3, blockA, rhs_base, depth, strideA, offsetA, strideB, row, rows,
2603 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2607#define MICRO_COMPLEX_EXTRA_COLS(N) \
2608 gemm_complex_cols<Scalar, Packet, Packetc, DataMapper, N, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, \
2609 RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, \
2610 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2612template <
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
const Index accCols,
2613 bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2614EIGEN_ALWAYS_INLINE
void gemm_complex_extra_cols(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB,
2615 Index depth, Index strideA, Index offsetA, Index strideB,
2616 Index offsetB, Index col, Index rows, Index cols, Index remaining_rows,
2617 const Packet& pAlphaReal,
const Packet& pAlphaImag,
2618 const Packet& pMask) {
2619 MICRO_EXTRA(MICRO_COMPLEX_EXTRA_COLS, cols - col,
true)
2622template <
typename LhsScalar,
typename RhsScalar,
typename Scalarc,
typename Scalar,
typename Packet,
typename Packetc,
2623 typename RhsPacket,
typename DataMapper,
const Index accRows,
const Index accCols,
bool ConjugateLhs,
2624 bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
2625EIGEN_STRONG_INLINE
void gemm_complex(
const DataMapper& res,
const LhsScalar* blockAc,
const RhsScalar* blockBc,
2626 Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB,
2627 Index offsetA, Index offsetB) {
2628 const Index remaining_rows = rows % accCols;
2630 if (strideA == -1) strideA = depth;
2631 if (strideB == -1) strideB = depth;
2633 const Packet pAlphaReal = pset1<Packet>(alpha.real());
2634 const Packet pAlphaImag = pset1<Packet>(alpha.imag());
2635 const Packet pMask = bmask<Packet>(remaining_rows);
2637 const Scalar* blockA = (Scalar*)blockAc;
2638 const Scalar* blockB = (Scalar*)blockBc;
2641 for (; col + accRows <= cols; col += accRows) {
2642 gemm_complex_cols<Scalar, Packet, Packetc, DataMapper, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
2643 RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows,
2644 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2648 gemm_complex_extra_cols<Scalar, Packet, Packetc, DataMapper, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal,
2649 RhsIsReal>(res, blockA, blockB, depth, strideA, offsetA, strideB, offsetB, col, rows, cols,
2650 remaining_rows, pAlphaReal, pAlphaImag, pMask);
2658EIGEN_ALWAYS_INLINE
bool supportsMMA() {
2659#if defined(EIGEN_ALTIVEC_MMA_ONLY)
2661#elif defined(EIGEN_ALTIVEC_MMA_DYNAMIC_DISPATCH) && defined(__BUILTIN_CPU_SUPPORTS__)
2662 return __builtin_cpu_supports(
"arch_3_1") && __builtin_cpu_supports(
"mma");
2668EIGEN_ALWAYS_INLINE Packet4f loadAndMultiplyF32(
const Packet4f& acc,
const Packet4f& pAlpha,
float* result) {
2669 Packet4f result_block = ploadu<Packet4f>(result);
2670 return pmadd(acc, pAlpha, result_block);
2673template <
bool lhsExtraRows>
2674EIGEN_ALWAYS_INLINE
void storeF32(
float*& result,
const Packet4f& result_block, Index rows, Index extra_rows) {
2675 EIGEN_IF_CONSTEXPR (lhsExtraRows) {
2676 pstoreu_partial(result, result_block, extra_rows);
2678 pstoreu(result, result_block);
2683template <
bool rhsExtraCols,
bool lhsExtraRows>
2684EIGEN_ALWAYS_INLINE
void storeResults(Packet4f (&acc)[4], Index rows,
const Packet4f& pAlpha,
float* result,
2685 Index extra_cols, Index extra_rows) {
2687 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
2689 Packet4f result_block = loadAndMultiplyF32(acc[x], pAlpha, result);
2690 storeF32<lhsExtraRows>(result, result_block, rows, extra_rows);
2691 }
while (++x < extra_cols);
2693 Packet4f result_block[4];
2694 float* result2 = result;
2696 result_block[x] = loadAndMultiplyF32(acc[x], pAlpha, result);
2701 storeF32<lhsExtraRows>(result2, result_block[x], rows, extra_rows);
2706EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Hi(
const Packet8us& data) {
2707 Packet8us z = pset1<Packet8us>(0);
2709 return reinterpret_cast<Packet4f
>(vec_mergeh(data, z));
2711 return reinterpret_cast<Packet4f
>(vec_mergeh(z, data));
2715EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Lo(
const Packet8us& data) {
2716 Packet8us z = pset1<Packet8us>(0);
2718 return reinterpret_cast<Packet4f
>(vec_mergel(data, z));
2720 return reinterpret_cast<Packet4f
>(vec_mergel(z, data));
2724template <Index N, Index M>
2725EIGEN_ALWAYS_INLINE
void storeConvertTwoBF16(
float* to, PacketBlock<Packet8bf, (N + 7) / 8>& block, Index extra = 0) {
2726 EIGEN_IF_CONSTEXPR (N < 4) {
2727 pstoreu_partial(to + 0, oneConvertBF16Hi(block.packet[0].m_val), extra);
2728 }
else EIGEN_IF_CONSTEXPR (N >= (M * 8 + 4)) {
2729 pstoreu(to + 0, oneConvertBF16Hi(block.packet[M].m_val));
2730 EIGEN_IF_CONSTEXPR (N >= 8) {
2731 pstoreu(to + 4, oneConvertBF16Lo(block.packet[M].m_val));
2737EIGEN_ALWAYS_INLINE
void storeConvertBlockBF16(
float* to, PacketBlock<Packet8bf, (N + 7) / 8>& block, Index extra) {
2738 storeConvertTwoBF16<N, 0>(to + 0, block, extra);
2739 EIGEN_IF_CONSTEXPR (N >= 16) {
2740 storeConvertTwoBF16<N, 1>(to + 8, block);
2742 EIGEN_IF_CONSTEXPR (N >= 32) {
2743 storeConvertTwoBF16<N, 2>(to + 16, block);
2744 storeConvertTwoBF16<N, 3>(to + 24, block);
2748template <
bool non_unit_str
ide, Index delta>
2749EIGEN_ALWAYS_INLINE Packet8bf loadBF16fromResult(bfloat16* src, Index resInc) {
2750 EIGEN_IF_CONSTEXPR (non_unit_stride) {
2751 return pgather<bfloat16, Packet8bf>(src + delta * resInc, resInc);
2753 return ploadu<Packet8bf>(src + delta);
2757static Packet16uc p16uc_MERGE16_32_1 = {0, 1, 16, 17, 2, 3, 18, 19, 0, 1, 16, 17, 2, 3, 18, 19};
2758static Packet16uc p16uc_MERGE16_32_2 = {4, 5, 20, 21, 6, 7, 22, 23, 4, 5, 20, 21, 6, 7, 22, 23};
2759static Packet16uc p16uc_MERGE16_32_3 = {8, 9, 24, 25, 10, 11, 26, 27, 8, 9, 24, 25, 10, 11, 26, 27};
2760static Packet16uc p16uc_MERGE16_32_4 = {12, 13, 28, 29, 14, 15, 30, 31, 12, 13, 28, 29, 14, 15, 30, 31};
2762static Packet16uc p16uc_MERGE16_32_5 = {0, 1, 16, 17, 16, 17, 16, 17, 0, 1, 16, 17, 16, 17, 16, 17};
2763static Packet16uc p16uc_MERGE16_32_6 = {2, 3, 18, 19, 18, 19, 18, 19, 2, 3, 18, 19, 18, 19, 18, 19};
2764static Packet16uc p16uc_MERGE16_32_7 = {4, 5, 20, 21, 20, 21, 20, 21, 4, 5, 20, 21, 20, 21, 20, 21};
2765static Packet16uc p16uc_MERGE16_32_8 = {6, 7, 22, 23, 22, 23, 22, 23, 6, 7, 22, 23, 22, 23, 22, 23};
2767EIGEN_ALWAYS_INLINE Packet4f oneConvertBF16Perm(
const Packet8us& data,
const Packet16uc& mask) {
2768 Packet8us z = pset1<Packet8us>(0);
2770 return reinterpret_cast<Packet4f
>(vec_perm(data, z, mask));
2772 return reinterpret_cast<Packet4f
>(vec_perm(z, data, mask));
2776template <
bool lhsExtraRows,
bool odd, Index size>
2777EIGEN_ALWAYS_INLINE
void convertArrayPointerBF16toF32DupOne(
float* result, Index rows,
const bfloat16* src,
2779 Packet4f dup[4 * 4];
2782 for (Index i = 0; i < size; i++) {
2783 data[i] = ploadu<Packet8bf>(src + rows * i);
2786 for (Index i = 0, j = 0; i < size; i++, j += 4) {
2787 dup[j + 0] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_5 : p16uc_MERGE16_32_1);
2788 dup[j + 1] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_6 : p16uc_MERGE16_32_2);
2789 dup[j + 2] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_7 : p16uc_MERGE16_32_3);
2790 dup[j + 3] = oneConvertBF16Perm(data[i].m_val, odd ? p16uc_MERGE16_32_8 : p16uc_MERGE16_32_4);
2793 for (Index j = 0; j < 4 * size; j += 4) {
2794 EIGEN_IF_CONSTEXPR (lhsExtraRows) {
2795 Packet4f z = pset1<Packet4f>(
float(0));
2798 pstoreu(result + (j + i) * 4, dup[j + i]);
2799 }
while (++i < extra_rows);
2801 pstoreu(result + (j + i) * 4, z);
2804 for (Index i = 0; i < 4; i++) {
2805 pstoreu(result + (j + i) * 4, dup[j + i]);
2811template <
bool lhsExtraRows>
2812EIGEN_ALWAYS_INLINE
void convertArrayPointerBF16toF32Dup(
float* result, Index cols, Index rows,
const bfloat16* src,
2813 Index delta, Index extra_rows) {
2816 for (; col + 4 * 2 <= cols; col += 4 * 2, result += 4 * 4 * 4, src += 4 * rows) {
2817 convertArrayPointerBF16toF32DupOne<lhsExtraRows, false, 4>(result, rows, src, extra_rows);
2819 for (; col + 2 <= cols; col += 2, result += 4 * 4, src += rows) {
2820 convertArrayPointerBF16toF32DupOne<lhsExtraRows, false, 1>(result, rows, src, extra_rows);
2823 convertArrayPointerBF16toF32DupOne<lhsExtraRows, true, 1>(result, rows, src - delta, extra_rows);
2827template <const Index size,
bool non_unit_str
ide>
2828EIGEN_ALWAYS_INLINE
void convertPointerBF16toF32(Index& i,
float* result, Index rows, bfloat16*& src, Index resInc) {
2829 while (i + size <= rows) {
2830 const Index count = size == 1 ? rows - i : size;
2831 PacketBlock<Packet8bf, (size + 7) / 8> r32;
2832 EIGEN_IF_CONSTEXPR (size < 8) {
2833 r32.packet[0] = pgather_partial<bfloat16, Packet8bf>(src, non_unit_stride ? resInc : 1, count);
2835 r32.packet[0] = loadBF16fromResult<non_unit_stride, 0>(src, resInc);
2837 EIGEN_IF_CONSTEXPR (size >= 16) {
2838 r32.packet[1] = loadBF16fromResult<non_unit_stride, 8>(src, resInc);
2840 EIGEN_IF_CONSTEXPR (size >= 32) {
2841 r32.packet[2] = loadBF16fromResult<non_unit_stride, 16>(src, resInc);
2842 r32.packet[3] = loadBF16fromResult<non_unit_stride, 24>(src, resInc);
2844 storeConvertBlockBF16<size>(result + i, r32, rows & 3);
2846 if (i < rows) src += count * resInc;
2847 EIGEN_IF_CONSTEXPR (size != 32) break;
2851template <
bool non_unit_stride>
2852EIGEN_ALWAYS_INLINE
void convertArrayPointerBF16toF32(
float* result, Index cols, Index rows, bfloat16* src,
2854 for (Index col = 0; col < cols; col++, result += rows) {
2856 bfloat16* src2 = src;
2857 convertPointerBF16toF32<32, non_unit_stride>(i, result, rows, src2, resInc);
2858 convertPointerBF16toF32<16, non_unit_stride>(i, result, rows, src2, resInc);
2859 convertPointerBF16toF32<8, non_unit_stride>(i, result, rows, src2, resInc);
2860 convertPointerBF16toF32<4, non_unit_stride>(i, result, rows, src2, resInc);
2861 convertPointerBF16toF32<1, non_unit_stride>(i, result, rows, src2, resInc);
2862 if (col + 1 < cols) src += rows * resInc;
2866template <Index num_acc, Index size = 4>
2867EIGEN_ALWAYS_INLINE
void zeroAccumulators(Packet4f (&acc)[num_acc][size]) {
2868 Packet4f z = pset1<Packet4f>(
float(0));
2870 for (Index k = 0; k < num_acc; k++) {
2871 for (Index j = 0; j < size; j++) {
2877template <Index num_acc>
2878EIGEN_ALWAYS_INLINE
void tranposeResults(Packet4f (&acc)[num_acc][4]) {
2879 for (Index i = 0; i < num_acc; i++) {
2880 Packet4ui t0, t1, t2, t3;
2881 t0 = vec_mergeh(
reinterpret_cast<Packet4ui
>(acc[i][0]),
reinterpret_cast<Packet4ui
>(acc[i][2]));
2882 t1 = vec_mergel(
reinterpret_cast<Packet4ui
>(acc[i][0]),
reinterpret_cast<Packet4ui
>(acc[i][2]));
2883 t2 = vec_mergeh(
reinterpret_cast<Packet4ui
>(acc[i][1]),
reinterpret_cast<Packet4ui
>(acc[i][3]));
2884 t3 = vec_mergel(
reinterpret_cast<Packet4ui
>(acc[i][1]),
reinterpret_cast<Packet4ui
>(acc[i][3]));
2885 acc[i][0] =
reinterpret_cast<Packet4f
>(vec_mergeh(t0, t2));
2886 acc[i][1] =
reinterpret_cast<Packet4f
>(vec_mergel(t0, t2));
2887 acc[i][2] =
reinterpret_cast<Packet4f
>(vec_mergeh(t1, t3));
2888 acc[i][3] =
reinterpret_cast<Packet4f
>(vec_mergel(t1, t3));
2892template <Index num_acc>
2893EIGEN_ALWAYS_INLINE
void addResults(Packet4f (&acc)[num_acc][4]) {
2894 for (Index i = 0, j = 0; j < num_acc; i++, j += 2) {
2895 for (Index x = 0, y = 0; x < 2; x++, y += 2) {
2896 for (Index w = 0, z = 0; w < 2; w++, z += 2) {
2897 acc[i][y + w] = acc[j + x][z + 0] + acc[j + x][z + 1];
2903template <Index num_acc,
bool rhsExtraCols,
bool lhsExtraRows, Index num_rhs>
2904EIGEN_ALWAYS_INLINE
void outputResultsVSX(Packet4f (&acc)[num_acc][4], Index rows,
const Packet4f& pAlpha,
2905 float* result,
const Index extra_cols, Index extra_rows) {
2906 tranposeResults<num_acc>(acc);
2907 addResults<num_acc>(acc);
2909 constexpr Index real_rhs = ((num_rhs / 2) - (rhsExtraCols ? 1 : 0));
2911 for (Index i = 0; i < real_rhs; i++, result += 4 * rows, k++) {
2912 storeResults<false, lhsExtraRows>(acc[k], rows, pAlpha, result, extra_cols, extra_rows);
2914 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
2915 storeResults<rhsExtraCols, lhsExtraRows>(acc[k], rows, pAlpha, result, extra_cols, extra_rows);
2920EIGEN_ALWAYS_INLINE
void loadTwoRhsFloat32(
const float* block, Index strideB, Index i, Packet4f& dhs0, Packet4f& dhs1) {
2921 dhs0 = ploadu<Packet4f>(block + strideB * i + 0);
2922 EIGEN_IF_CONSTEXPR (zero) {
2923 Packet4f dhs2 = pset1<Packet4f>(
float(0));
2924 dhs1 = vec_mergel(dhs0, dhs2);
2925 dhs0 = vec_mergeh(dhs0, dhs2);
2927 dhs1 = ploadu<Packet4f>(block + strideB * i + 4);
2931template <Index num_acc,
bool zero,
bool rhsExtraCols, Index num_rhs>
2932EIGEN_ALWAYS_INLINE
void KLoop(
const float* indexA,
const float* indexB, Packet4f (&acc)[num_acc][4], Index strideB,
2933 Index k, Index offsetB, Index extra_cols) {
2934 constexpr Index num_lhs = 4;
2935 Packet4f lhs[num_lhs], rhs[num_rhs];
2937 constexpr Index real_rhs = (num_rhs - (rhsExtraCols ? 2 : 0));
2938 for (Index i = 0; i < real_rhs; i += 2) {
2939 loadTwoRhsFloat32<zero>(indexB + k * 4, strideB, i, rhs[i + 0], rhs[i + 1]);
2941 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
2942 loadTwoRhsFloat32<zero>(indexB + k * extra_cols - offsetB, strideB, real_rhs, rhs[real_rhs + 0], rhs[real_rhs + 1]);
2945 indexA += 2 * k * 4;
2946 for (Index j = 0; j < num_lhs; j++) {
2947 lhs[j] = ploadu<Packet4f>(indexA + j * 4);
2950 for (Index j = 0; j < num_rhs; j++) {
2951 for (Index i = 0; i < num_lhs; i++) {
2952 acc[j][i] = pmadd(rhs[j], lhs[i], acc[j][i]);
2957template <const Index num_acc,
bool rhsExtraCols,
bool lhsExtraRows>
2958EIGEN_ALWAYS_INLINE
void colVSXLoopBodyIter(Index depth, Index rows,
const Packet4f& pAlpha,
const float* indexA,
2959 const float* indexB, Index strideB, Index offsetB,
float* result,
2960 const Index extra_cols,
const Index extra_rows) {
2961 constexpr Index num_rhs = num_acc;
2963 Packet4f acc[num_acc][4];
2965 zeroAccumulators<num_acc>(acc);
2968 for (k = 0; k + 2 <= depth; k += 2) {
2969 KLoop<num_acc, false, rhsExtraCols, num_rhs>(indexA, indexB, acc, strideB, k, offsetB, extra_cols);
2972 KLoop<num_acc, true, rhsExtraCols, num_rhs>(indexA, indexB, acc, strideB, k, offsetB, extra_cols);
2975 outputResultsVSX<num_acc, rhsExtraCols, lhsExtraRows, num_rhs>(acc, rows, pAlpha, result, extra_cols, extra_rows);
2979#define MAX_BFLOAT16_ACC_VSX 4
2981template <const Index num_acc,
bool rhsExtraCols,
bool lhsExtraRows>
2982void colVSXLoopBody(Index& col, Index depth, Index cols, Index rows,
const Packet4f& pAlpha,
const float* indexA,
2983 const float* indexB, Index strideB, Index offsetB,
float* result) {
2984 constexpr Index step = (num_acc * 4);
2985 const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0;
2986 const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0;
2987 constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC_VSX);
2990 colVSXLoopBodyIter<num_acc * 2, rhsExtraCols, lhsExtraRows>(depth, rows, pAlpha, indexA, indexB, strideB, offsetB,
2991 result, extra_cols, extra_rows);
2993 indexB += strideB * (num_acc * 2);
2994 result += rows * step;
2995 }
while (multiIters && (step <= cols - (col += step)));
2998template <const Index num_acc,
bool rhsExtraCols,
bool lhsExtraRows>
2999EIGEN_ALWAYS_INLINE
void colVSXLoopBodyExtraN(Index col, Index depth, Index cols, Index rows,
const Packet4f& pAlpha,
3000 const float* indexA,
const float* blockB, Index strideB, Index offsetB,
3002 EIGEN_IF_CONSTEXPR (MAX_BFLOAT16_ACC_VSX > num_acc) {
3003 colVSXLoopBody<num_acc + (rhsExtraCols ? 1 : 0), rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA,
3004 blockB, strideB, offsetB, result);
3008template <
bool rhsExtraCols,
bool lhsExtraRows>
3009void colVSXLoopBodyExtra(Index col, Index depth, Index cols, Index rows,
const Packet4f& pAlpha,
const float* indexA,
3010 const float* blockB, Index strideB, Index offsetB,
float* result) {
3011 switch ((cols - col) >> 2) {
3013 colVSXLoopBodyExtraN<3, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
3017 colVSXLoopBodyExtraN<2, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
3021 colVSXLoopBodyExtraN<1, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
3025 EIGEN_IF_CONSTEXPR (rhsExtraCols) {
3026 colVSXLoopBody<1, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result);
3032template <Index size,
bool lhsExtraRows = false>
3033EIGEN_ALWAYS_INLINE
void colVSXLoops(Index depth, Index cols, Index rows,
const Packet4f& pAlpha,
3034 const bfloat16* indexA,
const float* indexA2,
const float* blockB2, Index strideA,
3035 Index strideB, Index offsetB,
float* result2) {
3036 Index delta_rows = 2 * (lhsExtraRows ? (rows & 3) : size);
3037 for (Index row = 0; row < size; row += 4) {
3038 convertArrayPointerBF16toF32Dup<lhsExtraRows>(
const_cast<float*
>(indexA2), strideA, delta_rows, indexA, row,
3041 const float* blockB = blockB2;
3042 float* result = result2 + row;
3045 if (cols >= (MAX_BFLOAT16_ACC_VSX * 4)) {
3046 colVSXLoopBody<MAX_BFLOAT16_ACC_VSX, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB,
3047 strideB, 0, result);
3048 blockB += (strideB >> 1) * col;
3049 result += rows * col;
3052 colVSXLoopBodyExtra<true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, offsetB,
3055 colVSXLoopBodyExtra<false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA2, blockB, strideB, 0, result);
3060template <Index size>
3061EIGEN_ALWAYS_INLINE
void calcVSXColLoops(
const bfloat16*& indexA,
const float* indexA2, Index& row, Index depth,
3062 Index cols, Index rows,
const Packet4f& pAlpha,
const float* indexB,
3063 Index strideA, Index strideB, Index offsetA, Index offsetB, Index bigSuffix,
3065 if ((size == 16) || (rows & size)) {
3066 indexA += size * offsetA;
3067 colVSXLoops<size>(depth, cols, rows, pAlpha, indexA, indexA2, indexB, strideA, strideB, offsetB, result + row);
3069 indexA += bigSuffix * size / 16;
3073template <const Index size,
typename DataMapper,
typename MapperIndex>
3074EIGEN_ALWAYS_INLINE
void convertBF16toF32(Index& i,
float* result, Index rows,
const DataMapper& src) {
3075 constexpr Index extra = ((size < 4) ? 4 : size);
3076 while (i + size <= rows) {
3077 PacketBlock<Packet8bf, (size + 7) / 8> r32;
3078 const MapperIndex mapperRow = convert_index<MapperIndex>(i);
3079 r32.packet[0] = src.template loadPacket<Packet8bf>(mapperRow);
3080 EIGEN_IF_CONSTEXPR (size >= 16) {
3081 r32.packet[1] = src.template loadPacket<Packet8bf>(mapperRow + 8);
3083 EIGEN_IF_CONSTEXPR (size >= 32) {
3084 r32.packet[2] = src.template loadPacket<Packet8bf>(mapperRow + 16);
3085 r32.packet[3] = src.template loadPacket<Packet8bf>(mapperRow + 24);
3087 storeConvertBlockBF16<size>(result + i, r32, rows & 3);
3089 EIGEN_IF_CONSTEXPR (size != 32) break;
3093template <typename DataMapper>
3094EIGEN_ALWAYS_INLINE
void convertArrayBF16toF32(
float* result, Index cols, Index rows, const DataMapper& src) {
3095 using LinearMapper =
typename DataMapper::LinearMapper;
3096 using MapperIndex = mapper_index_t<DataMapper>;
3097 for (Index j = 0; j < cols; j++, result += rows) {
3098 const LinearMapper src2 = src.getLinearMapper(0, convert_index<MapperIndex>(j));
3100 convertBF16toF32<32, LinearMapper, MapperIndex>(i, result, rows, src2);
3101 convertBF16toF32<16, LinearMapper, MapperIndex>(i, result, rows, src2);
3102 convertBF16toF32<8, LinearMapper, MapperIndex>(i, result, rows, src2);
3103 convertBF16toF32<4, LinearMapper, MapperIndex>(i, result, rows, src2);
3104 convertBF16toF32<1, LinearMapper, MapperIndex>(i, result, rows, src2);
3108EIGEN_ALWAYS_INLINE Packet8bf convertF32toBF16VSX(
const float* res) {
3109 return F32ToBf16Both(ploadu<Packet4f>(res + 0), ploadu<Packet4f>(res + 4));
3112template <
typename DataMapper, const Index size>
3113EIGEN_ALWAYS_INLINE
void convertArrayF32toBF16ColVSX(
float* result, Index col, Index rows,
const DataMapper& res) {
3114 const DataMapper res2 = res.getSubMapper(0, convert_index<mapper_index_t<DataMapper>>(col));
3116 float* result2 = result + col * rows;
3117 for (row = 0; row + 8 <= rows; row += 8, result2 += 8) {
3119 PacketBlock<Packet8bf, size> block;
3120 for (Index j = 0; j < size; j++) {
3121 block.packet[j] = convertF32toBF16VSX(result2 + j * rows);
3123 res2.template storePacketBlock<Packet8bf, size>(convert_index<mapper_index_t<DataMapper>>(row), 0, block);
3127 for (
int j = 0; j < size; j++) {
3128 Packet8bf fp16 = convertF32toBF16VSX(result2 + j * rows);
3129 res2.template storePacketPartial<Packet8bf>(convert_index<mapper_index_t<DataMapper>>(row), j, fp16,
3130 convert_index<mapper_index_t<DataMapper>>(rows & 7));
3135template <
typename DataMapper>
3136EIGEN_ALWAYS_INLINE
void convertArrayF32toBF16VSX(
float* result, Index cols, Index rows,
const DataMapper& res) {
3138 for (col = 0; col + 4 <= cols; col += 4) {
3139 convertArrayF32toBF16ColVSX<DataMapper, 4>(result, col, rows, res);
3142 switch (cols - col) {
3144 convertArrayF32toBF16ColVSX<DataMapper, 1>(result, col, rows, res);
3147 convertArrayF32toBF16ColVSX<DataMapper, 2>(result, col, rows, res);
3150 convertArrayF32toBF16ColVSX<DataMapper, 3>(result, col, rows, res);
3155template <
typename DataMapper>
3156void gemmbfloat16(
const DataMapper& res,
const bfloat16* indexA,
const bfloat16* indexB, Index rows, Index depth,
3157 Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3158 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
3159 const Packet4f pAlpha = pset1<Packet4f>(falpha);
3161 if (strideA == -1) strideA = depth;
3162 if (strideB == -1) strideB = depth;
3164 ei_declare_aligned_stack_constructed_variable(
float, result, cols* rows, 0);
3165 ei_declare_aligned_stack_constructed_variable(
float, indexB2, strideB* cols, 0);
3166 ei_declare_aligned_stack_constructed_variable(
float, indexA2, ((strideA + 1) & -2) * 4 * 2, 0);
3168 convertArrayBF16toF32<DataMapper>(result, cols, rows, res);
3169 convertArrayPointerBF16toF32(indexB2, cols, strideB,
const_cast<bfloat16*
>(indexB));
3171 Index bigSuffix = 2 * 8 * (strideA - offsetA);
3172 float* indexBF32 = indexB2 + 4 * offsetB;
3178 while (row + 16 <= rows) {
3179 calcVSXColLoops<16>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
3183 calcVSXColLoops<8>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
3186 calcVSXColLoops<4>(indexA, indexA2, row, depth, cols, rows, pAlpha, indexBF32, strideA, strideB, offsetA, offsetB,
3191 colVSXLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexA2, indexBF32, strideA, strideB, offsetB,
3196 convertArrayF32toBF16VSX<DataMapper>(result, cols, rows, res);
3199#undef MAX_BFLOAT16_ACC_VSX
3201#include "MatrixVectorProduct.inc"
3206template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3207struct gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode> {
3208 void operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3211template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3212void gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>::operator()(
3213 double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3214 dhs_pack<double, DataMapper, Packet2d, ColMajor, PanelMode, true> pack;
3215 pack(blockA, lhs, depth, rows, stride, offset);
3218template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3219struct gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode> {
3220 void operator()(
double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3223template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3224void gemm_pack_lhs<double, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>::operator()(
3225 double* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3226 dhs_pack<double, DataMapper, Packet2d, RowMajor, PanelMode, true> pack;
3227 pack(blockA, lhs, depth, rows, stride, offset);
3230#if EIGEN_ALTIVEC_USE_CUSTOM_PACK
3231template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3232struct gemm_pack_rhs<double, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode> {
3233 void operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3236template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3237void gemm_pack_rhs<double, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3238 double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3239 dhs_pack<double, DataMapper, Packet2d, ColMajor, PanelMode, false> pack;
3240 pack(blockB, rhs, depth, cols, stride, offset);
3243template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3244struct gemm_pack_rhs<double, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode> {
3245 void operator()(
double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3248template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3249void gemm_pack_rhs<double, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3250 double* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3251 dhs_pack<double, DataMapper, Packet2d, RowMajor, PanelMode, false> pack;
3252 pack(blockB, rhs, depth, cols, stride, offset);
3255template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3256struct gemm_pack_rhs<bfloat16, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode> {
3257 void operator()(bfloat16* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3260template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3261void gemm_pack_rhs<bfloat16, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3262 bfloat16* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3263 dhs_pack<bfloat16, DataMapper, Packet8bf, ColMajor, PanelMode, false> pack;
3264 pack(blockB, rhs, depth, cols, stride, offset);
3267template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3268struct gemm_pack_rhs<bfloat16, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode> {
3269 void operator()(bfloat16* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3272template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3273void gemm_pack_rhs<bfloat16, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3274 bfloat16* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3275 dhs_pack<bfloat16, DataMapper, Packet8bf, RowMajor, PanelMode, false> pack;
3276 pack(blockB, rhs, depth, cols, stride, offset);
3280template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3281struct gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode> {
3282 void operator()(bfloat16* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3285template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3286void gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>::operator()(
3287 bfloat16* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3288 dhs_pack<bfloat16, DataMapper, Packet8bf, ColMajor, PanelMode, true> pack;
3289 pack(blockA, lhs, depth, rows, stride, offset);
3292template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3293struct gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode> {
3294 void operator()(bfloat16* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3297template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3298void gemm_pack_lhs<bfloat16, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>::operator()(
3299 bfloat16* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3300 dhs_pack<bfloat16, DataMapper, Packet8bf, RowMajor, PanelMode, true> pack;
3301 pack(blockA, lhs, depth, rows, stride, offset);
3304template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3305struct gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode> {
3306 void operator()(
float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3309template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3310void gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, RowMajor, Conjugate, PanelMode>::operator()(
3311 float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3312 dhs_pack<float, DataMapper, Packet4f, RowMajor, PanelMode, true> pack;
3313 pack(blockA, lhs, depth, rows, stride, offset);
3316template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3317struct gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode> {
3318 void operator()(
float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0, Index offset = 0);
3321template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3322void gemm_pack_lhs<float, Index, DataMapper, Pack1, Pack2, Packet, ColMajor, Conjugate, PanelMode>::operator()(
3323 float* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride, Index offset) {
3324 dhs_pack<float, DataMapper, Packet4f, ColMajor, PanelMode, true> pack;
3325 pack(blockA, lhs, depth, rows, stride, offset);
3328template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3329struct gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode> {
3330 void operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3334template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3335void gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate,
3336 PanelMode>::operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows,
3337 Index stride, Index offset) {
3338 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, RowMajor, Conjugate, PanelMode, true> pack;
3339 pack(blockA, lhs, depth, rows, stride, offset);
3342template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3343struct gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode> {
3344 void operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3348template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3349void gemm_pack_lhs<std::complex<float>, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate,
3350 PanelMode>::operator()(std::complex<float>* blockA,
const DataMapper& lhs, Index depth, Index rows,
3351 Index stride, Index offset) {
3352 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, ColMajor, Conjugate, PanelMode, true> pack;
3353 pack(blockA, lhs, depth, rows, stride, offset);
3356#if EIGEN_ALTIVEC_USE_CUSTOM_PACK
3357template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3358struct gemm_pack_rhs<float, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode> {
3359 void operator()(
float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3362template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3363void gemm_pack_rhs<float, Index, DataMapper, nr, ColMajor, Conjugate, PanelMode>::operator()(
3364 float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3365 dhs_pack<float, DataMapper, Packet4f, ColMajor, PanelMode, false> pack;
3366 pack(blockB, rhs, depth, cols, stride, offset);
3369template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3370struct gemm_pack_rhs<float, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode> {
3371 void operator()(
float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0, Index offset = 0);
3374template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3375void gemm_pack_rhs<float, Index, DataMapper, nr, RowMajor, Conjugate, PanelMode>::operator()(
3376 float* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3377 dhs_pack<float, DataMapper, Packet4f, RowMajor, PanelMode, false> pack;
3378 pack(blockB, rhs, depth, cols, stride, offset);
3382template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3383struct gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode> {
3384 void operator()(std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3388template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3389void gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode>::operator()(
3390 std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3391 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, ColMajor, Conjugate, PanelMode, false> pack;
3392 pack(blockB, rhs, depth, cols, stride, offset);
3395template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3396struct gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode> {
3397 void operator()(std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3401template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3402void gemm_pack_rhs<std::complex<float>, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode>::operator()(
3403 std::complex<float>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3404 dhs_cpack<float, DataMapper, Packet4f, Packet2cf, RowMajor, Conjugate, PanelMode, false> pack;
3405 pack(blockB, rhs, depth, cols, stride, offset);
3408template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3409struct gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate, PanelMode> {
3410 void operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3414template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3415void gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet,
RowMajor, Conjugate,
3416 PanelMode>::operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows,
3417 Index stride, Index offset) {
3418 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, RowMajor, Conjugate, PanelMode, true> pack;
3419 pack(blockA, lhs, depth, rows, stride, offset);
3422template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3423struct gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate, PanelMode> {
3424 void operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows, Index stride = 0,
3428template <
typename Index,
typename DataMapper,
int Pack1,
int Pack2,
typename Packet,
bool Conjugate,
bool PanelMode>
3429void gemm_pack_lhs<std::complex<double>, Index, DataMapper, Pack1, Pack2, Packet,
ColMajor, Conjugate,
3430 PanelMode>::operator()(std::complex<double>* blockA,
const DataMapper& lhs, Index depth, Index rows,
3431 Index stride, Index offset) {
3432 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, ColMajor, Conjugate, PanelMode, true> pack;
3433 pack(blockA, lhs, depth, rows, stride, offset);
3436template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3437struct gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode> {
3438 void operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3442template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3443void gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr,
ColMajor, Conjugate, PanelMode>::operator()(
3444 std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3445 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, ColMajor, Conjugate, PanelMode, false> pack;
3446 pack(blockB, rhs, depth, cols, stride, offset);
3449template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3450struct gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode> {
3451 void operator()(std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride = 0,
3455template <
typename Index,
typename DataMapper,
int nr,
bool Conjugate,
bool PanelMode>
3456void gemm_pack_rhs<std::complex<double>, Index, DataMapper, nr,
RowMajor, Conjugate, PanelMode>::operator()(
3457 std::complex<double>* blockB,
const DataMapper& rhs, Index depth, Index cols, Index stride, Index offset) {
3458 dhs_cpack<double, DataMapper, Packet2d, Packet1cd, RowMajor, Conjugate, PanelMode, false> pack;
3459 pack(blockB, rhs, depth, cols, stride, offset);
3463template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3464struct gebp_kernel<float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3465 using Packet =
typename quad_traits<float>::vectortype;
3466 using RhsPacket =
typename quad_traits<float>::rhstype;
3468 void operator()(
const DataMapper& res,
const float* blockA,
const float* blockB, Index rows, Index depth, Index cols,
3469 float alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0, Index offsetB = 0);
3472template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3473void gebp_kernel<float, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3474 const DataMapper& res,
const float* blockA,
const float* blockB, Index rows, Index depth, Index cols,
float alpha,
3475 Index strideA, Index strideB, Index offsetA, Index offsetB) {
3476 const Eigen::Index accRows = quad_traits<float>::rows;
3477 const Eigen::Index accCols = quad_traits<float>::size;
3480 static void (*gemm_function)(
const DataMapper&,
const float*,
const float*, Eigen::Index, Eigen::Index, Eigen::Index,
3481 float, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) =
3482#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3483 (supportsMMA()) ? &Eigen::internal::gemmMMA<float, Packet, RhsPacket, DataMapper, accRows, accCols> :
3485 &Eigen::internal::gemm<float, Packet, RhsPacket, DataMapper, accRows, accCols>;
3486 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3489template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3490struct gebp_kernel<std::complex<float>, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3491 using Packet = Packet4f;
3492 using Packetc = Packet2cf;
3493 using RhsPacket = Packet4f;
3495 void operator()(
const DataMapper& res,
const std::complex<float>* blockA,
const std::complex<float>* blockB,
3496 Index rows, Index depth, Index cols, std::complex<float> alpha, Index strideA = -1,
3497 Index strideB = -1, Index offsetA = 0, Index offsetB = 0);
3500template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3501void gebp_kernel<std::complex<float>, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs,
3502 ConjugateRhs>::operator()(
const DataMapper& res,
const std::complex<float>* blockA,
3503 const std::complex<float>* blockB, Index rows, Index depth, Index cols,
3504 std::complex<float> alpha, Index strideA, Index strideB, Index offsetA,
3506 const Eigen::Index accRows = quad_traits<float>::rows;
3507 const Eigen::Index accCols = quad_traits<float>::size;
3508 static void (*gemm_function)(
const DataMapper&,
const std::complex<float>*,
const std::complex<float>*, Eigen::Index,
3509 Eigen::Index, Eigen::Index, std::complex<float>, Eigen::Index, Eigen::Index,
3510 Eigen::Index, Eigen::Index) =
3511#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3512 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<std::complex<float>, std::complex<float>, std::complex<float>,
3513 float, Packet, Packetc, RhsPacket, DataMapper, accRows,
3514 accCols, ConjugateLhs, ConjugateRhs,
false,
false>
3517 &Eigen::internal::gemm_complex<std::complex<float>, std::complex<float>, std::complex<float>,
3518 float, Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3519 ConjugateLhs, ConjugateRhs,
false,
false>;
3520 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3523template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3524struct gebp_kernel<float, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3525 using Packet = Packet4f;
3526 using Packetc = Packet2cf;
3527 using RhsPacket = Packet4f;
3529 void operator()(
const DataMapper& res,
const float* blockA,
const std::complex<float>* blockB, Index rows,
3530 Index depth, Index cols, std::complex<float> alpha, Index strideA = -1, Index strideB = -1,
3531 Index offsetA = 0, Index offsetB = 0);
3534template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3535void gebp_kernel<float, std::complex<float>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3536 const DataMapper& res,
const float* blockA,
const std::complex<float>* blockB, Index rows, Index depth, Index cols,
3537 std::complex<float> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3538 const Eigen::Index accRows = quad_traits<float>::rows;
3539 const Eigen::Index accCols = quad_traits<float>::size;
3540 static void (*gemm_function)(
const DataMapper&,
const float*,
const std::complex<float>*, Eigen::Index, Eigen::Index,
3541 Eigen::Index, std::complex<float>, Eigen::Index, Eigen::Index, Eigen::Index,
3543#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3544 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<float, std::complex<float>, std::complex<float>, float,
3545 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3546 ConjugateLhs, ConjugateRhs,
true,
false>
3549 &Eigen::internal::gemm_complex<float, std::complex<float>, std::complex<float>, float, Packet,
3550 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3551 ConjugateRhs,
true,
false>;
3552 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3555template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3556struct gebp_kernel<std::complex<float>, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3557 using Packet = Packet4f;
3558 using Packetc = Packet2cf;
3559 using RhsPacket = Packet4f;
3561 void operator()(
const DataMapper& res,
const std::complex<float>* blockA,
const float* blockB, Index rows,
3562 Index depth, Index cols, std::complex<float> alpha, Index strideA = -1, Index strideB = -1,
3563 Index offsetA = 0, Index offsetB = 0);
3566template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3567void gebp_kernel<std::complex<float>, float, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3568 const DataMapper& res,
const std::complex<float>* blockA,
const float* blockB, Index rows, Index depth, Index cols,
3569 std::complex<float> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3570 const Eigen::Index accRows = quad_traits<float>::rows;
3571 const Eigen::Index accCols = quad_traits<float>::size;
3572 static void (*gemm_function)(
const DataMapper&,
const std::complex<float>*,
const float*, Eigen::Index, Eigen::Index,
3573 Eigen::Index, std::complex<float>, Eigen::Index, Eigen::Index, Eigen::Index,
3575#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3576 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<std::complex<float>, float, std::complex<float>, float,
3577 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3578 ConjugateLhs, ConjugateRhs,
false,
true>
3581 &Eigen::internal::gemm_complex<std::complex<float>, float, std::complex<float>, float, Packet,
3582 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3583 ConjugateRhs,
false,
true>;
3584 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3587template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3588struct gebp_kernel<double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3589 using Packet =
typename quad_traits<double>::vectortype;
3590 using RhsPacket =
typename quad_traits<double>::rhstype;
3592 void operator()(
const DataMapper& res,
const double* blockA,
const double* blockB, Index rows, Index depth,
3593 Index cols,
double alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0,
3597template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3598void gebp_kernel<double, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3599 const DataMapper& res,
const double* blockA,
const double* blockB, Index rows, Index depth, Index cols,
3600 double alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3601 const Eigen::Index accRows = quad_traits<double>::rows;
3602 const Eigen::Index accCols = quad_traits<double>::size;
3603 static void (*gemm_function)(
const DataMapper&,
const double*,
const double*, Eigen::Index, Eigen::Index,
3604 Eigen::Index, double, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) =
3605#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3606 (supportsMMA()) ? &Eigen::internal::gemmMMA<double, Packet, RhsPacket, DataMapper, accRows, accCols> :
3608 &Eigen::internal::gemm<double, Packet, RhsPacket, DataMapper, accRows, accCols>;
3609 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3612template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3613struct gebp_kernel<std::complex<double>, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3614 using Packet = quad_traits<double>::vectortype;
3615 using Packetc = Packet1cd;
3616 using RhsPacket = quad_traits<double>::rhstype;
3618 void operator()(
const DataMapper& res,
const std::complex<double>* blockA,
const std::complex<double>* blockB,
3619 Index rows, Index depth, Index cols, std::complex<double> alpha, Index strideA = -1,
3620 Index strideB = -1, Index offsetA = 0, Index offsetB = 0);
3623template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3624void gebp_kernel<std::complex<double>, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs,
3625 ConjugateRhs>::operator()(
const DataMapper& res,
const std::complex<double>* blockA,
3626 const std::complex<double>* blockB, Index rows, Index depth, Index cols,
3627 std::complex<double> alpha, Index strideA, Index strideB, Index offsetA,
3629 const Eigen::Index accRows = quad_traits<double>::rows;
3630 const Eigen::Index accCols = quad_traits<double>::size;
3631 static void (*gemm_function)(
const DataMapper&,
const std::complex<double>*,
const std::complex<double>*,
3632 Eigen::Index, Eigen::Index, Eigen::Index, std::complex<double>, Eigen::Index,
3633 Eigen::Index, Eigen::Index, Eigen::Index) =
3634#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3636 ? &Eigen::internal::gemm_complexMMA<std::complex<double>, std::complex<double>, std::complex<double>, double,
3637 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3638 ConjugateRhs,
false,
false>
3641 &Eigen::internal::gemm_complex<std::complex<double>, std::complex<double>, std::complex<double>, double,
3642 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3643 ConjugateRhs,
false,
false>;
3644 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3647template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3648struct gebp_kernel<std::complex<double>, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3649 using Packet = quad_traits<double>::vectortype;
3650 using Packetc = Packet1cd;
3651 using RhsPacket = quad_traits<double>::rhstype;
3653 void operator()(
const DataMapper& res,
const std::complex<double>* blockA,
const double* blockB, Index rows,
3654 Index depth, Index cols, std::complex<double> alpha, Index strideA = -1, Index strideB = -1,
3655 Index offsetA = 0, Index offsetB = 0);
3658template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3659void gebp_kernel<std::complex<double>, double, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3660 const DataMapper& res,
const std::complex<double>* blockA,
const double* blockB, Index rows, Index depth,
3661 Index cols, std::complex<double> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3662 const Eigen::Index accRows = quad_traits<double>::rows;
3663 const Eigen::Index accCols = quad_traits<double>::size;
3664 static void (*gemm_function)(
const DataMapper&,
const std::complex<double>*,
const double*, Eigen::Index,
3665 Eigen::Index, Eigen::Index, std::complex<double>, Eigen::Index, Eigen::Index,
3666 Eigen::Index, Eigen::Index) =
3667#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3668 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<std::complex<double>, double, std::complex<double>, double,
3669 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3670 ConjugateLhs, ConjugateRhs,
false,
true>
3673 &Eigen::internal::gemm_complex<std::complex<double>, double, std::complex<double>, double, Packet,
3674 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3675 ConjugateRhs,
false,
true>;
3676 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3679template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3680struct gebp_kernel<double, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3681 using Packet = quad_traits<double>::vectortype;
3682 using Packetc = Packet1cd;
3683 using RhsPacket = quad_traits<double>::rhstype;
3685 void operator()(
const DataMapper& res,
const double* blockA,
const std::complex<double>* blockB, Index rows,
3686 Index depth, Index cols, std::complex<double> alpha, Index strideA = -1, Index strideB = -1,
3687 Index offsetA = 0, Index offsetB = 0);
3690template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3691void gebp_kernel<double, std::complex<double>, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3692 const DataMapper& res,
const double* blockA,
const std::complex<double>* blockB, Index rows, Index depth,
3693 Index cols, std::complex<double> alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3694 const Eigen::Index accRows = quad_traits<double>::rows;
3695 const Eigen::Index accCols = quad_traits<double>::size;
3696 static void (*gemm_function)(
const DataMapper&,
const double*,
const std::complex<double>*, Eigen::Index,
3697 Eigen::Index, Eigen::Index, std::complex<double>, Eigen::Index, Eigen::Index,
3698 Eigen::Index, Eigen::Index) =
3699#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3700 (supportsMMA()) ? &Eigen::internal::gemm_complexMMA<double, std::complex<double>, std::complex<double>, double,
3701 Packet, Packetc, RhsPacket, DataMapper, accRows, accCols,
3702 ConjugateLhs, ConjugateRhs,
true,
false>
3705 &Eigen::internal::gemm_complex<double, std::complex<double>, std::complex<double>, double, Packet,
3706 Packetc, RhsPacket, DataMapper, accRows, accCols, ConjugateLhs,
3707 ConjugateRhs,
true,
false>;
3708 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
3711template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3712struct gebp_kernel<bfloat16, bfloat16, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs> {
3713 using Packet =
typename quad_traits<bfloat16>::vectortype;
3714 using RhsPacket =
typename quad_traits<bfloat16>::rhstype;
3716 void operator()(
const DataMapper& res,
const bfloat16* blockA,
const bfloat16* blockB, Index rows, Index depth,
3717 Index cols, bfloat16 alpha, Index strideA = -1, Index strideB = -1, Index offsetA = 0,
3721template <
typename Index,
typename DataMapper,
int mr,
int nr,
bool ConjugateLhs,
bool ConjugateRhs>
3722void gebp_kernel<bfloat16, bfloat16, Index, DataMapper, mr, nr, ConjugateLhs, ConjugateRhs>::operator()(
3723 const DataMapper& res,
const bfloat16* blockA,
const bfloat16* blockB, Index rows, Index depth, Index cols,
3724 bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
3725 static void (*gemm_function)(
const DataMapper&,
const bfloat16*,
const bfloat16*, Eigen::Index, Eigen::Index,
3726 Eigen::Index, bfloat16, Eigen::Index, Eigen::Index, Eigen::Index, Eigen::Index) =
3727#ifdef EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
3728 (supportsMMA()) ? &Eigen::internal::gemmMMAbfloat16<DataMapper> :
3730 &Eigen::internal::gemmbfloat16<DataMapper>;
3731 gemm_function(res, blockA, blockB, rows, depth, cols, alpha, strideA, strideB, offsetA, offsetB);
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321