5#ifndef EIGEN_MATRIX_PRODUCT_COMMON_ALTIVEC_H
6#define EIGEN_MATRIX_PRODUCT_COMMON_ALTIVEC_H
8#ifdef EIGEN_POWER_USE_PREFETCH
9#define EIGEN_POWER_PREFETCH(p) prefetch(p)
11#define EIGEN_POWER_PREFETCH(p)
15#include "../../InternalHeaderCheck.h"
21template <
typename Mapper>
22using mapper_index_t =
decltype(std::declval<const Mapper&>().stride());
24template <
typename Scalar,
typename Packet,
typename DataMapper, const Index accRows, const Index accCols>
25EIGEN_ALWAYS_INLINE
void gemm_extra_row(
const DataMapper& res,
const Scalar* lhs_base,
const Scalar* rhs_base,
26 Index depth, Index strideA, Index offsetA, Index strideB, Index row, Index rows,
27 Index remaining_rows,
const Packet& pAlpha,
const Packet& pMask);
29template <
typename Scalar,
typename Packet,
typename DataMapper, const Index accCols>
30EIGEN_ALWAYS_INLINE
void gemm_extra_cols(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB, Index depth,
31 Index strideA, Index offsetA, Index strideB, Index offsetB, Index col,
32 Index rows, Index cols, Index remaining_rows,
const Packet& pAlpha,
35template <
typename Packet>
36EIGEN_ALWAYS_INLINE Packet bmask(
const Index remaining_rows);
38template <
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
const Index accRows,
39 const Index accCols,
bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
40EIGEN_ALWAYS_INLINE
void gemm_complex_extra_row(
const DataMapper& res,
const Scalar* lhs_base,
const Scalar* rhs_base,
41 Index depth, Index strideA, Index offsetA, Index strideB, Index row,
42 Index rows, Index remaining_rows,
const Packet& pAlphaReal,
43 const Packet& pAlphaImag,
const Packet& pMask);
45template <
typename Scalar,
typename Packet,
typename Packetc,
typename DataMapper,
const Index accCols,
46 bool ConjugateLhs,
bool ConjugateRhs,
bool LhsIsReal,
bool RhsIsReal>
47EIGEN_ALWAYS_INLINE
void gemm_complex_extra_cols(
const DataMapper& res,
const Scalar* blockA,
const Scalar* blockB,
48 Index depth, Index strideA, Index offsetA, Index strideB,
49 Index offsetB, Index col, Index rows, Index cols, Index remaining_rows,
50 const Packet& pAlphaReal,
const Packet& pAlphaImag,
53template <
typename DataMapper>
54EIGEN_ALWAYS_INLINE
void convertArrayBF16toF32(
float* result, Index cols, Index rows,
const DataMapper& src);
56template <const Index size,
bool non_unit_str
ide, Index delta>
57EIGEN_ALWAYS_INLINE
void storeBF16fromResult(bfloat16* dst,
const Packet8bf& data, Index resInc, Index extra = 0);
59template <
bool non_unit_str
ide = false>
60EIGEN_ALWAYS_INLINE
void convertArrayPointerBF16toF32(
float* result, Index cols, Index rows, bfloat16* src,
63template <
bool rhsExtraCols,
bool lhsExtraRows>
64EIGEN_ALWAYS_INLINE
void storeResults(Packet4f (&acc)[4], Index rows,
const Packet4f& pAlpha,
float* result,
65 Index extra_cols, Index extra_rows);
67template <Index num_acc,
bool extraRows, Index size = 4>
68EIGEN_ALWAYS_INLINE
void outputVecColResults(Packet4f (&acc)[num_acc][size],
float* result,
const Packet4f& pAlpha,
71template <Index num_acc, Index size = 4>
72EIGEN_ALWAYS_INLINE
void outputVecResults(Packet4f (&acc)[num_acc][size],
float* result,
const Packet4f& pAlpha);
74template <
typename RhsMapper,
bool linear,
typename Index>
75EIGEN_ALWAYS_INLINE Packet8bf loadColData(RhsMapper& rhs, Index j);
77template <
typename Packet>
78EIGEN_ALWAYS_INLINE Packet ploadLhs(
const __UNPACK_TYPE__(Packet) * lhs);
80template <
typename DataMapper,
typename Packet,
const Index accCols,
int StorageOrder,
bool Complex,
int N,
82EIGEN_ALWAYS_INLINE
void bload(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc,
const DataMapper& res, Index row,
85template <
typename DataMapper,
typename Packet,
int N>
86EIGEN_ALWAYS_INLINE
void bstore(PacketBlock<Packet, N>& acc,
const DataMapper& res, Index row);
88template <
typename DataMapper,
typename Packet, const Index accCols,
bool Complex, Index N,
bool full = true>
89EIGEN_ALWAYS_INLINE
void bload_partial(PacketBlock<Packet, N*(Complex ? 2 : 1)>& acc,
const DataMapper& res, Index row,
92template <
typename DataMapper,
typename Packet, Index N>
93EIGEN_ALWAYS_INLINE
void bstore_partial(PacketBlock<Packet, N>& acc,
const DataMapper& res, Index row, Index elements);
95template <
typename Packet,
int N>
96EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ,
const Packet& pAlpha);
98template <
typename Packet,
int N,
bool mask>
99EIGEN_ALWAYS_INLINE
void bscale(PacketBlock<Packet, N>& acc, PacketBlock<Packet, N>& accZ,
const Packet& pAlpha,
100 const Packet& pMask);
102template <
typename Packet,
int N,
bool mask>
103EIGEN_ALWAYS_INLINE
void bscalec(PacketBlock<Packet, N>& aReal, PacketBlock<Packet, N>& aImag,
const Packet& bReal,
104 const Packet& bImag, PacketBlock<Packet, N>& cReal, PacketBlock<Packet, N>& cImag,
105 const Packet& pMask);
107template <
typename Packet,
typename Packetc,
int N,
bool full>
108EIGEN_ALWAYS_INLINE
void bcouple(PacketBlock<Packet, N>& taccReal, PacketBlock<Packet, N>& taccImag,
109 PacketBlock<Packetc, N * 2>& tRes, PacketBlock<Packetc, N>& acc1,
110 PacketBlock<Packetc, N>& acc2);
112#define MICRO_NORMAL(iter) (accCols == accCols2) || (unroll_factor != (iter + 1))
114#define MICRO_UNROLL_ITER1(func, N) \
115 switch (remaining_rows) { \
121 if (sizeof(Scalar) == sizeof(float)) { \
126 if (sizeof(Scalar) == sizeof(float)) { \
132#define MICRO_UNROLL_ITER(func, N) \
133 if (remaining_rows) { \
139#define MICRO_NORMAL_PARTIAL(iter) full || (unroll_factor != (iter + 1))
141#define MICRO_COMPLEX_UNROLL_ITER(func, N) MICRO_UNROLL_ITER1(func, N)
143#define MICRO_NORMAL_COLS(iter, a, b) ((MICRO_NORMAL(iter)) ? a : b)
145#define MICRO_LOAD1(lhs_ptr, iter) \
146 if (unroll_factor > iter) { \
147 lhsV##iter = ploadLhs<Packet>(lhs_ptr##iter); \
148 lhs_ptr##iter += MICRO_NORMAL_COLS(iter, accCols, accCols2); \
150 EIGEN_UNUSED_VARIABLE(lhsV##iter); \
153#define MICRO_LOAD_ONE(iter) MICRO_LOAD1(lhs_ptr, iter)
155#define MICRO_COMPLEX_LOAD_ONE(iter) \
156 if (!LhsIsReal && (unroll_factor > iter)) { \
157 lhsVi##iter = ploadLhs<Packet>(lhs_ptr_real##iter + MICRO_NORMAL_COLS(iter, imag_delta, imag_delta2)); \
159 EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
161 MICRO_LOAD1(lhs_ptr_real, iter)
163#define MICRO_LOAD1_PARTIAL(lhs_ptr, iter) \
164 if (unroll_factor > iter) { \
165 if (MICRO_NORMAL(iter)) { \
166 lhsV##iter = ploadLhs<Packet>(lhs_ptr##iter); \
167 lhs_ptr##iter += accCols; \
169 lhsV##iter = ploadu_partial<Packet>(lhs_ptr##iter, accCols2); \
170 lhs_ptr##iter += accCols2; \
173 EIGEN_UNUSED_VARIABLE(lhsV##iter); \
176#define MICRO_LOAD_PARTIAL_ONE(iter) MICRO_LOAD1_PARTIAL(lhs_ptr, iter)
178#define MICRO_COMPLEX_LOAD_PARTIAL_ONE(iter) \
179 if (!LhsIsReal && (unroll_factor > iter)) { \
180 if (MICRO_NORMAL(iter)) { \
181 lhsVi##iter = ploadLhs<Packet>(lhs_ptr_real##iter + imag_delta); \
183 lhsVi##iter = ploadu_partial<Packet>(lhs_ptr_real##iter + imag_delta2, accCols2); \
186 EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
188 MICRO_LOAD1_PARTIAL(lhs_ptr_real, iter)
190#define MICRO_SRC_PTR1(lhs_ptr, advRows, iter) \
191 if (unroll_factor > iter) { \
192 lhs_ptr##iter = lhs_base + (row + (iter * accCols)) * strideA * advRows - \
193 MICRO_NORMAL_COLS(iter, 0, (accCols - accCols2) * offsetA); \
195 EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \
198#define MICRO_SRC_PTR_ONE(iter) MICRO_SRC_PTR1(lhs_ptr, 1, iter)
200#define MICRO_COMPLEX_SRC_PTR_ONE(iter) MICRO_SRC_PTR1(lhs_ptr_real, advanceRows, iter)
202#define MICRO_PREFETCH1(lhs_ptr, iter) \
203 if (unroll_factor > iter) { \
204 EIGEN_POWER_PREFETCH(lhs_ptr##iter); \
207#define MICRO_PREFETCH_ONE(iter) MICRO_PREFETCH1(lhs_ptr, iter)
209#define MICRO_COMPLEX_PREFETCH_ONE(iter) MICRO_PREFETCH1(lhs_ptr_real, iter)
211#define MICRO_UPDATE \
212 if (accCols == accCols2) { \
213 EIGEN_UNUSED_VARIABLE(offsetA); \
214 row += unroll_factor * accCols; \
217#define MICRO_COMPLEX_UPDATE \
219 if (LhsIsReal || (accCols == accCols2)) { \
220 EIGEN_UNUSED_VARIABLE(imag_delta2); \