Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
MatrixProductMMAbfloat16.h
1// SPDX-FileCopyrightText: The Eigen Authors
2// SPDX-License-Identifier: MPL-2.0
3
4#ifndef EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H
5#define EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H
6
7#if EIGEN_COMP_LLVM
8#define BFLOAT16_UNROLL _Pragma("unroll 8")
9#else
10#define BFLOAT16_UNROLL _Pragma("GCC unroll(8)")
11#endif
12
13namespace Eigen {
14
15namespace internal {
16
17template <bool zero>
18EIGEN_ALWAYS_INLINE Packet8bf loadBfloat16(const bfloat16* indexA) {
19 Packet8bf lhs1 = ploadu<Packet8bf>(indexA);
20 if (zero) {
21 Packet8bf lhs2 = pset1<Packet8bf>(Eigen::bfloat16(0));
22 return vec_mergeh(lhs1.m_val, lhs2.m_val);
23 } else {
24 return lhs1;
25 }
26}
27
28template <bool zero>
29EIGEN_ALWAYS_INLINE Packet8bf loadRhsBfloat16(const bfloat16* blockB, Index strideB, Index i) {
30 return loadBfloat16<zero>(blockB + strideB * i);
31}
32
33template <Index num_acc, Index num_packets, bool zero, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs,
34 Index num_lhs>
35EIGEN_ALWAYS_INLINE void KLoop(const bfloat16* indexA, const bfloat16* indexB, __vector_quad (&quad_acc)[num_acc],
36 Index strideB, Index k, Index offsetB, Index extra_cols, Index extra_rows) {
37 Packet8bf lhs[num_lhs], rhs[num_rhs];
38
39 BFLOAT16_UNROLL
40 for (Index i = 0; i < (num_rhs - (rhsExtraCols ? 1 : 0)); i++) {
41 rhs[i] = loadRhsBfloat16<zero>(indexB + k * 4, strideB, i);
42 }
43 if (rhsExtraCols) {
44 rhs[num_rhs - 1] = loadRhsBfloat16<zero>(indexB + k * extra_cols - offsetB, strideB, num_rhs - 1);
45 }
46
47 indexA += k * (lhsExtraRows ? extra_rows : num_packets);
48 if (num_lhs == 1) {
49 lhs[0] = loadBfloat16<zero>(indexA);
50 } else {
51 BFLOAT16_UNROLL
52 for (Index j = 0; j < num_lhs; j += 2) {
53 Packet8bf lhs1 = ploadu<Packet8bf>(indexA + (j + 0) * (zero ? 4 : 8));
54 if (zero) {
55 Packet8bf lhs2 = pset1<Packet8bf>(Eigen::bfloat16(0));
56 lhs[j + 0] = vec_mergeh(lhs1.m_val, lhs2.m_val);
57 lhs[j + 1] = vec_mergel(lhs1.m_val, lhs2.m_val);
58 } else {
59 lhs[j + 0] = lhs1;
60 lhs[j + 1] = ploadu<Packet8bf>(indexA + (j + 1) * 8);
61 }
62 }
63 }
64
65 BFLOAT16_UNROLL
66 for (Index i = 0, x = 0; i < num_rhs; i++) {
67 BFLOAT16_UNROLL
68 for (Index j = 0; j < num_lhs; j++, x++) {
69 __builtin_mma_xvbf16ger2pp(&(quad_acc[x]), reinterpret_cast<Packet16uc>(rhs[i].m_val),
70 reinterpret_cast<Packet16uc>(lhs[j].m_val));
71 }
72 }
73}
74
75template <Index num_acc>
76EIGEN_ALWAYS_INLINE void zeroAccumulators(__vector_quad (&quad_acc)[num_acc]) {
77 BFLOAT16_UNROLL
78 for (Index k = 0; k < num_acc; k++) __builtin_mma_xxsetaccz(&(quad_acc[k]));
79}
80
81template <Index num_acc>
82EIGEN_ALWAYS_INLINE void disassembleAccumulators(__vector_quad (&quad_acc)[num_acc], Packet4f (&acc)[num_acc][4]) {
83 BFLOAT16_UNROLL
84 for (Index k = 0; k < num_acc; k++) __builtin_mma_disassemble_acc((void*)acc[k], &(quad_acc[k]));
85}
86
87template <Index num_acc, bool rhsExtraCols, bool lhsExtraRows, Index num_rhs, Index num_lhs>
88EIGEN_ALWAYS_INLINE void outputResults(Packet4f (&acc)[num_acc][4], Index rows, const Packet4f& pAlpha, float* result,
89 const Index extra_cols, Index extra_rows) {
90 BFLOAT16_UNROLL
91 for (Index i = 0, k = 0; i < num_rhs - (rhsExtraCols ? 1 : 0); i++, result += 4 * rows) {
92 BFLOAT16_UNROLL
93 for (Index j = 0; j < num_lhs; j++, k++) {
94 storeResults<false, lhsExtraRows>(acc[k], rows, pAlpha, result + j * 4, extra_cols, extra_rows);
95 }
96 }
97 if (rhsExtraCols) {
98 storeResults<rhsExtraCols, lhsExtraRows>(acc[num_acc - 1], rows, pAlpha, result, extra_cols, extra_rows);
99 }
100}
101
102template <const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows, bool multiIter = false>
103EIGEN_ALWAYS_INLINE void colLoopBodyIter(Index depth, Index rows, const Packet4f& pAlpha, const bfloat16* indexA,
104 const bfloat16* indexB, Index strideB, Index offsetB, float* result,
105 const Index extra_cols, const Index extra_rows) {
106 constexpr Index num_lhs = multiIter ? (num_packets / 4) : 1;
107 constexpr Index num_rhs = (num_acc + num_lhs - 1) / num_lhs;
108
109 for (Index offset_row = 0; offset_row < num_packets; offset_row += 4, indexA += (multiIter ? 0 : 8),
110 indexB += (multiIter ? (num_rhs * strideB) : 0), result += (multiIter ? (4 * rows * num_rhs) : 4)) {
111 Packet4f acc[num_acc][4];
112 __vector_quad quad_acc[num_acc];
113
114 zeroAccumulators<num_acc>(quad_acc);
115
116 Index k;
117 for (k = 0; k + 2 <= depth; k += 2) {
118 KLoop<num_acc, num_packets, false, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(
119 indexA, indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows);
120 }
121 if (depth & 1) {
122 KLoop<num_acc, num_packets, true, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(
123 indexA - (multiIter ? 0 : offset_row), indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows);
124 }
125
126 disassembleAccumulators<num_acc>(quad_acc, acc);
127
128 outputResults<num_acc, rhsExtraCols, lhsExtraRows, num_rhs, num_lhs>(acc, rows, pAlpha, result, extra_cols,
129 extra_rows);
130 }
131}
132
133#define MAX_BFLOAT16_ACC 8
134
135template <const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
136void colLoopBody(Index& col, Index depth, Index cols, Index rows, const Packet4f& pAlpha, const bfloat16* indexA,
137 const bfloat16* indexB, Index strideB, Index offsetB, float* result) {
138 constexpr Index step = (num_acc * 4); // each accumulator has 4 elements
139 const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0;
140 const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0;
141 constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC);
142 constexpr bool normIters = multiIters && ((num_acc % (num_packets / 4)) == 0);
143
144 do {
145 colLoopBodyIter<num_acc, num_packets, rhsExtraCols, lhsExtraRows, normIters>(
146 depth, rows, pAlpha, indexA, indexB, strideB, offsetB, result, extra_cols, extra_rows);
147
148 indexB += strideB * num_acc;
149 result += rows * step;
150 } while (multiIters && (step <= cols - (col += step)));
151}
152
153template <const Index num_acc, const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
154EIGEN_ALWAYS_INLINE void colLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f& pAlpha,
155 const bfloat16* indexA, const bfloat16* blockB, Index strideB, Index offsetB,
156 float* result) {
157 if (MAX_BFLOAT16_ACC > num_acc) {
158 colLoopBody<num_acc + (rhsExtraCols ? 1 : 0), num_packets, rhsExtraCols, lhsExtraRows>(
159 col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result);
160 }
161}
162
163template <const Index num_packets, bool rhsExtraCols, bool lhsExtraRows>
164void colLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const Packet4f& pAlpha, const bfloat16* indexA,
165 const bfloat16* blockB, Index strideB, Index offsetB, float* result) {
166 switch ((cols - col) >> 2) {
167 case 7:
168 colLoopBodyExtraN<7, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
169 strideB, offsetB, result);
170 break;
171 case 6:
172 colLoopBodyExtraN<6, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
173 strideB, offsetB, result);
174 break;
175 case 5:
176 colLoopBodyExtraN<5, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
177 strideB, offsetB, result);
178 break;
179 case 4:
180 colLoopBodyExtraN<4, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
181 strideB, offsetB, result);
182 break;
183 case 3:
184 colLoopBodyExtraN<3, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
185 strideB, offsetB, result);
186 break;
187 case 2:
188 colLoopBodyExtraN<2, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
189 strideB, offsetB, result);
190 break;
191 case 1:
192 colLoopBodyExtraN<1, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
193 strideB, offsetB, result);
194 break;
195 default:
196 if (rhsExtraCols) {
197 colLoopBody<1, num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB,
198 offsetB, result);
199 }
200 break;
201 }
202}
203
204template <const Index num_packets, bool lhsExtraRows = false>
205EIGEN_ALWAYS_INLINE void colLoops(Index depth, Index cols, Index rows, const Packet4f& pAlpha, const bfloat16* indexA,
206 const bfloat16* blockB, Index strideB, Index offsetB, float* result) {
207 Index col = 0;
208 if (cols >= (MAX_BFLOAT16_ACC * 4)) {
209 colLoopBody<MAX_BFLOAT16_ACC, num_packets, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB,
210 strideB, 0, result);
211 blockB += (strideB >> 2) * col;
212 result += rows * col;
213 }
214 if (cols & 3) {
215 colLoopBodyExtra<num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB,
216 result);
217 } else {
218 colLoopBodyExtra<num_packets, false, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, 0,
219 result);
220 }
221}
222
223EIGEN_ALWAYS_INLINE Packet8bf convertF32toBF16(const float* res) {
224 Packet16uc fp16[2];
225 __vector_pair fp16_vp = *reinterpret_cast<__vector_pair*>(const_cast<float*>(res));
226 __builtin_vsx_disassemble_pair(reinterpret_cast<void*>(fp16), &fp16_vp);
227 fp16[0] = __builtin_vsx_xvcvspbf16(fp16[0]);
228 fp16[1] = __builtin_vsx_xvcvspbf16(fp16[1]);
229 return vec_pack(reinterpret_cast<Packet4ui>(fp16[0]), reinterpret_cast<Packet4ui>(fp16[1]));
230}
231
232template <typename DataMapper, const Index size>
233EIGEN_ALWAYS_INLINE void convertArrayF32toBF16Col(float* result, Index col, Index rows, const DataMapper& res) {
234 const DataMapper res2 = res.getSubMapper(0, convert_index<mapper_index_t<DataMapper>>(col));
235 Index row;
236 float* result2 = result + col * rows;
237 for (row = 0; row + 8 <= rows; row += 8, result2 += 8) {
238 // get and save block
239 PacketBlock<Packet8bf, size> block;
240 BFLOAT16_UNROLL
241 for (Index j = 0; j < size; j++) {
242 block.packet[j] = convertF32toBF16(result2 + j * rows);
243 }
244 res2.template storePacketBlock<Packet8bf, size>(convert_index<mapper_index_t<DataMapper>>(row), 0, block);
245 }
246 // extra rows
247 if (row < rows) {
248 BFLOAT16_UNROLL
249 for (int j = 0; j < size; j++) {
250 Packet8bf fp16 = convertF32toBF16(result2 + j * rows);
251 res2.template storePacketPartial<Packet8bf>(convert_index<mapper_index_t<DataMapper>>(row), j, fp16,
252 convert_index<mapper_index_t<DataMapper>>(rows & 7));
253 }
254 }
255}
256
257template <const Index size, bool non_unit_stride = false>
258EIGEN_ALWAYS_INLINE void convertPointerF32toBF16(Index& i, float* result, Index rows, bfloat16*& dst,
259 Index resInc = 1) {
260 constexpr Index extra = ((size < 8) ? 8 : size);
261 while (i + size <= rows) {
262 PacketBlock<Packet8bf, (size + 7) / 8> r32;
263 r32.packet[0] = convertF32toBF16(result + i + 0);
264 if (size >= 16) {
265 r32.packet[1] = convertF32toBF16(result + i + 8);
266 }
267 if (size >= 32) {
268 r32.packet[2] = convertF32toBF16(result + i + 16);
269 r32.packet[3] = convertF32toBF16(result + i + 24);
270 }
271 storeBF16fromResult<size, non_unit_stride, 0>(dst, r32.packet[0], resInc, rows & 7);
272 if (size >= 16) {
273 storeBF16fromResult<size, non_unit_stride, 8>(dst, r32.packet[1], resInc);
274 }
275 if (size >= 32) {
276 storeBF16fromResult<size, non_unit_stride, 16>(dst, r32.packet[2], resInc);
277 storeBF16fromResult<size, non_unit_stride, 24>(dst, r32.packet[3], resInc);
278 }
279 i += extra;
280 dst += extra * resInc;
281 if (size != 32) break;
282 }
283}
284
285template <bool non_unit_stride = false>
286EIGEN_ALWAYS_INLINE void convertArrayPointerF32toBF16(float* result, Index rows, bfloat16* dst, Index resInc = 1) {
287 Index i = 0;
288 convertPointerF32toBF16<32, non_unit_stride>(i, result, rows, dst, resInc);
289 convertPointerF32toBF16<16, non_unit_stride>(i, result, rows, dst, resInc);
290 convertPointerF32toBF16<8, non_unit_stride>(i, result, rows, dst, resInc);
291 convertPointerF32toBF16<1, non_unit_stride>(i, result, rows, dst, resInc);
292}
293
294template <typename DataMapper>
295EIGEN_ALWAYS_INLINE void convertArrayF32toBF16(float* result, Index cols, Index rows, const DataMapper& res) {
296 Index col;
297 for (col = 0; col + 4 <= cols; col += 4) {
298 convertArrayF32toBF16Col<DataMapper, 4>(result, col, rows, res);
299 }
300 // extra cols
301 switch (cols - col) {
302 case 1:
303 convertArrayF32toBF16Col<DataMapper, 1>(result, col, rows, res);
304 break;
305 case 2:
306 convertArrayF32toBF16Col<DataMapper, 2>(result, col, rows, res);
307 break;
308 case 3:
309 convertArrayF32toBF16Col<DataMapper, 3>(result, col, rows, res);
310 break;
311 }
312}
313
314template <Index size>
315EIGEN_ALWAYS_INLINE void calcColLoops(const bfloat16*& indexA, Index& row, Index depth, Index cols, Index rows,
316 const Packet4f& pAlpha, const bfloat16* indexB, Index strideB, Index offsetA,
317 Index offsetB, Index bigSuffix, float* result) {
318 if ((size == 16) || (rows & size)) {
319 indexA += size * offsetA;
320 colLoops<size>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row);
321 row += size;
322 indexA += bigSuffix * size / 16;
323 }
324}
325
326template <typename DataMapper>
327void gemmMMAbfloat16(const DataMapper& res, const bfloat16* indexA, const bfloat16* indexB, Index rows, Index depth,
328 Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) {
329 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
330 const Packet4f pAlpha = pset1<Packet4f>(falpha);
331 ei_declare_aligned_stack_constructed_variable(float, result, cols* rows, 0);
332
333 convertArrayBF16toF32<DataMapper>(result, cols, rows, res);
334
335 if (strideA == -1) strideA = depth;
336 if (strideB == -1) strideB = depth;
337 // Packing is done in blocks.
338 // There are 4 possible sizes of blocks
339 // Blocks of 8 columns with 16 elements (8x16)
340 // Blocks of 8 columns with 8 elements (8x8). This happens when there's 16 > rows >= 8
341 // Blocks of 8 columns with 4 elements (8x4). This happens when there's 8 > rows >= 4
342 // Blocks of 8 columns with < 4 elements. This happens when there's less than 4 remaining rows
343
344 // Loop for LHS standard block (8x16)
345 Index bigSuffix = (2 * 8) * (strideA - offsetA);
346 indexB += 4 * offsetB;
347 strideB *= 4;
348 offsetB *= 3;
349
350 Index row = 0;
351 while (row + 16 <= rows) {
352 calcColLoops<16>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
353 }
354 // LHS (8x8) block
355 calcColLoops<8>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
356 // LHS (8x4) block
357 calcColLoops<4>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result);
358 // extra rows
359 if (rows & 3) {
360 // This index is the beginning of remaining block.
361 colLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row);
362 }
363
364 // Convert back to bfloat16
365 convertArrayF32toBF16<DataMapper>(result, cols, rows, res);
366}
367
368#undef MAX_BFLOAT16_ACC
369
370#if !EIGEN_ALTIVEC_DISABLE_MMA
371template <Index num_acc, typename LhsMapper, bool zero>
372EIGEN_ALWAYS_INLINE void loadVecLoop(Index k, LhsMapper& lhs, Packet8bf (&a0)[num_acc], Packet8bf b1) {
373 const auto row = convert_index<mapper_index_t<LhsMapper>>(k * 4);
374 a0[k + 0] = lhs.template loadPacket<Packet8bf>(row, 0);
375 if (!zero) {
376 b1 = lhs.template loadPacket<Packet8bf>(row, 1);
377 }
378 if (num_acc > (k + 1)) {
379 a0[k + 1] = vec_mergel(a0[k + 0].m_val, b1.m_val);
380 }
381 a0[k + 0] = vec_mergeh(a0[k + 0].m_val, b1.m_val);
382}
383
384template <Index num_acc>
385EIGEN_ALWAYS_INLINE void multVec(__vector_quad (&quad_acc)[num_acc], const Packet8bf (&a0)[num_acc],
386 const Packet8bf& b0) {
387 BFLOAT16_UNROLL
388 for (Index k = 0; k < num_acc; k++) {
389 __builtin_mma_xvbf16ger2pp(&(quad_acc[k]), reinterpret_cast<Packet16uc>(b0.m_val),
390 reinterpret_cast<Packet16uc>(a0[k].m_val));
391 }
392}
393
394template <Index num_acc, typename LhsMapper, typename RhsMapper, bool zero, bool linear>
395EIGEN_ALWAYS_INLINE void vecColLoop(Index j, LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc]) {
396 Packet8bf a0[num_acc];
397 Packet8bf b1 = pset1<Packet8bf>(Eigen::bfloat16(0));
398 const auto col = convert_index<mapper_index_t<LhsMapper>>(j);
399 Packet8bf b0 = loadColData<RhsMapper, linear>(rhs, col);
400
401 if (zero) {
402 b0 = vec_mergeh(b0.m_val, b1.m_val);
403 }
404
405 using LhsSubMapper = typename LhsMapper::SubMapper;
406
407 LhsSubMapper lhs2 = lhs.getSubMapper(0, col);
408 BFLOAT16_UNROLL
409 for (Index k = 0; k < num_acc; k += 2) {
410 loadVecLoop<num_acc, LhsSubMapper, zero>(k, lhs2, a0, b1);
411 }
412
413 multVec<num_acc>(quad_acc, a0, b0);
414}
415
416#define MAX_BFLOAT16_VEC_ACC 8
417
418template <const Index num_acc, typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
419void colVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f& pAlpha,
420 float* result) {
421 constexpr Index step = (num_acc * 4);
422 const Index extra_rows = (extraRows) ? (rows & 3) : 0;
423 constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC);
424
425 do {
426 Packet4f acc[num_acc][4];
427 __vector_quad quad_acc[num_acc];
428
429 zeroAccumulators<num_acc>(quad_acc);
430
431 using LhsSubMapper = typename LhsMapper::SubMapper;
432
433 LhsSubMapper lhs2 = lhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(row), 0);
434 for (Index j = 0; j + 2 <= cend; j += 2) {
435 vecColLoop<num_acc, LhsSubMapper, RhsMapper, false, linear>(j, lhs2, rhs, quad_acc);
436 }
437 if (cend & 1) {
438 vecColLoop<num_acc, LhsSubMapper, RhsMapper, true, linear>(cend - 1, lhs2, rhs, quad_acc);
439 }
440
441 disassembleAccumulators<num_acc>(quad_acc, acc);
442
443 outputVecColResults<num_acc, extraRows>(acc, result, pAlpha, extra_rows);
444
445 result += step;
446 } while (multiIters && (step <= rows - (row += step)));
447}
448
449template <const Index num_acc, typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
450EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
451 const Packet4f& pAlpha, float* result) {
452 if (MAX_BFLOAT16_VEC_ACC > num_acc) {
453 colVecColLoopBody<num_acc + (extraRows ? 1 : 0), LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs,
454 pAlpha, result);
455 }
456}
457
458template <typename LhsMapper, typename RhsMapper, bool extraRows, bool linear>
459EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
460 const Packet4f& pAlpha, float* result) {
461 switch ((rows - row) >> 2) {
462 case 7:
463 colVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
464 break;
465 case 6:
466 colVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
467 break;
468 case 5:
469 colVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
470 break;
471 case 4:
472 colVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
473 break;
474 case 3:
475 colVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
476 break;
477 case 2:
478 colVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
479 break;
480 case 1:
481 colVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
482 break;
483 default:
484 if (extraRows) {
485 colVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
486 }
487 break;
488 }
489}
490
491template <typename LhsMapper, typename RhsMapper, bool linear>
492EIGEN_ALWAYS_INLINE void calcVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f& pAlpha,
493 float* result) {
494 Index row = 0;
495 if (rows >= (MAX_BFLOAT16_VEC_ACC * 4)) {
496 colVecColLoopBody<MAX_BFLOAT16_VEC_ACC, LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha,
497 result);
498 result += row;
499 }
500 if (rows & 3) {
501 colVecColLoopBodyExtra<LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
502 } else {
503 colVecColLoopBodyExtra<LhsMapper, RhsMapper, false, linear>(row, cend, rows, lhs, rhs, pAlpha, result);
504 }
505}
506
507template <typename RhsMapper, typename LhsMapper, typename = void>
508struct UseMMAStride : std::false_type {
509 static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
510 const Packet4f& pAlpha, float* result) {
511 using RhsSubMapper = typename RhsMapper::SubMapper;
512
513 RhsSubMapper rhs2 = rhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(j2), 0);
514 calcVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
515 }
516};
517
518template <typename RhsMapper, typename LhsMapper>
519struct UseMMAStride<RhsMapper, LhsMapper,
520 std::enable_if_t<std::is_member_function_pointer<decltype(&RhsMapper::stride)>::value>>
521 : std::true_type {
522 static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs,
523 const Packet4f& pAlpha, float* result) {
524 using RhsSubMapper = typename RhsMapper::SubMapper;
525
526 RhsSubMapper rhs2 = rhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(j2), 0);
527 if (rhs.stride() == 1) {
528 calcVecColLoops<LhsMapper, RhsSubMapper, true>(jend - j2, rows, lhs, rhs2, pAlpha, result);
529 } else {
530 calcVecColLoops<LhsMapper, RhsSubMapper, false>(jend - j2, rows, lhs, rhs2, pAlpha, result);
531 }
532 }
533};
534
535template <typename LhsMapper, typename RhsMapper>
536void gemvMMA_bfloat16_col(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, bfloat16* res,
537 Index resIncr, bfloat16 alpha) {
538 EIGEN_UNUSED_VARIABLE(resIncr);
539 eigen_internal_assert(resIncr == 1);
540
541 // The following copy tells the compiler that lhs's attributes are not modified outside this function
542 // This helps GCC to generate proper code.
543 LhsMapper lhs(alhs);
544 RhsMapper rhs2(rhs);
545
546 const Index lhsStride = lhs.stride();
547
548 // TODO: improve the following heuristic:
549 const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8);
550 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
551 Packet4f pAlpha = pset1<Packet4f>(falpha);
552
553 ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
554
555 convertArrayPointerBF16toF32(result, 1, rows, res);
556
557 for (Index j2 = 0; j2 < cols; j2 += block_cols) {
558 Index jend = numext::mini(j2 + block_cols, cols);
559
560 using LhsSubMapper = typename LhsMapper::SubMapper;
561
562 LhsSubMapper lhs2 = lhs.getSubMapper(0, convert_index<mapper_index_t<LhsMapper>>(j2));
563 UseMMAStride<RhsMapper, LhsSubMapper>::run(j2, jend, rows, lhs2, rhs2, pAlpha, result);
564 }
565
566 convertArrayPointerF32toBF16(result, rows, res);
567}
568
569static Packet16uc p16uc_ELEMENT_VEC3 = {0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f,
570 0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f};
571
572template <Index num_acc>
573EIGEN_ALWAYS_INLINE void preduxVecResults2(Packet4f (&acc)[num_acc][4], Index k) {
574 if (num_acc > (k + 1)) {
575 acc[k][0] = vec_mergeh(acc[k][0], acc[k + 1][0]);
576 acc[k][1] = vec_mergeo(acc[k][1], acc[k + 1][1]);
577 acc[k][2] = vec_mergel(acc[k][2], acc[k + 1][2]);
578 acc[k][3] = vec_perm(acc[k][3], acc[k + 1][3], p16uc_ELEMENT_VEC3);
579
580 acc[k][0] = (acc[k][0] + acc[k][2]) + (acc[k][1] + acc[k][3]);
581 } else {
582 acc[k][0] = vec_mergeh(acc[k][0], acc[k][1]);
583 acc[k][0] += vec_mergel(acc[k][2], acc[k][3]);
584#ifdef _BIG_ENDIAN
585 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12);
586#else
587 acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4);
588#endif
589 }
590}
591
592template <Index num_acc>
593EIGEN_ALWAYS_INLINE void preduxVecResults(Packet4f (&acc)[num_acc][4]) {
594 BFLOAT16_UNROLL
595 for (Index k = 0; k < num_acc; k += 4) {
596 preduxVecResults2<num_acc>(acc, k + 0);
597 if (num_acc > (k + 2)) {
598 preduxVecResults2<num_acc>(acc, k + 2);
599 acc[k + 0][0] = reinterpret_cast<Packet4f>(
600 vec_mergeh(reinterpret_cast<Packet2ul>(acc[k + 0][0]), reinterpret_cast<Packet2ul>(acc[k + 2][0])));
601 }
602 }
603}
604
605template <Index num_acc, typename LhsMapper, typename RhsMapper, bool extra>
606EIGEN_ALWAYS_INLINE void multVecLoop(__vector_quad (&quad_acc)[num_acc], const LhsMapper& lhs, RhsMapper& rhs, Index j,
607 Index extra_cols) {
608 Packet8bf a0[num_acc], b0;
609 using MapperIndex = mapper_index_t<LhsMapper>;
610 const MapperIndex col = convert_index<MapperIndex>(j);
611 const MapperIndex extraCols = convert_index<MapperIndex>(extra_cols);
612
613 if (extra) {
614 b0 = rhs.template loadPacketPartial<Packet8bf>(col, extraCols);
615 } else {
616 b0 = rhs.template loadPacket<Packet8bf>(col);
617 }
618
619 const LhsMapper lhs2 = lhs.getSubMapper(0, col);
620 BFLOAT16_UNROLL
621 for (int k = 0; k < num_acc; k++) {
622 if (extra) {
623 a0[k] = lhs2.template loadPacketPartial<Packet8bf>(k, 0, extraCols);
624 } else {
625 a0[k] = lhs2.template loadPacket<Packet8bf>(k, 0);
626 }
627 }
628
629 multVec<num_acc>(quad_acc, a0, b0);
630}
631
632template <Index num_acc, typename LhsMapper, typename RhsMapper>
633EIGEN_ALWAYS_INLINE void vecLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc],
634 Index extra_cols) {
635 Index j = 0;
636 for (; j + 8 <= cols; j += 8) {
637 multVecLoop<num_acc, LhsMapper, RhsMapper, false>(quad_acc, lhs, rhs, j, extra_cols);
638 }
639
640 if (extra_cols) {
641 multVecLoop<num_acc, LhsMapper, RhsMapper, true>(quad_acc, lhs, rhs, j, extra_cols);
642 }
643}
644
645template <const Index num_acc, typename LhsMapper, typename RhsMapper>
646void colVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f& pAlpha,
647 float* result) {
648 constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC);
649 const Index extra_cols = (cols & 7);
650
651 do {
652 Packet4f acc[num_acc][4];
653 __vector_quad quad_acc[num_acc];
654
655 zeroAccumulators<num_acc>(quad_acc);
656
657 const LhsMapper lhs2 = lhs.getSubMapper(convert_index<mapper_index_t<LhsMapper>>(row), 0);
658 vecLoop<num_acc, LhsMapper, RhsMapper>(cols, lhs2, rhs, quad_acc, extra_cols);
659
660 disassembleAccumulators<num_acc>(quad_acc, acc);
661
662 preduxVecResults<num_acc>(acc);
663
664 outputVecResults<num_acc>(acc, result, pAlpha);
665
666 result += num_acc;
667 } while (multiIters && (num_acc <= rows - (row += num_acc)));
668}
669
670template <const Index num_acc, typename LhsMapper, typename RhsMapper>
671EIGEN_ALWAYS_INLINE void colVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
672 const Packet4f& pAlpha, float* result) {
673 if (MAX_BFLOAT16_VEC_ACC > num_acc) {
674 colVecLoopBody<num_acc, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
675 }
676}
677
678template <typename LhsMapper, typename RhsMapper>
679EIGEN_ALWAYS_INLINE void colVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs,
680 const Packet4f& pAlpha, float* result) {
681 switch (rows - row) {
682 case 7:
683 colVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
684 break;
685 case 6:
686 colVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
687 break;
688 case 5:
689 colVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
690 break;
691 case 4:
692 colVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
693 break;
694 case 3:
695 colVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
696 break;
697 case 2:
698 colVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
699 break;
700 case 1:
701 colVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
702 break;
703 }
704}
705
706template <typename LhsMapper, typename RhsMapper>
707EIGEN_ALWAYS_INLINE void calcVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f& pAlpha,
708 float* result) {
709 Index row = 0;
710 if (rows >= MAX_BFLOAT16_VEC_ACC) {
711 colVecLoopBody<MAX_BFLOAT16_VEC_ACC, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
712 result += row;
713 }
714 colVecLoopBodyExtra<LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result);
715}
716
717template <typename LhsMapper, typename RhsMapper>
718EIGEN_STRONG_INLINE void gemvMMA_bfloat16_row(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs,
719 bfloat16* res, Index resIncr, bfloat16 alpha) {
720 using LinearMapper = typename RhsMapper::LinearMapper;
721
722 // The following copy tells the compiler that lhs's attributes are not modified outside this function
723 // This helps GCC to generate proper code.
724 LhsMapper lhs(alhs);
725 LinearMapper rhs2 = rhs.getLinearMapper(0, 0);
726
727 eigen_internal_assert(rhs.stride() == 1);
728
729 float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha);
730 const Packet4f pAlpha = pset1<Packet4f>(falpha);
731
732 ei_declare_aligned_stack_constructed_variable(float, result, rows, 0);
733 if (resIncr == 1) {
734 convertArrayPointerBF16toF32(result, 1, rows, res);
735 } else {
736 convertArrayPointerBF16toF32<true>(result, 1, rows, res, resIncr);
737 }
738 calcVecLoops<LhsMapper, LinearMapper>(cols, rows, lhs, rhs2, pAlpha, result);
739 if (resIncr == 1) {
740 convertArrayPointerF32toBF16(result, rows, res);
741 } else {
742 convertArrayPointerF32toBF16<true>(result, rows, res, resIncr);
743 }
744}
745#endif
746
747#undef MAX_BFLOAT16_VEC_ACC
748#undef BFLOAT16_UNROLL
749
750} // namespace internal
751} // namespace Eigen
752#endif // EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H