Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GeneralMatrixMatrix_BLAS.h
1/*
2 Copyright (c) 2011, Intel Corporation. All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without modification,
5 are permitted provided that the following conditions are met:
6
7 * Redistributions of source code must retain the above copyright notice, this
8 list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright notice,
10 this list of conditions and the following disclaimer in the documentation
11 and/or other materials provided with the distribution.
12 * Neither the name of Intel Corporation nor the names of its contributors may
13 be used to endorse or promote products derived from this software without
14 specific prior written permission.
15
16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
23 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26
27 ********************************************************************************
28 * Content : Eigen bindings to BLAS F77
29 * General matrix-matrix product functionality based on ?GEMM.
30 ********************************************************************************
31*/
32// SPDX-License-Identifier: BSD-3-Clause
33
34#ifndef EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
35#define EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
36
37// IWYU pragma: private
38#include "../InternalHeaderCheck.h"
39
40namespace Eigen {
41
42namespace internal {
43
44/**********************************************************************
45 * This file implements general matrix-matrix multiplication using BLAS
46 * gemm function via partial specialization of
47 * general_matrix_matrix_product::run(..) method for float, double,
48 * std::complex<float> and std::complex<double> types
49 **********************************************************************/
50
51// gemm specialization
52
53// Temporaries are ColMajor, also under EIGEN_DEFAULT_TO_ROW_MAJOR: BLAS takes column-major storage.
54#define EIGEN_BLAS_GEMM_SPECIALIZATION(EIGTYPE, EIGPREFIX, BLASTYPE, BLASFUNC) \
55 template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs> \
56 struct general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
57 ConjugateRhs, ColMajor, 1> { \
58 using Traits = gebp_traits<EIGTYPE, EIGTYPE>; \
59 using Scalar = EIGTYPE; \
60 \
61 static void run(Index rows, Index cols, Index depth, const EIGTYPE* lhs_, Index lhsStride, const EIGTYPE* rhs_, \
62 Index rhsStride, Scalar* res, Index resIncr, Index resStride, EIGTYPE alpha, \
63 level3_blocking<EIGTYPE, EIGTYPE>& blocking, GemmParallelInfo<Index>* info) { \
64 if (rows == 0 || cols == 0 || depth == 0 || numext::is_exactly_zero(alpha)) return; \
65 if (!blas_compatible_stride<LhsStorageOrder>(rows, depth, lhsStride) || \
66 !blas_compatible_stride<RhsStorageOrder>(depth, cols, rhsStride) || \
67 !blas_compatible_stride<ColMajor>(rows, cols, resStride)) { \
68 general_matrix_matrix_product<Index, EIGTYPE, LhsStorageOrder, ConjugateLhs, EIGTYPE, RhsStorageOrder, \
69 ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, depth, lhs_, lhsStride, rhs_, \
70 rhsStride, res, resIncr, resStride, alpha, \
71 blocking, info); \
72 return; \
73 } \
74 EIGEN_ONLY_USED_FOR_DEBUG(resIncr); \
75 eigen_assert(resIncr == 1); \
76 char transa, transb; \
77 BlasIndex m, n, k, lda, ldb, ldc; \
78 const EIGTYPE *a, *b; \
79 EIGTYPE beta(1); \
80 using ColMajorMatrix = Matrix<EIGTYPE, Dynamic, Dynamic, ColMajor>; \
81 ColMajorMatrix a_tmp, b_tmp; \
82 \
83 /* Set transpose options */ \
84 transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N'; \
85 transb = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N'; \
86 \
87 /* Set m, n, k */ \
88 m = convert_index<BlasIndex>(rows); \
89 n = convert_index<BlasIndex>(cols); \
90 k = convert_index<BlasIndex>(depth); \
91 \
92 /* Set lda, ldb, ldc */ \
93 lda = blas_leading_dim<LhsStorageOrder>(rows, depth, lhsStride); \
94 ldb = blas_leading_dim<RhsStorageOrder>(depth, cols, rhsStride); \
95 ldc = blas_leading_dim<ColMajor>(rows, cols, resStride); \
96 \
97 /* Set a, b, c */ \
98 EIGEN_IF_CONSTEXPR ((LhsStorageOrder == ColMajor) && (ConjugateLhs)) { \
99 Map<const ColMajorMatrix, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride)); \
100 a_tmp = lhs.conjugate(); \
101 a = a_tmp.data(); \
102 lda = convert_index<BlasIndex>(a_tmp.outerStride()); \
103 } else \
104 a = lhs_; \
105 \
106 EIGEN_IF_CONSTEXPR ((RhsStorageOrder == ColMajor) && (ConjugateRhs)) { \
107 Map<const ColMajorMatrix, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride)); \
108 b_tmp = rhs.conjugate(); \
109 b = b_tmp.data(); \
110 ldb = convert_index<BlasIndex>(b_tmp.outerStride()); \
111 } else \
112 b = rhs_; \
113 \
114 BLASFUNC(&transa, &transb, &m, &n, &k, (const BLASTYPE*)&numext::real_ref(alpha), (const BLASTYPE*)a, &lda, \
115 (const BLASTYPE*)b, &ldb, (const BLASTYPE*)&numext::real_ref(beta), (BLASTYPE*)res, &ldc); \
116 } \
117 };
118
119#ifdef EIGEN_USE_MKL
120EIGEN_BLAS_GEMM_SPECIALIZATION(double, d, double, dgemm)
121EIGEN_BLAS_GEMM_SPECIALIZATION(float, f, float, sgemm)
122EIGEN_BLAS_GEMM_SPECIALIZATION(dcomplex, cd, MKL_Complex16, zgemm)
123EIGEN_BLAS_GEMM_SPECIALIZATION(scomplex, cf, MKL_Complex8, cgemm)
124#else
125EIGEN_BLAS_GEMM_SPECIALIZATION(double, d, double, EIGEN_BLAS_SYM(dgemm))
126EIGEN_BLAS_GEMM_SPECIALIZATION(float, f, float, EIGEN_BLAS_SYM(sgemm))
127EIGEN_BLAS_GEMM_SPECIALIZATION(dcomplex, cd, double, EIGEN_BLAS_SYM(zgemm))
128EIGEN_BLAS_GEMM_SPECIALIZATION(scomplex, cf, float, EIGEN_BLAS_SYM(cgemm))
129#endif
130
131// If OpenBLAS with BUILD_BFLOAT16=1 support is available,
132// use sbgemm for bfloat16.
133#if defined(EIGEN_USE_OPENBLAS_BFLOAT16) && EIGEN_USE_OPENBLAS_BFLOAT16
134
135extern "C" {
136// OpenBLAS prototype.
137void EIGEN_BLAS_SYM(sbgemm)(const char* trans_a, const char* trans_b, const EIGEN_BLAS_INT* M, const EIGEN_BLAS_INT* N,
138 const EIGEN_BLAS_INT* K, const float* alpha, const Eigen::bfloat16* A,
139 const EIGEN_BLAS_INT* lda, const Eigen::bfloat16* B, const EIGEN_BLAS_INT* ldb,
140 const float* beta, float* C, const EIGEN_BLAS_INT* ldc);
141} // extern "C"
142
143template <typename Index, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder, bool ConjugateRhs>
144struct general_matrix_matrix_product<Index, Eigen::bfloat16, LhsStorageOrder, ConjugateLhs, Eigen::bfloat16,
145 RhsStorageOrder, ConjugateRhs, ColMajor, 1> {
146 using Traits = gebp_traits<Eigen::bfloat16, Eigen::bfloat16>;
147
148 static void run(Index rows, Index cols, Index depth, const Eigen::bfloat16* lhs_, Index lhsStride,
149 const Eigen::bfloat16* rhs_, Index rhsStride, Eigen::bfloat16* res, Index resIncr, Index resStride,
150 Eigen::bfloat16 alpha, level3_blocking<Eigen::bfloat16, Eigen::bfloat16>& blocking,
151 GemmParallelInfo<Index>* info) {
152 if (rows == 0 || cols == 0 || depth == 0 || numext::is_exactly_zero(alpha)) return;
153 if (!blas_compatible_stride<LhsStorageOrder>(rows, depth, lhsStride) ||
154 !blas_compatible_stride<RhsStorageOrder>(depth, cols, rhsStride)) {
155 general_matrix_matrix_product<Index, Eigen::bfloat16, LhsStorageOrder, ConjugateLhs, Eigen::bfloat16,
156 RhsStorageOrder, ConjugateRhs, ColMajor, Dynamic>::run(rows, cols, depth, lhs_,
157 lhsStride, rhs_, rhsStride,
158 res, resIncr, resStride,
159 alpha, blocking, info);
160 return;
161 }
162 EIGEN_ONLY_USED_FOR_DEBUG(resIncr);
163 eigen_assert(resIncr == 1);
164 char transa, transb;
165 BlasIndex m, n, k, lda, ldb, ldc;
166 const Eigen::bfloat16 *a, *b;
167
168 float falpha = static_cast<float>(alpha);
169 float fbeta = float(1.0);
170
171 // ColMajor, also under EIGEN_DEFAULT_TO_ROW_MAJOR: the strides below are BLAS's.
172 using MatrixXbf16 = Matrix<Eigen::bfloat16, Dynamic, Dynamic, ColMajor>;
173 MatrixXbf16 a_tmp, b_tmp;
174
175 /* Set transpose options */
176 transa = (LhsStorageOrder == RowMajor) ? ((ConjugateLhs) ? 'C' : 'T') : 'N';
177 transb = (RhsStorageOrder == RowMajor) ? ((ConjugateRhs) ? 'C' : 'T') : 'N';
178
179 /* Set m, n, k */
180 m = convert_index<BlasIndex>(rows);
181 n = convert_index<BlasIndex>(cols);
182 k = convert_index<BlasIndex>(depth);
183
184 /* Set lda, ldb, ldc */
185 lda = blas_leading_dim<LhsStorageOrder>(rows, depth, lhsStride);
186 ldb = blas_leading_dim<RhsStorageOrder>(depth, cols, rhsStride);
187 ldc = convert_index<BlasIndex>(m);
188
189 /* Set a, b, c */
190 EIGEN_IF_CONSTEXPR ((LhsStorageOrder == ColMajor) && (ConjugateLhs)) {
191 Map<const MatrixXbf16, 0, OuterStride<> > lhs(lhs_, m, k, OuterStride<>(lhsStride));
192 a_tmp = lhs.conjugate();
193 a = a_tmp.data();
194 lda = convert_index<BlasIndex>(a_tmp.outerStride());
195 } else {
196 a = lhs_;
197 }
198
199 EIGEN_IF_CONSTEXPR ((RhsStorageOrder == ColMajor) && (ConjugateRhs)) {
200 Map<const MatrixXbf16, 0, OuterStride<> > rhs(rhs_, k, n, OuterStride<>(rhsStride));
201 b_tmp = rhs.conjugate();
202 b = b_tmp.data();
203 ldb = convert_index<BlasIndex>(b_tmp.outerStride());
204 } else {
205 b = rhs_;
206 }
207
208 // sbgemm updates a float C, so res <- bf16(float(res) + alpha * A * B), rounded once.
209 Map<MatrixXbf16, 0, OuterStride<> > result(res, m, n, OuterStride<>(resStride));
210 Matrix<float, Dynamic, Dynamic, ColMajor> r_tmp = result.cast<float>();
211
212 EIGEN_BLAS_SYM(sbgemm)
213 (&transa, &transb, &m, &n, &k, (const float*)&numext::real_ref(falpha), a, &lda, b, &ldb,
214 (const float*)&numext::real_ref(fbeta), r_tmp.data(), &ldc);
215
216 result = r_tmp.cast<Eigen::bfloat16>();
217 }
218};
219
220#endif // EIGEN_USE_OPENBLAS_BFLOAT16
221
222#undef EIGEN_BLAS_GEMM_SPECIALIZATION
223} // namespace internal
224
225} // end namespace Eigen
226
227#endif // EIGEN_GENERAL_MATRIX_MATRIX_BLAS_H
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321