Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TriangularMatrixMatrix.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TRIANGULAR_MATRIX_MATRIX_H
12#define EIGEN_TRIANGULAR_MATRIX_MATRIX_H
13
14// IWYU pragma: private
15#include "../InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21/* Optimized triangular matrix * matrix (_TRMM++) product built on top of
22 * the general matrix matrix product.
23 */
24template <typename Scalar, typename Index, int Mode, bool LhsIsTriangular, int LhsStorageOrder, bool ConjugateLhs,
25 int RhsStorageOrder, bool ConjugateRhs, int ResStorageOrder, int ResInnerStride, int Version = Specialized>
26struct product_triangular_matrix_matrix;
27
28template <typename Scalar, typename Index, int Mode, bool LhsIsTriangular, int LhsStorageOrder, bool ConjugateLhs,
29 int RhsStorageOrder, bool ConjugateRhs, int ResInnerStride, int Version>
30struct product_triangular_matrix_matrix<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs,
31 RhsStorageOrder, ConjugateRhs, RowMajor, ResInnerStride, Version> {
32 static EIGEN_STRONG_INLINE void run(Index rows, Index cols, Index depth, const Scalar* lhs, Index lhsStride,
33 const Scalar* rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
34 const Scalar& alpha, level3_blocking<Scalar, Scalar>& blocking) {
35 product_triangular_matrix_matrix<Scalar, Index, (Mode & (UnitDiag | ZeroDiag)) | ((Mode & Upper) ? Lower : Upper),
36 (!LhsIsTriangular), RhsStorageOrder == RowMajor ? ColMajor : RowMajor,
37 ConjugateRhs, LhsStorageOrder == RowMajor ? ColMajor : RowMajor, ConjugateLhs,
38 ColMajor, ResInnerStride>::run(cols, rows, depth, rhs, rhsStride, lhs, lhsStride,
39 res, resIncr, resStride, alpha, blocking);
40 }
41};
42
43// implements col-major += alpha * op(triangular) * op(general)
44template <typename Scalar, typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder,
45 bool ConjugateRhs, int ResInnerStride, int Version>
46struct product_triangular_matrix_matrix<Scalar, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder,
47 ConjugateRhs, ColMajor, ResInnerStride, Version> {
48 using Traits = gebp_traits<Scalar, Scalar>;
49 enum {
50 SmallPanelWidth = 2 * plain_enum_max(Traits::mr, Traits::nr),
51 // A multiple of mr, so that horizontal panels start on lhs micro-panel boundaries. It exceeds SmallPanelWidth only
52 // if nr > mr, e.g. for NEON double (18 vs 16), whose vertical panels ran 1-3% slower at 18 on Cortex-A76.
53 HorizontalPanelWidth = numext::div_ceil(int(SmallPanelWidth), int(Traits::mr)) * int(Traits::mr),
54 IsLower = (Mode & Lower) == Lower,
55 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1
56 };
57
58 static EIGEN_DONT_INLINE void run(Index _rows, Index _cols, Index _depth, const Scalar* lhs_, Index lhsStride,
59 const Scalar* rhs_, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
60 const Scalar& alpha, level3_blocking<Scalar, Scalar>& blocking);
61};
62
63template <typename Scalar, typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder,
64 bool ConjugateRhs, int ResInnerStride, int Version>
65EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
66 Scalar, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, ConjugateRhs, ColMajor, ResInnerStride,
67 Version>::run(Index _rows, Index _cols, Index _depth, const Scalar* lhs_, Index lhsStride, const Scalar* rhs_,
68 Index rhsStride, Scalar* res_, Index resIncr, Index resStride, const Scalar& alpha,
69 level3_blocking<Scalar, Scalar>& blocking) {
70 // strip zeros
71 Index diagSize = (std::min)(_rows, _depth);
72 Index rows = IsLower ? _rows : diagSize;
73 Index depth = IsLower ? diagSize : _depth;
74 Index cols = _cols;
75
76 using LhsMapper = const_blas_data_mapper<Scalar, Index, LhsStorageOrder>;
77 using RhsMapper = const_blas_data_mapper<Scalar, Index, RhsStorageOrder>;
78 using ResMapper = blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>;
79 LhsMapper lhs(lhs_, lhsStride);
80 RhsMapper rhs(rhs_, rhsStride);
81 ResMapper res(res_, resStride, resIncr);
82
83 Index kc = blocking.kc(); // cache block size along the K direction
84 Index mc = (std::min)(rows, blocking.mc()); // cache block size along the M direction
85#ifdef EIGEN_VECTORIZE_SME
86 // The SME kernel moves each output block through ZA on every call, which a panelWidth-deep update does not amortize.
87 constexpr bool HorizontalBelowNr = sme_has_gebp_kernel<Scalar, Scalar>::value;
88#else
89 constexpr bool HorizontalBelowNr = false;
90#endif
91 // P horizontal panels update each output panel once instead of (P+1)/2 times on average, with the same packed
92 // coefficients and flops. Below nr columns, the generic kernel's partial register blocks run faster over the short
93 // depth of vertical panels.
94 const bool horizontal =
95 (HorizontalBelowNr || cols >= Traits::nr) && Index(HorizontalPanelWidth) <= (std::min)(kc, mc);
96 // The small panel size must not be larger than blocking size.
97 // Usually this should never be the case because SmallPanelWidth^2 is very small
98 // compared to L2 cache size, but let's be safe:
99 const Index panelWidth =
100 horizontal ? Index(HorizontalPanelWidth) : (std::min)(Index(SmallPanelWidth), (std::min)(kc, mc));
101
102 std::size_t sizeA = kc * mc;
103 std::size_t sizeB = kc * cols;
104
105 ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA());
106 ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB());
107
108 static constexpr int PanelCapacity =
109 (EIGEN_STACK_ALLOCATION_LIMIT == 0 ||
110 HorizontalPanelWidth * HorizontalPanelWidth * sizeof(Scalar) <= EIGEN_STACK_ALLOCATION_LIMIT)
111 ? int(HorizontalPanelWidth)
112 : Dynamic;
113 Matrix<Scalar, PanelCapacity, PanelCapacity, LhsStorageOrder> triangularBuffer(HorizontalPanelWidth,
114 HorizontalPanelWidth);
115 triangularBuffer.topLeftCorner(panelWidth, panelWidth).setZero();
116 EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag)
117 triangularBuffer.diagonal().setZero();
118 else
119 triangularBuffer.diagonal().setOnes();
120
121 gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
122 // Panel mode; with stride == depth and offset 0 it packs exactly as plain mode does.
123 gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
124 LhsStorageOrder, false, true>
125 pack_lhs;
126 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
127
128 for (Index k2 = IsLower ? depth : 0; IsLower ? k2 > 0 : k2 < depth; IsLower ? k2 -= kc : k2 += kc) {
129 Index actual_kc = (std::min)(IsLower ? k2 : depth - k2, kc);
130 Index actual_k2 = IsLower ? k2 - actual_kc : k2;
131
132 // align blocks with the end of the triangular part for trapezoidal lhs
133 EIGEN_IF_CONSTEXPR (!IsLower) {
134 if ((k2 < rows) && (k2 + actual_kc > rows)) {
135 actual_kc = rows - k2;
136 k2 = k2 + actual_kc - kc;
137 }
138 }
139
140 pack_rhs(blockB, rhs.getSubMapper(actual_k2, 0), actual_kc, cols);
141
142 // the selected lhs's panel has to be split in three different parts:
143 // 1 - the part which is zero => skip it
144 // 2 - the diagonal block => special kernel
145 // 3 - the dense panel below (lower case) or above (upper case) the diagonal block => GEPP
146
147 // the block diagonal, if any:
148 if (IsLower || actual_k2 < rows) {
149 // The trick is to pack each micro triangular block while filling the opposite triangular part with zeros.
150 // To this end we do an extra triangular copy to a small temporary buffer
151 const auto packTriangle = [&](Scalar* dst, Index k1, Index width, Index stride, Index offset) {
152 const Index start = actual_k2 + k1;
153 for (Index k = 0; k < width; ++k) {
154 EIGEN_IF_CONSTEXPR (SetDiag) triangularBuffer.coeffRef(k, k) = lhs(start + k, start + k);
155 for (Index i = IsLower ? k + 1 : 0; IsLower ? i < width : i < k; ++i)
156 triangularBuffer.coeffRef(i, k) = lhs(start + i, start + k);
157 }
158 pack_lhs(dst, LhsMapper(triangularBuffer.data(), triangularBuffer.outerStride()), width, width, stride, offset);
159 };
160 // Each group of horizontal panels that fits blockA is packed in the vertical order, whose tall column segments
161 // stream from memory, at stride actual_kc.
162 if (horizontal) {
163 const Index groupRows = mc / panelWidth * panelWidth;
164 for (Index g0 = 0; g0 < actual_kc; g0 += groupRows) {
165 const Index g1 = (std::min)(actual_kc, g0 + groupRows);
166 for (Index k1 = IsLower ? 0 : g0; k1 < (IsLower ? g1 : actual_kc); k1 += panelWidth) {
167 const Index width = (std::min)(actual_kc - k1, panelWidth);
168 if (k1 >= g0 && k1 < g1) packTriangle(blockA + (k1 - g0) * actual_kc, k1, width, actual_kc, k1);
169 const Index begin = IsLower ? (std::max)(k1 + width, g0) : g0;
170 const Index end = IsLower ? g1 : (std::min)(k1, g1);
171 if (end > begin)
172 pack_lhs(blockA + (begin - g0) * actual_kc, lhs.getSubMapper(actual_k2 + begin, actual_k2 + k1), width,
173 end - begin, actual_kc, k1);
174 }
175 for (Index k1 = g0; k1 < g1; k1 += panelWidth) {
176 const Index width = (std::min)(actual_kc - k1, panelWidth);
177 const Index offset = IsLower ? 0 : k1;
178 gebp_kernel(res.getSubMapper(actual_k2 + k1, 0), blockA + (k1 - g0) * actual_kc, blockB, width,
179 IsLower ? k1 + width : actual_kc - k1, cols, alpha, actual_kc, actual_kc, offset, offset);
180 }
181 }
182 } else {
183 // for each small vertical panels of lhs
184 for (Index k1 = 0; k1 < actual_kc; k1 += panelWidth) {
185 Index actualPanelWidth = std::min<Index>(actual_kc - k1, panelWidth);
186 Index lengthTarget = IsLower ? actual_kc - k1 - actualPanelWidth : k1;
187 Index startBlock = actual_k2 + k1;
188 Index blockBOffset = k1;
189
190 // => GEBP with the micro triangular block
191 packTriangle(blockA, k1, actualPanelWidth, actualPanelWidth, 0);
192
193 gebp_kernel(res.getSubMapper(startBlock, 0), blockA, blockB, actualPanelWidth, actualPanelWidth, cols, alpha,
194 actualPanelWidth, actual_kc, 0, blockBOffset);
195
196 // GEBP with remaining micro panel
197 if (lengthTarget > 0) {
198 Index startTarget = IsLower ? actual_k2 + k1 + actualPanelWidth : actual_k2;
199
200 pack_lhs(blockA, lhs.getSubMapper(startTarget, startBlock), actualPanelWidth, lengthTarget,
201 actualPanelWidth, 0);
202
203 gebp_kernel(res.getSubMapper(startTarget, 0), blockA, blockB, lengthTarget, actualPanelWidth, cols, alpha,
204 actualPanelWidth, actual_kc, 0, blockBOffset);
205 }
206 }
207 }
208 }
209 // the part below (lower case) or above (upper case) the diagonal => GEPP
210 {
211 Index start = IsLower ? k2 : 0;
212 Index end = IsLower ? rows : (std::min)(actual_k2, rows);
213 for (Index i2 = start; i2 < end; i2 += mc) {
214 const Index actual_mc = (std::min)(i2 + mc, end) - i2;
215 gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
216 LhsStorageOrder, false>()(blockA, lhs.getSubMapper(i2, actual_k2), actual_kc, actual_mc);
217
218 gebp_kernel(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, cols, alpha, -1, -1, 0, 0);
219 }
220 }
221 }
222}
223
224// implements col-major += alpha * op(general) * op(triangular)
225template <typename Scalar, typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder,
226 bool ConjugateRhs, int ResInnerStride, int Version>
227struct product_triangular_matrix_matrix<Scalar, Index, Mode, false, LhsStorageOrder, ConjugateLhs, RhsStorageOrder,
228 ConjugateRhs, ColMajor, ResInnerStride, Version> {
229 using Traits = gebp_traits<Scalar, Scalar>;
230 enum {
231 SmallPanelWidth = plain_enum_max(Traits::mr, Traits::nr),
232 IsLower = (Mode & Lower) == Lower,
233 SetDiag = (Mode & (ZeroDiag | UnitDiag)) ? 0 : 1
234 };
235
236 static EIGEN_DONT_INLINE void run(Index _rows, Index _cols, Index _depth, const Scalar* lhs_, Index lhsStride,
237 const Scalar* rhs_, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
238 const Scalar& alpha, level3_blocking<Scalar, Scalar>& blocking);
239};
240
241template <typename Scalar, typename Index, int Mode, int LhsStorageOrder, bool ConjugateLhs, int RhsStorageOrder,
242 bool ConjugateRhs, int ResInnerStride, int Version>
243EIGEN_DONT_INLINE void product_triangular_matrix_matrix<
244 Scalar, Index, Mode, false, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, ConjugateRhs, ColMajor, ResInnerStride,
245 Version>::run(Index _rows, Index _cols, Index _depth, const Scalar* lhs_, Index lhsStride, const Scalar* rhs_,
246 Index rhsStride, Scalar* res_, Index resIncr, Index resStride, const Scalar& alpha,
247 level3_blocking<Scalar, Scalar>& blocking) {
248 const Index PacketBytes = packet_traits<Scalar>::size * sizeof(Scalar);
249 // strip zeros
250 Index diagSize = (std::min)(_cols, _depth);
251 Index rows = _rows;
252 Index depth = IsLower ? _depth : diagSize;
253 Index cols = IsLower ? diagSize : _cols;
254
255 using LhsMapper = const_blas_data_mapper<Scalar, Index, LhsStorageOrder>;
256 using RhsMapper = const_blas_data_mapper<Scalar, Index, RhsStorageOrder>;
257 using ResMapper = blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>;
258 LhsMapper lhs(lhs_, lhsStride);
259 RhsMapper rhs(rhs_, rhsStride);
260 ResMapper res(res_, resStride, resIncr);
261
262 Index kc = blocking.kc(); // cache block size along the K direction
263 Index mc = (std::min)(rows, blocking.mc()); // cache block size along the M direction
264
265 std::size_t sizeA = kc * mc;
266 // Room for realigning geb below, which moves it by less than one packet: more than EIGEN_MAX_ALIGN_BYTES when
267 // packets are wider than the allocation alignment (RVV with LMUL > 1, or a lowered EIGEN_MAX_ALIGN_BYTES).
268 std::size_t sizeB = kc * cols + PacketBytes / sizeof(Scalar);
269
270 ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA());
271 ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB());
272
273 static constexpr int PanelCapacity =
274 (EIGEN_STACK_ALLOCATION_LIMIT == 0 ||
275 SmallPanelWidth * SmallPanelWidth * sizeof(Scalar) <= EIGEN_STACK_ALLOCATION_LIMIT)
276 ? int(SmallPanelWidth)
277 : Dynamic;
278 Matrix<Scalar, PanelCapacity, PanelCapacity, RhsStorageOrder> triangularBuffer(SmallPanelWidth, SmallPanelWidth);
279 triangularBuffer.setZero();
280 EIGEN_IF_CONSTEXPR ((Mode & ZeroDiag) == ZeroDiag)
281 triangularBuffer.diagonal().setZero();
282 else
283 triangularBuffer.diagonal().setOnes();
284
285 gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
286 gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
287 LhsStorageOrder>
288 pack_lhs;
289 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
290 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder, false, true> pack_rhs_panel;
291
292 for (Index k2 = IsLower ? 0 : depth; IsLower ? k2 < depth : k2 > 0; IsLower ? k2 += kc : k2 -= kc) {
293 Index actual_kc = (std::min)(IsLower ? depth - k2 : k2, kc);
294 Index actual_k2 = IsLower ? k2 : k2 - actual_kc;
295
296 // align blocks with the end of the triangular part for trapezoidal rhs
297 EIGEN_IF_CONSTEXPR (IsLower) {
298 if ((k2 < cols) && (actual_k2 + actual_kc > cols)) {
299 actual_kc = cols - k2;
300 k2 = actual_k2 + actual_kc - kc;
301 }
302 }
303
304 // remaining size
305 Index rs = IsLower ? (std::min)(cols, actual_k2) : cols - k2;
306 // size of the triangular part
307 Index ts = (IsLower && actual_k2 >= cols) ? 0 : actual_kc;
308
309 Scalar* geb = blockB + ts * ts;
310 geb = geb + internal::first_aligned<PacketBytes>(geb, PacketBytes / sizeof(Scalar));
311
312 pack_rhs(geb, rhs.getSubMapper(actual_k2, IsLower ? 0 : k2), actual_kc, rs);
313
314 // pack the triangular part of the rhs padding the unrolled blocks with zeros
315 if (ts > 0) {
316 for (Index j2 = 0; j2 < actual_kc; j2 += SmallPanelWidth) {
317 Index actualPanelWidth = std::min<Index>(actual_kc - j2, SmallPanelWidth);
318 Index actual_j2 = actual_k2 + j2;
319 Index panelOffset = IsLower ? j2 + actualPanelWidth : 0;
320 Index panelLength = IsLower ? actual_kc - j2 - actualPanelWidth : j2;
321 // general part
322 pack_rhs_panel(blockB + j2 * actual_kc, rhs.getSubMapper(actual_k2 + panelOffset, actual_j2), panelLength,
323 actualPanelWidth, actual_kc, panelOffset);
324
325 // append the triangular part via a temporary buffer
326 for (Index j = 0; j < actualPanelWidth; ++j) {
327 EIGEN_IF_CONSTEXPR (SetDiag) triangularBuffer.coeffRef(j, j) = rhs(actual_j2 + j, actual_j2 + j);
328 for (Index k = IsLower ? j + 1 : 0; IsLower ? k < actualPanelWidth : k < j; ++k)
329 triangularBuffer.coeffRef(k, j) = rhs(actual_j2 + k, actual_j2 + j);
330 }
331
332 pack_rhs_panel(blockB + j2 * actual_kc, RhsMapper(triangularBuffer.data(), triangularBuffer.outerStride()),
333 actualPanelWidth, actualPanelWidth, actual_kc, j2);
334 }
335 }
336
337 for (Index i2 = 0; i2 < rows; i2 += mc) {
338 const Index actual_mc = (std::min)(mc, rows - i2);
339 pack_lhs(blockA, lhs.getSubMapper(i2, actual_k2), actual_kc, actual_mc);
340
341 // triangular kernel
342 if (ts > 0) {
343 for (Index j2 = 0; j2 < actual_kc; j2 += SmallPanelWidth) {
344 Index actualPanelWidth = std::min<Index>(actual_kc - j2, SmallPanelWidth);
345 Index panelLength = IsLower ? actual_kc - j2 : j2 + actualPanelWidth;
346 Index blockOffset = IsLower ? j2 : 0;
347
348 gebp_kernel(res.getSubMapper(i2, actual_k2 + j2), blockA, blockB + j2 * actual_kc, actual_mc, panelLength,
349 actualPanelWidth, alpha, actual_kc, actual_kc, // strides
350 blockOffset, blockOffset); // offsets
351 }
352 }
353 gebp_kernel(res.getSubMapper(i2, IsLower ? 0 : k2), blockA, geb, actual_mc, actual_kc, rs, alpha, -1, -1, 0, 0);
354 }
355 }
356}
357
358/***************************************************************************
359 * Wrapper to product_triangular_matrix_matrix
360 ***************************************************************************/
361
362} // end namespace internal
363
364namespace internal {
365template <int Mode, bool LhsIsTriangular, typename Lhs, typename Rhs>
366struct triangular_product_impl<Mode, LhsIsTriangular, Lhs, false, Rhs, false> {
367 template <typename Dest>
368 static void run(Dest& dst, const Lhs& a_lhs, const Rhs& a_rhs, const typename Dest::Scalar& alpha) {
369 using LhsScalar = typename Lhs::Scalar;
370 using RhsScalar = typename Rhs::Scalar;
371 using Scalar = typename Dest::Scalar;
372
373 if (a_lhs.rows() == 0 || a_lhs.cols() == 0 || a_rhs.cols() == 0) return;
374 // Reuse GEMM's generic crossover: below it, packing costs more than a coefficient-based product.
375 if (a_lhs.rows() + a_lhs.cols() + a_rhs.cols() < EIGEN_GEMM_TO_COEFFBASED_THRESHOLD) {
376 constexpr bool lower = (Mode & Lower) != 0;
377 constexpr bool implicitDiag = (Mode & (UnitDiag | ZeroDiag)) != 0;
378 evaluator<Lhs> lhs(a_lhs);
379 evaluator<Rhs> rhs(a_rhs);
380 const Index depth = a_lhs.cols();
381 for (Index j = 0; j < a_rhs.cols(); ++j) {
382 for (Index i = 0; i < a_lhs.rows(); ++i) {
383 const Index diagonal = LhsIsTriangular ? i : j;
384 const bool prefix = LhsIsTriangular ? lower : !lower;
385 const Index begin = prefix ? 0 : diagonal + Index(implicitDiag);
386 const Index end = prefix ? (numext::mini)(depth, diagonal + Index(!implicitDiag)) : depth;
387 Scalar sum(0);
388 for (Index k = begin; k < end; ++k) sum += lhs.coeff(i, k) * rhs.coeff(k, j);
389 EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) != 0) {
390 if (diagonal < depth) sum += LhsIsTriangular ? rhs.coeff(i, j) : lhs.coeff(i, j);
391 }
392 dst.coeffRef(i, j) += alpha * sum;
393 }
394 }
395 return;
396 }
397
398 using LhsBlasTraits = internal::blas_traits<Lhs>;
399 using ActualLhsType = typename LhsBlasTraits::DirectLinearAccessType;
400 using ActualLhsTypeCleaned = internal::remove_all_t<ActualLhsType>;
401 using RhsBlasTraits = internal::blas_traits<Rhs>;
402 using ActualRhsType = typename RhsBlasTraits::DirectLinearAccessType;
403 using ActualRhsTypeCleaned = internal::remove_all_t<ActualRhsType>;
404
405 internal::add_const_on_value_type_t<ActualLhsType> lhs = LhsBlasTraits::extract(a_lhs);
406 internal::add_const_on_value_type_t<ActualRhsType> rhs = RhsBlasTraits::extract(a_rhs);
407
408 LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(a_lhs);
409 RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(a_rhs);
410 Scalar actualAlpha = alpha * lhs_alpha * rhs_alpha;
411
412 using BlockingType = internal::gemm_blocking_space<(Dest::Flags & RowMajorBit) ? RowMajor : ColMajor, Scalar,
413 Scalar, Lhs::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime,
414 Lhs::MaxColsAtCompileTime, 4>;
415
416 enum { IsLower = (Mode & Lower) == Lower };
417 Index stripedRows = ((!LhsIsTriangular) || (IsLower)) ? lhs.rows() : (std::min)(lhs.rows(), lhs.cols());
418 Index stripedCols = ((LhsIsTriangular) || (!IsLower)) ? rhs.cols() : (std::min)(rhs.cols(), rhs.rows());
419 Index stripedDepth = LhsIsTriangular ? ((!IsLower) ? lhs.cols() : (std::min)(lhs.cols(), lhs.rows()))
420 : ((IsLower) ? rhs.rows() : (std::min)(rhs.rows(), rhs.cols()));
421
422 using Trmm = internal::product_triangular_matrix_matrix<
423 Scalar, Index, Mode, LhsIsTriangular,
424 (internal::traits<ActualLhsTypeCleaned>::Flags & RowMajorBit) ? RowMajor : ColMajor,
425 LhsBlasTraits::NeedToConjugate,
426 (internal::traits<ActualRhsTypeCleaned>::Flags & RowMajorBit) ? RowMajor : ColMajor,
427 RhsBlasTraits::NeedToConjugate, (internal::traits<Dest>::Flags & RowMajorBit) ? RowMajor : ColMajor,
428 Dest::InnerStrideAtCompileTime>;
429 const auto runWith = [&](level3_blocking<Scalar, Scalar>& blocking) {
430 Trmm::run(stripedRows, stripedCols, stripedDepth, // sizes
431 &lhs.coeffRef(0, 0), lhs.outerStride(), // lhs info
432 &rhs.coeffRef(0, 0), rhs.outerStride(), // rhs info
433 &dst.coeffRef(0, 0), dst.innerStride(), dst.outerStride(), // result info
434 actualAlpha, blocking);
435 };
436
437#ifdef EIGEN_VECTORIZE_AVX512
438 // The AVX512 physical-L1 cap on kc already keeps the micro-panels in L1, so on large outputs BlockingType's
439 // extra KcFactor = 4 makes depth blocks too short to amortize result loads/stores; small and narrow products
440 // still favor it. Fixed-size blocking packs the whole product into inline buffers and ignores KcFactor, so only
441 // dynamic blocking can change; a second fixed-size blocking object would double the stack frame.
442 constexpr bool DynamicBlocking = Lhs::MaxRowsAtCompileTime == Dynamic || Rhs::MaxColsAtCompileTime == Dynamic ||
443 Lhs::MaxColsAtCompileTime == Dynamic;
444 if (DynamicBlocking && (is_same<Scalar, float>::value || is_same<Scalar, double>::value) &&
445 gebp_traits<Scalar, Scalar>::nr == 8 && stripedRows >= 512 && stripedCols >= 512) {
446 using LargeBlockingType = internal::gemm_blocking_space<(Dest::Flags & RowMajorBit) ? RowMajor : ColMajor, Scalar,
447 Scalar, Dynamic, Dynamic, Dynamic, 1>;
448 LargeBlockingType blocking(stripedRows, stripedCols, stripedDepth, 1, false);
449 runWith(blocking);
450 } else
451#endif
452 {
453 BlockingType blocking(stripedRows, stripedCols, stripedDepth, 1, false);
454 runWith(blocking);
455 }
456
457 // A factor s nested in a unit triangle reached the kernel through actualAlpha, so the unit diagonal
458 // contributed alpha * s * other instead of alpha * other: subtract alpha * (s - 1) * other.
459 EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == UnitDiag) {
460 EIGEN_IF_CONSTEXPR (LhsIsTriangular) {
461 if (!numext::is_exactly_one(lhs_alpha)) {
462 Index diagSize = (std::min)(lhs.rows(), lhs.cols());
463 dst.matrix().topRows(diagSize) -= ((alpha * (lhs_alpha - LhsScalar(1))) * a_rhs).topRows(diagSize);
464 }
465 } else {
466 if (!numext::is_exactly_one(rhs_alpha)) {
467 Index diagSize = (std::min)(rhs.rows(), rhs.cols());
468 dst.matrix().leftCols(diagSize) -= (alpha * (rhs_alpha - RhsScalar(1))) * a_lhs.leftCols(diagSize);
469 }
470 }
471 }
472 }
473};
474
475} // end namespace internal
476
477} // end namespace Eigen
478
479#endif // EIGEN_TRIANGULAR_MATRIX_MATRIX_H
@ UnitDiag
Definition Constants.h:216
@ ZeroDiag
Definition Constants.h:218
@ Lower
Definition Constants.h:212
@ Upper
Definition Constants.h:214
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
constexpr unsigned int RowMajorBit
Definition Constants.h:71