11#ifndef EIGEN_TRIANGULAR_MATRIX_MATRIX_H
12#define EIGEN_TRIANGULAR_MATRIX_MATRIX_H
15#include "../InternalHeaderCheck.h"
24template <
typename Scalar,
typename Index,
int Mode,
bool LhsIsTriangular,
int LhsStorageOrder,
bool ConjugateLhs,
25 int RhsStorageOrder,
bool ConjugateRhs,
int ResStorageOrder,
int ResInnerStride,
int Version = Specialized>
26struct product_triangular_matrix_matrix;
28template <
typename Scalar,
typename Index,
int Mode,
bool LhsIsTriangular,
int LhsStorageOrder,
bool ConjugateLhs,
29 int RhsStorageOrder,
bool ConjugateRhs,
int ResInnerStride,
int Version>
30struct product_triangular_matrix_matrix<Scalar, Index, Mode, LhsIsTriangular, LhsStorageOrder, ConjugateLhs,
31 RhsStorageOrder, ConjugateRhs,
RowMajor, ResInnerStride, Version> {
32 static EIGEN_STRONG_INLINE
void run(Index rows, Index cols, Index depth,
const Scalar* lhs, Index lhsStride,
33 const Scalar* rhs, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
34 const Scalar& alpha, level3_blocking<Scalar, Scalar>& blocking) {
38 ColMajor, ResInnerStride>::run(cols, rows, depth, rhs, rhsStride, lhs, lhsStride,
39 res, resIncr, resStride, alpha, blocking);
44template <
typename Scalar,
typename Index,
int Mode,
int LhsStorageOrder,
bool ConjugateLhs,
int RhsStorageOrder,
45 bool ConjugateRhs,
int ResInnerStride,
int Version>
46struct product_triangular_matrix_matrix<Scalar, Index, Mode, true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder,
47 ConjugateRhs,
ColMajor, ResInnerStride, Version> {
48 using Traits = gebp_traits<Scalar, Scalar>;
50 SmallPanelWidth = 2 * plain_enum_max(Traits::mr, Traits::nr),
53 HorizontalPanelWidth = numext::div_ceil(
int(SmallPanelWidth),
int(Traits::mr)) * int(Traits::mr),
58 static EIGEN_DONT_INLINE
void run(Index _rows, Index _cols, Index _depth,
const Scalar* lhs_, Index lhsStride,
59 const Scalar* rhs_, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
60 const Scalar& alpha, level3_blocking<Scalar, Scalar>& blocking);
63template <
typename Scalar,
typename Index,
int Mode,
int LhsStorageOrder,
bool ConjugateLhs,
int RhsStorageOrder,
64 bool ConjugateRhs,
int ResInnerStride,
int Version>
65EIGEN_DONT_INLINE
void product_triangular_matrix_matrix<
66 Scalar, Index, Mode,
true, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, ConjugateRhs,
ColMajor, ResInnerStride,
67 Version>::run(Index _rows, Index _cols, Index _depth,
const Scalar* lhs_, Index lhsStride,
const Scalar* rhs_,
68 Index rhsStride, Scalar* res_, Index resIncr, Index resStride,
const Scalar& alpha,
69 level3_blocking<Scalar, Scalar>& blocking) {
71 Index diagSize = (std::min)(_rows, _depth);
72 Index rows = IsLower ? _rows : diagSize;
73 Index depth = IsLower ? diagSize : _depth;
76 using LhsMapper = const_blas_data_mapper<Scalar, Index, LhsStorageOrder>;
77 using RhsMapper = const_blas_data_mapper<Scalar, Index, RhsStorageOrder>;
78 using ResMapper = blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>;
79 LhsMapper lhs(lhs_, lhsStride);
80 RhsMapper rhs(rhs_, rhsStride);
81 ResMapper res(res_, resStride, resIncr);
83 Index kc = blocking.kc();
84 Index mc = (std::min)(rows, blocking.mc());
85#ifdef EIGEN_VECTORIZE_SME
87 constexpr bool HorizontalBelowNr = sme_has_gebp_kernel<Scalar, Scalar>::value;
89 constexpr bool HorizontalBelowNr =
false;
94 const bool horizontal =
95 (HorizontalBelowNr || cols >= Traits::nr) && Index(HorizontalPanelWidth) <= (std::min)(kc, mc);
99 const Index panelWidth =
100 horizontal ? Index(HorizontalPanelWidth) : (std::min)(Index(SmallPanelWidth), (std::min)(kc, mc));
102 std::size_t sizeA = kc * mc;
103 std::size_t sizeB = kc * cols;
105 ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA());
106 ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB());
108 static constexpr int PanelCapacity =
109 (EIGEN_STACK_ALLOCATION_LIMIT == 0 ||
110 HorizontalPanelWidth * HorizontalPanelWidth *
sizeof(Scalar) <= EIGEN_STACK_ALLOCATION_LIMIT)
111 ?
int(HorizontalPanelWidth)
113 Matrix<Scalar, PanelCapacity, PanelCapacity, LhsStorageOrder> triangularBuffer(HorizontalPanelWidth,
114 HorizontalPanelWidth);
115 triangularBuffer.topLeftCorner(panelWidth, panelWidth).setZero();
117 triangularBuffer.diagonal().setZero();
119 triangularBuffer.diagonal().setOnes();
121 gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
123 gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress,
typename Traits::LhsPacket4Packing,
124 LhsStorageOrder,
false,
true>
126 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
128 for (Index k2 = IsLower ? depth : 0; IsLower ? k2 > 0 : k2 < depth; IsLower ? k2 -= kc : k2 += kc) {
129 Index actual_kc = (std::min)(IsLower ? k2 : depth - k2, kc);
130 Index actual_k2 = IsLower ? k2 - actual_kc : k2;
133 EIGEN_IF_CONSTEXPR (!IsLower) {
134 if ((k2 < rows) && (k2 + actual_kc > rows)) {
135 actual_kc = rows - k2;
136 k2 = k2 + actual_kc - kc;
140 pack_rhs(blockB, rhs.getSubMapper(actual_k2, 0), actual_kc, cols);
148 if (IsLower || actual_k2 < rows) {
151 const auto packTriangle = [&](Scalar* dst, Index k1, Index width, Index stride, Index offset) {
152 const Index start = actual_k2 + k1;
153 for (Index k = 0; k < width; ++k) {
154 EIGEN_IF_CONSTEXPR (SetDiag) triangularBuffer.coeffRef(k, k) = lhs(start + k, start + k);
155 for (Index i = IsLower ? k + 1 : 0; IsLower ? i < width : i < k; ++i)
156 triangularBuffer.coeffRef(i, k) = lhs(start + i, start + k);
158 pack_lhs(dst, LhsMapper(triangularBuffer.data(), triangularBuffer.outerStride()), width, width, stride, offset);
163 const Index groupRows = mc / panelWidth * panelWidth;
164 for (Index g0 = 0; g0 < actual_kc; g0 += groupRows) {
165 const Index g1 = (std::min)(actual_kc, g0 + groupRows);
166 for (Index k1 = IsLower ? 0 : g0; k1 < (IsLower ? g1 : actual_kc); k1 += panelWidth) {
167 const Index width = (std::min)(actual_kc - k1, panelWidth);
168 if (k1 >= g0 && k1 < g1) packTriangle(blockA + (k1 - g0) * actual_kc, k1, width, actual_kc, k1);
169 const Index begin = IsLower ? (std::max)(k1 + width, g0) : g0;
170 const Index end = IsLower ? g1 : (std::min)(k1, g1);
172 pack_lhs(blockA + (begin - g0) * actual_kc, lhs.getSubMapper(actual_k2 + begin, actual_k2 + k1), width,
173 end - begin, actual_kc, k1);
175 for (Index k1 = g0; k1 < g1; k1 += panelWidth) {
176 const Index width = (std::min)(actual_kc - k1, panelWidth);
177 const Index offset = IsLower ? 0 : k1;
178 gebp_kernel(res.getSubMapper(actual_k2 + k1, 0), blockA + (k1 - g0) * actual_kc, blockB, width,
179 IsLower ? k1 + width : actual_kc - k1, cols, alpha, actual_kc, actual_kc, offset, offset);
184 for (Index k1 = 0; k1 < actual_kc; k1 += panelWidth) {
185 Index actualPanelWidth = std::min<Index>(actual_kc - k1, panelWidth);
186 Index lengthTarget = IsLower ? actual_kc - k1 - actualPanelWidth : k1;
187 Index startBlock = actual_k2 + k1;
188 Index blockBOffset = k1;
191 packTriangle(blockA, k1, actualPanelWidth, actualPanelWidth, 0);
193 gebp_kernel(res.getSubMapper(startBlock, 0), blockA, blockB, actualPanelWidth, actualPanelWidth, cols, alpha,
194 actualPanelWidth, actual_kc, 0, blockBOffset);
197 if (lengthTarget > 0) {
198 Index startTarget = IsLower ? actual_k2 + k1 + actualPanelWidth : actual_k2;
200 pack_lhs(blockA, lhs.getSubMapper(startTarget, startBlock), actualPanelWidth, lengthTarget,
201 actualPanelWidth, 0);
203 gebp_kernel(res.getSubMapper(startTarget, 0), blockA, blockB, lengthTarget, actualPanelWidth, cols, alpha,
204 actualPanelWidth, actual_kc, 0, blockBOffset);
211 Index start = IsLower ? k2 : 0;
212 Index end = IsLower ? rows : (std::min)(actual_k2, rows);
213 for (Index i2 = start; i2 < end; i2 += mc) {
214 const Index actual_mc = (std::min)(i2 + mc, end) - i2;
215 gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress,
typename Traits::LhsPacket4Packing,
216 LhsStorageOrder,
false>()(blockA, lhs.getSubMapper(i2, actual_k2), actual_kc, actual_mc);
218 gebp_kernel(res.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, cols, alpha, -1, -1, 0, 0);
225template <
typename Scalar,
typename Index,
int Mode,
int LhsStorageOrder,
bool ConjugateLhs,
int RhsStorageOrder,
226 bool ConjugateRhs,
int ResInnerStride,
int Version>
227struct product_triangular_matrix_matrix<Scalar, Index, Mode, false, LhsStorageOrder, ConjugateLhs, RhsStorageOrder,
228 ConjugateRhs,
ColMajor, ResInnerStride, Version> {
229 using Traits = gebp_traits<Scalar, Scalar>;
231 SmallPanelWidth = plain_enum_max(Traits::mr, Traits::nr),
236 static EIGEN_DONT_INLINE
void run(Index _rows, Index _cols, Index _depth,
const Scalar* lhs_, Index lhsStride,
237 const Scalar* rhs_, Index rhsStride, Scalar* res, Index resIncr, Index resStride,
238 const Scalar& alpha, level3_blocking<Scalar, Scalar>& blocking);
241template <
typename Scalar,
typename Index,
int Mode,
int LhsStorageOrder,
bool ConjugateLhs,
int RhsStorageOrder,
242 bool ConjugateRhs,
int ResInnerStride,
int Version>
243EIGEN_DONT_INLINE
void product_triangular_matrix_matrix<
244 Scalar, Index, Mode,
false, LhsStorageOrder, ConjugateLhs, RhsStorageOrder, ConjugateRhs,
ColMajor, ResInnerStride,
245 Version>::run(Index _rows, Index _cols, Index _depth,
const Scalar* lhs_, Index lhsStride,
const Scalar* rhs_,
246 Index rhsStride, Scalar* res_, Index resIncr, Index resStride,
const Scalar& alpha,
247 level3_blocking<Scalar, Scalar>& blocking) {
248 const Index PacketBytes = packet_traits<Scalar>::size *
sizeof(Scalar);
250 Index diagSize = (std::min)(_cols, _depth);
252 Index depth = IsLower ? _depth : diagSize;
253 Index cols = IsLower ? diagSize : _cols;
255 using LhsMapper = const_blas_data_mapper<Scalar, Index, LhsStorageOrder>;
256 using RhsMapper = const_blas_data_mapper<Scalar, Index, RhsStorageOrder>;
257 using ResMapper = blas_data_mapper<typename Traits::ResScalar, Index, ColMajor, Unaligned, ResInnerStride>;
258 LhsMapper lhs(lhs_, lhsStride);
259 RhsMapper rhs(rhs_, rhsStride);
260 ResMapper res(res_, resStride, resIncr);
262 Index kc = blocking.kc();
263 Index mc = (std::min)(rows, blocking.mc());
265 std::size_t sizeA = kc * mc;
268 std::size_t sizeB = kc * cols + PacketBytes /
sizeof(Scalar);
270 ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA());
271 ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB());
273 static constexpr int PanelCapacity =
274 (EIGEN_STACK_ALLOCATION_LIMIT == 0 ||
275 SmallPanelWidth * SmallPanelWidth *
sizeof(Scalar) <= EIGEN_STACK_ALLOCATION_LIMIT)
276 ?
int(SmallPanelWidth)
278 Matrix<Scalar, PanelCapacity, PanelCapacity, RhsStorageOrder> triangularBuffer(SmallPanelWidth, SmallPanelWidth);
279 triangularBuffer.setZero();
281 triangularBuffer.diagonal().setZero();
283 triangularBuffer.diagonal().setOnes();
285 gebp_kernel<Scalar, Scalar, Index, ResMapper, Traits::mr, Traits::nr, ConjugateLhs, ConjugateRhs> gebp_kernel;
286 gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress,
typename Traits::LhsPacket4Packing,
289 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
290 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder, false, true> pack_rhs_panel;
292 for (Index k2 = IsLower ? 0 : depth; IsLower ? k2 < depth : k2 > 0; IsLower ? k2 += kc : k2 -= kc) {
293 Index actual_kc = (std::min)(IsLower ? depth - k2 : k2, kc);
294 Index actual_k2 = IsLower ? k2 : k2 - actual_kc;
297 EIGEN_IF_CONSTEXPR (IsLower) {
298 if ((k2 < cols) && (actual_k2 + actual_kc > cols)) {
299 actual_kc = cols - k2;
300 k2 = actual_k2 + actual_kc - kc;
305 Index rs = IsLower ? (std::min)(cols, actual_k2) : cols - k2;
307 Index ts = (IsLower && actual_k2 >= cols) ? 0 : actual_kc;
309 Scalar* geb = blockB + ts * ts;
310 geb = geb + internal::first_aligned<PacketBytes>(geb, PacketBytes /
sizeof(Scalar));
312 pack_rhs(geb, rhs.getSubMapper(actual_k2, IsLower ? 0 : k2), actual_kc, rs);
316 for (Index j2 = 0; j2 < actual_kc; j2 += SmallPanelWidth) {
317 Index actualPanelWidth = std::min<Index>(actual_kc - j2, SmallPanelWidth);
318 Index actual_j2 = actual_k2 + j2;
319 Index panelOffset = IsLower ? j2 + actualPanelWidth : 0;
320 Index panelLength = IsLower ? actual_kc - j2 - actualPanelWidth : j2;
322 pack_rhs_panel(blockB + j2 * actual_kc, rhs.getSubMapper(actual_k2 + panelOffset, actual_j2), panelLength,
323 actualPanelWidth, actual_kc, panelOffset);
326 for (Index j = 0; j < actualPanelWidth; ++j) {
327 EIGEN_IF_CONSTEXPR (SetDiag) triangularBuffer.coeffRef(j, j) = rhs(actual_j2 + j, actual_j2 + j);
328 for (Index k = IsLower ? j + 1 : 0; IsLower ? k < actualPanelWidth : k < j; ++k)
329 triangularBuffer.coeffRef(k, j) = rhs(actual_j2 + k, actual_j2 + j);
332 pack_rhs_panel(blockB + j2 * actual_kc, RhsMapper(triangularBuffer.data(), triangularBuffer.outerStride()),
333 actualPanelWidth, actualPanelWidth, actual_kc, j2);
337 for (Index i2 = 0; i2 < rows; i2 += mc) {
338 const Index actual_mc = (std::min)(mc, rows - i2);
339 pack_lhs(blockA, lhs.getSubMapper(i2, actual_k2), actual_kc, actual_mc);
343 for (Index j2 = 0; j2 < actual_kc; j2 += SmallPanelWidth) {
344 Index actualPanelWidth = std::min<Index>(actual_kc - j2, SmallPanelWidth);
345 Index panelLength = IsLower ? actual_kc - j2 : j2 + actualPanelWidth;
346 Index blockOffset = IsLower ? j2 : 0;
348 gebp_kernel(res.getSubMapper(i2, actual_k2 + j2), blockA, blockB + j2 * actual_kc, actual_mc, panelLength,
349 actualPanelWidth, alpha, actual_kc, actual_kc,
350 blockOffset, blockOffset);
353 gebp_kernel(res.getSubMapper(i2, IsLower ? 0 : k2), blockA, geb, actual_mc, actual_kc, rs, alpha, -1, -1, 0, 0);
365template <
int Mode,
bool LhsIsTriangular,
typename Lhs,
typename Rhs>
366struct triangular_product_impl<Mode, LhsIsTriangular, Lhs, false, Rhs, false> {
367 template <
typename Dest>
368 static void run(Dest& dst,
const Lhs& a_lhs,
const Rhs& a_rhs,
const typename Dest::Scalar& alpha) {
369 using LhsScalar =
typename Lhs::Scalar;
370 using RhsScalar =
typename Rhs::Scalar;
371 using Scalar =
typename Dest::Scalar;
373 if (a_lhs.rows() == 0 || a_lhs.cols() == 0 || a_rhs.cols() == 0)
return;
375 if (a_lhs.rows() + a_lhs.cols() + a_rhs.cols() < EIGEN_GEMM_TO_COEFFBASED_THRESHOLD) {
376 constexpr bool lower = (Mode &
Lower) != 0;
378 evaluator<Lhs> lhs(a_lhs);
379 evaluator<Rhs> rhs(a_rhs);
380 const Index depth = a_lhs.cols();
381 for (Index j = 0; j < a_rhs.cols(); ++j) {
382 for (Index i = 0; i < a_lhs.rows(); ++i) {
383 const Index diagonal = LhsIsTriangular ? i : j;
384 const bool prefix = LhsIsTriangular ? lower : !lower;
385 const Index begin = prefix ? 0 : diagonal + Index(implicitDiag);
386 const Index end = prefix ? (numext::mini)(depth, diagonal + Index(!implicitDiag)) : depth;
388 for (Index k = begin; k < end; ++k) sum += lhs.coeff(i, k) * rhs.coeff(k, j);
389 EIGEN_IF_CONSTEXPR ((Mode &
UnitDiag) != 0) {
390 if (diagonal < depth) sum += LhsIsTriangular ? rhs.coeff(i, j) : lhs.coeff(i, j);
392 dst.coeffRef(i, j) += alpha * sum;
398 using LhsBlasTraits = internal::blas_traits<Lhs>;
399 using ActualLhsType =
typename LhsBlasTraits::DirectLinearAccessType;
400 using ActualLhsTypeCleaned = internal::remove_all_t<ActualLhsType>;
401 using RhsBlasTraits = internal::blas_traits<Rhs>;
402 using ActualRhsType =
typename RhsBlasTraits::DirectLinearAccessType;
403 using ActualRhsTypeCleaned = internal::remove_all_t<ActualRhsType>;
405 internal::add_const_on_value_type_t<ActualLhsType> lhs = LhsBlasTraits::extract(a_lhs);
406 internal::add_const_on_value_type_t<ActualRhsType> rhs = RhsBlasTraits::extract(a_rhs);
408 LhsScalar lhs_alpha = LhsBlasTraits::extractScalarFactor(a_lhs);
409 RhsScalar rhs_alpha = RhsBlasTraits::extractScalarFactor(a_rhs);
410 Scalar actualAlpha = alpha * lhs_alpha * rhs_alpha;
413 Scalar, Lhs::MaxRowsAtCompileTime, Rhs::MaxColsAtCompileTime,
414 Lhs::MaxColsAtCompileTime, 4>;
417 Index stripedRows = ((!LhsIsTriangular) || (IsLower)) ? lhs.rows() : (std::min)(lhs.rows(), lhs.cols());
418 Index stripedCols = ((LhsIsTriangular) || (!IsLower)) ? rhs.cols() : (std::min)(rhs.cols(), rhs.rows());
419 Index stripedDepth = LhsIsTriangular ? ((!IsLower) ? lhs.cols() : (std::min)(lhs.cols(), lhs.rows()))
420 : ((IsLower) ? rhs.rows() : (std::min)(rhs.rows(), rhs.cols()));
422 using Trmm = internal::product_triangular_matrix_matrix<
423 Scalar, Index, Mode, LhsIsTriangular,
425 LhsBlasTraits::NeedToConjugate,
428 Dest::InnerStrideAtCompileTime>;
429 const auto runWith = [&](level3_blocking<Scalar, Scalar>& blocking) {
430 Trmm::run(stripedRows, stripedCols, stripedDepth,
431 &lhs.coeffRef(0, 0), lhs.outerStride(),
432 &rhs.coeffRef(0, 0), rhs.outerStride(),
433 &dst.coeffRef(0, 0), dst.innerStride(), dst.outerStride(),
434 actualAlpha, blocking);
437#ifdef EIGEN_VECTORIZE_AVX512
442 constexpr bool DynamicBlocking = Lhs::MaxRowsAtCompileTime == Dynamic || Rhs::MaxColsAtCompileTime == Dynamic ||
443 Lhs::MaxColsAtCompileTime == Dynamic;
444 if (DynamicBlocking && (is_same<Scalar, float>::value || is_same<Scalar, double>::value) &&
445 gebp_traits<Scalar, Scalar>::nr == 8 && stripedRows >= 512 && stripedCols >= 512) {
447 Scalar, Dynamic, Dynamic, Dynamic, 1>;
448 LargeBlockingType blocking(stripedRows, stripedCols, stripedDepth, 1,
false);
453 BlockingType blocking(stripedRows, stripedCols, stripedDepth, 1,
false);
460 EIGEN_IF_CONSTEXPR (LhsIsTriangular) {
461 if (!numext::is_exactly_one(lhs_alpha)) {
462 Index diagSize = (std::min)(lhs.rows(), lhs.cols());
463 dst.matrix().topRows(diagSize) -= ((alpha * (lhs_alpha - LhsScalar(1))) * a_rhs).topRows(diagSize);
466 if (!numext::is_exactly_one(rhs_alpha)) {
467 Index diagSize = (std::min)(rhs.rows(), rhs.cols());
468 dst.matrix().leftCols(diagSize) -= (alpha * (rhs_alpha - RhsScalar(1))) * a_lhs.leftCols(diagSize);
@ UnitDiag
Definition Constants.h:216
@ ZeroDiag
Definition Constants.h:218
@ Lower
Definition Constants.h:212
@ Upper
Definition Constants.h:214
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
constexpr unsigned int RowMajorBit
Definition Constants.h:71