Eigen  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TriangularSolverMatrix.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5// Modifications Copyright (C) 2022 Intel Corporation
6//
7// This Source Code Form is subject to the terms of the Mozilla
8// Public License v. 2.0. If a copy of the MPL was not distributed
9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
10// SPDX-License-Identifier: MPL-2.0
11
12#ifndef EIGEN_TRIANGULAR_SOLVER_MATRIX_H
13#define EIGEN_TRIANGULAR_SOLVER_MATRIX_H
14
15// IWYU pragma: private
16#include "../InternalHeaderCheck.h"
17
18namespace Eigen {
19
20namespace internal {
21
22template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride,
23 bool Specialized>
24struct trsmKernelL {
25 // Generic Implementation of triangular solve for triangular matrix on left and multiple rhs.
26 // Handles non-packed matrices.
27 //
28 // A lower-triangular panel is addressed from its top-left element with indices in [0, size);
29 // an upper-triangular one is addressed from its bottom-right element with indices in
30 // (-size, 0]. Both origins are elements of the panel, so callers never form a pointer outside
31 // the matrix they solve in. The AVX-512 specializations take both by the top-left element and
32 // convert when they delegate here.
33 static void kernel(Index size, Index otherSize, const Scalar* _tri, Index triStride, Scalar* _other, Index otherIncr,
34 Index otherStride);
35};
36
37template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride,
38 bool Specialized>
39struct trsmKernelR {
40 // Generic Implementation of triangular solve for triangular matrix on right and multiple lhs.
41 // Handles non-packed matrices.
42 static void kernel(Index size, Index otherSize, const Scalar* _tri, Index triStride, Scalar* _other, Index otherIncr,
43 Index otherStride);
44};
45
46// The packet lanes are independent right-hand sides. Half the registers hold output accumulators,
47// enough independent multiply-add chains to cover the FMA latency on 16- and 32-register targets;
48// the rest hold the RHS packets and coefficient broadcasts.
49template <typename Scalar>
50struct triangular_solve_packet_traits {
51 static constexpr bool Enabled = packet_traits<Scalar>::Vectorizable &&
52 (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value) &&
53 std::numeric_limits<Scalar>::is_iec559 && std::numeric_limits<Scalar>::radix == 2;
54 static constexpr int PacketSize = packet_traits<Scalar>::size;
55 static constexpr int RegisterRows = 4;
56 static constexpr int NumberOfRegisters = gebp_traits<Scalar, Scalar>::NumberOfRegisters;
57 static constexpr int RhsPackets = plain_enum_max(
58 1, plain_enum_min((NumberOfRegisters - 2) / (RegisterRows + 1), NumberOfRegisters / (2 * RegisterRows)));
59 // Rows solved per step for a tile of the given RHS packets: a narrow tile takes twice the rows while
60 // the extra accumulators fit in a full tile's registers, keeping more multiply-add chains in flight.
61 static constexpr int tile_rows(int packets) { return packets * 2 <= RhsPackets ? 2 * RegisterRows : RegisterRows; }
62 // Allocation bound, independent of the cache-dependent direct-solve cutoff.
63 static constexpr int WorkspaceRows = 128;
64#if defined(EIGEN_VECTORIZE_AVX512) && EIGEN_USE_AVX512_TRSM_L_KERNELS
65 static constexpr bool UseUnblocked = false;
66#else
67 static constexpr bool UseUnblocked = Enabled;
68#endif
69
70 template <typename Index>
71 static EIGEN_STRONG_INLINE bool use_unblocked(Index size, Index cols, std::ptrdiff_t l1) {
72 if (!UseUnblocked || size < RegisterRows || size > WorkspaceRows || cols < PacketSize) return false;
73 const int packets = cols >= RhsPackets * PacketSize ? RhsPackets : 1;
74 // Like GEBP's L1 depth model: the RHS tile, reciprocals, and RegisterRows rows
75 // of A are reused along k. Leave half of L1 for streaming and associativity.
76 const std::ptrdiff_t rowBytes = (packets * PacketSize + RegisterRows + 1) * sizeof(Scalar);
77 const std::ptrdiff_t transposeBytes = PacketSize * PacketSize * sizeof(Scalar);
78 return std::ptrdiff_t(size) * rowBytes + transposeBytes <= l1 / 2;
79 }
80};
81
82template <typename Scalar, typename Index, int Mode, int TriStorageOrder>
83struct triangular_solve_packet_kernel {
84 using Traits = triangular_solve_packet_traits<Scalar>;
85 using Packet = typename packet_traits<Scalar>::type;
86 using TriMapper = const_blas_data_mapper<Scalar, Index, TriStorageOrder>;
87 static constexpr int PacketSize = Traits::PacketSize;
88 static constexpr bool IsLower = (Mode & Lower) != 0;
89
90 template <int RhsPackets>
91 static EIGEN_STRONG_INLINE bool all_finite(const PacketBlock<Packet, RhsPackets>& x, bool_constant<true>) {
92 // Classify bits: fast-math may assume the arithmetic results are finite.
93 using FloatTraits = binary_floating_point_traits<Scalar>;
94 bool nonfinite = false;
95 for (int p = 0; p < RhsPackets; ++p) {
96 Scalar values[PacketSize];
97 pstoreu(values, x.packet[p]);
98 EIGEN_FAST_MATH_CONSTANT_BARRIER(values);
99 for (int c = 0; c < PacketSize; ++c)
100 nonfinite |= (FloatTraits::bits(values[c]) & FloatTraits::kExponentMask) == FloatTraits::kExponentMask;
101 }
102 return !nonfinite;
103 }
104
105 // Keep disabled scalar instantiations valid in C++14.
106 template <int RhsPackets>
107 static EIGEN_STRONG_INLINE bool all_finite(const PacketBlock<Packet, RhsPackets>&, bool_constant<false>) {
108 return true;
109 }
110
111 template <int RhsPackets>
112 static EIGEN_STRONG_INLINE void update(PacketBlock<Packet, RhsPackets>& x, const PacketBlock<Packet, RhsPackets>& y,
113 Scalar a) {
114 const Packet pa = pset1<Packet>(a);
115 for (int p = 0; p < RhsPackets; ++p) x.packet[p] = pnmadd(pa, y.packet[p], x.packet[p]);
116 }
117
118 template <int RhsPackets>
119 static EIGEN_STRONG_INLINE void scale(PacketBlock<Packet, RhsPackets>& x, Scalar a) {
120 EIGEN_IF_CONSTEXPR (!(Mode & UnitDiag)) {
121 const Packet pa = pset1<Packet>(a);
122 for (int p = 0; p < RhsPackets; ++p) x.packet[p] = pmul(x.packet[p], pa);
123 }
124 }
125
126 template <std::size_t Row, int RhsPackets, std::size_t... Next>
127 static EIGEN_STRONG_INLINE void solve_row(PacketBlock<Packet, RhsPackets>* x, const TriMapper& a,
128 const Scalar* inverse, Index r0, Index step, std::index_sequence<Next...>) {
129 const Index row = r0 + Index(Row) * step;
130 scale(x[Row], inverse[row]);
131 int unroll[] = {0, (update(x[Row + 1 + Next], x[Row], a(r0 + Index(Row + 1 + Next) * step, row)), 0)...};
132 EIGEN_UNUSED_VARIABLE(unroll);
133 }
134
135 // GCC generates extra instructions when the accumulator indices come from a loop.
136 template <int RhsPackets, std::size_t... Rows>
137 static EIGEN_STRONG_INLINE void solve_block(Index i, const TriMapper& a, const Scalar* inverse,
138 PacketBlock<Packet, RhsPackets>* work, Index r0, Index step,
139 std::index_sequence<Rows...>) {
140 PacketBlock<Packet, RhsPackets> x[] = {work[r0 + Index(Rows) * step]...};
141 for (Index k = 0; k < i; ++k) {
142 const Index c = IsLower ? k : r0 + i - k;
143 const PacketBlock<Packet, RhsPackets> y = work[c];
144 int unroll[] = {0, (update(x[Rows], y, a(r0 + Index(Rows) * step, c)), 0)...};
145 EIGEN_UNUSED_VARIABLE(unroll);
146 }
147 // The braced expansion orders the dependent solves by row.
148 int solve_rows[] = {
149 0, (solve_row<Rows>(x, a, inverse, r0, step, std::make_index_sequence<sizeof...(Rows) - Rows - 1>{}), 0)...};
150 EIGEN_UNUSED_VARIABLE(solve_rows);
151 int store_rows[] = {0, (work[r0 + Index(Rows) * step] = x[Rows], 0)...};
152 EIGEN_UNUSED_VARIABLE(store_rows);
153 }
154
155 template <int RhsPackets>
156 static EIGEN_STRONG_INLINE void solve(Index size, const TriMapper& a, const Scalar* inverse, Scalar* other,
157 Index otherStride) {
158 PacketBlock<Packet, RhsPackets> work[Traits::WorkspaceRows];
159 for (int p = 0; p < RhsPackets; ++p) {
160 Index row = 0;
161 for (; row + PacketSize <= size; row += PacketSize) {
162 PacketBlock<Packet, PacketSize> block;
163 for (int c = 0; c < PacketSize; ++c)
164 block.packet[c] = ploadu<Packet>(other + row + (p * PacketSize + c) * otherStride);
165 ptranspose(block);
166 for (int r = 0; r < PacketSize; ++r) work[row + r].packet[p] = block.packet[r];
167 }
168 for (; row < size; ++row)
169 work[row].packet[p] = pgather<Scalar, Packet>(other + row + p * PacketSize * otherStride, otherStride);
170 }
171 Index i = 0;
172 const Index step = IsLower ? 1 : -1;
173 constexpr int TileRows = Traits::tile_rows(RhsPackets);
174 EIGEN_IF_CONSTEXPR (TileRows != Traits::RegisterRows) {
175 for (; i + TileRows <= size; i += TileRows)
176 solve_block(i, a, inverse, work, IsLower ? i : size - i - 1, step, std::make_index_sequence<TileRows>{});
177 }
178 for (; i + Traits::RegisterRows <= size; i += Traits::RegisterRows) {
179 const Index r0 = IsLower ? i : size - i - 1;
180 // Preserve the four-row schedule that keeps GCC and Clang's hot loops compact.
181 EIGEN_IF_CONSTEXPR (Traits::RegisterRows == 4) {
182 const Index r1 = r0 + step, r2 = r1 + step, r3 = r2 + step;
183 PacketBlock<Packet, RhsPackets> x0 = work[r0], x1 = work[r1], x2 = work[r2], x3 = work[r3];
184 for (Index k = 0; k < i; ++k) {
185 const Index c = IsLower ? k : size - k - 1;
186 const PacketBlock<Packet, RhsPackets> y = work[c];
187 update(x0, y, a(r0, c));
188 update(x1, y, a(r1, c));
189 update(x2, y, a(r2, c));
190 update(x3, y, a(r3, c));
191 }
192 scale(x0, inverse[r0]);
193 update(x1, x0, a(r1, r0));
194 update(x2, x0, a(r2, r0));
195 update(x3, x0, a(r3, r0));
196 scale(x1, inverse[r1]);
197 update(x2, x1, a(r2, r1));
198 update(x3, x1, a(r3, r1));
199 scale(x2, inverse[r2]);
200 update(x3, x2, a(r3, r2));
201 scale(x3, inverse[r3]);
202 work[r0] = x0;
203 work[r1] = x1;
204 work[r2] = x2;
205 work[r3] = x3;
206 } else {
207 solve_block(i, a, inverse, work, r0, step, std::make_index_sequence<Traits::RegisterRows>{});
208 }
209 }
210 for (; i < size; ++i) {
211 const Index r = IsLower ? i : size - i - 1;
212 PacketBlock<Packet, RhsPackets> x = work[r];
213 for (Index k = 0; k < i; ++k) {
214 const Index c = IsLower ? k : size - k - 1;
215 update(x, work[c], a(r, c));
216 }
217 scale(x, inverse[r]);
218 work[r] = x;
219 }
220 EIGEN_IF_CONSTEXPR (TriStorageOrder == RowMajor) {
221 // Successive RHS updates can overflow before cancellation in a row's dot product.
222 // Every later row uses all solved rows, propagating nonfinite lanes to the final row.
223 // Retry with the original accumulation order before overwriting any RHS coefficient.
224 if (!all_finite(work[IsLower ? size - 1 : 0], bool_constant<Traits::Enabled>{})) {
225 const Index origin = IsLower ? 0 : size - 1;
226 trsmKernelL<Scalar, Index, Mode, false, TriStorageOrder, 1, false>::kernel(
227 size, Index(RhsPackets * PacketSize), &a(origin, origin), a.stride(), other + origin, Index(1),
228 otherStride);
229 return;
230 }
231 }
232 for (int p = 0; p < RhsPackets; ++p) {
233 Index row = 0;
234 for (; row + PacketSize <= size; row += PacketSize) {
235 PacketBlock<Packet, PacketSize> block;
236 for (int r = 0; r < PacketSize; ++r) block.packet[r] = work[row + r].packet[p];
237 ptranspose(block);
238 for (int c = 0; c < PacketSize; ++c) pstoreu(other + row + (p * PacketSize + c) * otherStride, block.packet[c]);
239 }
240 for (; row < size; ++row)
241 pscatter<Scalar, Packet>(other + row + p * PacketSize * otherStride, work[row].packet[p], otherStride);
242 }
243 }
244
245 // The last cols < PacketSize columns, zero-padded to a packet in a copy. For a row-major triangle this costs
246 // one packet solve instead of a scalar dot product per coefficient; a column-major triangle's scalar solve
247 // is a contiguous axpy per column, which compilers vectorize. Padded lanes are dropped. Inlined, its buffer
248 // and second solve<1> copy measured slower even for solves that never pad.
249 static EIGEN_DONT_INLINE void solve_padded(Index size, const TriMapper& a, const Scalar* inverse, Scalar* other,
250 Index otherStride, Index cols) {
251 Map<Matrix<Scalar, Dynamic, Dynamic, ColMajor>, Unaligned, OuterStride<>> rest(other, size, cols,
252 OuterStride<>(otherStride));
253 static constexpr int PaddedBytes = Traits::WorkspaceRows * PacketSize * int(sizeof(Scalar));
254 static constexpr int PaddedMaxRows =
255 (EIGEN_STACK_ALLOCATION_LIMIT == 0 || PaddedBytes <= EIGEN_STACK_ALLOCATION_LIMIT) ? int(Traits::WorkspaceRows)
256 : Dynamic;
257 static constexpr int PaddedMaxCols =
258 (EIGEN_STACK_ALLOCATION_LIMIT == 0 || PaddedBytes <= EIGEN_STACK_ALLOCATION_LIMIT) ? int(PacketSize) : Dynamic;
259 Matrix<Scalar, Dynamic, Dynamic, ColMajor, PaddedMaxRows, PaddedMaxCols> padded(size, Index(PacketSize));
260 padded.leftCols(cols) = rest;
261 padded.rightCols(PacketSize - cols).setZero();
262 solve<1>(size, a, inverse, padded.data(), size);
263 rest = padded.leftCols(cols);
264 }
265
266 static EIGEN_DONT_INLINE void kernel(Index size, Index cols, const Scalar* tri, Index triStride, Scalar* other,
267 Index otherStride) {
268 eigen_internal_assert(size <= Traits::WorkspaceRows && cols >= PacketSize &&
269 (TriStorageOrder == RowMajor || cols % PacketSize == 0));
270 EIGEN_IF_CONSTEXPR (!IsLower) {
271 tri -= (size - 1) * (triStride + 1);
272 other -= size - 1;
273 }
274 TriMapper a(tri, triStride);
275 Scalar inverse[Traits::WorkspaceRows];
276 Map<Vector<Scalar, Dynamic>> mapped(inverse, size);
277 EIGEN_IF_CONSTEXPR (Mode & UnitDiag) {
278 mapped.setOnes();
279 } else {
280 const Map<const Vector<Scalar, Dynamic>, Unaligned, InnerStride<Dynamic>> diagonal(
281 tri, size, InnerStride<Dynamic>(triStride + 1));
282 mapped = diagonal.cwiseInverse();
283 }
284 Index j = 0;
285 EIGEN_IF_CONSTEXPR (Traits::RhsPackets > 1) {
286 for (; j + Traits::RhsPackets * PacketSize <= cols; j += Traits::RhsPackets * PacketSize)
287 solve<Traits::RhsPackets>(size, a, inverse, other + j * otherStride, otherStride);
288 }
289 EIGEN_IF_CONSTEXPR (Traits::RhsPackets > 2) {
290 for (; j + 2 * PacketSize <= cols; j += 2 * PacketSize)
291 solve<2>(size, a, inverse, other + j * otherStride, otherStride);
292 }
293 for (; j + PacketSize <= cols; j += PacketSize) solve<1>(size, a, inverse, other + j * otherStride, otherStride);
294 EIGEN_IF_CONSTEXPR (TriStorageOrder == RowMajor) {
295 if (j < cols) solve_padded(size, a, inverse, other + j * otherStride, otherStride, cols - j);
296 }
297 }
298};
299
300template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride,
301 bool Specialized>
302EIGEN_STRONG_INLINE void trsmKernelL<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride,
303 Specialized>::kernel(Index size, Index otherSize, const Scalar* _tri,
304 Index triStride, Scalar* _other, Index otherIncr,
305 Index otherStride) {
306 EIGEN_IF_CONSTEXPR ((Specialized && OtherInnerStride == 1 && triangular_solve_packet_traits<Scalar>::Enabled)) {
307 if (size >= triangular_solve_packet_traits<Scalar>::RegisterRows &&
308 size <= triangular_solve_packet_traits<Scalar>::WorkspaceRows && otherSize >= packet_traits<Scalar>::size) {
309 // The packet kernel pads the last columns only for a row-major triangle; see solve_padded.
310 const Index packetCols =
311 TriStorageOrder == RowMajor ? otherSize : numext::round_down(otherSize, Index(packet_traits<Scalar>::size));
312 triangular_solve_packet_kernel<Scalar, Index, Mode, TriStorageOrder>::kernel(size, packetCols, _tri, triStride,
313 _other, otherStride);
314 if (packetCols == otherSize) return;
315 otherSize -= packetCols;
316 _other += packetCols * otherStride;
317 }
318 }
319 using TriMapper = const_blas_data_mapper<Scalar, Index, TriStorageOrder>;
320 using OtherMapper = blas_data_mapper<Scalar, Index, ColMajor, Unaligned, OtherInnerStride>;
321 TriMapper tri(_tri, triStride);
322 OtherMapper other(_other, otherStride, otherIncr);
323
324 enum { IsLower = (Mode & Lower) == Lower };
325 conj_if<Conjugate> conj;
326
327 // tr solve
328 for (Index k = 0; k < size; ++k) {
329 // TODO: write a small kernel handling this (can be shared with trsv)
330 Index i = IsLower ? k : -k;
331 Index rs = size - k - 1; // remaining size
332 Index s = TriStorageOrder == RowMajor ? (IsLower ? 0 : i + 1) : IsLower ? i + 1 : i - rs;
333
334 Scalar a = (Mode & UnitDiag) ? Scalar(1) : Scalar(Scalar(1) / conj(tri(i, i)));
335 for (Index j = 0; j < otherSize; ++j) {
336 EIGEN_IF_CONSTEXPR (TriStorageOrder == RowMajor) {
337 Scalar b(0);
338 const Scalar* l = &tri(i, s);
339 typename OtherMapper::LinearMapper r = other.getLinearMapper(s, j);
340 for (Index i3 = 0; i3 < k; ++i3) b += conj(l[i3]) * r(i3);
341
342 other(i, j) = (other(i, j) - b) * a;
343 } else {
344 Scalar& otherij = other(i, j);
345 otherij *= a;
346 Scalar b = otherij;
347 typename OtherMapper::LinearMapper r = other.getLinearMapper(s, j);
348 typename TriMapper::LinearMapper l = tri.getLinearMapper(s, i);
349 for (Index i3 = 0; i3 < rs; ++i3) r(i3) -= b * conj(l(i3));
350 }
351 }
352 }
353}
354
355template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride,
356 bool Specialized>
357EIGEN_STRONG_INLINE void trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride,
358 Specialized>::kernel(Index size, Index otherSize, const Scalar* _tri,
359 Index triStride, Scalar* _other, Index otherIncr,
360 Index otherStride) {
361 using RealScalar = typename NumTraits<Scalar>::Real;
362 using LhsMapper = blas_data_mapper<Scalar, Index, ColMajor, Unaligned, OtherInnerStride>;
363 using RhsMapper = const_blas_data_mapper<Scalar, Index, TriStorageOrder>;
364 LhsMapper lhs(_other, otherStride, otherIncr);
365 RhsMapper rhs(_tri, triStride);
366
367 enum { IsLower = (Mode & Lower) == Lower };
368 conj_if<Conjugate> conj;
369
370 for (Index k = 0; k < size; ++k) {
371 Index j = IsLower ? size - k - 1 : k;
372
373 typename LhsMapper::LinearMapper r = lhs.getLinearMapper(0, j);
374 EIGEN_IF_CONSTEXPR (OtherInnerStride == 1 && packet_traits<Scalar>::Vectorizable) {
375 using Packet = typename packet_traits<Scalar>::type;
376 constexpr Index PS = unpacket_traits<Packet>::size;
377 // Unrolled k3 loop by 4 to reduce r load/store traffic.
378 Index k3 = 0;
379 for (; k3 + 3 < k; k3 += 4) {
380 Index col0 = IsLower ? j + 1 + k3 : k3;
381 Scalar b0 = conj(rhs(col0, j));
382 Scalar b1 = conj(rhs(col0 + 1, j));
383 Scalar b2 = conj(rhs(col0 + 2, j));
384 Scalar b3 = conj(rhs(col0 + 3, j));
385 Packet neg_pb0 = pset1<Packet>(-b0);
386 Packet neg_pb1 = pset1<Packet>(-b1);
387 Packet neg_pb2 = pset1<Packet>(-b2);
388 Packet neg_pb3 = pset1<Packet>(-b3);
389 typename LhsMapper::LinearMapper a0 = lhs.getLinearMapper(0, col0);
390 typename LhsMapper::LinearMapper a1 = lhs.getLinearMapper(0, col0 + 1);
391 typename LhsMapper::LinearMapper a2 = lhs.getLinearMapper(0, col0 + 2);
392 typename LhsMapper::LinearMapper a3 = lhs.getLinearMapper(0, col0 + 3);
393 Index i = 0;
394 for (; i + PS <= otherSize; i += PS) {
395 Packet pr = r.template loadPacket<Packet>(i);
396 pr = pmadd(a0.template loadPacket<Packet>(i), neg_pb0, pr);
397 pr = pmadd(a1.template loadPacket<Packet>(i), neg_pb1, pr);
398 pr = pmadd(a2.template loadPacket<Packet>(i), neg_pb2, pr);
399 pr = pmadd(a3.template loadPacket<Packet>(i), neg_pb3, pr);
400 r.template storePacket<Packet>(i, pr);
401 }
402 for (; i < otherSize; ++i) {
403 r(i) -= a0(i) * b0 + a1(i) * b1 + a2(i) * b2 + a3(i) * b3;
404 }
405 }
406 // Handle remaining k3 iterations with vectorized inner loop.
407 for (; k3 < k; ++k3) {
408 Scalar b = conj(rhs(IsLower ? j + 1 + k3 : k3, j));
409 typename LhsMapper::LinearMapper a = lhs.getLinearMapper(0, IsLower ? j + 1 + k3 : k3);
410 Packet neg_pb = pset1<Packet>(-b);
411 Index i = 0;
412 for (; i + PS <= otherSize; i += PS) {
413 Packet pr = r.template loadPacket<Packet>(i);
414 pr = pmadd(a.template loadPacket<Packet>(i), neg_pb, pr);
415 r.template storePacket<Packet>(i, pr);
416 }
417 for (; i < otherSize; ++i) r(i) -= a(i) * b;
418 }
419 // Vectorized diagonal scaling.
420 EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) {
421 Scalar inv_rjj = RealScalar(1) / conj(rhs(j, j));
422 Packet pinv = pset1<Packet>(inv_rjj);
423 Index i = 0;
424 for (; i + PS <= otherSize; i += PS) {
425 r.template storePacket<Packet>(i, pmul(r.template loadPacket<Packet>(i), pinv));
426 }
427 for (; i < otherSize; ++i) r(i) *= inv_rjj;
428 }
429 } else {
430 for (Index k3 = 0; k3 < k; ++k3) {
431 Scalar b = conj(rhs(IsLower ? j + 1 + k3 : k3, j));
432 typename LhsMapper::LinearMapper a = lhs.getLinearMapper(0, IsLower ? j + 1 + k3 : k3);
433 for (Index i = 0; i < otherSize; ++i) r(i) -= a(i) * b;
434 }
435 EIGEN_IF_CONSTEXPR ((Mode & UnitDiag) == 0) {
436 Scalar inv_rjj = RealScalar(1) / conj(rhs(j, j));
437 for (Index i = 0; i < otherSize; ++i) r(i) *= inv_rjj;
438 }
439 }
440 }
441}
442
443// if the rhs is row major, let's transpose the product
444template <typename Scalar, typename Index, int Side, int Mode, bool Conjugate, int TriStorageOrder,
445 int OtherInnerStride>
446struct triangular_solve_matrix<Scalar, Index, Side, Mode, Conjugate, TriStorageOrder, RowMajor, OtherInnerStride> {
447 static void run(Index size, Index cols, const Scalar* tri, Index triStride, Scalar* _other, Index otherIncr,
448 Index otherStride, level3_blocking<Scalar, Scalar>& blocking) {
449 triangular_solve_matrix<
450 Scalar, Index, Side == OnTheLeft ? OnTheRight : OnTheLeft, (Mode & UnitDiag) | ((Mode & Upper) ? Lower : Upper),
451 NumTraits<Scalar>::IsComplex && Conjugate, TriStorageOrder == RowMajor ? ColMajor : RowMajor, ColMajor,
452 OtherInnerStride>::run(size, cols, tri, triStride, _other, otherIncr, otherStride, blocking);
453 }
454};
455
459template <typename Scalar>
460std::ptrdiff_t triangular_solve_budget(std::ptrdiff_t l2, std::ptrdiff_t l3) {
461 return (numext::maxi)(l3 / 4, l2) / std::ptrdiff_t(sizeof(Scalar));
462}
463
468template <typename Index>
469Index triangular_solve_panel_columns(Index size, Index cols, std::ptrdiff_t budget, Index nr) {
470 eigen_internal_assert(size > 0);
471 const std::ptrdiff_t width = numext::round_down<std::ptrdiff_t>(budget / size, nr);
472 return width >= 512 && width < cols ? Index(width) : cols;
473}
474
485template <typename Scalar, typename Index>
486Index triangular_solve_kc(Index size, Index otherSize, Index extent, std::ptrdiff_t budget, bool slabRuns,
487 level3_blocking<Scalar, Scalar>& blocking) {
488 EIGEN_UNUSED_VARIABLE(extent);
489 const bool deep = std::ptrdiff_t(size) * otherSize > budget || (slabRuns && std::ptrdiff_t(size) * size / 2 > budget);
490 if (!deep || blocking.blockA() != nullptr) return blocking.kc();
491 Index kc = size, mc = size, nc = otherSize;
492 computeProductBlockingSizes<Scalar, Scalar>(kc, mc, nc);
493 kc = (numext::mini)(kc, numext::round_down((numext::mini)(size / 8, Index(160)), Index(8)));
494#if defined(EIGEN_ALLOCA) && !defined(EIGEN_NO_ALLOCA)
495 const std::ptrdiff_t stackKc =
496 std::ptrdiff_t(EIGEN_STACK_ALLOCATION_LIMIT) / (std::ptrdiff_t(sizeof(Scalar)) * extent);
497 if (blocking.kc() <= stackKc) kc = Index((numext::mini)(std::ptrdiff_t(kc), stackKc));
498#endif
499 return (numext::maxi)(kc, blocking.kc());
500}
501
502/* Optimized triangular solver with multiple right hand side and the triangular matrix on the left
503 */
504template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride>
505struct triangular_solve_matrix<Scalar, Index, OnTheLeft, Mode, Conjugate, TriStorageOrder, ColMajor, OtherInnerStride> {
506 static EIGEN_DONT_INLINE void run(Index size, Index otherSize, const Scalar* _tri, Index triStride, Scalar* _other,
507 Index otherIncr, Index otherStride, level3_blocking<Scalar, Scalar>& blocking);
508};
509
510template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride>
511EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheLeft, Mode, Conjugate, TriStorageOrder, ColMajor,
512 OtherInnerStride>::run(Index size, Index otherSize, const Scalar* _tri,
513 Index triStride, Scalar* _other, Index otherIncr,
514 Index otherStride,
515 level3_blocking<Scalar, Scalar>& blocking) {
516 std::ptrdiff_t l1, l2, l3;
517 manage_caching_sizes(GetAction, &l1, &l2, &l3);
518 EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 && triangular_solve_packet_traits<Scalar>::Enabled)) {
519 using PacketTraits = triangular_solve_packet_traits<Scalar>;
520 if (PacketTraits::use_unblocked(size, otherSize, l1)) {
521 const Index origin = (Mode & Lower) ? 0 : size - 1;
522 trsmKernelL<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride, true>::kernel(
523 size, otherSize, _tri + origin * (triStride + 1), triStride, _other + origin, otherIncr, otherStride);
524 return;
525 }
526 }
527#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) && EIGEN_USE_AVX512_TRSM_L_KERNELS && \
528 EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS
529 EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
530 (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
531 // Very rough cutoffs to determine when to call trsm w/o packing
532 // For small problem sizes trsmKernel compiled with clang is generally faster.
533 // TODO: Investigate better heuristics for cutoffs.
534 double L2Cap = 0.5; // 50% of L2 size
535 if (size < avx512_trsm_cutoff<Scalar>(l2, otherSize, L2Cap)) {
536 trsmKernelL<Scalar, Index, Mode, Conjugate, TriStorageOrder, 1, /*Specialized=*/true>::kernel(
537 size, otherSize, _tri, triStride, _other, 1, otherStride);
538 return;
539 }
540 }
541#endif
542
543 using TriMapper = const_blas_data_mapper<Scalar, Index, TriStorageOrder>;
544 using OtherMapper = blas_data_mapper<Scalar, Index, ColMajor, Unaligned, OtherInnerStride>;
545 TriMapper tri(_tri, triStride);
546
547 using Traits = gebp_traits<Scalar, Scalar>;
548
549 // The diagonal panels are updated by gebp with a depth of SmallPanelWidth. The DoublePacket kernel that FMA targets
550 // with 16 registers use is one packet tall; where that is narrower than nr (complex<double> with AVX2+FMA), such
551 // shallow updates run slowly, so its panels are twice as wide.
552#ifdef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
553 constexpr bool WidePanels = is_double_packet<typename Traits::AccPacket>::value && Traits::NumberOfRegisters < 32 &&
554 int(Traits::mr) < int(Traits::nr);
555#else
556 constexpr bool WidePanels = false;
557#endif
558 enum {
559 SmallPanelWidth = WidePanels ? 2 * Traits::nr : plain_enum_max(Traits::mr, Traits::nr),
560 IsLower = (Mode & Lower) == Lower
561 };
562
563 // Every k-block updates the rows of the right-hand side beyond it through gebp, so a right-hand side
564 // solved whole streams through the caches size/kc times (issue #3162); column panels that stay in
565 // cache keep those sweeps out of memory. This kernel packs the slabs of the triangle by rows, which
566 // costs more the deeper they are, so a large triangle alone does not deepen it.
567 const std::ptrdiff_t budget = triangular_solve_budget<Scalar>(l2, l3);
568 const Index nc = triangular_solve_panel_columns(size, otherSize, budget, Index(Traits::nr));
569 const Index mc = (numext::mini)(size, blocking.mc()); // cache block size along the M direction
570 // The tr solve below packs up to panelWidth x kc entries of the triangle into blockA, where panelWidth is
571 // SmallPanelWidth or, with the packet kernel, at most blockARows.
572 const Index blockARows = (numext::maxi)(mc, Index(SmallPanelWidth));
573 // cache block size along the K direction
574 const Index kc = triangular_solve_kc<Scalar>(size, otherSize, (numext::maxi)(blockARows, nc), budget,
575 /*slabRuns=*/false, blocking);
576
577 std::size_t sizeA = kc * blockARows;
578 std::size_t sizeB = kc * nc;
579
580 ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA());
581 ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB());
582
583 gebp_kernel<Scalar, Scalar, Index, OtherMapper, Traits::mr, Traits::nr, Conjugate, false> gebp_kernel;
584 gemm_pack_lhs<Scalar, Index, TriMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing,
585 TriStorageOrder>
586 pack_lhs;
587 gemm_pack_rhs<Scalar, Index, OtherMapper, Traits::nr, ColMajor, false, true> pack_rhs;
588
589 // the goal here is to subdivide the Rhs panels such that we keep some cache
590 // coherence when accessing the rhs elements
591 Index subcols = otherSize > 0 ? l2 / (4 * sizeof(Scalar) * numext::maxi<Index>(otherStride, size)) : 0;
592 Index colStep = Traits::nr;
593 // Width of the diagonal panels solved between gebp updates of their depth.
594 Index panelWidth = SmallPanelWidth;
595 EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 && triangular_solve_packet_traits<Scalar>::UseUnblocked)) {
596 // The packet kernel solves a whole diagonal block faster than narrow panels do, since those
597 // interleave slower rank-SmallPanelWidth gebp updates; a block deeper than its workspace is split
598 // evenly. That needs a tile of two packets of right-hand sides where the registers allow: a single
599 // packet measured slower. pack_rhs reads each kc x subcols chunk right after its solve, so a chunk should
600 // stay in L2; packet-aligned chunks confine any column tail to an Rhs panel's last chunk.
601 using PacketTraits = triangular_solve_packet_traits<Scalar>;
602 const Index rows = Index(PacketTraits::RegisterRows), packet = Index(PacketTraits::PacketSize);
603 if (otherSize >= Index(plain_enum_min(2, PacketTraits::RhsPackets)) * packet) {
604 const Index panels = numext::div_ceil(kc, Index(PacketTraits::WorkspaceRows));
605 panelWidth = (numext::mini)(blockARows, rows * numext::div_ceil(numext::div_ceil(kc, panels), rows));
606 colStep = colStep % packet == 0 ? colStep : packet % colStep == 0 ? packet : colStep * packet;
607 subcols = Index(l2 / (2 * std::ptrdiff_t(sizeof(Scalar)) * kc));
608 }
609 }
610 subcols = numext::maxi<Index>(numext::round_down(subcols, colStep), colStep);
611
612 for (Index j0 = 0; j0 < otherSize; j0 += nc) {
613 const Index cols = (numext::mini)(otherSize - j0, nc);
614 Scalar* _panel = _other + j0 * otherStride;
615 OtherMapper other(_panel, otherStride, otherIncr);
616
617 for (Index k2 = IsLower ? 0 : size; IsLower ? k2 < size : k2 > 0; IsLower ? k2 += kc : k2 -= kc) {
618 const Index actual_kc = (numext::mini)(IsLower ? size - k2 : k2, kc);
619
620 // We have selected and packed a big horizontal panel R1 of rhs. Let B be the packed copy of this panel,
621 // and R2 the remaining part of rhs. The corresponding vertical panel of lhs is split into
622 // A11 (the triangular part) and A21 the remaining rectangular part.
623 // Then the high level algorithm is:
624 // - B = R1 => general block copy (done during the next step)
625 // - R1 = A11^-1 B => tricky part
626 // - update B from the new R1 => actually this has to be performed continuously during the above step
627 // - R2 -= A21 * B => GEPP
628
629 // The tricky part: compute R1 = A11^-1 B while updating B from R1
630 // The idea is to split A11 into multiple small vertical panels.
631 // Each panel can be split into a small triangular part T1k which is processed without optimization,
632 // and the remaining small part T2k which is processed using gebp with appropriate block strides
633 for (Index j2 = 0; j2 < cols; j2 += subcols) {
634 Index actual_cols = (numext::mini)(cols - j2, subcols);
635 // for each small vertical panels [T1k^T, T2k^T]^T of lhs
636 for (Index k1 = 0; k1 < actual_kc; k1 += panelWidth) {
637 Index actualPanelWidth = numext::mini<Index>(actual_kc - k1, panelWidth);
638 // tr solve
639 {
640 Index i = IsLower ? k2 + k1 : k2 - k1 - 1;
641#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) && EIGEN_USE_AVX512_TRSM_L_KERNELS
642 EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
643 (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
644 i = IsLower ? k2 + k1 : k2 - k1 - actualPanelWidth;
645 }
646#endif
647 using DiagonalKernel =
648 trsmKernelL<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride, /*Specialized=*/true>;
649 const Scalar* diagonal = _tri + i + i * triStride;
650 Scalar* rhs = _panel + i * otherIncr + j2 * otherStride;
651 // Narrow panels keep their compile-time width bound, which lets compilers shorten the scalar
652 // kernel's loops; the packet kernel solves wider ones.
653 if (panelWidth == Index(SmallPanelWidth))
654 DiagonalKernel::kernel(numext::mini<Index>(actualPanelWidth, SmallPanelWidth), actual_cols, diagonal,
655 triStride, rhs, otherIncr, otherStride);
656 else
657 DiagonalKernel::kernel(actualPanelWidth, actual_cols, diagonal, triStride, rhs, otherIncr, otherStride);
658 }
659
660 Index lengthTarget = actual_kc - k1 - actualPanelWidth;
661 Index startBlock = IsLower ? k2 + k1 : k2 - k1 - actualPanelWidth;
662 Index blockBOffset = IsLower ? k1 : lengthTarget;
663
664 // update the respective rows of B from other
665 pack_rhs(blockB + actual_kc * j2, other.getSubMapper(startBlock, j2), actualPanelWidth, actual_cols,
666 actual_kc, blockBOffset);
667
668 // GEBP
669 if (lengthTarget > 0) {
670 Index startTarget = IsLower ? k2 + k1 + actualPanelWidth : k2 - actual_kc;
671
672 pack_lhs(blockA, tri.getSubMapper(startTarget, startBlock), actualPanelWidth, lengthTarget);
673
674 gebp_kernel(other.getSubMapper(startTarget, j2), blockA, blockB + actual_kc * j2, lengthTarget,
675 actualPanelWidth, actual_cols, Scalar(-1), actualPanelWidth, actual_kc, 0, blockBOffset);
676 }
677 }
678 }
679
680 // R2 -= A21 * B => GEPP
681 {
682 Index start = IsLower ? k2 + kc : 0;
683 Index end = IsLower ? size : k2 - kc;
684 for (Index i2 = start; i2 < end; i2 += mc) {
685 const Index actual_mc = (numext::mini)(mc, end - i2);
686 if (actual_mc > 0) {
687 pack_lhs(blockA, tri.getSubMapper(i2, IsLower ? k2 : k2 - kc), actual_kc, actual_mc);
688
689 gebp_kernel(other.getSubMapper(i2, 0), blockA, blockB, actual_mc, actual_kc, cols, Scalar(-1), -1, -1, 0,
690 0);
691 }
692 }
693 }
694 }
695 }
696}
697
698/* Optimized triangular solver with multiple left hand sides and the triangular matrix on the right
699 */
700template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride>
701struct triangular_solve_matrix<Scalar, Index, OnTheRight, Mode, Conjugate, TriStorageOrder, ColMajor,
702 OtherInnerStride> {
703 static EIGEN_DONT_INLINE void run(Index size, Index otherSize, const Scalar* _tri, Index triStride, Scalar* _other,
704 Index otherIncr, Index otherStride, level3_blocking<Scalar, Scalar>& blocking);
705};
706
707template <typename Scalar, typename Index, int Mode, bool Conjugate, int TriStorageOrder, int OtherInnerStride>
708EIGEN_DONT_INLINE void triangular_solve_matrix<Scalar, Index, OnTheRight, Mode, Conjugate, TriStorageOrder, ColMajor,
709 OtherInnerStride>::run(Index size, Index otherSize, const Scalar* _tri,
710 Index triStride, Scalar* _other, Index otherIncr,
711 Index otherStride,
712 level3_blocking<Scalar, Scalar>& blocking) {
713 Index rows = otherSize;
714
715 std::ptrdiff_t l1, l2, l3;
716 manage_caching_sizes(GetAction, &l1, &l2, &l3);
717
718#if defined(EIGEN_VECTORIZE_AVX512) && defined(EIGEN_USE_AVX512_TRSM_R_KERNELS) && EIGEN_USE_AVX512_TRSM_R_KERNELS && \
719 EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS
720 EIGEN_IF_CONSTEXPR ((OtherInnerStride == 1 &&
721 (std::is_same<Scalar, float>::value || std::is_same<Scalar, double>::value))) {
722 // TODO: Investigate better heuristics for cutoffs.
723 double L2Cap = 0.5; // 50% of L2 size
724 if (size < avx512_trsm_cutoff<Scalar>(l2, rows, L2Cap)) {
725 trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride, /*Specialized=*/true>::kernel(
726 size, rows, _tri, triStride, _other, 1, otherStride);
727 return;
728 }
729 }
730#endif
731
732 using LhsMapper = blas_data_mapper<Scalar, Index, ColMajor, Unaligned, OtherInnerStride>;
733 using RhsMapper = const_blas_data_mapper<Scalar, Index, TriStorageOrder>;
734 LhsMapper lhs(_other, otherStride, otherIncr);
735 RhsMapper rhs(_tri, triStride);
736
737 using Traits = gebp_traits<Scalar, Scalar>;
738 enum {
739 RhsStorageOrder = TriStorageOrder,
740 SmallPanelWidth = plain_enum_max(Traits::mr, Traits::nr),
741 IsLower = (Mode & Lower) == Lower
742 };
743
744 // Every k-block sweeps all rows of the left-hand side through gebp and packs its slab of a
745 // column-major triangle one column run at a time, so a large triangle alone deepens this kernel
746 // (issue #3162); a row-major triangle is packed by rows, as on the left.
747 const std::ptrdiff_t budget = triangular_solve_budget<Scalar>(l2, l3);
748 Index mc = (numext::mini)(rows, blocking.mc()); // cache block size along the M direction
749 // cache block size along the K direction
750 const Index kc = triangular_solve_kc<Scalar>(size, rows, (numext::maxi)(mc, size), budget,
751 /*slabRuns=*/TriStorageOrder == ColMajor, blocking);
752 // blockA packs kc x mc entries of the left-hand side, and rows can far exceed size. Past half the
753 // budget a deeper kc takes proportionally fewer rows per pass, so blockA does not outgrow the buffer
754 // the blocking chose.
755 if (kc > blocking.kc()) {
756 const std::ptrdiff_t maxA = (numext::maxi)(std::ptrdiff_t(blocking.kc()) * mc, budget / 2);
757 mc = (numext::mini)(mc, (numext::maxi)(Index(Traits::mr), numext::round_down(Index(maxA / kc), Index(Traits::mr))));
758 }
759
760 std::size_t sizeA = kc * mc;
761 std::size_t sizeB = kc * size;
762
763 ei_declare_aligned_stack_constructed_variable(Scalar, blockA, sizeA, blocking.blockA());
764 ei_declare_aligned_stack_constructed_variable(Scalar, blockB, sizeB, blocking.blockB());
765
766 gebp_kernel<Scalar, Scalar, Index, LhsMapper, Traits::mr, Traits::nr, false, Conjugate> gebp_kernel;
767 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder> pack_rhs;
768 gemm_pack_rhs<Scalar, Index, RhsMapper, Traits::nr, RhsStorageOrder, false, true> pack_rhs_panel;
769 gemm_pack_lhs<Scalar, Index, LhsMapper, Traits::mr, Traits::LhsProgress, typename Traits::LhsPacket4Packing, ColMajor,
770 false, true>
771 pack_lhs_panel;
772
773 for (Index k2 = IsLower ? size : 0; IsLower ? k2 > 0 : k2 < size; IsLower ? k2 -= kc : k2 += kc) {
774 const Index actual_kc = (numext::mini)(IsLower ? k2 : size - k2, kc);
775 Index actual_k2 = IsLower ? k2 - actual_kc : k2;
776
777 Index startPanel = IsLower ? 0 : k2 + actual_kc;
778 Index rs = IsLower ? actual_k2 : size - actual_k2 - actual_kc;
779 Scalar* geb = blockB + actual_kc * actual_kc;
780
781 if (rs > 0) pack_rhs(geb, rhs.getSubMapper(actual_k2, startPanel), actual_kc, rs);
782
783 // triangular packing (we only pack the panels off the diagonal,
784 // neglecting the blocks overlapping the diagonal
785 {
786 for (Index j2 = 0; j2 < actual_kc; j2 += SmallPanelWidth) {
787 Index actualPanelWidth = numext::mini<Index>(actual_kc - j2, SmallPanelWidth);
788 Index actual_j2 = actual_k2 + j2;
789 Index panelOffset = IsLower ? j2 + actualPanelWidth : 0;
790 Index panelLength = IsLower ? actual_kc - j2 - actualPanelWidth : j2;
791
792 if (panelLength > 0)
793 pack_rhs_panel(blockB + j2 * actual_kc, rhs.getSubMapper(actual_k2 + panelOffset, actual_j2), panelLength,
794 actualPanelWidth, actual_kc, panelOffset);
795 }
796 }
797
798 for (Index i2 = 0; i2 < rows; i2 += mc) {
799 const Index actual_mc = (numext::mini)(mc, rows - i2);
800
801 // triangular solver kernel
802 {
803 // for each small block of the diagonal (=> vertical panels of rhs)
804 for (Index j2 = IsLower ? (actual_kc - ((actual_kc % SmallPanelWidth) ? Index(actual_kc % SmallPanelWidth)
805 : Index(SmallPanelWidth)))
806 : 0;
807 IsLower ? j2 >= 0 : j2 < actual_kc; IsLower ? j2 -= SmallPanelWidth : j2 += SmallPanelWidth) {
808 Index actualPanelWidth = numext::mini<Index>(actual_kc - j2, SmallPanelWidth);
809 Index absolute_j2 = actual_k2 + j2;
810 Index panelOffset = IsLower ? j2 + actualPanelWidth : 0;
811 Index panelLength = IsLower ? actual_kc - j2 - actualPanelWidth : j2;
812
813 // GEBP
814 if (panelLength > 0) {
815 gebp_kernel(lhs.getSubMapper(i2, absolute_j2), blockA, blockB + j2 * actual_kc, actual_mc, panelLength,
816 actualPanelWidth, Scalar(-1), actual_kc, actual_kc, // strides
817 panelOffset, panelOffset); // offsets
818 }
819
820 {
821 // unblocked triangular solve
822 trsmKernelR<Scalar, Index, Mode, Conjugate, TriStorageOrder, OtherInnerStride,
823 /*Specialized=*/true>::kernel(actualPanelWidth, actual_mc,
824 _tri + absolute_j2 + absolute_j2 * triStride, triStride,
825 _other + i2 * otherIncr + absolute_j2 * otherStride, otherIncr,
826 otherStride);
827 }
828 // pack the just computed part of lhs to A
829 pack_lhs_panel(blockA, lhs.getSubMapper(i2, absolute_j2), actualPanelWidth, actual_mc, actual_kc, j2);
830 }
831 }
832
833 if (rs > 0)
834 gebp_kernel(lhs.getSubMapper(i2, startPanel), blockA, geb, actual_mc, actual_kc, rs, Scalar(-1), -1, -1, 0, 0);
835 }
836 }
837}
838} // end namespace internal
839
840} // end namespace Eigen
841
842#endif // EIGEN_TRIANGULAR_SOLVER_MATRIX_H
@ UnitDiag
Definition Constants.h:216
@ Lower
Definition Constants.h:212
@ Upper
Definition Constants.h:214
@ Unaligned
Definition Constants.h:236
@ ColMajor
Definition Constants.h:319
@ RowMajor
Definition Constants.h:321
@ OnTheLeft
Definition Constants.h:332
@ OnTheRight
Definition Constants.h:334