Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GpuLU.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Eigen Authors
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// GPU partial-pivoting LU decomposition using cuSOLVER, wrapping
12// cusolverDnXgetrf and cusolverDnXgetrs.
13
14#ifndef EIGEN_GPU_LU_H
15#define EIGEN_GPU_LU_H
16
17// IWYU pragma: private
18#include "./InternalHeaderCheck.h"
19
20#include "./GpuSolverContext.h"
21
22namespace Eigen {
23namespace gpu {
36template <typename Scalar_>
37class LU {
38 public:
39 using Scalar = Scalar_;
40 using RealScalar = typename NumTraits<Scalar>::Real;
42
43 LU() = default;
44
48 explicit LU(Context& ctx) : solver_ctx_(ctx) {}
49
50 template <typename InputType>
51 explicit LU(const DenseBase<InputType>& A) {
52 compute(A);
53 }
54
56 explicit LU(const DeviceMatrix<Scalar>& d_A) { compute(d_A); }
57
59 explicit LU(DeviceMatrix<Scalar>&& d_A) { compute(std::move(d_A)); }
60
62 template <typename InputType>
63 LU(Context& ctx, const DenseBase<InputType>& A) : solver_ctx_(ctx) {
64 compute(A);
65 }
66
68 LU(Context& ctx, const DeviceMatrix<Scalar>& d_A) : solver_ctx_(ctx) { compute(d_A); }
69
71 LU(Context& ctx, DeviceMatrix<Scalar>&& d_A) : solver_ctx_(ctx) { compute(std::move(d_A)); }
72
73 ~LU() = default;
74
75 LU(const LU&) = delete;
76 LU& operator=(const LU&) = delete;
77
78 LU(LU&& o) noexcept
79 : solver_ctx_(std::move(o.solver_ctx_)),
80 d_lu_(std::move(o.d_lu_)),
81 d_ipiv_(std::move(o.d_ipiv_)),
82 n_(o.n_),
83 lda_(o.lda_) {
84 o.n_ = 0;
85 o.lda_ = 0;
86 }
87
88 LU& operator=(LU&& o) noexcept {
89 if (this != &o) {
90 solver_ctx_ = std::move(o.solver_ctx_);
91 d_lu_ = std::move(o.d_lu_);
92 d_ipiv_ = std::move(o.d_ipiv_);
93 n_ = o.n_;
94 lda_ = o.lda_;
95 o.n_ = 0;
96 o.lda_ = 0;
97 }
98 return *this;
99 }
100
103 template <typename InputType>
105 eigen_assert(A.rows() == A.cols() && "LU requires a square matrix");
106 if (!begin_compute(A.rows())) return *this;
107
108 // Ref binds column-major direct-access input in place (no host copy);
109 // row-major layouts and expressions evaluate into its temporary.
110 const Ref<const PlainMatrix> mat(A.derived());
111 lda_ = static_cast<int64_t>(mat.rows());
112 allocate_lu_storage();
113 internal::upload_host_matrix(static_cast<Scalar*>(d_lu_.get()), mat.rows(), mat.data(), mat.outerStride(),
114 mat.rows(), mat.cols(), solver_ctx_.stream());
115 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
116
117 factorize();
118 return *this;
119 }
120
123 eigen_assert(d_A.rows() == d_A.cols() && "LU requires a square matrix");
124 if (!begin_compute(d_A.rows())) return *this;
125
126 lda_ = static_cast<int64_t>(d_A.rows());
127 d_A.waitReady(solver_ctx_.stream());
128 allocate_lu_storage();
129 EIGEN_CUDA_RUNTIME_CHECK(
130 cudaMemcpyAsync(d_lu_.get(), d_A.data(), matrixBytes(), cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
131
132 factorize();
133 return *this;
134 }
135
139 if (d_A.isView()) return compute(static_cast<const DeviceMatrix<Scalar>&>(d_A));
140 eigen_assert(d_A.rows() == d_A.cols() && "LU requires a square matrix");
141 if (!begin_compute(d_A.rows())) return *this;
142
143 lda_ = static_cast<int64_t>(d_A.rows());
144 d_A.waitReady(solver_ctx_.stream());
145 d_lu_ = internal::DeviceBuffer::adopt(static_cast<void*>(d_A.release()), matrixBytes());
146
147 factorize();
148 return *this;
149 }
150
156 template <typename Rhs>
157 PlainMatrix solve(const MatrixBase<Rhs>& B, GpuOp op = GpuOp::NoTrans) const {
158 // Debug builds verify the factorization (info() synchronizes the stream on
159 // the first call after compute()); release builds skip both the check and
160 // the sync — use info() explicitly when failure must be detected.
161 eigen_assert(solver_ctx_.info() == Success && "LU::solve called on a failed or uninitialized factorization");
162 eigen_assert(B.rows() == n_);
163 if (n_ == 0) return PlainMatrix(0, B.cols());
164
165 const Ref<const PlainMatrix> rhs(B.derived());
166 const int64_t nrhs = static_cast<int64_t>(rhs.cols());
167 const int64_t ldb = static_cast<int64_t>(rhs.rows());
168 internal::DeviceBuffer d_x(matrixBytes(nrhs, ldb));
169 internal::upload_host_matrix(static_cast<Scalar*>(d_x.get()), ldb, rhs.data(), rhs.outerStride(), rhs.rows(),
170 rhs.cols(), solver_ctx_.stream());
171 DeviceMatrix<Scalar> d_X = solve_impl(nrhs, ldb, op, std::move(d_x));
172
173 PlainMatrix X(n_, B.cols());
174 int solve_info = 0;
175 EIGEN_CUDA_RUNTIME_CHECK(
176 cudaMemcpyAsync(X.data(), d_X.data(), matrixBytes(nrhs, ldb), cudaMemcpyDeviceToHost, solver_ctx_.stream()));
177 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(&solve_info, solver_ctx_.scratch_info(), sizeof(int),
178 cudaMemcpyDeviceToHost, solver_ctx_.stream()));
179 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
180
181 eigen_assert(solve_info == 0 && "cusolverDnXgetrs reported an error");
182 return X;
183 }
184
190 DeviceMatrix<Scalar> solve(const DeviceMatrix<Scalar>& d_B, GpuOp op = GpuOp::NoTrans) const {
191 eigen_assert(solver_ctx_.info() == Success && "LU::solve called on a failed or uninitialized factorization");
192 eigen_assert(d_B.rows() == n_);
193 d_B.waitReady(solver_ctx_.stream());
194 const int64_t nrhs = static_cast<int64_t>(d_B.cols());
195 const int64_t ldb = static_cast<int64_t>(d_B.rows());
196 internal::DeviceBuffer d_x(matrixBytes(nrhs, ldb));
197 EIGEN_CUDA_RUNTIME_CHECK(
198 cudaMemcpyAsync(d_x.get(), d_B.data(), matrixBytes(nrhs, ldb), cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
199 return solve_impl(nrhs, ldb, op, std::move(d_x));
200 }
201
205 DeviceMatrix<Scalar> solve(DeviceMatrix<Scalar>&& d_B, GpuOp op = GpuOp::NoTrans) const {
206 if (d_B.isView()) return solve(static_cast<const DeviceMatrix<Scalar>&>(d_B), op);
207 eigen_assert(solver_ctx_.info() == Success && "LU::solve called on a failed or uninitialized factorization");
208 eigen_assert(d_B.rows() == n_);
209 d_B.waitReady(solver_ctx_.stream());
210 const int64_t nrhs = static_cast<int64_t>(d_B.cols());
211 const int64_t ldb = static_cast<int64_t>(d_B.rows());
213 internal::DeviceBuffer::adopt(static_cast<void*>(d_B.release()), matrixBytes(nrhs, ldb));
214 return solve_impl(nrhs, ldb, op, std::move(d_x));
215 }
216
217 ComputationInfo info() const { return solver_ctx_.info(); }
218 Index rows() const { return n_; }
219 Index cols() const { return n_; }
220 cudaStream_t stream() const { return solver_ctx_.stream(); }
221
222 private:
223 mutable internal::GpuSolverContext solver_ctx_;
224 internal::DeviceBuffer d_lu_; // adopted from an rvalue input, else grow-only
225 internal::DeviceBuffer d_ipiv_; // grow-only
226 int64_t n_ = 0;
227 int64_t lda_ = 0;
228
229 bool begin_compute(Index rows) {
230 n_ = rows;
231 return solver_ctx_.begin_compute(n_ != 0);
232 }
233
234 size_t matrixBytes() const { return matrixBytes(n_, lda_); }
235
236 static size_t matrixBytes(int64_t cols, int64_t ld) {
237 return static_cast<size_t>(ld) * static_cast<size_t>(cols) * sizeof(Scalar);
238 }
239
240 void allocate_lu_storage() { internal::ensure_sized(d_lu_, matrixBytes()); }
241
242 // Solve in place on `d_x` (which already holds B), then re-wrap as a typed
243 // DeviceMatrix carrying shape and a ready event. The release/adopt hop hands
244 // ownership of the raw cudaMalloc pointer from the untyped DeviceBuffer to
245 // the typed DeviceMatrix without copying.
246 DeviceMatrix<Scalar> solve_impl(int64_t nrhs, int64_t ldb, GpuOp op, internal::DeviceBuffer&& d_x) const {
247 constexpr cudaDataType_t dtype = internal::cusolver_data_type<Scalar>::value;
248 const cublasOperation_t trans = internal::to_cublas_op(op);
249
250 // cuSOLVER rejects the empty problem's leading dimensions.
251 if (n_ > 0)
252 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrs(solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, trans, n_, nrhs, dtype,
253 d_lu_.get(), lda_, static_cast<const int64_t*>(d_ipiv_.get()), dtype,
254 d_x.get(), ldb, solver_ctx_.scratch_info()));
255
256 DeviceMatrix<Scalar> result =
257 DeviceMatrix<Scalar>::adopt(static_cast<Scalar*>(d_x.release()), n_, static_cast<Index>(nrhs));
258 result.recordReady(solver_ctx_.stream());
259 return result;
260 }
261
262 void factorize() {
263 constexpr cudaDataType_t dtype = internal::cusolver_data_type<Scalar>::value;
264 const size_t ipiv_bytes = static_cast<size_t>(n_) * sizeof(int64_t);
265
266 solver_ctx_.mark_pending();
267
268 internal::ensure_sized(d_ipiv_, ipiv_bytes);
269
270 size_t dev_ws_bytes = 0, host_ws_bytes = 0;
271 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrf_bufferSize(solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, n_, n_, dtype,
272 d_lu_.get(), lda_, dtype, &dev_ws_bytes, &host_ws_bytes));
273
274 solver_ctx_.ensure_scratch(dev_ws_bytes);
275 solver_ctx_.h_workspace_.resize(host_ws_bytes);
276
277 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrf(
278 solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, n_, n_, dtype, d_lu_.get(), lda_,
279 static_cast<int64_t*>(d_ipiv_.get()), dtype, solver_ctx_.scratch_workspace(), dev_ws_bytes,
280 host_ws_bytes > 0 ? solver_ctx_.h_workspace_.data() : nullptr, host_ws_bytes, solver_ctx_.scratch_info()));
281
282 solver_ctx_.enqueue_info_copy();
283 }
284};
285} // namespace gpu
286} // namespace Eigen
287
288#endif // EIGEN_GPU_LU_H
constexpr Scalar * data()
Unified GPU execution context owning a CUDA stream and library handles.
Definition GpuContext.h:81
RAII wrapper for a dense column-major matrix in GPU device memory.
Definition DeviceMatrix.h:122
static DeviceMatrix adopt(Scalar *device_ptr, Index rows, Index cols)
Definition DeviceMatrix.h:562
void waitReady(cudaStream_t stream) const
Definition DeviceMatrix.h:374
GPU LU decomposition with partial pivoting via cuSOLVER.
Definition GpuLU.h:37
LU & compute(const DenseBase< InputType > &A)
Definition GpuLU.h:104
LU & compute(DeviceMatrix< Scalar > &&d_A)
Definition GpuLU.h:138
LU & compute(const DeviceMatrix< Scalar > &d_A)
Definition GpuLU.h:122
PlainMatrix solve(const MatrixBase< Rhs > &B, GpuOp op=GpuOp::NoTrans) const
Definition GpuLU.h:157
LU(const DeviceMatrix< Scalar > &d_A)
Definition GpuLU.h:56
LU(Context &ctx)
Definition GpuLU.h:48
LU(Context &ctx, const DenseBase< InputType > &A)
Definition GpuLU.h:63
LU(DeviceMatrix< Scalar > &&d_A)
Definition GpuLU.h:59
LU(Context &ctx, DeviceMatrix< Scalar > &&d_A)
Definition GpuLU.h:71
LU(Context &ctx, const DeviceMatrix< Scalar > &d_A)
Definition GpuLU.h:68
DeviceMatrix< Scalar > solve(DeviceMatrix< Scalar > &&d_B, GpuOp op=GpuOp::NoTrans) const
Definition GpuLU.h:205
DeviceMatrix< Scalar > solve(const DeviceMatrix< Scalar > &d_B, GpuOp op=GpuOp::NoTrans) const
Definition GpuLU.h:190
Internal RAII owner for an untyped GPU device allocation.
Definition GpuSupport.h:296
ComputationInfo
Namespace containing all symbols from the Eigen library.