23#include "./InternalHeaderCheck.h"
25#include "./GpuSolverContext.h"
29template <
typename Scalar_>
32 using Scalar = Scalar_;
33 using RealScalar =
typename NumTraits<Scalar>::Real;
34 using PlainMatrix = Eigen::Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
41 explicit QR(Context& ctx) : solver_ctx_(ctx) {}
43 template <
typename InputType>
44 explicit QR(
const DenseBase<InputType>& A) {
48 explicit QR(
const DeviceMatrix<Scalar>& d_A) { compute(d_A); }
51 explicit QR(DeviceMatrix<Scalar>&& d_A) { compute(std::move(d_A)); }
54 template <
typename InputType>
55 QR(Context& ctx,
const DenseBase<InputType>& A) : solver_ctx_(ctx) {
60 QR(Context& ctx,
const DeviceMatrix<Scalar>& d_A) : solver_ctx_(ctx) { compute(d_A); }
63 QR(Context& ctx, DeviceMatrix<Scalar>&& d_A) : solver_ctx_(ctx) { compute(std::move(d_A)); }
67 QR(
const QR&) =
delete;
68 QR& operator=(
const QR&) =
delete;
71 : solver_ctx_(std::move(o.solver_ctx_)),
72 d_qr_(std::move(o.d_qr_)),
73 d_tau_(std::move(o.d_tau_)),
77 transposed_(o.transposed_) {
81 o.transposed_ =
false;
84 QR& operator=(QR&& o)
noexcept {
86 solver_ctx_ = std::move(o.solver_ctx_);
87 d_qr_ = std::move(o.d_qr_);
88 d_tau_ = std::move(o.d_tau_);
92 transposed_ = o.transposed_;
96 o.transposed_ =
false;
101 template <
typename InputType>
102 QR& compute(
const DenseBase<InputType>& A) {
110 QR& compute(
const DeviceMatrix<Scalar>& d_A) {
111 if (!begin_compute(d_A))
return *
this;
114 transpose_into_factor(d_A);
116 const size_t mat_bytes = factorBytes();
117 allocate_factor_storage(mat_bytes);
118 EIGEN_CUDA_RUNTIME_CHECK(
119 cudaMemcpyAsync(d_qr_.get(), d_A.data(), mat_bytes, cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
129 QR& compute(DeviceMatrix<Scalar>&& d_A) {
130 if (d_A.isView())
return compute(
static_cast<const DeviceMatrix<Scalar>&
>(d_A));
131 if (!begin_compute(d_A))
return *
this;
134 transpose_into_factor(d_A);
136 d_qr_ = internal::DeviceBuffer::adopt(
static_cast<void*
>(d_A.release()), factorBytes());
146 template <
typename Rhs>
147 PlainMatrix solve(
const MatrixBase<Rhs>& B)
const {
151 eigen_assert(solver_ctx_.info() ==
Success &&
"QR::solve called on a failed or uninitialized factorization");
152 eigen_assert(B.rows() == m_);
154 if (m_ == 0 || n_ == 0)
return PlainMatrix::Zero(n_, B.cols());
156 const Ref<const PlainMatrix> rhs(B.derived());
157 const Index nrhs = rhs.cols();
160 return solve_overdetermined_host(rhs);
162 return solve_underdetermined_host(rhs, nrhs);
167 DeviceMatrix<Scalar> solve(
const DeviceMatrix<Scalar>& d_B)
const {
168 eigen_assert(solver_ctx_.info() ==
Success &&
"QR::solve called on a failed or uninitialized factorization");
169 eigen_assert(d_B.rows() == m_);
170 if (m_ == 0 || n_ == 0) {
171 DeviceMatrix<Scalar> X(n_, d_B.cols());
172 X.setZero(solver_ctx_.stream());
175 d_B.waitReady(solver_ctx_.stream());
178 return solve_overdetermined_device(d_B);
180 return solve_underdetermined_device(d_B);
185 Index rows()
const {
return m_; }
186 Index cols()
const {
return n_; }
187 cudaStream_t stream()
const {
return solver_ctx_.stream(); }
190 PlainMatrix matrixR()
const {
191 eigen_assert(solver_ctx_.info() ==
Success);
192 eigen_assert(!transposed_ &&
"matrixR() not available when m < n (we factored A^H internally)");
193 PlainMatrix qr_full(m_, n_);
194 if (m_ > 0 && n_ > 0) {
195 solver_ctx_.download(qr_full.data(), d_qr_.get(),
196 static_cast<size_t>(lda_) *
static_cast<size_t>(n_) *
sizeof(Scalar));
198 PlainMatrix R = qr_full.topRows(k()).template triangularView<Upper>();
203 mutable internal::GpuSolverContext solver_ctx_;
206 internal::DeviceBuffer d_qr_;
207 internal::DeviceBuffer d_tau_;
211 bool transposed_ =
false;
214 int64_t factor_rows()
const {
return transposed_ ? n_ : m_; }
215 int64_t factor_cols()
const {
return transposed_ ? m_ : n_; }
216 int64_t k()
const {
return (std::min)(m_, n_); }
218 size_t factorBytes()
const {
return static_cast<size_t>(lda_) *
static_cast<size_t>(factor_cols()) *
sizeof(Scalar); }
222 bool begin_compute(
const DeviceMatrix<Scalar>& d_A) {
225 transposed_ = (m_ < n_);
226 lda_ = (std::max)(int64_t(1),
static_cast<int64_t
>(transposed_ ? n_ : m_));
227 if (!solver_ctx_.begin_compute(m_ != 0 && n_ != 0)) {
228 d_qr_ = internal::DeviceBuffer();
229 d_tau_ = internal::DeviceBuffer();
232 d_A.waitReady(solver_ctx_.stream());
236 void allocate_factor_storage(
size_t mat_bytes) { internal::ensure_sized(d_qr_, mat_bytes); }
239 void transpose_into_factor(
const DeviceMatrix<Scalar>& d_A) {
240 allocate_factor_storage(factorBytes());
241 Scalar alpha_one(1), beta_zero(0);
242 EIGEN_CUBLAS_CHECK(internal::cublasXgeam(solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, n_, m_, &alpha_one,
243 d_A.data(), d_A.rows(), &beta_zero,
static_cast<const Scalar*
>(
nullptr),
244 n_,
static_cast<Scalar*
>(d_qr_.get()), n_));
248 constexpr cudaDataType_t dtype = internal::cusolver_data_type<Scalar>::value;
250 solver_ctx_.mark_pending();
252 internal::ensure_sized(d_tau_,
static_cast<size_t>(k()) *
sizeof(Scalar));
254 const int64_t fm = factor_rows();
255 const int64_t fn = factor_cols();
256 size_t dev_ws = 0, host_ws = 0;
257 EIGEN_CUSOLVER_CHECK(cusolverDnXgeqrf_bufferSize(solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, fm, fn, dtype,
258 d_qr_.get(), lda_, dtype, d_tau_.get(), dtype, &dev_ws, &host_ws));
260 solver_ctx_.ensure_scratch(dev_ws);
261 solver_ctx_.h_workspace_.resize(host_ws);
263 EIGEN_CUSOLVER_CHECK(
264 cusolverDnXgeqrf(solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, fm, fn, dtype, d_qr_.get(), lda_, dtype,
265 d_tau_.get(), dtype, solver_ctx_.scratch_workspace(), dev_ws,
266 host_ws > 0 ? solver_ctx_.h_workspace_.data() :
nullptr, host_ws, solver_ctx_.scratch_info()));
268 solver_ctx_.enqueue_info_copy();
274 void apply_Q(cublasOperation_t op,
void* d_B, int64_t ldb, int64_t nrhs)
const {
275 const int im = internal::to_blas_int(factor_rows());
276 const int in = internal::to_blas_int(nrhs);
277 const int ik = internal::to_blas_int(k());
278 const int ilda = internal::to_blas_int(lda_);
279 const int ildb = internal::to_blas_int(ldb);
282 EIGEN_CUSOLVER_CHECK(internal::cusolverDnXormqr_bufferSize(
283 solver_ctx_.cusolverHandle(), CUBLAS_SIDE_LEFT, op, im, in, ik,
static_cast<const Scalar*
>(d_qr_.get()), ilda,
284 static_cast<const Scalar*
>(d_tau_.get()),
static_cast<const Scalar*
>(d_B), ildb, &lwork));
286 solver_ctx_.ensure_scratch(
static_cast<size_t>(lwork) *
sizeof(Scalar));
288 EIGEN_CUSOLVER_CHECK(internal::cusolverDnXormqr(
289 solver_ctx_.cusolverHandle(), CUBLAS_SIDE_LEFT, op, im, in, ik,
static_cast<const Scalar*
>(d_qr_.get()), ilda,
290 static_cast<const Scalar*
>(d_tau_.get()),
static_cast<Scalar*
>(d_B), ildb,
291 static_cast<Scalar*
>(solver_ctx_.scratch_workspace()), lwork, solver_ctx_.scratch_info()));
294 void apply_QH(
void* d_B, int64_t ldb, int64_t nrhs)
const {
295 constexpr cublasOperation_t trans = NumTraits<Scalar>::IsComplex ? CUBLAS_OP_C : CUBLAS_OP_T;
296 apply_Q(trans, d_B, ldb, nrhs);
299 PlainMatrix solve_overdetermined_host(
const Ref<const PlainMatrix>& rhs)
const {
300 const Index nrhs = rhs.cols();
301 const size_t b_bytes =
static_cast<size_t>(m_) *
static_cast<size_t>(nrhs) *
sizeof(Scalar);
303 internal::DeviceBuffer d_B(b_bytes);
304 internal::upload_host_matrix(
static_cast<Scalar*
>(d_B.get()), m_, rhs.data(), rhs.outerStride(), m_, nrhs,
305 solver_ctx_.stream());
307 apply_QH(d_B.get(), m_, nrhs);
308 trsm_R(d_B.get(), m_, nrhs, CUBLAS_OP_N);
310 PlainMatrix X(n_, nrhs);
312 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(X.data(), d_B.get(),
313 static_cast<size_t>(n_) *
static_cast<size_t>(nrhs) *
sizeof(Scalar),
314 cudaMemcpyDeviceToHost, solver_ctx_.stream()));
316 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy2DAsync(X.data(),
static_cast<size_t>(n_) *
sizeof(Scalar), d_B.get(),
317 static_cast<size_t>(m_) *
sizeof(Scalar),
318 static_cast<size_t>(n_) *
sizeof(Scalar),
static_cast<size_t>(nrhs),
319 cudaMemcpyDeviceToHost, solver_ctx_.stream()));
321 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
325 DeviceMatrix<Scalar> solve_overdetermined_device(
const DeviceMatrix<Scalar>& d_B)
const {
326 const Index nrhs = d_B.cols();
327 const size_t b_bytes =
static_cast<size_t>(m_) *
static_cast<size_t>(nrhs) *
sizeof(Scalar);
329 internal::DeviceBuffer d_work(b_bytes);
330 EIGEN_CUDA_RUNTIME_CHECK(
331 cudaMemcpyAsync(d_work.get(), d_B.data(), b_bytes, cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
333 apply_QH(d_work.get(), m_, nrhs);
334 trsm_R(d_work.get(), m_, nrhs, CUBLAS_OP_N);
337 DeviceMatrix<Scalar> result =
339 result.recordReady(solver_ctx_.stream());
342 DeviceMatrix<Scalar> result(n_,
static_cast<Index
>(nrhs));
343 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy2DAsync(result.data(),
static_cast<size_t>(n_) *
sizeof(Scalar), d_work.get(),
344 static_cast<size_t>(m_) *
sizeof(Scalar),
345 static_cast<size_t>(n_) *
sizeof(Scalar),
static_cast<size_t>(nrhs),
346 cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
347 result.recordReady(solver_ctx_.stream());
356 PlainMatrix solve_underdetermined_host(
const Ref<const PlainMatrix>& rhs, Index nrhs)
const {
357 const size_t x_bytes =
static_cast<size_t>(n_) *
static_cast<size_t>(nrhs) *
sizeof(Scalar);
359 internal::DeviceBuffer d_X(x_bytes);
361 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(d_X.get(), 0, x_bytes, solver_ctx_.stream()));
364 internal::upload_host_matrix(
static_cast<Scalar*
>(d_X.get()), n_, rhs.data(), rhs.outerStride(), m_, nrhs,
365 solver_ctx_.stream());
367 trsm_R(d_X.get(), n_, nrhs, trsm_op_conj_trans());
368 apply_Q(CUBLAS_OP_N, d_X.get(), n_, nrhs);
370 PlainMatrix X(n_, nrhs);
371 EIGEN_CUDA_RUNTIME_CHECK(
372 cudaMemcpyAsync(X.data(), d_X.get(), x_bytes, cudaMemcpyDeviceToHost, solver_ctx_.stream()));
373 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
377 DeviceMatrix<Scalar> solve_underdetermined_device(
const DeviceMatrix<Scalar>& d_B)
const {
378 const Index nrhs = d_B.cols();
379 const size_t x_bytes =
static_cast<size_t>(n_) *
static_cast<size_t>(nrhs) *
sizeof(Scalar);
381 internal::DeviceBuffer d_X(x_bytes);
382 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(d_X.get(), 0, x_bytes, solver_ctx_.stream()));
384 if (m_ > 0 && nrhs > 0) {
385 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy2DAsync(d_X.get(),
static_cast<size_t>(n_) *
sizeof(Scalar), d_B.data(),
386 static_cast<size_t>(m_) *
sizeof(Scalar),
387 static_cast<size_t>(m_) *
sizeof(Scalar),
static_cast<size_t>(nrhs),
388 cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
391 trsm_R(d_X.get(), n_, nrhs, trsm_op_conj_trans());
392 apply_Q(CUBLAS_OP_N, d_X.get(), n_, nrhs);
394 DeviceMatrix<Scalar> result =
396 result.recordReady(solver_ctx_.stream());
400 static cublasOperation_t trsm_op_conj_trans() {
return NumTraits<Scalar>::IsComplex ? CUBLAS_OP_C : CUBLAS_OP_T; }
404 void trsm_R(
void* d_B, int64_t ldb, int64_t nrhs, cublasOperation_t op)
const {
406 EIGEN_CUBLAS_CHECK(internal::cublasXtrsm(
407 solver_ctx_.cublasHandle(), CUBLAS_SIDE_LEFT, CUBLAS_FILL_MODE_UPPER, op, CUBLAS_DIAG_NON_UNIT, k(), nrhs,
408 &alpha,
static_cast<const Scalar*
>(d_qr_.get()), lda_,
static_cast<Scalar*
>(d_B), ldb));
static DeviceMatrix fromHost(const DenseBase< Derived > &host, cudaStream_t stream=nullptr)
Definition DeviceMatrix.h:226
static DeviceMatrix adopt(Scalar *device_ptr, Index rows, Index cols)
Definition DeviceMatrix.h:562
Namespace containing all symbols from the Eigen library.