17#ifndef EIGEN_GPU_SVD_H
18#define EIGEN_GPU_SVD_H
21#include "./InternalHeaderCheck.h"
23#include "./GpuSolverContext.h"
27template <
typename Scalar_>
30 using Scalar = Scalar_;
31 using RealScalar =
typename NumTraits<Scalar>::Real;
32 using PlainMatrix = Eigen::Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
33 using PlainVector = Eigen::Matrix<Scalar, Dynamic, 1>;
34 using RealVector = Eigen::Matrix<RealScalar, Dynamic, 1>;
41 explicit SVD(Context& ctx) : solver_ctx_(ctx) {}
43 template <
typename InputType>
49 compute(d_A, options);
54 compute(std::move(d_A), options);
58 template <
typename InputType>
67 compute(d_A, options);
71 SVD(Context& ctx, DeviceMatrix<Scalar>&& d_A,
unsigned int options =
ComputeThinU |
ComputeThinV) : solver_ctx_(ctx) {
72 compute(std::move(d_A), options);
77 SVD(
const SVD&) =
delete;
78 SVD& operator=(
const SVD&) =
delete;
81 : solver_ctx_(std::move(o.solver_ctx_)),
82 d_A_(std::move(o.d_A_)),
83 d_U_(std::move(o.d_U_)),
84 d_S_(std::move(o.d_S_)),
85 d_VT_(std::move(o.d_VT_)),
86 d_D_(std::move(o.d_D_)),
87 cached_diag_kk_(o.cached_diag_kk_),
88 cached_diag_lambda_(o.cached_diag_lambda_),
89 diag_valid_(o.diag_valid_),
94 transposed_(o.transposed_) {
95 o.diag_valid_ =
false;
100 o.transposed_ =
false;
103 SVD& operator=(SVD&& o)
noexcept {
105 solver_ctx_ = std::move(o.solver_ctx_);
106 d_A_ = std::move(o.d_A_);
107 d_U_ = std::move(o.d_U_);
108 d_S_ = std::move(o.d_S_);
109 d_VT_ = std::move(o.d_VT_);
110 d_D_ = std::move(o.d_D_);
111 cached_diag_kk_ = o.cached_diag_kk_;
112 cached_diag_lambda_ = o.cached_diag_lambda_;
113 diag_valid_ = o.diag_valid_;
114 options_ = o.options_;
118 transposed_ = o.transposed_;
119 o.diag_valid_ =
false;
124 o.transposed_ =
false;
129 template <
typename InputType>
139 if (!begin_compute(d_A, options))
return *
this;
142 transpose_into_input(d_A);
144 const size_t mat_bytes =
static_cast<size_t>(lda_) *
static_cast<size_t>(n_) *
sizeof(Scalar);
145 d_A_ = internal::DeviceBuffer(mat_bytes);
146 EIGEN_CUDA_RUNTIME_CHECK(
147 cudaMemcpyAsync(d_A_.get(), d_A.data(), mat_bytes, cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
158 if (d_A.isView())
return compute(
static_cast<const DeviceMatrix<Scalar>&
>(d_A), options);
159 if (!begin_compute(d_A, options))
return *
this;
162 transpose_into_input(d_A);
164 const size_t a_bytes = d_A.sizeInBytes();
165 d_A_ = internal::DeviceBuffer::adopt(
static_cast<void*
>(d_A.release()), a_bytes);
174 Index rows()
const {
return transposed_ ? n_ : m_; }
175 Index cols()
const {
return transposed_ ? m_ : n_; }
178 RealVector singularValues()
const {
179 eigen_assert(solver_ctx_.info() ==
Success);
180 const Index k = (std::min)(m_, n_);
182 solver_ctx_.download(S.data(), d_S_.get(),
static_cast<size_t>(k) *
sizeof(RealScalar));
187 PlainMatrix matrixU()
const {
188 eigen_assert(solver_ctx_.info() ==
Success);
190 const Index m_orig = transposed_ ? n_ : m_;
191 const Index n_orig = transposed_ ? m_ : n_;
192 const Index k = (std::min)(m_orig, n_orig);
196 if (m_ == 0 || n_ == 0)
return PlainMatrix::Identity(m_, ucols);
197 PlainMatrix U(m_, ucols);
198 solver_ctx_.download(U.data(), d_U_.get(),
static_cast<size_t>(m_) *
static_cast<size_t>(ucols) *
sizeof(Scalar));
201 const Index vtrows = (options_ &
ComputeFullU) ? m_orig : k;
202 PlainMatrix VT_stored(vtrows, n_);
203 solver_ctx_.download(VT_stored.data(), d_VT_.get(),
204 static_cast<size_t>(vtrows) *
static_cast<size_t>(n_) *
sizeof(Scalar));
205 return VT_stored.adjoint();
210 PlainMatrix matrixV()
const {
return matrixVT().adjoint(); }
213 PlainMatrix matrixVT()
const {
214 eigen_assert(solver_ctx_.info() ==
Success);
216 const Index m_orig = transposed_ ? n_ : m_;
217 const Index n_orig = transposed_ ? m_ : n_;
218 const Index k = (std::min)(m_orig, n_orig);
220 const Index vtrows = (options_ &
ComputeFullV) ? n_ : k;
221 if (m_ == 0 || n_ == 0)
return PlainMatrix::Identity(vtrows, n_);
222 PlainMatrix VT(vtrows, n_);
223 solver_ctx_.download(VT.data(), d_VT_.get(),
224 static_cast<size_t>(vtrows) *
static_cast<size_t>(n_) *
sizeof(Scalar));
227 const Index ucols = (options_ &
ComputeFullV) ? n_orig : k;
228 PlainMatrix U_stored(m_, ucols);
229 solver_ctx_.download(U_stored.data(), d_U_.get(),
230 static_cast<size_t>(m_) *
static_cast<size_t>(ucols) *
sizeof(Scalar));
231 return U_stored.adjoint();
246 DeviceMatrix<RealScalar> d_singularValues()
const {
247 eigen_assert(solver_ctx_.info() ==
Success);
248 const Index k = (std::min)(m_, n_);
250 v.recordReady(solver_ctx_.stream());
256 DeviceMatrix<Scalar> d_matrixU()
const {
257 eigen_assert(solver_ctx_.info() ==
Success);
259 const Index m_orig = transposed_ ? n_ : m_;
260 const Index n_orig = transposed_ ? m_ : n_;
261 const Index k = (std::min)(m_orig, n_orig);
264 if (m_ == 0 || n_ == 0) {
268 v.recordReady(solver_ctx_.stream());
272 const Index vtrows_stored = (options_ &
ComputeFullU) ? n_ : k;
273 DeviceMatrix<Scalar> result(n_, vtrows_stored);
274 if (n_ > 0 && vtrows_stored > 0) {
275 Scalar alpha_one(1), beta_zero(0);
276 EIGEN_CUBLAS_CHECK(internal::cublasXgeam(solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, n_, vtrows_stored,
277 &alpha_one,
static_cast<const Scalar*
>(d_VT_.get()), vtrows_stored,
278 &beta_zero,
static_cast<const Scalar*
>(
nullptr), n_, result.data(), n_));
279 result.recordReady(solver_ctx_.stream());
286 DeviceMatrix<Scalar> d_matrixVT()
const {
287 eigen_assert(solver_ctx_.info() ==
Success);
289 const Index m_orig = transposed_ ? n_ : m_;
290 const Index n_orig = transposed_ ? m_ : n_;
291 const Index k = (std::min)(m_orig, n_orig);
293 const Index vtrows = (options_ &
ComputeFullV) ? n_ : k;
294 if (m_ == 0 || n_ == 0) {
298 v.recordReady(solver_ctx_.stream());
302 const Index ucols = (options_ &
ComputeFullV) ? n_orig : k;
303 DeviceMatrix<Scalar> result(ucols, m_);
304 if (ucols > 0 && m_ > 0) {
305 Scalar alpha_one(1), beta_zero(0);
306 EIGEN_CUBLAS_CHECK(internal::cublasXgeam(solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, ucols, m_,
307 &alpha_one,
static_cast<const Scalar*
>(d_U_.get()), m_, &beta_zero,
308 static_cast<const Scalar*
>(
nullptr), ucols, result.data(), ucols));
309 result.recordReady(solver_ctx_.stream());
315 Index rank(RealScalar threshold = RealScalar(-1))
const {
316 RealVector S = singularValues();
317 if (S.size() == 0)
return 0;
319 threshold = (std::max)(m_, n_) * S(0) * NumTraits<RealScalar>::epsilon();
321 return (S.array() > threshold).count();
325 template <
typename Rhs>
326 PlainMatrix solve(
const MatrixBase<Rhs>& B)
const {
327 return solve_impl(B, (std::min)(m_, n_), RealScalar(0));
334 template <typename Rhs, typename Trunc, std::enable_if_t<std::is_integral<Trunc>::value,
int> = 0>
335 PlainMatrix solve(
const MatrixBase<Rhs>& B, Trunc trunc)
const {
336 eigen_assert(trunc > 0 && trunc <= (std::min)(m_, n_));
337 return solve_impl(B,
static_cast<Index
>(trunc), RealScalar(0));
341 template <typename Rhs, typename Lambda, std::enable_if_t<std::is_floating_point<Lambda>::value,
int> = 0>
342 PlainMatrix solve(
const MatrixBase<Rhs>& B, Lambda lambda)
const {
343 eigen_assert(lambda > 0);
344 return solve_impl(B, (std::min)(m_, n_),
static_cast<RealScalar
>(lambda));
352 DeviceMatrix<Scalar> solve(
const DeviceMatrix<Scalar>& d_B)
const {
353 return solve_device_impl(d_B, (std::min)(m_, n_), RealScalar(0));
357 template <typename Trunc, std::enable_if_t<std::is_integral<Trunc>::value,
int> = 0>
358 DeviceMatrix<Scalar> solve(
const DeviceMatrix<Scalar>& d_B, Trunc trunc)
const {
359 eigen_assert(trunc > 0 && trunc <= (std::min)(m_, n_));
360 return solve_device_impl(d_B,
static_cast<Index
>(trunc), RealScalar(0));
364 template <typename Lambda, std::enable_if_t<std::is_floating_point<Lambda>::value,
int> = 0>
365 DeviceMatrix<Scalar> solve(
const DeviceMatrix<Scalar>& d_B, Lambda lambda)
const {
366 eigen_assert(lambda > 0);
367 return solve_device_impl(d_B, (std::min)(m_, n_),
static_cast<RealScalar
>(lambda));
370 cudaStream_t stream()
const {
return solver_ctx_.stream(); }
373 mutable internal::GpuSolverContext solver_ctx_;
374 internal::DeviceBuffer d_A_;
375 internal::DeviceBuffer d_U_;
376 internal::DeviceBuffer d_S_;
377 internal::DeviceBuffer d_VT_;
379 mutable internal::DeviceBuffer d_D_;
380 mutable Index cached_diag_kk_ = -1;
381 mutable RealScalar cached_diag_lambda_ = RealScalar(-1);
382 mutable bool diag_valid_ =
false;
383 unsigned int options_ = 0;
387 bool transposed_ =
false;
391 bool begin_compute(
const DeviceMatrix<Scalar>& d_A,
unsigned int options) {
398 if (!solver_ctx_.begin_compute(m_ != 0 && n_ != 0)) {
399 d_A_ = internal::DeviceBuffer();
400 d_U_ = internal::DeviceBuffer();
401 d_S_ = internal::DeviceBuffer();
402 d_VT_ = internal::DeviceBuffer();
405 transposed_ = (m_ < n_);
410 lda_ =
static_cast<int64_t
>(d_A.rows());
412 d_A.waitReady(solver_ctx_.stream());
417 void transpose_into_input(
const DeviceMatrix<Scalar>& d_A) {
418 const size_t mat_bytes =
static_cast<size_t>(lda_) *
static_cast<size_t>(n_) *
sizeof(Scalar);
419 d_A_ = internal::DeviceBuffer(mat_bytes);
421 Scalar alpha_one(1), beta_zero(0);
422 EIGEN_CUBLAS_CHECK(internal::cublasXgeam(solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, m_, n_, &alpha_one,
423 d_A.data(), d_A.rows(), &beta_zero,
static_cast<const Scalar*
>(
nullptr),
424 m_,
static_cast<Scalar*
>(d_A_.get()), m_));
428 static unsigned int swap_uv_options(
unsigned int opts) {
429 unsigned int result = 0;
437 static signed char jobu(
unsigned int opts) {
443 static signed char jobvt(
unsigned int opts) {
450 constexpr cudaDataType_t dtype = internal::cusolver_data_type<Scalar>::value;
451 constexpr cudaDataType_t rtype = internal::cuda_data_type<RealScalar>::value;
452 const Index k = (std::min)(m_, n_);
454 solver_ctx_.mark_pending();
456 internal::ensure_sized(d_S_,
static_cast<size_t>(k) *
sizeof(RealScalar));
458 const unsigned int int_opts = transposed_ ? swap_uv_options(options_) : options_;
462 const int64_t ldu = m_;
463 const int64_t ldvt = vtrows > 0 ? vtrows : 1;
466 internal::ensure_sized(d_U_,
static_cast<size_t>(m_) *
static_cast<size_t>(ucols) *
sizeof(Scalar));
469 internal::ensure_sized(d_VT_,
static_cast<size_t>(vtrows) *
static_cast<size_t>(n_) *
sizeof(Scalar));
472 eigen_assert(m_ >= n_ &&
"Internal error: m_ < n_ should have been handled by transpose in compute()");
473 size_t dev_ws = 0, host_ws = 0;
474 EIGEN_CUSOLVER_CHECK(cusolverDnXgesvd_bufferSize(
475 solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, jobu(int_opts), jobvt(int_opts), m_, n_, dtype, d_A_.get(),
476 lda_, rtype, d_S_.get(), dtype, ucols > 0 ? d_U_.get() :
nullptr, ldu, dtype,
477 vtrows > 0 ? d_VT_.get() :
nullptr, ldvt, dtype, &dev_ws, &host_ws));
479 solver_ctx_.ensure_scratch(dev_ws);
480 solver_ctx_.h_workspace_.resize(host_ws);
482 EIGEN_CUSOLVER_CHECK(cusolverDnXgesvd(
483 solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, jobu(int_opts), jobvt(int_opts), m_, n_, dtype, d_A_.get(),
484 lda_, rtype, d_S_.get(), dtype, ucols > 0 ? d_U_.get() :
nullptr, ldu, dtype,
485 vtrows > 0 ? d_VT_.get() :
nullptr, ldvt, dtype, solver_ctx_.scratch_workspace(), dev_ws,
486 host_ws > 0 ? solver_ctx_.h_workspace_.data() :
nullptr, host_ws, solver_ctx_.scratch_info()));
488 solver_ctx_.enqueue_info_copy();
494 d_A_ = internal::DeviceBuffer();
507 void build_diag(Index kk, RealScalar lambda)
const {
508 if (diag_valid_ && cached_diag_kk_ == kk && cached_diag_lambda_ == lambda)
return;
510 const Index k = (std::min)(m_, n_);
512 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(S.data(), d_S_.get(),
static_cast<size_t>(k) *
sizeof(RealScalar),
513 cudaMemcpyDeviceToHost, solver_ctx_.stream()));
514 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
516 const RealScalar drop_threshold = S(0) * RealScalar(k) * NumTraits<RealScalar>::epsilon();
517 auto S_head = S.head(kk).array();
519 if (lambda == RealScalar(0)) {
520 D = (S_head > drop_threshold).select(S_head.inverse(), RealScalar(0)).matrix().template cast<Scalar>();
522 D = (S_head / (S_head.square() + lambda * lambda)).matrix().template cast<Scalar>();
525 const size_t d_bytes =
static_cast<size_t>(kk) *
sizeof(Scalar);
526 internal::ensure_sized(d_D_, d_bytes);
527 EIGEN_CUDA_RUNTIME_CHECK(
528 cudaMemcpyAsync(d_D_.get(), D.data(), d_bytes, cudaMemcpyHostToDevice, solver_ctx_.stream()));
529 cached_diag_kk_ = kk;
530 cached_diag_lambda_ = lambda;
537 void apply_pinv(
const Scalar* B_dev, Index kk, Index nrhs, Scalar* X_dev)
const {
538 const Index m_orig = transposed_ ? n_ : m_;
539 const Index n_orig = transposed_ ? m_ : n_;
540 const Index k = (std::min)(m_, n_);
542 auto* U_dev =
static_cast<const Scalar*
>(d_U_.get());
543 auto* VT_dev =
static_cast<const Scalar*
>(d_VT_.get());
545 Scalar scalars[2] = {Scalar(1), Scalar(0)};
548 internal::DeviceBuffer d_tmp(
static_cast<size_t>(kk) *
static_cast<size_t>(nrhs) *
sizeof(Scalar));
549 auto* tmp_dev =
static_cast<Scalar*
>(d_tmp.get());
551 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, kk,
552 nrhs, m_, &scalars[0], U_dev, m_, B_dev, m_orig, &scalars[1], tmp_dev, kk,
553 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
554 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
556 const Index vtrows_stored = (swap_uv_options(options_) &
ComputeFullV) ? n_ : k;
557 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N, kk,
558 nrhs, m_orig, &scalars[0], VT_dev, vtrows_stored, B_dev, m_orig, &scalars[1], tmp_dev, kk,
559 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
560 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
564 EIGEN_CUBLAS_CHECK(internal::cublasXdgmm(solver_ctx_.cublasHandle(), CUBLAS_SIDE_LEFT, kk, nrhs, tmp_dev, kk,
565 static_cast<const Scalar*
>(d_D_.get()), 1, tmp_dev, kk));
569 const Index vtrows = (options_ &
ComputeFullV) ? n_ : k;
570 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N,
571 n_orig, nrhs, kk, &scalars[0], VT_dev, vtrows, tmp_dev, kk, &scalars[1], X_dev, n_orig,
572 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
573 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
575 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N,
576 n_orig, nrhs, kk, &scalars[0], U_dev, m_, tmp_dev, kk, &scalars[1], X_dev, n_orig,
577 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
578 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
582 template <
typename Rhs>
583 PlainMatrix solve_impl(
const MatrixBase<Rhs>& B, Index trunc, RealScalar lambda)
const {
584 eigen_assert(solver_ctx_.info() ==
Success &&
"SVD::solve called on a failed or uninitialized decomposition");
588 const Index m_orig = transposed_ ? n_ : m_;
589 const Index n_orig = transposed_ ? m_ : n_;
590 eigen_assert(B.rows() == m_orig);
592 const Index k = (std::min)(m_, n_);
593 const Index kk = (std::min)(trunc, k);
594 const Index nrhs = B.cols();
597 if (kk == 0 || nrhs == 0 || n_orig == 0) {
598 return PlainMatrix::Zero(n_orig, nrhs);
604 const Ref<const PlainMatrix> rhs(B.derived());
605 internal::DeviceBuffer d_B(
static_cast<size_t>(m_orig) *
static_cast<size_t>(nrhs) *
sizeof(Scalar));
606 internal::upload_host_matrix(
static_cast<Scalar*
>(d_B.get()), m_orig, rhs.data(), rhs.outerStride(), m_orig, nrhs,
607 solver_ctx_.stream());
608 build_diag(kk, lambda);
610 PlainMatrix X(n_orig, nrhs);
611 internal::DeviceBuffer d_X(
static_cast<size_t>(n_orig) *
static_cast<size_t>(nrhs) *
sizeof(Scalar));
612 apply_pinv(
static_cast<const Scalar*
>(d_B.get()), kk, nrhs,
static_cast<Scalar*
>(d_X.get()));
614 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(X.data(), d_X.get(),
615 static_cast<size_t>(n_orig) *
static_cast<size_t>(nrhs) *
sizeof(Scalar),
616 cudaMemcpyDeviceToHost, solver_ctx_.stream()));
617 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
622 DeviceMatrix<Scalar> solve_device_impl(
const DeviceMatrix<Scalar>& d_B, Index trunc, RealScalar lambda)
const {
623 eigen_assert(solver_ctx_.info() ==
Success &&
"SVD::solve called on a failed or uninitialized decomposition");
627 const Index m_orig = transposed_ ? n_ : m_;
628 const Index n_orig = transposed_ ? m_ : n_;
629 eigen_assert(d_B.rows() == m_orig);
630 EIGEN_UNUSED_VARIABLE(m_orig);
632 const Index k = (std::min)(m_, n_);
633 const Index kk = (std::min)(trunc, k);
634 const Index nrhs = d_B.cols();
636 if (kk == 0 || nrhs == 0 || n_orig == 0) {
637 DeviceMatrix<Scalar> X(n_orig, nrhs);
638 X.setZero(solver_ctx_.stream());
642 d_B.waitReady(solver_ctx_.stream());
643 build_diag(kk, lambda);
645 DeviceMatrix<Scalar> X(n_orig, nrhs);
646 apply_pinv(d_B.data(), kk, nrhs, X.data());
647 X.recordReady(solver_ctx_.stream());
static DeviceMatrix fromHost(const DenseBase< Derived > &host, cudaStream_t stream=nullptr)
Definition DeviceMatrix.h:226
static DeviceMatrix view(Scalar *device_ptr, Index rows, Index cols)
Definition DeviceMatrix.h:578
Namespace containing all symbols from the Eigen library.