Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GpuSVD.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// GPU SVD using cuSOLVER's divide-and-conquer cusolverDnXgesvd. U, S, and VT
12// stay on device; solve() forms X = V * diag(D) * U^H * B with cuBLAS GEMM.
13//
14// cuSOLVER returns VT rather than V, so VT is what is stored; matrixV() adjoints
15// it to match JacobiSVD and BDCSVD.
16
17#ifndef EIGEN_GPU_SVD_H
18#define EIGEN_GPU_SVD_H
19
20// IWYU pragma: private
21#include "./InternalHeaderCheck.h"
22
23#include "./GpuSolverContext.h"
24
25namespace Eigen {
26namespace gpu {
27template <typename Scalar_>
28class SVD {
29 public:
30 using Scalar = Scalar_;
31 using RealScalar = typename NumTraits<Scalar>::Real;
32 using PlainMatrix = Eigen::Matrix<Scalar, Dynamic, Dynamic, ColMajor>;
33 using PlainVector = Eigen::Matrix<Scalar, Dynamic, 1>;
34 using RealVector = Eigen::Matrix<RealScalar, Dynamic, 1>;
35
36 SVD() = default;
37
41 explicit SVD(Context& ctx) : solver_ctx_(ctx) {}
42
43 template <typename InputType>
44 explicit SVD(const DenseBase<InputType>& A, unsigned int options = ComputeThinU | ComputeThinV) {
45 compute(A, options);
46 }
47
48 explicit SVD(const DeviceMatrix<Scalar>& d_A, unsigned int options = ComputeThinU | ComputeThinV) {
49 compute(d_A, options);
50 }
51
53 explicit SVD(DeviceMatrix<Scalar>&& d_A, unsigned int options = ComputeThinU | ComputeThinV) {
54 compute(std::move(d_A), options);
55 }
56
58 template <typename InputType>
59 SVD(Context& ctx, const DenseBase<InputType>& A, unsigned int options = ComputeThinU | ComputeThinV)
60 : solver_ctx_(ctx) {
61 compute(A, options);
62 }
63
65 SVD(Context& ctx, const DeviceMatrix<Scalar>& d_A, unsigned int options = ComputeThinU | ComputeThinV)
66 : solver_ctx_(ctx) {
67 compute(d_A, options);
68 }
69
71 SVD(Context& ctx, DeviceMatrix<Scalar>&& d_A, unsigned int options = ComputeThinU | ComputeThinV) : solver_ctx_(ctx) {
72 compute(std::move(d_A), options);
73 }
74
75 ~SVD() = default;
76
77 SVD(const SVD&) = delete;
78 SVD& operator=(const SVD&) = delete;
79
80 SVD(SVD&& o) noexcept
81 : solver_ctx_(std::move(o.solver_ctx_)),
82 d_A_(std::move(o.d_A_)),
83 d_U_(std::move(o.d_U_)),
84 d_S_(std::move(o.d_S_)),
85 d_VT_(std::move(o.d_VT_)),
86 d_D_(std::move(o.d_D_)),
87 cached_diag_kk_(o.cached_diag_kk_),
88 cached_diag_lambda_(o.cached_diag_lambda_),
89 diag_valid_(o.diag_valid_),
90 options_(o.options_),
91 m_(o.m_),
92 n_(o.n_),
93 lda_(o.lda_),
94 transposed_(o.transposed_) {
95 o.diag_valid_ = false;
96 o.options_ = 0;
97 o.m_ = 0;
98 o.n_ = 0;
99 o.lda_ = 0;
100 o.transposed_ = false;
101 }
102
103 SVD& operator=(SVD&& o) noexcept {
104 if (this != &o) {
105 solver_ctx_ = std::move(o.solver_ctx_);
106 d_A_ = std::move(o.d_A_);
107 d_U_ = std::move(o.d_U_);
108 d_S_ = std::move(o.d_S_);
109 d_VT_ = std::move(o.d_VT_);
110 d_D_ = std::move(o.d_D_);
111 cached_diag_kk_ = o.cached_diag_kk_;
112 cached_diag_lambda_ = o.cached_diag_lambda_;
113 diag_valid_ = o.diag_valid_;
114 options_ = o.options_;
115 m_ = o.m_;
116 n_ = o.n_;
117 lda_ = o.lda_;
118 transposed_ = o.transposed_;
119 o.diag_valid_ = false;
120 o.options_ = 0;
121 o.m_ = 0;
122 o.n_ = 0;
123 o.lda_ = 0;
124 o.transposed_ = false;
125 }
126 return *this;
127 }
128
129 template <typename InputType>
130 SVD& compute(const DenseBase<InputType>& A, unsigned int options = ComputeThinU | ComputeThinV) {
131 // Upload to device, then delegate to the adopting overload — the freshly
132 // uploaded matrix is consumed in place by gesvd, so no second device copy.
133 // The wide-matrix transpose runs on the GPU (via cublasXgeam) inside the
134 // device-input path; no host transpose.
135 return compute(DeviceMatrix<Scalar>::fromHost(A.derived(), solver_ctx_.stream()), options);
136 }
137
138 SVD& compute(const DeviceMatrix<Scalar>& d_A, unsigned int options = ComputeThinU | ComputeThinV) {
139 if (!begin_compute(d_A, options)) return *this;
140
141 if (transposed_) {
142 transpose_into_input(d_A);
143 } else {
144 const size_t mat_bytes = static_cast<size_t>(lda_) * static_cast<size_t>(n_) * sizeof(Scalar);
145 d_A_ = internal::DeviceBuffer(mat_bytes);
146 EIGEN_CUDA_RUNTIME_CHECK(
147 cudaMemcpyAsync(d_A_.get(), d_A.data(), mat_bytes, cudaMemcpyDeviceToDevice, solver_ctx_.stream()));
148 }
149
150 factorize();
151 return *this;
152 }
153
157 SVD& compute(DeviceMatrix<Scalar>&& d_A, unsigned int options = ComputeThinU | ComputeThinV) {
158 if (d_A.isView()) return compute(static_cast<const DeviceMatrix<Scalar>&>(d_A), options);
159 if (!begin_compute(d_A, options)) return *this;
160
161 if (transposed_) {
162 transpose_into_input(d_A);
163 } else {
164 const size_t a_bytes = d_A.sizeInBytes();
165 d_A_ = internal::DeviceBuffer::adopt(static_cast<void*>(d_A.release()), a_bytes);
166 }
167
168 factorize();
169 return *this;
170 }
171
172 ComputationInfo info() const { return solver_ctx_.info(); }
173
174 Index rows() const { return transposed_ ? n_ : m_; }
175 Index cols() const { return transposed_ ? m_ : n_; }
176
178 RealVector singularValues() const {
179 eigen_assert(solver_ctx_.info() == Success);
180 const Index k = (std::min)(m_, n_);
181 RealVector S(k);
182 solver_ctx_.download(S.data(), d_S_.get(), static_cast<size_t>(k) * sizeof(RealScalar));
183 return S;
184 }
185
187 PlainMatrix matrixU() const {
188 eigen_assert(solver_ctx_.info() == Success);
189 eigen_assert((options_ & (ComputeThinU | ComputeFullU)) && "matrixU() requires ComputeThinU or ComputeFullU");
190 const Index m_orig = transposed_ ? n_ : m_;
191 const Index n_orig = transposed_ ? m_ : n_;
192 const Index k = (std::min)(m_orig, n_orig);
193 if (!transposed_) {
194 const Index ucols = (options_ & ComputeFullU) ? m_ : k;
195 // An empty input runs no factorization, so there is no device buffer: the factors are identities.
196 if (m_ == 0 || n_ == 0) return PlainMatrix::Identity(m_, ucols);
197 PlainMatrix U(m_, ucols);
198 solver_ctx_.download(U.data(), d_U_.get(), static_cast<size_t>(m_) * static_cast<size_t>(ucols) * sizeof(Scalar));
199 return U;
200 } else {
201 const Index vtrows = (options_ & ComputeFullU) ? m_orig : k;
202 PlainMatrix VT_stored(vtrows, n_);
203 solver_ctx_.download(VT_stored.data(), d_VT_.get(),
204 static_cast<size_t>(vtrows) * static_cast<size_t>(n_) * sizeof(Scalar));
205 return VT_stored.adjoint();
206 }
207 }
208
210 PlainMatrix matrixV() const { return matrixVT().adjoint(); }
211
213 PlainMatrix matrixVT() const {
214 eigen_assert(solver_ctx_.info() == Success);
215 eigen_assert((options_ & (ComputeThinV | ComputeFullV)) && "matrixVT() requires ComputeThinV or ComputeFullV");
216 const Index m_orig = transposed_ ? n_ : m_;
217 const Index n_orig = transposed_ ? m_ : n_;
218 const Index k = (std::min)(m_orig, n_orig);
219 if (!transposed_) {
220 const Index vtrows = (options_ & ComputeFullV) ? n_ : k;
221 if (m_ == 0 || n_ == 0) return PlainMatrix::Identity(vtrows, n_);
222 PlainMatrix VT(vtrows, n_);
223 solver_ctx_.download(VT.data(), d_VT_.get(),
224 static_cast<size_t>(vtrows) * static_cast<size_t>(n_) * sizeof(Scalar));
225 return VT;
226 } else {
227 const Index ucols = (options_ & ComputeFullV) ? n_orig : k;
228 PlainMatrix U_stored(m_, ucols);
229 solver_ctx_.download(U_stored.data(), d_U_.get(),
230 static_cast<size_t>(m_) * static_cast<size_t>(ucols) * sizeof(Scalar));
231 return U_stored.adjoint();
232 }
233 }
234
235 //
236 // These return non-owning DeviceMatrix views over the SVD's internal device storage.
237 // The view borrows the pointer: destruction does not free; the SVD object must outlive
238 // any view derived from it. For the common case (m >= n) all three accessors are pure
239 // metadata: zero kernel launches, zero allocations.
240 //
241 // For wide matrices (m < n, internally factored as A^H), original U and V^T are the
242 // adjoints of the stored buffers, so d_matrixU() / d_matrixVT() build them via a
243 // cublasXgeam into an owning temporary. d_singularValues() remains zero-copy.
244
246 DeviceMatrix<RealScalar> d_singularValues() const {
247 eigen_assert(solver_ctx_.info() == Success);
248 const Index k = (std::min)(m_, n_);
249 auto v = DeviceMatrix<RealScalar>::view(static_cast<RealScalar*>(d_S_.get()), k, 1);
250 v.recordReady(solver_ctx_.stream());
251 return v;
252 }
253
256 DeviceMatrix<Scalar> d_matrixU() const {
257 eigen_assert(solver_ctx_.info() == Success);
258 eigen_assert((options_ & (ComputeThinU | ComputeFullU)) && "d_matrixU() requires ComputeThinU or ComputeFullU");
259 const Index m_orig = transposed_ ? n_ : m_;
260 const Index n_orig = transposed_ ? m_ : n_;
261 const Index k = (std::min)(m_orig, n_orig);
262 if (!transposed_) {
263 const Index ucols = (options_ & ComputeFullU) ? m_ : k;
264 if (m_ == 0 || n_ == 0) {
265 return DeviceMatrix<Scalar>::fromHost(PlainMatrix::Identity(m_, ucols), solver_ctx_.stream());
266 }
267 auto v = DeviceMatrix<Scalar>::view(static_cast<Scalar*>(d_U_.get()), m_, ucols);
268 v.recordReady(solver_ctx_.stream());
269 return v;
270 }
271 // transposed: U_orig = VT_stored^H -> conjugate-transpose via cublasXgeam.
272 const Index vtrows_stored = (options_ & ComputeFullU) ? n_ : k;
273 DeviceMatrix<Scalar> result(n_, vtrows_stored);
274 if (n_ > 0 && vtrows_stored > 0) {
275 Scalar alpha_one(1), beta_zero(0);
276 EIGEN_CUBLAS_CHECK(internal::cublasXgeam(solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, n_, vtrows_stored,
277 &alpha_one, static_cast<const Scalar*>(d_VT_.get()), vtrows_stored,
278 &beta_zero, static_cast<const Scalar*>(nullptr), n_, result.data(), n_));
279 result.recordReady(solver_ctx_.stream());
280 }
281 return result;
282 }
283
286 DeviceMatrix<Scalar> d_matrixVT() const {
287 eigen_assert(solver_ctx_.info() == Success);
288 eigen_assert((options_ & (ComputeThinV | ComputeFullV)) && "d_matrixVT() requires ComputeThinV or ComputeFullV");
289 const Index m_orig = transposed_ ? n_ : m_;
290 const Index n_orig = transposed_ ? m_ : n_;
291 const Index k = (std::min)(m_orig, n_orig);
292 if (!transposed_) {
293 const Index vtrows = (options_ & ComputeFullV) ? n_ : k;
294 if (m_ == 0 || n_ == 0) {
295 return DeviceMatrix<Scalar>::fromHost(PlainMatrix::Identity(vtrows, n_), solver_ctx_.stream());
296 }
297 auto v = DeviceMatrix<Scalar>::view(static_cast<Scalar*>(d_VT_.get()), vtrows, n_);
298 v.recordReady(solver_ctx_.stream());
299 return v;
300 }
301 // transposed: VT_orig = U_stored^H.
302 const Index ucols = (options_ & ComputeFullV) ? n_orig : k;
303 DeviceMatrix<Scalar> result(ucols, m_);
304 if (ucols > 0 && m_ > 0) {
305 Scalar alpha_one(1), beta_zero(0);
306 EIGEN_CUBLAS_CHECK(internal::cublasXgeam(solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, ucols, m_,
307 &alpha_one, static_cast<const Scalar*>(d_U_.get()), m_, &beta_zero,
308 static_cast<const Scalar*>(nullptr), ucols, result.data(), ucols));
309 result.recordReady(solver_ctx_.stream());
310 }
311 return result;
312 }
313
315 Index rank(RealScalar threshold = RealScalar(-1)) const {
316 RealVector S = singularValues();
317 if (S.size() == 0) return 0;
318 if (threshold < 0) {
319 threshold = (std::max)(m_, n_) * S(0) * NumTraits<RealScalar>::epsilon();
320 }
321 return (S.array() > threshold).count();
322 }
323
325 template <typename Rhs>
326 PlainMatrix solve(const MatrixBase<Rhs>& B) const {
327 return solve_impl(B, (std::min)(m_, n_), RealScalar(0));
328 }
329
330 // The second argument selects the solve by its kind: an integer is a truncation, a floating-point value a
331 // regularization. Overloads on Index and RealScalar would be ambiguous for a literal such as 2 or 0.1.
332
334 template <typename Rhs, typename Trunc, std::enable_if_t<std::is_integral<Trunc>::value, int> = 0>
335 PlainMatrix solve(const MatrixBase<Rhs>& B, Trunc trunc) const {
336 eigen_assert(trunc > 0 && trunc <= (std::min)(m_, n_));
337 return solve_impl(B, static_cast<Index>(trunc), RealScalar(0));
338 }
339
341 template <typename Rhs, typename Lambda, std::enable_if_t<std::is_floating_point<Lambda>::value, int> = 0>
342 PlainMatrix solve(const MatrixBase<Rhs>& B, Lambda lambda) const {
343 eigen_assert(lambda > 0);
344 return solve_impl(B, (std::min)(m_, n_), static_cast<RealScalar>(lambda));
345 }
346
352 DeviceMatrix<Scalar> solve(const DeviceMatrix<Scalar>& d_B) const {
353 return solve_device_impl(d_B, (std::min)(m_, n_), RealScalar(0));
354 }
355
357 template <typename Trunc, std::enable_if_t<std::is_integral<Trunc>::value, int> = 0>
358 DeviceMatrix<Scalar> solve(const DeviceMatrix<Scalar>& d_B, Trunc trunc) const {
359 eigen_assert(trunc > 0 && trunc <= (std::min)(m_, n_));
360 return solve_device_impl(d_B, static_cast<Index>(trunc), RealScalar(0));
361 }
362
364 template <typename Lambda, std::enable_if_t<std::is_floating_point<Lambda>::value, int> = 0>
365 DeviceMatrix<Scalar> solve(const DeviceMatrix<Scalar>& d_B, Lambda lambda) const {
366 eigen_assert(lambda > 0);
367 return solve_device_impl(d_B, (std::min)(m_, n_), static_cast<RealScalar>(lambda));
368 }
369
370 cudaStream_t stream() const { return solver_ctx_.stream(); }
371
372 private:
373 mutable internal::GpuSolverContext solver_ctx_;
374 internal::DeviceBuffer d_A_; // gesvd input scratch; released after factorize()
375 internal::DeviceBuffer d_U_; // grow-only
376 internal::DeviceBuffer d_S_; // grow-only
377 internal::DeviceBuffer d_VT_; // grow-only
378 // Cached inverse-diagonal for solve (built lazily, reused across solves; grow-only).
379 mutable internal::DeviceBuffer d_D_;
380 mutable Index cached_diag_kk_ = -1;
381 mutable RealScalar cached_diag_lambda_ = RealScalar(-1);
382 mutable bool diag_valid_ = false;
383 unsigned int options_ = 0;
384 int64_t m_ = 0;
385 int64_t n_ = 0;
386 int64_t lda_ = 0;
387 bool transposed_ = false;
388
389 // Common compute() prologue: record shape/options, reset info and cached
390 // diagonal, wait on input. Returns false (clearing state) for empty input.
391 bool begin_compute(const DeviceMatrix<Scalar>& d_A, unsigned int options) {
392 options_ = options;
393 m_ = d_A.rows();
394 n_ = d_A.cols();
395 lda_ = 0;
396 transposed_ = false;
397 diag_valid_ = false;
398 if (!solver_ctx_.begin_compute(m_ != 0 && n_ != 0)) {
399 d_A_ = internal::DeviceBuffer();
400 d_U_ = internal::DeviceBuffer();
401 d_S_ = internal::DeviceBuffer();
402 d_VT_ = internal::DeviceBuffer();
403 return false;
404 }
405 transposed_ = (m_ < n_);
406 if (transposed_) {
407 std::swap(m_, n_);
408 lda_ = m_;
409 } else {
410 lda_ = static_cast<int64_t>(d_A.rows());
411 }
412 d_A.waitReady(solver_ctx_.stream());
413 return true;
414 }
415
416 // Wide input (m < n): produce d_A_ = A^H on device via cuBLAS geam.
417 void transpose_into_input(const DeviceMatrix<Scalar>& d_A) {
418 const size_t mat_bytes = static_cast<size_t>(lda_) * static_cast<size_t>(n_) * sizeof(Scalar);
419 d_A_ = internal::DeviceBuffer(mat_bytes);
420 // geam: C(m×n) = alpha * op(A) + beta * op(B). beta=0, B=nullptr.
421 Scalar alpha_one(1), beta_zero(0);
422 EIGEN_CUBLAS_CHECK(internal::cublasXgeam(solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, m_, n_, &alpha_one,
423 d_A.data(), d_A.rows(), &beta_zero, static_cast<const Scalar*>(nullptr),
424 m_, static_cast<Scalar*>(d_A_.get()), m_));
425 }
426
427 // Swap U↔V flags for the transposed case.
428 static unsigned int swap_uv_options(unsigned int opts) {
429 unsigned int result = 0;
430 if (opts & ComputeThinU) result |= ComputeThinV;
431 if (opts & ComputeFullU) result |= ComputeFullV;
432 if (opts & ComputeThinV) result |= ComputeThinU;
433 if (opts & ComputeFullV) result |= ComputeFullU;
434 return result;
435 }
436
437 static signed char jobu(unsigned int opts) {
438 if (opts & ComputeFullU) return 'A';
439 if (opts & ComputeThinU) return 'S';
440 return 'N';
441 }
442
443 static signed char jobvt(unsigned int opts) {
444 if (opts & ComputeFullV) return 'A';
445 if (opts & ComputeThinV) return 'S';
446 return 'N';
447 }
448
449 void factorize() {
450 constexpr cudaDataType_t dtype = internal::cusolver_data_type<Scalar>::value;
451 constexpr cudaDataType_t rtype = internal::cuda_data_type<RealScalar>::value;
452 const Index k = (std::min)(m_, n_);
453
454 solver_ctx_.mark_pending();
455
456 internal::ensure_sized(d_S_, static_cast<size_t>(k) * sizeof(RealScalar));
457
458 const unsigned int int_opts = transposed_ ? swap_uv_options(options_) : options_;
459
460 const Index ucols = (int_opts & ComputeFullU) ? m_ : ((int_opts & ComputeThinU) ? k : 0);
461 const Index vtrows = (int_opts & ComputeFullV) ? n_ : ((int_opts & ComputeThinV) ? k : 0);
462 const int64_t ldu = m_;
463 const int64_t ldvt = vtrows > 0 ? vtrows : 1;
464
465 if (ucols > 0) {
466 internal::ensure_sized(d_U_, static_cast<size_t>(m_) * static_cast<size_t>(ucols) * sizeof(Scalar));
467 }
468 if (vtrows > 0) {
469 internal::ensure_sized(d_VT_, static_cast<size_t>(vtrows) * static_cast<size_t>(n_) * sizeof(Scalar));
470 }
471
472 eigen_assert(m_ >= n_ && "Internal error: m_ < n_ should have been handled by transpose in compute()");
473 size_t dev_ws = 0, host_ws = 0;
474 EIGEN_CUSOLVER_CHECK(cusolverDnXgesvd_bufferSize(
475 solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, jobu(int_opts), jobvt(int_opts), m_, n_, dtype, d_A_.get(),
476 lda_, rtype, d_S_.get(), dtype, ucols > 0 ? d_U_.get() : nullptr, ldu, dtype,
477 vtrows > 0 ? d_VT_.get() : nullptr, ldvt, dtype, &dev_ws, &host_ws));
478
479 solver_ctx_.ensure_scratch(dev_ws);
480 solver_ctx_.h_workspace_.resize(host_ws);
481
482 EIGEN_CUSOLVER_CHECK(cusolverDnXgesvd(
483 solver_ctx_.cusolverHandle(), solver_ctx_.params_.p, jobu(int_opts), jobvt(int_opts), m_, n_, dtype, d_A_.get(),
484 lda_, rtype, d_S_.get(), dtype, ucols > 0 ? d_U_.get() : nullptr, ldu, dtype,
485 vtrows > 0 ? d_VT_.get() : nullptr, ldvt, dtype, solver_ctx_.scratch_workspace(), dev_ws,
486 host_ws > 0 ? solver_ctx_.h_workspace_.data() : nullptr, host_ws, solver_ctx_.scratch_info()));
487
488 solver_ctx_.enqueue_info_copy();
489
490 // The input copy is pure gesvd scratch — release it now. The free is
491 // stream-ordered (or synchronous on the fallback allocator), so it waits
492 // for gesvd to retire; the memory returns to the pool instead of staying
493 // resident for the solver's lifetime.
494 d_A_ = internal::DeviceBuffer();
495 }
496
497 // Ensure d_D_ holds the kk-entry inverse diagonal for (kk, lambda).
498 // Downloads S and synchronizes only when the cached diagonal doesn't match;
499 // repeated solves with the same truncation/regularization are then free of
500 // host syncs and H2D traffic for the diagonal.
501 //
502 // For lambda == 0 we mirror Eigen's SVDBase::_solve_impl: drop singular
503 // values below S(0) * k * eps (numerical-rank truncation), so this
504 // pseudoinverse solve agrees with CPU BDCSVD::solve on near-singular A.
505 // dgmm wants the diagonal in the matrix scalar type — for complex Scalar
506 // the diagonal is still real, so we build the real values then cast.
507 void build_diag(Index kk, RealScalar lambda) const {
508 if (diag_valid_ && cached_diag_kk_ == kk && cached_diag_lambda_ == lambda) return;
509
510 const Index k = (std::min)(m_, n_);
511 RealVector S(k);
512 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(S.data(), d_S_.get(), static_cast<size_t>(k) * sizeof(RealScalar),
513 cudaMemcpyDeviceToHost, solver_ctx_.stream()));
514 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
515
516 const RealScalar drop_threshold = S(0) * RealScalar(k) * NumTraits<RealScalar>::epsilon();
517 auto S_head = S.head(kk).array();
518 PlainVector D(kk);
519 if (lambda == RealScalar(0)) {
520 D = (S_head > drop_threshold).select(S_head.inverse(), RealScalar(0)).matrix().template cast<Scalar>();
521 } else {
522 D = (S_head / (S_head.square() + lambda * lambda)).matrix().template cast<Scalar>();
523 }
524
525 const size_t d_bytes = static_cast<size_t>(kk) * sizeof(Scalar);
526 internal::ensure_sized(d_D_, d_bytes);
527 EIGEN_CUDA_RUNTIME_CHECK(
528 cudaMemcpyAsync(d_D_.get(), D.data(), d_bytes, cudaMemcpyHostToDevice, solver_ctx_.stream()));
529 cached_diag_kk_ = kk;
530 cached_diag_lambda_ = lambda;
531 diag_valid_ = true;
532 }
533
534 // Shared pseudoinverse application: X = V_orig * diag(D) * U_orig^H * B,
535 // entirely on device. Assumes build_diag(kk, ...) has run and B_dev/X_dev
536 // are device pointers with leading dimensions m_orig / n_orig.
537 void apply_pinv(const Scalar* B_dev, Index kk, Index nrhs, Scalar* X_dev) const {
538 const Index m_orig = transposed_ ? n_ : m_;
539 const Index n_orig = transposed_ ? m_ : n_;
540 const Index k = (std::min)(m_, n_);
541
542 auto* U_dev = static_cast<const Scalar*>(d_U_.get());
543 auto* VT_dev = static_cast<const Scalar*>(d_VT_.get());
544
545 Scalar scalars[2] = {Scalar(1), Scalar(0)};
546
547 // Step 1: tmp = U_orig^H * B (kk × nrhs).
548 internal::DeviceBuffer d_tmp(static_cast<size_t>(kk) * static_cast<size_t>(nrhs) * sizeof(Scalar));
549 auto* tmp_dev = static_cast<Scalar*>(d_tmp.get());
550 if (!transposed_) {
551 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N, kk,
552 nrhs, m_, &scalars[0], U_dev, m_, B_dev, m_orig, &scalars[1], tmp_dev, kk,
553 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
554 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
555 } else {
556 const Index vtrows_stored = (swap_uv_options(options_) & ComputeFullV) ? n_ : k;
557 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N, kk,
558 nrhs, m_orig, &scalars[0], VT_dev, vtrows_stored, B_dev, m_orig, &scalars[1], tmp_dev, kk,
559 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
560 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
561 }
562
563 // Step 2: tmp = diag(D) * tmp on device via cublasXdgmm.
564 EIGEN_CUBLAS_CHECK(internal::cublasXdgmm(solver_ctx_.cublasHandle(), CUBLAS_SIDE_LEFT, kk, nrhs, tmp_dev, kk,
565 static_cast<const Scalar*>(d_D_.get()), 1, tmp_dev, kk));
566
567 // Step 3: X = V_orig * tmp (n_orig × nrhs).
568 if (!transposed_) {
569 const Index vtrows = (options_ & ComputeFullV) ? n_ : k;
570 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_C, CUBLAS_OP_N,
571 n_orig, nrhs, kk, &scalars[0], VT_dev, vtrows, tmp_dev, kk, &scalars[1], X_dev, n_orig,
572 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
573 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
574 } else {
575 internal::cublaslt_gemm(solver_ctx_.cublasLtHandle(), solver_ctx_.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N,
576 n_orig, nrhs, kk, &scalars[0], U_dev, m_, tmp_dev, kk, &scalars[1], X_dev, n_orig,
577 solver_ctx_.gemmWorkspace(), solver_ctx_.gemmPlanCache(),
578 solver_ctx_.cublasLtMaxWorkspaceBytes(), solver_ctx_.stream());
579 }
580 }
581
582 template <typename Rhs>
583 PlainMatrix solve_impl(const MatrixBase<Rhs>& B, Index trunc, RealScalar lambda) const {
584 eigen_assert(solver_ctx_.info() == Success && "SVD::solve called on a failed or uninitialized decomposition");
585 eigen_assert((options_ & (ComputeThinU | ComputeFullU)) && "solve requires U");
586 eigen_assert((options_ & (ComputeThinV | ComputeFullV)) && "solve requires V");
587
588 const Index m_orig = transposed_ ? n_ : m_;
589 const Index n_orig = transposed_ ? m_ : n_;
590 eigen_assert(B.rows() == m_orig);
591
592 const Index k = (std::min)(m_, n_);
593 const Index kk = (std::min)(trunc, k);
594 const Index nrhs = B.cols();
595
596 // Empty problem: no rank, no RHS, or zero domain -> result is the zero matrix.
597 if (kk == 0 || nrhs == 0 || n_orig == 0) {
598 return PlainMatrix::Zero(n_orig, nrhs);
599 }
600
601 // Enqueue the B upload before build_diag: when the diagonal must be
602 // (re)built, its S-download sync then also covers the in-flight upload —
603 // one blocking wait instead of two.
604 const Ref<const PlainMatrix> rhs(B.derived());
605 internal::DeviceBuffer d_B(static_cast<size_t>(m_orig) * static_cast<size_t>(nrhs) * sizeof(Scalar));
606 internal::upload_host_matrix(static_cast<Scalar*>(d_B.get()), m_orig, rhs.data(), rhs.outerStride(), m_orig, nrhs,
607 solver_ctx_.stream());
608 build_diag(kk, lambda);
609
610 PlainMatrix X(n_orig, nrhs);
611 internal::DeviceBuffer d_X(static_cast<size_t>(n_orig) * static_cast<size_t>(nrhs) * sizeof(Scalar));
612 apply_pinv(static_cast<const Scalar*>(d_B.get()), kk, nrhs, static_cast<Scalar*>(d_X.get()));
613
614 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(X.data(), d_X.get(),
615 static_cast<size_t>(n_orig) * static_cast<size_t>(nrhs) * sizeof(Scalar),
616 cudaMemcpyDeviceToHost, solver_ctx_.stream()));
617 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(solver_ctx_.stream()));
618
619 return X;
620 }
621
622 DeviceMatrix<Scalar> solve_device_impl(const DeviceMatrix<Scalar>& d_B, Index trunc, RealScalar lambda) const {
623 eigen_assert(solver_ctx_.info() == Success && "SVD::solve called on a failed or uninitialized decomposition");
624 eigen_assert((options_ & (ComputeThinU | ComputeFullU)) && "solve requires U");
625 eigen_assert((options_ & (ComputeThinV | ComputeFullV)) && "solve requires V");
626
627 const Index m_orig = transposed_ ? n_ : m_;
628 const Index n_orig = transposed_ ? m_ : n_;
629 eigen_assert(d_B.rows() == m_orig);
630 EIGEN_UNUSED_VARIABLE(m_orig);
631
632 const Index k = (std::min)(m_, n_);
633 const Index kk = (std::min)(trunc, k);
634 const Index nrhs = d_B.cols();
635
636 if (kk == 0 || nrhs == 0 || n_orig == 0) {
637 DeviceMatrix<Scalar> X(n_orig, nrhs);
638 X.setZero(solver_ctx_.stream());
639 return X;
640 }
641
642 d_B.waitReady(solver_ctx_.stream());
643 build_diag(kk, lambda);
644
645 DeviceMatrix<Scalar> X(n_orig, nrhs);
646 apply_pinv(d_B.data(), kk, nrhs, X.data());
647 X.recordReady(solver_ctx_.stream());
648 return X;
649 }
650};
651} // namespace gpu
652} // namespace Eigen
653
654#endif // EIGEN_GPU_SVD_H
static DeviceMatrix fromHost(const DenseBase< Derived > &host, cudaStream_t stream=nullptr)
Definition DeviceMatrix.h:226
static DeviceMatrix view(Scalar *device_ptr, Index rows, Index cols)
Definition DeviceMatrix.h:578
ComputationInfo
Namespace containing all symbols from the Eigen library.