Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GpuSparseSolverBase.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// Common base for the GPU sparse direct solvers (LLT, LDLT, LU) via cuDSS.
12//
13// All three share one analyzePattern → factorize → solve workflow and differ only
14// in the cudssMatrixType_t and cudssMatrixViewType_t they pass to cuDSS, so this
15// CRTP base implements the workflow and derived classes supply the two types.
16//
17// Not thread-safe: concurrent calls race on the cuDSS handle, the bound stream,
18// and the cached scratch buffers. This includes concurrent solve() calls on one
19// instance, const though solve() is. Use one solver per thread.
20
21#ifndef EIGEN_GPU_SPARSE_SOLVER_BASE_H
22#define EIGEN_GPU_SPARSE_SOLVER_BASE_H
23
24// IWYU pragma: private
25#include "./InternalHeaderCheck.h"
26
27#include "./CuSparseSupport.h"
28#include "./CuDssSupport.h"
29
30namespace Eigen {
31namespace gpu {
32
33// The named algorithms carry the cuDSS values, so apply_config() can
34// static_cast them. cuDSS < 0.8 names none of them, so there they are not
35// declared at all: selecting an algorithm the linked cuDSS cannot honor is a
36// compile error rather than a runtime rejection, and EIGEN_HAS_CUDSS_SOLVER_CONFIG
37// lets a caller branch on which case a build is in.
38// Default is declared either way and is a sentinel rather than a cuDSS value:
39// apply_config() skips its cudssConfigSet(), leaving the cuDSS default in place
40// without depending on its numeric value.
41
46enum class SparseReordering : int {
47 Default = -1,
48#if EIGEN_HAS_CUDSS_SOLVER_CONFIG
49 BtfColamd = CUDSS_REORDERING_ALG_BTF_COLAMD,
50 Colamd = CUDSS_REORDERING_ALG_COLAMD,
51 Amd = CUDSS_REORDERING_ALG_AMD,
52 NestedDissection = CUDSS_REORDERING_ALG_NESTED_DISSECTION,
53 Natural = CUDSS_REORDERING_ALG_NONE,
54#endif
55};
56
63enum class SparseMatching : int {
64 Default = -1,
65#if EIGEN_HAS_CUDSS_SOLVER_CONFIG
66 None = CUDSS_MATCHING_ALG_NONE,
67 MaxDiagCount = CUDSS_MATCHING_ALG_MAX_DIAG_COUNT,
68 MaxMinDiag = CUDSS_MATCHING_ALG_MAX_MIN_DIAG,
69 MaxMinDiagAlt = CUDSS_MATCHING_ALG_MAX_MIN_DIAG_ALT,
70 MaxDiagSum = CUDSS_MATCHING_ALG_MAX_DIAG_SUM,
71 MaxDiagProduct = CUDSS_MATCHING_ALG_MAX_DIAG_PRODUCT,
72 Auto = CUDSS_MATCHING_ALG_AUTO,
73#endif
74};
75
80enum class SparsePivoting : int {
81 Default = -1,
82#if EIGEN_HAS_CUDSS_SOLVER_CONFIG
83 None = CUDSS_PIVOT_NONE,
84 GlobalCol = CUDSS_PIVOT_GLOBAL_COL,
85 GlobalRow = CUDSS_PIVOT_GLOBAL_ROW,
86 Diagonal = CUDSS_PIVOT_DIAGONAL,
87 LocalBlock = CUDSS_PIVOT_LOCAL_BLOCK,
88#endif
89};
90
99 SparseReordering reordering = SparseReordering::Default;
100 SparseMatching matching = SparseMatching::Default;
101 SparsePivoting pivoting = SparsePivoting::Default;
103 double pivotThreshold = -1.0;
105 double pivotEpsilon = -1.0;
109 double refinementTolerance = -1.0;
112 bool hybridMemory = false;
119 bool hybridExecute = false;
120
122 bool isDefault() const {
123 return reordering == SparseReordering::Default && matching == SparseMatching::Default &&
124 pivoting == SparsePivoting::Default && pivotThreshold < 0 && pivotEpsilon < 0 && refinementSteps < 0 &&
126 }
127};
128
129namespace internal {
130
140template <typename Scalar_, typename Derived>
141class SparseSolverBase {
142 public:
143 using Scalar = Scalar_;
144 using RealScalar = typename NumTraits<Scalar>::Real;
145 using StorageIndex = int;
148 using DenseVector = Matrix<Scalar, Dynamic, 1>;
150
151 SparseSolverBase() { init_context(/*stream=*/nullptr, /*borrow_stream=*/false); }
152
157 explicit SparseSolverBase(Context& ctx) { init_context(ctx.stream(), /*borrow_stream=*/true); }
158
160 destroy_cudss_objects();
161 if (handle_) (void)cudssDestroy(handle_);
162 if (owns_stream_ && stream_) (void)cudaStreamDestroy(stream_);
163 }
164
165 SparseSolverBase(const SparseSolverBase&) = delete;
166 SparseSolverBase& operator=(const SparseSolverBase&) = delete;
167
188 Derived& setConfig(const SparseSolverConfig& cfg) {
189#if !EIGEN_HAS_CUDSS_SOLVER_CONFIG
190 eigen_assert(cfg.isDefault() && "SparseSolverConfig knobs require cuDSS >= 0.8");
191 if (!cfg.isDefault()) {
192 // The assert above is compiled out in release builds. Record what was
193 // asked for and fail through info() rather than factorizing with the
194 // cuDSS defaults this call meant to replace. analyzePattern() keeps
195 // failing while the stored config is one this cuDSS cannot honor.
196 config_opts_ = cfg;
197 analysis_done_ = false;
198 info_ = InvalidInput;
199 info_synced_ = true;
200 return derived();
201 }
202#endif
203
204 const bool hybrid_mode_changed =
205 cfg.hybridMemory != config_opts_.hybridMemory || cfg.hybridExecute != config_opts_.hybridExecute;
206 eigen_assert((!hybrid_mode_changed || !analysis_done_) &&
207 "hybridMemory/hybridExecute change what analyzePattern() builds, so they must be set before it");
208
209 config_opts_ = cfg;
210 apply_config();
211
212 if (hybrid_mode_changed && analysis_done_) {
213 // The assert above is compiled out in release builds. Fail through
214 // info() rather than factorizing against the wrong analysis.
215 analysis_done_ = false;
216 info_ = InvalidInput;
217 info_synced_ = true;
218 }
219 return derived();
220 }
221
223 const SparseSolverConfig& config() const { return config_opts_; }
224
226 template <typename InputType>
229 if (info_ == Success) {
230 factorize(A);
231 }
232 return derived();
233 }
234
237 template <typename InputType>
239#if !EIGEN_HAS_CUDSS_SOLVER_CONFIG
240 if (!config_opts_.isDefault()) {
241 // setConfig() refused this config; analyzing as if it had been applied
242 // would hand factorize() state built for settings the caller replaced.
243 info_ = InvalidInput;
244 info_synced_ = true;
245 analysis_done_ = false;
246 return derived();
247 }
248#endif
249
250 const InputType& input = A.derived();
251 check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
252 eigen_assert(input.rows() == input.cols() && "GpuSparseSolver requires a square matrix");
253
254 n_ = input.rows();
255 info_ = InvalidInput;
256 info_synced_ = true;
257 analysis_done_ = false;
258
259 if (n_ == 0) {
260 nnz_ = 0;
261 info_ = Success;
262 info_synced_ = true;
263 analysis_done_ = true;
264 return derived();
265 }
266
267 // For symmetric solvers, ColMajor CSC can be reinterpreted as CSR with
268 // swapped triangle view — zero copy, except that Hermitian matrix types
269 // need the value array conjugated (see csc_upload_values). For general
270 // solvers, we must convert to actual RowMajor CSR so cuDSS sees the
271 // correct matrix, not A^T.
272 if (Derived::needs_csr_conversion()) {
273 const CsrMat csr(input);
274 nnz_ = csr.nonZeros();
275 upload_compressed(csr.outerIndexPtr(), csr.innerIndexPtr(), csr.valuePtr());
276 } else {
277 // Bind by reference when the input is already a compressed CSC matrix —
278 // no host copy.
279 SpMat storage;
280 const SpMat& csc = bind_sparse<SpMat>(input, storage);
281 nnz_ = csc.nonZeros();
282 upload_compressed(csc.outerIndexPtr(), csc.innerIndexPtr(), csc_upload_values(csc.valuePtr(), nnz_));
283 }
284 create_cudss_matrix();
285
286 if (data_) {
287 cudssData_t old = data_;
288 data_ = nullptr;
289 EIGEN_CUDSS_CHECK(cudssDataDestroy(handle_, old));
290 }
291 EIGEN_CUDSS_CHECK(cudssDataCreate(handle_, &data_));
292
293 create_placeholder_dense();
294
295 EIGEN_CUDSS_CHECK(cudssExecute(handle_, CUDSS_PHASE_ANALYSIS, config_, data_, d_A_cudss_, d_x_cudss_, d_b_cudss_));
296
297 analysis_done_ = true;
298 info_ = Success;
299 info_synced_ = true;
300 return derived();
301 }
302
311 template <typename InputType>
313 eigen_assert(analysis_done_ && "factorize() requires analyzePattern() first");
314 if (!analysis_done_) {
315 // Reachable in release builds whenever setConfig() invalidated the
316 // analysis rather than applying the request; see setConfig().
317 info_ = InvalidInput;
318 info_synced_ = true;
319 return derived();
320 }
321
322 if (n_ == 0) {
323 info_ = Success;
324 info_synced_ = true;
325 return derived();
326 }
327
328 // Convert to the same format used in analyzePattern. When the input is
329 // already a compressed CSC matrix (symmetric solvers), it is bound by
330 // reference — no host copy or conversion on refactorize (Hermitian types
331 // re-conjugate the value array; the pattern stays zero-copy).
332 // The temporaries must outlive the async memcpy (pageable H2D is actually
333 // synchronous w.r.t. the host, but keep them alive for clarity).
334 const InputType& input = A.derived();
335 check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
336 eigen_assert(input.rows() == n_ && input.cols() == n_);
337
338 const Scalar* value_ptr;
339 Index value_nnz;
340 CsrMat csr_tmp;
341 SpMat csc_storage;
342 if (Derived::needs_csr_conversion()) {
343 csr_tmp = CsrMat(input);
344 value_ptr = csr_tmp.valuePtr();
345 value_nnz = csr_tmp.nonZeros();
346 } else {
347 const SpMat& csc = bind_sparse<SpMat>(input, csc_storage);
348 value_ptr = csc_upload_values(csc.valuePtr(), csc.nonZeros());
349 value_nnz = csc.nonZeros();
350 }
351 eigen_assert(value_nnz == nnz_);
352 EIGEN_UNUSED_VARIABLE(value_nnz);
353
354 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.get(), value_ptr, static_cast<size_t>(nnz_) * sizeof(Scalar),
355 cudaMemcpyHostToDevice, stream_));
356
357 EIGEN_CUDSS_CHECK(cudssMatrixSetValues(d_A_cudss_, d_values_.get()));
358
359 info_ = InvalidInput;
360 info_synced_ = false;
361 EIGEN_CUDSS_CHECK(
362 cudssExecute(handle_, CUDSS_PHASE_FACTORIZATION, config_, data_, d_A_cudss_, d_x_cudss_, d_b_cudss_));
363
364 return derived();
365 }
366
369 template <typename Rhs>
370 DenseMatrix solve(const MatrixBase<Rhs>& B) const {
371 // Debug builds verify the factorization (info() synchronizes on the first
372 // call after factorize()); release builds skip both the check and the
373 // sync — use info() explicitly when failure must be detected.
374 eigen_assert(info() == Success && "GpuSparseSolver::solve requires a successful factorization");
375 eigen_assert(B.rows() == n_);
376
377 if (n_ == 0 || B.cols() == 0) return DenseMatrix(n_, B.cols());
378
379 const Ref<const DenseMatrix> rhs(B.derived());
380 const int64_t nrhs = static_cast<int64_t>(rhs.cols());
381
382 // Reuse cached d_b/d_x scratch to avoid cudaMalloc/cudaFree per solve.
383 const size_t rhs_bytes = static_cast<size_t>(n_) * static_cast<size_t>(nrhs) * sizeof(Scalar);
384 ensure_solve_buffer(d_b_solve_, rhs_bytes);
385 ensure_solve_buffer(d_x_solve_, rhs_bytes);
386 internal::upload_host_matrix(static_cast<Scalar*>(d_b_solve_.get()), n_, rhs.data(), rhs.outerStride(), n_, nrhs,
387 stream_);
388
389 update_solve_descriptors(nrhs, d_b_solve_.get(), d_x_solve_.get());
390 EIGEN_CUDSS_CHECK(
391 cudssExecute(handle_, CUDSS_PHASE_SOLVE, config_, data_, d_A_cudss_, x_solve_cudss_, b_solve_cudss_));
392
393 DenseMatrix X(n_, rhs.cols());
394 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(X.data(), d_x_solve_.get(), rhs_bytes, cudaMemcpyDeviceToHost, stream_));
395 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
396
397 return X;
398 }
399
406 eigen_assert(info() == Success && "GpuSparseSolver::solve requires a successful factorization");
407 eigen_assert(d_B.rows() == n_);
408
409 const int64_t nrhs = static_cast<int64_t>(d_B.cols());
410 DeviceMatrix<Scalar> X(n_, d_B.cols());
411 if (n_ == 0 || nrhs == 0) return X;
412
413 d_B.waitReady(stream_);
414 update_solve_descriptors(nrhs, const_cast<Scalar*>(d_B.data()), X.data());
415 EIGEN_CUDSS_CHECK(
416 cudssExecute(handle_, CUDSS_PHASE_SOLVE, config_, data_, d_A_cudss_, x_solve_cudss_, b_solve_cudss_));
417 X.recordReady(stream_);
418 return X;
419 }
420
421 ComputationInfo info() const {
422 sync_info();
423 return info_;
424 }
425 Index rows() const { return n_; }
426 Index cols() const { return n_; }
427
428 cudaStream_t stream() const { return stream_; }
429
430 protected:
431 cudaStream_t stream_ = nullptr;
432 bool owns_stream_ = true;
433 cudssHandle_t handle_ = nullptr;
434 cudssConfig_t config_ = nullptr;
435 cudssData_t data_ = nullptr;
436 cudssMatrix_t d_A_cudss_ = nullptr;
437 cudssMatrix_t d_x_cudss_ = nullptr;
438 cudssMatrix_t d_b_cudss_ = nullptr;
439
440 DeviceBuffer d_rowPtr_;
441 DeviceBuffer d_colIdx_;
442 DeviceBuffer d_values_;
443
444 // Host staging for the conjugated value array (Hermitian zero-copy path
445 // only; see csc_upload_values). Kept as a member so it outlives the async
446 // H2D copy.
447 DenseVector conj_values_;
448
449 mutable DeviceBuffer d_b_solve_;
450 mutable DeviceBuffer d_x_solve_;
451
452 // Cached cuDSS dense descriptors for solve, re-pointed per call and
453 // recreated only when nrhs changes.
454 mutable cudssMatrix_t b_solve_cudss_ = nullptr;
455 mutable cudssMatrix_t x_solve_cudss_ = nullptr;
456 mutable int64_t solve_desc_nrhs_ = -1;
457
458 int64_t n_ = 0;
459 int64_t nnz_ = 0;
460 mutable ComputationInfo info_ = InvalidInput;
461 mutable bool info_synced_ = true;
462 bool analysis_done_ = false;
463 SparseSolverConfig config_opts_;
464
465 private:
466 Derived& derived() { return static_cast<Derived&>(*this); }
467 const Derived& derived() const { return static_cast<const Derived&>(*this); }
468
469 void init_context(cudaStream_t stream, bool borrow_stream) {
470 if (borrow_stream) {
471 // nullptr is CUDA's valid legacy default stream, so ownership cannot be
472 // inferred from the stream value.
473 stream_ = stream;
474 owns_stream_ = false;
475 } else {
476 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&stream_));
477 owns_stream_ = true;
478 }
479 EIGEN_CUDSS_CHECK(cudssCreate(&handle_));
480 EIGEN_CUDSS_CHECK(cudssSetStream(handle_, stream_));
481 EIGEN_CUDSS_CHECK(cudssConfigCreate(&config_));
482 }
483
484 // Rebuild config_ from config_opts_. Recreating rather than mutating lets a
485 // field reset to Default restore the cuDSS default, whose numeric value is
486 // not part of the cuDSS API contract.
487 void apply_config() {
488#if EIGEN_HAS_CUDSS_SOLVER_CONFIG
489 if (config_) {
490 // cuDSS reads the config during execution, and both factorize() and the
491 // device-resident solve() return with their phase still queued. Retire
492 // that work before the config it is reading goes away.
493 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
494 (void)cudssConfigDestroy(config_);
495 }
496 config_ = nullptr;
497 EIGEN_CUDSS_CHECK(cudssConfigCreate(&config_));
498 const SparseSolverConfig& c = config_opts_;
499 if (c.reordering != SparseReordering::Default) {
500 const cudssReorderingAlg_t v = static_cast<cudssReorderingAlg_t>(c.reordering);
501 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_REORDERING_ALG, &v, sizeof(v)));
502 }
503 if (c.matching != SparseMatching::Default) {
504 const cudssMatchingAlg_t v = static_cast<cudssMatchingAlg_t>(c.matching);
505 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_MATCHING_ALG, &v, sizeof(v)));
506 }
507 if (c.pivoting != SparsePivoting::Default) {
508 const cudssPivotType_t v = static_cast<cudssPivotType_t>(c.pivoting);
509 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_PIVOT_TYPE, &v, sizeof(v)));
510 }
511 if (c.pivotThreshold >= 0) {
512 EIGEN_CUDSS_CHECK(
513 cudssConfigSet(config_, CUDSS_CONFIG_PIVOT_THRESHOLD, &c.pivotThreshold, sizeof(c.pivotThreshold)));
514 }
515 if (c.pivotEpsilon >= 0) {
516 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_PIVOT_EPSILON, &c.pivotEpsilon, sizeof(c.pivotEpsilon)));
517 }
518 if (c.refinementSteps >= 0) {
519 EIGEN_CUDSS_CHECK(
520 cudssConfigSet(config_, CUDSS_CONFIG_IR_N_STEPS, &c.refinementSteps, sizeof(c.refinementSteps)));
521 }
522 if (c.refinementTolerance >= 0) {
523 EIGEN_CUDSS_CHECK(
524 cudssConfigSet(config_, CUDSS_CONFIG_IR_TOL, &c.refinementTolerance, sizeof(c.refinementTolerance)));
525 }
526 if (c.hybridMemory) {
527 const int v = 1;
528 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_HYBRID_MEMORY_MODE, &v, sizeof(v)));
529 if (c.hybridMemoryDeviceLimit >= 0) {
530 const int64_t limit = c.hybridMemoryDeviceLimit;
531 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_HYBRID_DEVICE_MEMORY_LIMIT, &limit, sizeof(limit)));
532 }
533 }
534 if (c.hybridExecute) {
535 const int v = 1;
536 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_HYBRID_EXECUTE_MODE, &v, sizeof(v)));
537 }
538#else
539 // setConfig() returns early for anything else, so there is nothing to apply.
540 eigen_internal_assert(config_opts_.isDefault());
541#endif
542 }
543
544 void ensure_solve_buffer(DeviceBuffer& buf, size_t needed) const {
545 if (needed > buf.size()) {
546 if (buf) EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
547 buf = DeviceBuffer(needed);
548 }
549 }
550
551 // Recreate the solve descriptors when nrhs changes; otherwise just re-point
552 // them (cudssMatrixSetValues is a host-side pointer update).
553 void update_solve_descriptors(int64_t nrhs, void* b_ptr, void* x_ptr) const {
554 constexpr cudss_value_type_t dtype = to_cudss_data_type(cuda_data_type<Scalar>::value);
555 if (!b_solve_cudss_ || solve_desc_nrhs_ != nrhs) {
556 destroy_solve_descriptors();
557 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&b_solve_cudss_, n_, nrhs, n_, b_ptr, dtype, CUDSS_LAYOUT_COL_MAJOR));
558 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&x_solve_cudss_, n_, nrhs, n_, x_ptr, dtype, CUDSS_LAYOUT_COL_MAJOR));
559 solve_desc_nrhs_ = nrhs;
560 } else {
561 EIGEN_CUDSS_CHECK(cudssMatrixSetValues(b_solve_cudss_, b_ptr));
562 EIGEN_CUDSS_CHECK(cudssMatrixSetValues(x_solve_cudss_, x_ptr));
563 }
564 }
565
566 void destroy_solve_descriptors() const {
567 if (b_solve_cudss_) {
568 (void)cudssMatrixDestroy(b_solve_cudss_);
569 b_solve_cudss_ = nullptr;
570 }
571 if (x_solve_cudss_) {
572 (void)cudssMatrixDestroy(x_solve_cudss_);
573 x_solve_cudss_ = nullptr;
574 }
575 solve_desc_nrhs_ = -1;
576 }
577
578 void sync_info() const {
579 if (!info_synced_) {
580 // cudssDataGet for CUDSS_DATA_INFO synchronizes the stream internally,
581 // so an explicit cudaStreamSynchronize would be redundant.
582 int cudss_info = 0;
583 EIGEN_CUDSS_CHECK(cudssDataGet(handle_, data_, CUDSS_DATA_INFO, &cudss_info, sizeof(cudss_info), nullptr));
584 info_ = (cudss_info == 0) ? Success : NumericalIssue;
585 info_synced_ = true;
586 }
587 }
588
589 // Destructor-only cleanup: there is no useful recovery path for failures.
590 void destroy_cudss_objects() {
591 // Same lifetime rule as apply_config(): the queued phase reads the data
592 // handle and the matrix descriptors as well as the config. Unchecked
593 // because this runs from the destructor.
594 (void)cudaStreamSynchronize(stream_);
595 destroy_solve_descriptors();
596 if (d_A_cudss_) {
597 (void)cudssMatrixDestroy(d_A_cudss_);
598 d_A_cudss_ = nullptr;
599 }
600 if (d_x_cudss_) {
601 (void)cudssMatrixDestroy(d_x_cudss_);
602 d_x_cudss_ = nullptr;
603 }
604 if (d_b_cudss_) {
605 (void)cudssMatrixDestroy(d_b_cudss_);
606 d_b_cudss_ = nullptr;
607 }
608 if (data_) {
609 (void)cudssDataDestroy(handle_, data_);
610 data_ = nullptr;
611 }
612 if (config_) {
613 (void)cudssConfigDestroy(config_);
614 config_ = nullptr;
615 }
616 }
617
618 // The zero-copy CSC-as-CSR reinterpretation hands cuDSS the stored triangle
619 // of A^T. Real symmetric types are unaffected (A^T = A), but the Hermitian
620 // types need the triangle of A, whose entries are the conjugates of A^T's.
621 static constexpr bool needs_value_conjugation() {
622 return Derived::cudss_matrix_type() == CUDSS_MTYPE_HPD || Derived::cudss_matrix_type() == CUDSS_MTYPE_HERMITIAN;
623 }
624
625 // Value array for the zero-copy CSC-as-CSR path: conjugated into
626 // conj_values_ for Hermitian matrix types, passed through otherwise. The
627 // pattern arrays remain zero-copy either way.
628 const Scalar* csc_upload_values(const Scalar* values, Index nnz) {
629 if (!needs_value_conjugation()) return values;
630 conj_values_ = Map<const DenseVector>(values, nnz).conjugate();
631 return conj_values_.data();
632 }
633
634 void upload_compressed(const StorageIndex* outer, const StorageIndex* inner, const Scalar* values) {
635 const size_t rowptr_bytes = static_cast<size_t>(n_ + 1) * sizeof(StorageIndex);
636 const size_t colidx_bytes = static_cast<size_t>(nnz_) * sizeof(StorageIndex);
637 const size_t values_bytes = static_cast<size_t>(nnz_) * sizeof(Scalar);
638
639 d_rowPtr_ = DeviceBuffer(rowptr_bytes);
640 d_colIdx_ = DeviceBuffer(colidx_bytes);
641 d_values_ = DeviceBuffer(values_bytes);
642
643 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_rowPtr_.get(), outer, rowptr_bytes, cudaMemcpyHostToDevice, stream_));
644 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_colIdx_.get(), inner, colidx_bytes, cudaMemcpyHostToDevice, stream_));
645 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.get(), values, values_bytes, cudaMemcpyHostToDevice, stream_));
646 }
647
648 void create_cudss_matrix() {
649 if (d_A_cudss_) {
650 // Clear the member first: if the destroy or the create below throws, the destructor must not destroy it again.
651 cudssMatrix_t old = d_A_cudss_;
652 d_A_cudss_ = nullptr;
653 EIGEN_CUDSS_CHECK(cudssMatrixDestroy(old));
654 }
655
656 constexpr cudss_value_type_t idx_type = to_cudss_data_type(cudss_index_type<StorageIndex>::value);
657 constexpr cudss_value_type_t val_type = to_cudss_data_type(cuda_data_type<Scalar>::value);
658 constexpr cudssMatrixType_t mtype = Derived::cudss_matrix_type();
659 constexpr cudssMatrixViewType_t mview = Derived::cudss_matrix_view();
660
661#if defined(CUDSS_VERSION) && CUDSS_VERSION >= 800
662 // cuDSS 0.8 split the index type into separate offset/index type params.
663 EIGEN_CUDSS_CHECK(cudssMatrixCreateCsr(&d_A_cudss_, n_, n_, nnz_, d_rowPtr_.get(),
664 /*rowEnd=*/nullptr, d_colIdx_.get(), d_values_.get(), idx_type, idx_type,
665 val_type, mtype, mview, CUDSS_BASE_ZERO));
666#else
667 EIGEN_CUDSS_CHECK(cudssMatrixCreateCsr(&d_A_cudss_, n_, n_, nnz_, d_rowPtr_.get(),
668 /*rowEnd=*/nullptr, d_colIdx_.get(), d_values_.get(), idx_type, val_type,
669 mtype, mview, CUDSS_BASE_ZERO));
670#endif
671 }
672
673 void create_placeholder_dense() {
674 // A new analysis may change n_, so the cached solve descriptors are stale.
675 destroy_solve_descriptors();
676 // Clear each member before its destroy so a throwing check cannot leave a handle for the destructor to destroy
677 // again.
678 if (d_x_cudss_) {
679 cudssMatrix_t old = d_x_cudss_;
680 d_x_cudss_ = nullptr;
681 EIGEN_CUDSS_CHECK(cudssMatrixDestroy(old));
682 }
683 if (d_b_cudss_) {
684 cudssMatrix_t old = d_b_cudss_;
685 d_b_cudss_ = nullptr;
686 EIGEN_CUDSS_CHECK(cudssMatrixDestroy(old));
687 }
688 constexpr cudss_value_type_t dtype = to_cudss_data_type(cuda_data_type<Scalar>::value);
689 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&d_x_cudss_, n_, 1, n_, nullptr, dtype, CUDSS_LAYOUT_COL_MAJOR));
690 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&d_b_cudss_, n_, 1, n_, nullptr, dtype, CUDSS_LAYOUT_COL_MAJOR));
691 }
692};
693} // namespace internal
694} // namespace gpu
695} // namespace Eigen
696
697#endif // EIGEN_GPU_SPARSE_SOLVER_BASE_H
constexpr Scalar * data()
StorageIndex * innerIndexPtr()
StorageIndex * outerIndexPtr()
Index nonZeros() const
Unified GPU execution context owning a CUDA stream and library handles.
Definition GpuContext.h:81
RAII wrapper for a dense column-major matrix in GPU device memory.
Definition DeviceMatrix.h:122
void recordReady(cudaStream_t stream)
Definition DeviceMatrix.h:365
void waitReady(cudaStream_t stream) const
Definition DeviceMatrix.h:374
Definition GpuSparseSolverBase.h:141
Derived & analyzePattern(const SparseMatrixBase< InputType > &A)
Definition GpuSparseSolverBase.h:238
DenseMatrix solve(const MatrixBase< Rhs > &B) const
Definition GpuSparseSolverBase.h:370
SparseSolverBase(Context &ctx)
Definition GpuSparseSolverBase.h:157
Derived & setConfig(const SparseSolverConfig &cfg)
Definition GpuSparseSolverBase.h:188
Derived & factorize(const SparseMatrixBase< InputType > &A)
Definition GpuSparseSolverBase.h:312
Derived & compute(const SparseMatrixBase< InputType > &A)
Definition GpuSparseSolverBase.h:227
DeviceMatrix< Scalar > solve(const DeviceMatrix< Scalar > &d_B) const
Definition GpuSparseSolverBase.h:405
const SparseSolverConfig & config() const
Definition GpuSparseSolverBase.h:223
ComputationInfo
NumericalIssue
Namespace containing all symbols from the Eigen library.
constexpr Derived & derived()
Definition GpuSparseSolverBase.h:98
bool hybridMemory
Definition GpuSparseSolverBase.h:112
bool hybridExecute
Definition GpuSparseSolverBase.h:119
double refinementTolerance
Definition GpuSparseSolverBase.h:109
int refinementSteps
Definition GpuSparseSolverBase.h:107
double pivotEpsilon
Definition GpuSparseSolverBase.h:105
bool isDefault() const
Definition GpuSparseSolverBase.h:122
double pivotThreshold
Definition GpuSparseSolverBase.h:103
int64_t hybridMemoryDeviceLimit
Definition GpuSparseSolverBase.h:116