141class SparseSolverBase {
143 using Scalar = Scalar_;
145 using StorageIndex = int;
151 SparseSolverBase() { init_context(
nullptr,
false); }
160 destroy_cudss_objects();
161 if (handle_) (void)cudssDestroy(handle_);
162 if (owns_stream_ && stream_) (void)cudaStreamDestroy(stream_);
189#if !EIGEN_HAS_CUDSS_SOLVER_CONFIG
190 eigen_assert(cfg.
isDefault() &&
"SparseSolverConfig knobs require cuDSS >= 0.8");
197 analysis_done_ =
false;
204 const bool hybrid_mode_changed =
206 eigen_assert((!hybrid_mode_changed || !analysis_done_) &&
207 "hybridMemory/hybridExecute change what analyzePattern() builds, so they must be set before it");
212 if (hybrid_mode_changed && analysis_done_) {
215 analysis_done_ =
false;
226 template <
typename InputType>
237 template <
typename InputType>
239#if !EIGEN_HAS_CUDSS_SOLVER_CONFIG
240 if (!config_opts_.isDefault()) {
245 analysis_done_ =
false;
250 const InputType& input = A.
derived();
251 check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
252 eigen_assert(input.rows() == input.cols() &&
"GpuSparseSolver requires a square matrix");
257 analysis_done_ =
false;
263 analysis_done_ =
true;
272 if (Derived::needs_csr_conversion()) {
273 const CsrMat csr(input);
280 const SpMat& csc = bind_sparse<SpMat>(input, storage);
284 create_cudss_matrix();
287 cudssData_t old = data_;
289 EIGEN_CUDSS_CHECK(cudssDataDestroy(handle_, old));
291 EIGEN_CUDSS_CHECK(cudssDataCreate(handle_, &data_));
293 create_placeholder_dense();
295 EIGEN_CUDSS_CHECK(cudssExecute(handle_, CUDSS_PHASE_ANALYSIS, config_, data_, d_A_cudss_, d_x_cudss_, d_b_cudss_));
297 analysis_done_ =
true;
311 template <
typename InputType>
313 eigen_assert(analysis_done_ &&
"factorize() requires analyzePattern() first");
314 if (!analysis_done_) {
334 const InputType& input = A.
derived();
335 check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
336 eigen_assert(input.rows() == n_ && input.cols() == n_);
338 const Scalar* value_ptr;
342 if (Derived::needs_csr_conversion()) {
343 csr_tmp = CsrMat(input);
347 const SpMat& csc = bind_sparse<SpMat>(input, csc_storage);
351 eigen_assert(value_nnz == nnz_);
352 EIGEN_UNUSED_VARIABLE(value_nnz);
354 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.get(), value_ptr,
static_cast<size_t>(nnz_) *
sizeof(Scalar),
355 cudaMemcpyHostToDevice, stream_));
357 EIGEN_CUDSS_CHECK(cudssMatrixSetValues(d_A_cudss_, d_values_.get()));
360 info_synced_ =
false;
362 cudssExecute(handle_, CUDSS_PHASE_FACTORIZATION, config_, data_, d_A_cudss_, d_x_cudss_, d_b_cudss_));
369 template <
typename Rhs>
374 eigen_assert(info() ==
Success &&
"GpuSparseSolver::solve requires a successful factorization");
375 eigen_assert(B.rows() == n_);
377 if (n_ == 0 || B.cols() == 0)
return DenseMatrix(n_, B.cols());
380 const int64_t nrhs =
static_cast<int64_t
>(rhs.cols());
383 const size_t rhs_bytes =
static_cast<size_t>(n_) *
static_cast<size_t>(nrhs) *
sizeof(Scalar);
384 ensure_solve_buffer(d_b_solve_, rhs_bytes);
385 ensure_solve_buffer(d_x_solve_, rhs_bytes);
386 internal::upload_host_matrix(
static_cast<Scalar*
>(d_b_solve_.get()), n_, rhs.data(), rhs.outerStride(), n_, nrhs,
389 update_solve_descriptors(nrhs, d_b_solve_.get(), d_x_solve_.get());
391 cudssExecute(handle_, CUDSS_PHASE_SOLVE, config_, data_, d_A_cudss_, x_solve_cudss_, b_solve_cudss_));
393 DenseMatrix X(n_, rhs.cols());
394 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(X.
data(), d_x_solve_.get(), rhs_bytes, cudaMemcpyDeviceToHost, stream_));
395 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
406 eigen_assert(info() ==
Success &&
"GpuSparseSolver::solve requires a successful factorization");
407 eigen_assert(d_B.rows() == n_);
409 const int64_t nrhs =
static_cast<int64_t
>(d_B.cols());
411 if (n_ == 0 || nrhs == 0)
return X;
414 update_solve_descriptors(nrhs,
const_cast<Scalar*
>(d_B.data()), X.data());
416 cudssExecute(handle_, CUDSS_PHASE_SOLVE, config_, data_, d_A_cudss_, x_solve_cudss_, b_solve_cudss_));
425 Index rows()
const {
return n_; }
426 Index cols()
const {
return n_; }
428 cudaStream_t stream()
const {
return stream_; }
431 cudaStream_t stream_ =
nullptr;
432 bool owns_stream_ =
true;
433 cudssHandle_t handle_ =
nullptr;
434 cudssConfig_t config_ =
nullptr;
435 cudssData_t data_ =
nullptr;
436 cudssMatrix_t d_A_cudss_ =
nullptr;
437 cudssMatrix_t d_x_cudss_ =
nullptr;
438 cudssMatrix_t d_b_cudss_ =
nullptr;
440 DeviceBuffer d_rowPtr_;
441 DeviceBuffer d_colIdx_;
442 DeviceBuffer d_values_;
447 DenseVector conj_values_;
449 mutable DeviceBuffer d_b_solve_;
450 mutable DeviceBuffer d_x_solve_;
454 mutable cudssMatrix_t b_solve_cudss_ =
nullptr;
455 mutable cudssMatrix_t x_solve_cudss_ =
nullptr;
456 mutable int64_t solve_desc_nrhs_ = -1;
461 mutable bool info_synced_ =
true;
462 bool analysis_done_ =
false;
463 SparseSolverConfig config_opts_;
466 Derived& derived() {
return static_cast<Derived&
>(*this); }
467 const Derived& derived()
const {
return static_cast<const Derived&
>(*this); }
469 void init_context(cudaStream_t stream,
bool borrow_stream) {
474 owns_stream_ =
false;
476 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&stream_));
479 EIGEN_CUDSS_CHECK(cudssCreate(&handle_));
480 EIGEN_CUDSS_CHECK(cudssSetStream(handle_, stream_));
481 EIGEN_CUDSS_CHECK(cudssConfigCreate(&config_));
487 void apply_config() {
488#if EIGEN_HAS_CUDSS_SOLVER_CONFIG
493 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
494 (void)cudssConfigDestroy(config_);
497 EIGEN_CUDSS_CHECK(cudssConfigCreate(&config_));
498 const SparseSolverConfig& c = config_opts_;
499 if (c.reordering != SparseReordering::Default) {
500 const cudssReorderingAlg_t v =
static_cast<cudssReorderingAlg_t
>(c.reordering);
501 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_REORDERING_ALG, &v,
sizeof(v)));
503 if (c.matching != SparseMatching::Default) {
504 const cudssMatchingAlg_t v =
static_cast<cudssMatchingAlg_t
>(c.matching);
505 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_MATCHING_ALG, &v,
sizeof(v)));
507 if (c.pivoting != SparsePivoting::Default) {
508 const cudssPivotType_t v =
static_cast<cudssPivotType_t
>(c.pivoting);
509 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_PIVOT_TYPE, &v,
sizeof(v)));
511 if (c.pivotThreshold >= 0) {
513 cudssConfigSet(config_, CUDSS_CONFIG_PIVOT_THRESHOLD, &c.pivotThreshold,
sizeof(c.pivotThreshold)));
515 if (c.pivotEpsilon >= 0) {
516 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_PIVOT_EPSILON, &c.pivotEpsilon,
sizeof(c.pivotEpsilon)));
518 if (c.refinementSteps >= 0) {
520 cudssConfigSet(config_, CUDSS_CONFIG_IR_N_STEPS, &c.refinementSteps,
sizeof(c.refinementSteps)));
522 if (c.refinementTolerance >= 0) {
524 cudssConfigSet(config_, CUDSS_CONFIG_IR_TOL, &c.refinementTolerance,
sizeof(c.refinementTolerance)));
526 if (c.hybridMemory) {
528 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_HYBRID_MEMORY_MODE, &v,
sizeof(v)));
529 if (c.hybridMemoryDeviceLimit >= 0) {
530 const int64_t limit = c.hybridMemoryDeviceLimit;
531 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_HYBRID_DEVICE_MEMORY_LIMIT, &limit,
sizeof(limit)));
534 if (c.hybridExecute) {
536 EIGEN_CUDSS_CHECK(cudssConfigSet(config_, CUDSS_CONFIG_HYBRID_EXECUTE_MODE, &v,
sizeof(v)));
540 eigen_internal_assert(config_opts_.isDefault());
544 void ensure_solve_buffer(DeviceBuffer& buf,
size_t needed)
const {
545 if (needed > buf.size()) {
546 if (buf) EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
547 buf = DeviceBuffer(needed);
553 void update_solve_descriptors(int64_t nrhs,
void* b_ptr,
void* x_ptr)
const {
554 constexpr cudss_value_type_t dtype = to_cudss_data_type(cuda_data_type<Scalar>::value);
555 if (!b_solve_cudss_ || solve_desc_nrhs_ != nrhs) {
556 destroy_solve_descriptors();
557 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&b_solve_cudss_, n_, nrhs, n_, b_ptr, dtype, CUDSS_LAYOUT_COL_MAJOR));
558 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&x_solve_cudss_, n_, nrhs, n_, x_ptr, dtype, CUDSS_LAYOUT_COL_MAJOR));
559 solve_desc_nrhs_ = nrhs;
561 EIGEN_CUDSS_CHECK(cudssMatrixSetValues(b_solve_cudss_, b_ptr));
562 EIGEN_CUDSS_CHECK(cudssMatrixSetValues(x_solve_cudss_, x_ptr));
566 void destroy_solve_descriptors()
const {
567 if (b_solve_cudss_) {
568 (void)cudssMatrixDestroy(b_solve_cudss_);
569 b_solve_cudss_ =
nullptr;
571 if (x_solve_cudss_) {
572 (void)cudssMatrixDestroy(x_solve_cudss_);
573 x_solve_cudss_ =
nullptr;
575 solve_desc_nrhs_ = -1;
578 void sync_info()
const {
583 EIGEN_CUDSS_CHECK(cudssDataGet(handle_, data_, CUDSS_DATA_INFO, &cudss_info,
sizeof(cudss_info),
nullptr));
590 void destroy_cudss_objects() {
594 (void)cudaStreamSynchronize(stream_);
595 destroy_solve_descriptors();
597 (void)cudssMatrixDestroy(d_A_cudss_);
598 d_A_cudss_ =
nullptr;
601 (void)cudssMatrixDestroy(d_x_cudss_);
602 d_x_cudss_ =
nullptr;
605 (void)cudssMatrixDestroy(d_b_cudss_);
606 d_b_cudss_ =
nullptr;
609 (void)cudssDataDestroy(handle_, data_);
613 (void)cudssConfigDestroy(config_);
621 static constexpr bool needs_value_conjugation() {
622 return Derived::cudss_matrix_type() == CUDSS_MTYPE_HPD || Derived::cudss_matrix_type() == CUDSS_MTYPE_HERMITIAN;
628 const Scalar* csc_upload_values(
const Scalar* values, Index nnz) {
629 if (!needs_value_conjugation())
return values;
630 conj_values_ = Map<const DenseVector>(values, nnz).conjugate();
631 return conj_values_.data();
634 void upload_compressed(
const StorageIndex* outer,
const StorageIndex* inner,
const Scalar* values) {
635 const size_t rowptr_bytes =
static_cast<size_t>(n_ + 1) *
sizeof(StorageIndex);
636 const size_t colidx_bytes =
static_cast<size_t>(nnz_) *
sizeof(StorageIndex);
637 const size_t values_bytes =
static_cast<size_t>(nnz_) *
sizeof(Scalar);
639 d_rowPtr_ = DeviceBuffer(rowptr_bytes);
640 d_colIdx_ = DeviceBuffer(colidx_bytes);
641 d_values_ = DeviceBuffer(values_bytes);
643 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_rowPtr_.get(), outer, rowptr_bytes, cudaMemcpyHostToDevice, stream_));
644 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_colIdx_.get(), inner, colidx_bytes, cudaMemcpyHostToDevice, stream_));
645 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.get(), values, values_bytes, cudaMemcpyHostToDevice, stream_));
648 void create_cudss_matrix() {
651 cudssMatrix_t old = d_A_cudss_;
652 d_A_cudss_ =
nullptr;
653 EIGEN_CUDSS_CHECK(cudssMatrixDestroy(old));
656 constexpr cudss_value_type_t idx_type = to_cudss_data_type(cudss_index_type<StorageIndex>::value);
657 constexpr cudss_value_type_t val_type = to_cudss_data_type(cuda_data_type<Scalar>::value);
658 constexpr cudssMatrixType_t mtype = Derived::cudss_matrix_type();
659 constexpr cudssMatrixViewType_t mview = Derived::cudss_matrix_view();
661#if defined(CUDSS_VERSION) && CUDSS_VERSION >= 800
663 EIGEN_CUDSS_CHECK(cudssMatrixCreateCsr(&d_A_cudss_, n_, n_, nnz_, d_rowPtr_.get(),
664 nullptr, d_colIdx_.get(), d_values_.get(), idx_type, idx_type,
665 val_type, mtype, mview, CUDSS_BASE_ZERO));
667 EIGEN_CUDSS_CHECK(cudssMatrixCreateCsr(&d_A_cudss_, n_, n_, nnz_, d_rowPtr_.get(),
668 nullptr, d_colIdx_.get(), d_values_.get(), idx_type, val_type,
669 mtype, mview, CUDSS_BASE_ZERO));
673 void create_placeholder_dense() {
675 destroy_solve_descriptors();
679 cudssMatrix_t old = d_x_cudss_;
680 d_x_cudss_ =
nullptr;
681 EIGEN_CUDSS_CHECK(cudssMatrixDestroy(old));
684 cudssMatrix_t old = d_b_cudss_;
685 d_b_cudss_ =
nullptr;
686 EIGEN_CUDSS_CHECK(cudssMatrixDestroy(old));
688 constexpr cudss_value_type_t dtype = to_cudss_data_type(cuda_data_type<Scalar>::value);
689 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&d_x_cudss_, n_, 1, n_,
nullptr, dtype, CUDSS_LAYOUT_COL_MAJOR));
690 EIGEN_CUDSS_CHECK(cudssMatrixCreateDn(&d_b_cudss_, n_, 1, n_,
nullptr, dtype, CUDSS_LAYOUT_COL_MAJOR));