541 cusparseOperation_t op) {
542 eigen_assert(A.isCompressed());
544 const Index m = A.rows();
545 const Index n = A.cols();
546 const Index nnz = A.nonZeros();
547 const Index x_size = (op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? n : m;
548 const Index y_size = (op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? m : n;
550 eigen_assert(x.size() == x_size);
551 eigen_assert(y.size() == y_size);
553 if (m == 0 || n == 0 || nnz == 0) {
559 spmv_host_cached(x, y, x_size, y_size, alpha, beta, op);
566 template <
typename Bsm,
typename Rhs>
567 DenseVector multiply_host_return_block(
const Bsm& A,
const MatrixBase<Rhs>& x, GpuOp op) {
568 DenseVector y((op == GpuOp::NoTrans) ? A.rows() : A.cols());
570 multiply_host_block(A, x.derived(), y, Scalar(1), Scalar(0), op,
575 template <
typename Bsm>
576 DeviceSparseView<Scalar> device_view_block(
const Bsm& A, std::false_type) {
577 return deviceView(SpMat(A.toSparse()));
580 template <
typename Bsm,
typename RhsDerived,
typename DestDerived>
581 void multiply_host_block(
const Bsm& A,
const RhsDerived& x, DestDerived& y, Scalar alpha, Scalar beta, GpuOp op,
583 multiply(A.toSparse(), x, y, alpha, beta, op);
586 template <
typename Bsm>
587 void multiply_device_block(
const Bsm& A,
const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y, Scalar alpha,
588 Scalar beta, GpuOp op, std::false_type) {
589 multiply(A.toSparse(), d_x, d_y, alpha, beta, op);
592 template <
typename Bsm,
typename Rhs>
593 DenseMatrix multiply_mat_block(
const Bsm& A,
const MatrixBase<Rhs>& X, GpuOp op, std::false_type) {
594 return multiplyMat(A.toSparse(), X, op);
597#if EIGEN_HAS_CUSPARSE_BSR
598 template <
typename Bsm>
599 DeviceSparseView<Scalar> device_view_block(
const Bsm& A, std::true_type) {
600 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
601 const internal::BsrBinding<Bsm> bound(A, GpuOp::NoTrans);
602 upload_bsr(bound.arrays());
603 return DeviceSparseView<Scalar>(*
this, A.rows(), A.cols(), generation_);
606 template <
typename Bsm,
typename RhsDerived,
typename DestDerived>
607 void multiply_host_block(
const Bsm& A,
const RhsDerived& x, DestDerived& y, Scalar alpha, Scalar beta, GpuOp op,
609 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
610 const internal::BsrBinding<Bsm> bound(A, op);
611 const internal::BsrArrays<Scalar, StorageIndex>& opA = bound.arrays();
613 eigen_assert(x.size() == opA.cols());
614 eigen_assert(y.size() == opA.rows());
616 if (opA.rows() == 0 || opA.cols() == 0 || opA.bnnz == 0) {
622 spmv_host_cached(x, y, opA.cols(), opA.rows(), alpha, beta, CUSPARSE_OPERATION_NON_TRANSPOSE);
625 template <
typename Bsm>
626 void multiply_device_block(
const Bsm& A,
const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y, Scalar alpha,
627 Scalar beta, GpuOp op, std::true_type) {
628 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
629 const internal::BsrBinding<Bsm> bound(A, op);
630 upload_bsr(bound.arrays());
631 spmv_device_exec(d_x, d_y, alpha, beta, GpuOp::NoTrans);
634 template <
typename Bsm,
typename Rhs>
635 DenseMatrix multiply_mat_block(
const Bsm& A,
const MatrixBase<Rhs>& X, GpuOp op, std::true_type) {
636 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
637 const internal::BsrBinding<Bsm> bound(A, op);
638 const internal::BsrArrays<Scalar, StorageIndex>& opA = bound.arrays();
639 const DenseMatrix rhs(X.derived());
640 eigen_assert(opA.cols() == rhs.rows());
642 const Index n = rhs.cols();
643 if (opA.rows() == 0 || opA.cols() == 0 || n == 0 || opA.bnnz == 0)
return DenseMatrix::Zero(opA.rows(), n);
645 DenseMatrix Y = DenseMatrix::Zero(opA.rows(), n);
647 spmm_host_cached(rhs, Y, opA.rows(), opA.cols(), Scalar(1), Scalar(0), CUSPARSE_OPERATION_NON_TRANSPOSE);
655 template <
typename RhsDerived,
typename DestDerived>
656 void spmv_host_cached(
const RhsDerived& x, DestDerived& y, Index x_size, Index y_size, Scalar alpha, Scalar beta,
657 cusparseOperation_t op) {
658 ensure_buffer(d_x_,
static_cast<size_t>(x_size) *
sizeof(Scalar));
661 const Ref<const DenseVector> x_ref(x);
662 EIGEN_CUDA_RUNTIME_CHECK(
663 cudaMemcpyAsync(d_x_.get(), x_ref.data(), x_size *
sizeof(Scalar), cudaMemcpyHostToDevice, stream_));
665 ensure_buffer(d_y_,
static_cast<size_t>(y_size) *
sizeof(Scalar));
666 if (beta != Scalar(0)) {
667 const Ref<const DenseVector> y_ref(y);
668 EIGEN_CUDA_RUNTIME_CHECK(
669 cudaMemcpyAsync(d_y_.get(), y_ref.data(), y_size *
sizeof(Scalar), cudaMemcpyHostToDevice, stream_));
672 exec_spmv(x_size, y_size, d_x_.get(), d_y_.get(), alpha, beta, op);
676 const Index incy = y.cols() == 1 ? y.rowStride() : y.colStride();
678 EIGEN_CUDA_RUNTIME_CHECK(
679 cudaMemcpyAsync(y.data(), d_y_.get(), y_size *
sizeof(Scalar), cudaMemcpyDeviceToHost, stream_));
682 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy2DAsync(y.data(),
static_cast<size_t>(incy) *
sizeof(Scalar), d_y_.get(),
683 sizeof(Scalar),
sizeof(Scalar),
static_cast<size_t>(y_size),
684 cudaMemcpyDeviceToHost, stream_));
686 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
695 void spmv_device_exec(
const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y, Scalar alpha = Scalar(1),
696 Scalar beta = Scalar(0), GpuOp op = GpuOp::NoTrans)
const {
697 eigen_assert(spmat_desc_ &&
"sparse matrix not uploaded — call deviceView() or multiply() first");
698 check_op_against_upload(op);
700 eigen_assert(d_x.data() != d_y.data() &&
"SpMV: output aliases input vector");
702 const cusparseOperation_t cu_op = internal::to_cusparse_op<Scalar>(op);
703 const Index m = cached_rows_;
704 const Index n = cached_cols_;
705 const Index x_size = (cu_op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? n : m;
706 const Index y_size = (cu_op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? m : n;
708 eigen_assert(d_x.rows() * d_x.cols() == x_size);
710 if (m == 0 || n == 0 || cached_nnz_ == 0) {
714 if (d_y.rows() * d_y.cols() != y_size) {
715 d_y.resize(y_size, 1);
716 d_y.setZero(stream_);
717 }
else if (beta == Scalar(0)) {
718 d_y.setZero(stream_);
719 }
else if (beta != Scalar(1)) {
726 if (d_y.rows() * d_y.cols() != y_size) {
727 d_y.resize(y_size, 1);
731 d_x.waitReady(stream_);
732 d_y.waitReady(stream_);
734 exec_spmv(x_size, y_size,
const_cast<void*
>(
static_cast<const void*
>(d_x.data())),
static_cast<void*
>(d_y.data()),
737 d_y.recordReady(stream_);
744 void spmm_device_exec(
const DeviceMatrix<Scalar>& d_X, DeviceMatrix<Scalar>& d_Y, Scalar alpha = Scalar(1),
745 Scalar beta = Scalar(0), GpuOp op = GpuOp::NoTrans)
const {
746 eigen_assert(spmat_desc_ &&
"sparse matrix not uploaded — call deviceView() or multiply() first");
747 check_op_against_upload(op);
748 eigen_assert(d_X.data() != d_Y.data() &&
"SpMM: output aliases input matrix");
750 const cusparseOperation_t cu_op = internal::to_cusparse_op<Scalar>(op);
751 const bool transposed = (cu_op != CUSPARSE_OPERATION_NON_TRANSPOSE);
752 const Index m_op = transposed ? cached_cols_ : cached_rows_;
753 const Index k_op = transposed ? cached_rows_ : cached_cols_;
754 const Index n = d_X.cols();
756 eigen_assert(d_X.rows() == k_op);
758 if (m_op == 0 || n == 0 || cached_nnz_ == 0) {
759 if (d_Y.rows() != m_op || d_Y.cols() != n) {
761 d_Y.setZero(stream_);
762 }
else if (beta == Scalar(0)) {
763 d_Y.setZero(stream_);
764 }
else if (beta != Scalar(1)) {
770 if (d_Y.rows() != m_op || d_Y.cols() != n) {
774 d_X.waitReady(stream_);
775 d_Y.waitReady(stream_);
777 exec_spmm(m_op, k_op, n,
const_cast<void*
>(
static_cast<const void*
>(d_X.data())),
static_cast<void*
>(d_Y.data()),
780 d_Y.recordReady(stream_);
788#if !defined(CUSPARSE_VERSION) || CUSPARSE_VERSION < 12000
789 static constexpr bool kUseCsrOfTranspose =
true;
790 static constexpr cusparseSpMMAlg_t kSpMMAlg = CUSPARSE_SPMM_CSR_ALG2;
792 static constexpr bool kUseCsrOfTranspose =
false;
793 static constexpr cusparseSpMMAlg_t kSpMMAlg = CUSPARSE_SPMM_ALG_DEFAULT;
799 static cusparseOperation_t descriptor_op(cusparseOperation_t user_op) {
800 EIGEN_IF_CONSTEXPR (!kUseCsrOfTranspose)
return user_op;
802 case CUSPARSE_OPERATION_NON_TRANSPOSE:
803 return CUSPARSE_OPERATION_TRANSPOSE;
804 case CUSPARSE_OPERATION_TRANSPOSE:
805 return CUSPARSE_OPERATION_NON_TRANSPOSE;
809 eigen_assert(
false &&
"CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE not supported on cuSPARSE < 12.0");
814 static int op_index(cusparseOperation_t op) {
816 case CUSPARSE_OPERATION_TRANSPOSE:
818 case CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE:
825 void invalidate_ws_caches()
const {
826 for (
int i = 0; i < 3; ++i) {
827 spmv_ws_size_[i] = kWsUnknown;
828 spmm_ws_size_[i] = kWsUnknown;
835 void update_dnvec(cusparseDnVecDescr_t& desc, int64_t& cur_size, int64_t size,
void* ptr)
const {
836 if (!desc || cur_size != size) {
839 cusparseDnVecDescr_t old = desc;
841 EIGEN_CUSPARSE_CHECK(cusparseDestroyDnVec(old));
843 EIGEN_CUSPARSE_CHECK(cusparseCreateDnVec(&desc, size, ptr, kValueType));
845 invalidate_ws_caches();
847 EIGEN_CUSPARSE_CHECK(cusparseDnVecSetValues(desc, ptr));
851 void update_dnmat(cusparseDnMatDescr_t& desc, int64_t& cur_rows, int64_t& cur_cols, int64_t rows, int64_t cols,
853 if (!desc || cur_rows != rows || cur_cols != cols) {
855 cusparseDnMatDescr_t old = desc;
857 EIGEN_CUSPARSE_CHECK(cusparseDestroyDnMat(old));
860 EIGEN_CUSPARSE_CHECK(cusparseCreateDnMat(&desc, rows, cols, rows, ptr, kValueType, CUSPARSE_ORDER_COL));
863 invalidate_ws_caches();
865 EIGEN_CUSPARSE_CHECK(cusparseDnMatSetValues(desc, ptr));
869 void destroy_dense_descriptors() {
870 if (x_vec_desc_) (void)cusparseDestroyDnVec(x_vec_desc_);
871 if (y_vec_desc_) (void)cusparseDestroyDnVec(y_vec_desc_);
872 if (x_mat_desc_) (void)cusparseDestroyDnMat(x_mat_desc_);
873 if (y_mat_desc_) (void)cusparseDestroyDnMat(y_mat_desc_);
874 x_vec_desc_ = y_vec_desc_ =
nullptr;
875 x_mat_desc_ = y_mat_desc_ =
nullptr;
878 void exec_spmv(Index x_size, Index y_size,
void* d_x_ptr,
void* d_y_ptr, Scalar alpha, Scalar beta,
879 cusparseOperation_t op)
const {
880 const cusparseOperation_t cu_op = descriptor_op(op);
881 update_dnvec(x_vec_desc_, x_vec_size_, x_size, d_x_ptr);
882 update_dnvec(y_vec_desc_, y_vec_size_, y_size, d_y_ptr);
884 size_t& ws_size = spmv_ws_size_[op_index(cu_op)];
885 if (ws_size == kWsUnknown) {
886 EIGEN_CUSPARSE_CHECK(cusparseSpMV_bufferSize(handle_, cu_op, &alpha, spmat_desc_, x_vec_desc_, &beta, y_vec_desc_,
887 kValueType, CUSPARSE_SPMV_ALG_DEFAULT, &ws_size));
889 ensure_buffer(d_workspace_, ws_size);
891 EIGEN_CUSPARSE_CHECK(cusparseSpMV(handle_, cu_op, &alpha, spmat_desc_, x_vec_desc_, &beta, y_vec_desc_, kValueType,
892 CUSPARSE_SPMV_ALG_DEFAULT, d_workspace_.get()));
895 void exec_spmm(Index m_op, Index k_op, Index n,
void* d_x_ptr,
void* d_y_ptr, Scalar alpha, Scalar beta,
896 cusparseOperation_t op)
const {
897 const cusparseOperation_t cu_op = descriptor_op(op);
899 update_dnmat(x_mat_desc_, x_mat_rows_, x_mat_cols_, k_op, n, d_x_ptr);
900 update_dnmat(y_mat_desc_, y_mat_rows_, y_mat_cols_, m_op, n, d_y_ptr);
902 size_t& ws_size = spmm_ws_size_[op_index(cu_op)];
903 if (ws_size == kWsUnknown) {
904 EIGEN_CUSPARSE_CHECK(cusparseSpMM_bufferSize(handle_, cu_op, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
905 spmat_desc_, x_mat_desc_, &beta, y_mat_desc_, kValueType, kSpMMAlg,
908 ensure_buffer(d_workspace_, ws_size);
910 EIGEN_CUSPARSE_CHECK(cusparseSpMM(handle_, cu_op, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, spmat_desc_,
911 x_mat_desc_, &beta, y_mat_desc_, kValueType, kSpMMAlg, d_workspace_.get()));
914 void spmm_impl(
const SpMat& A,
const DenseMatrix& X, DenseMatrix& Y, Scalar alpha, Scalar beta,
915 cusparseOperation_t op) {
916 eigen_assert(A.isCompressed());
921 const bool transposed = (op != CUSPARSE_OPERATION_NON_TRANSPOSE);
922 const Index m_op = transposed ? A.cols() : A.rows();
923 const Index k_op = transposed ? A.rows() : A.cols();
924 const Index n = X.cols();
925 const Index nnz = A.nonZeros();
927 if (m_op == 0 || n == 0 || k_op == 0 || nnz == 0) {
933 spmm_host_cached(X, Y, m_op, k_op, alpha, beta, op);
938 void spmm_host_cached(
const DenseMatrix& X, DenseMatrix& Y, Index m_op, Index k_op, Scalar alpha, Scalar beta,
939 cusparseOperation_t op) {
940 const Index n = X.cols();
941 const size_t x_bytes =
static_cast<size_t>(k_op) *
static_cast<size_t>(n) *
sizeof(Scalar);
942 const size_t y_bytes =
static_cast<size_t>(m_op) *
static_cast<size_t>(n) *
sizeof(Scalar);
943 ensure_buffer(d_x_, x_bytes);
944 ensure_buffer(d_y_, y_bytes);
945 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_x_.get(), X.data(), x_bytes, cudaMemcpyHostToDevice, stream_));
946 if (beta != Scalar(0)) {
947 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_y_.get(), Y.data(), y_bytes, cudaMemcpyHostToDevice, stream_));
950 exec_spmm(m_op, k_op, n, d_x_.get(), d_y_.get(), alpha, beta, op);
952 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(Y.data(), d_y_.get(), y_bytes, cudaMemcpyDeviceToHost, stream_));
953 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
956 void upload_sparse(
const SpMat& A) {
962 const Index m = A.rows();
963 const Index n = A.cols();
964 const Index nnz = A.nonZeros();
965 upload_arrays(n + 1, nnz, nnz, A.outerIndexPtr(), A.innerIndexPtr(),
967 if (descriptor_key_matches(m, n, nnz, 0))
return;
969 destroy_spmat_descriptor(
true);
970 EIGEN_IF_CONSTEXPR (kUseCsrOfTranspose) {
974 EIGEN_CUSPARSE_CHECK(cusparseCreateCsr(&spmat_desc_, n, m, nnz, d_outerPtr_.get(), d_innerIdx_.get(),
975 d_values_.get(), kIndexType, kIndexType, CUSPARSE_INDEX_BASE_ZERO,
978 EIGEN_CUSPARSE_CHECK(cusparseCreateCsc(&spmat_desc_, m, n, nnz, d_outerPtr_.get(), d_innerIdx_.get(),
979 d_values_.get(), kIndexType, kIndexType, CUSPARSE_INDEX_BASE_ZERO,
982 set_descriptor_key(m, n, nnz, 0);
985#if EIGEN_HAS_CUSPARSE_BSR
986 void upload_bsr(
const internal::BsrArrays<Scalar, StorageIndex>& opA) {
987 upload_arrays(opA.brows + 1, opA.bnnz, opA.nonZeros(), opA.outer,
988 opA.inner, opA.values);
989 if (descriptor_key_matches(opA.rows(), opA.cols(), opA.nonZeros(), opA.blockSize))
return;
991 destroy_spmat_descriptor(
true);
993 EIGEN_CUSPARSE_CHECK(cusparseCreateBsr(&spmat_desc_, opA.brows, opA.bcols, opA.bnnz, opA.blockSize, opA.blockSize,
994 d_outerPtr_.get(), d_innerIdx_.get(), d_values_.get(), kIndexType,
995 kIndexType, CUSPARSE_INDEX_BASE_ZERO, kValueType, CUSPARSE_ORDER_ROW));
996 set_descriptor_key(opA.rows(), opA.cols(), opA.nonZeros(), opA.blockSize);
1000 void upload_arrays(Index outer_count, Index inner_count, Index value_count,
const StorageIndex* host_outer,
1001 const StorageIndex* host_inner,
const Scalar* host_values) {
1002 const size_t outer_bytes =
static_cast<size_t>(outer_count) *
sizeof(StorageIndex);
1003 const size_t inner_bytes =
static_cast<size_t>(inner_count) *
sizeof(StorageIndex);
1004 const size_t val_bytes =
static_cast<size_t>(value_count) *
sizeof(Scalar);
1014 ensure_buffer(d_values_, val_bytes);
1015 ensure_buffer(d_outerPtr_, outer_bytes);
1016 ensure_buffer(d_innerIdx_, inner_bytes);
1017 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.get(), host_values, val_bytes, cudaMemcpyHostToDevice, stream_));
1018 EIGEN_CUDA_RUNTIME_CHECK(
1019 cudaMemcpyAsync(d_outerPtr_.get(), host_outer, outer_bytes, cudaMemcpyHostToDevice, stream_));
1020 EIGEN_CUDA_RUNTIME_CHECK(
1021 cudaMemcpyAsync(d_innerIdx_.get(), host_inner, inner_bytes, cudaMemcpyHostToDevice, stream_));
1027 bool descriptor_key_matches(Index m, Index n, Index nnz, Index block_size)
const {
1028 return m == cached_rows_ && n == cached_cols_ && nnz == cached_nnz_ && block_size == cached_block_size_;
1031 void set_descriptor_key(Index m, Index n, Index nnz, Index block_size) {
1035 cached_block_size_ = block_size;
1040 void check_op_against_upload(GpuOp op)
const {
1041 eigen_assert((cached_block_size_ == 0 || op == GpuOp::NoTrans) &&
1042 "cuSPARSE runs BSR products only with op == NoTrans; pass op to multiply(A, d_x, d_y, ...) instead");
1043 EIGEN_UNUSED_VARIABLE(op);
1049 void destroy_spmat_descriptor(
bool checked) {
1051 cusparseSpMatDescr_t old = spmat_desc_;
1052 spmat_desc_ =
nullptr;
1054 EIGEN_CUSPARSE_CHECK(cusparseDestroySpMat(old));
1056 (
void)cusparseDestroySpMat(old);
1058 set_descriptor_key(-1, -1, -1, 0);
1059 invalidate_ws_caches();
1062 void ensure_buffer(internal::DeviceBuffer& buf,
size_t needed)
const {
1063 if (needed > buf.size()) {
1064 if (buf) EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
1065 buf = internal::DeviceBuffer(needed);
1072template <
typename Scalar_>
1077 eigen_assert(expr.view().
generation() == expr.view().context().uploadGeneration() &&
1078 "DeviceSparseView is stale: its SparseContext has since uploaded another sparse matrix");
1079 if (expr.x().cols() <= 1) {
1080 expr.view().context().spmv_device_exec(expr.x(), *
this, Scalar_(1), Scalar_(0), GpuOp::NoTrans);
1082 expr.view().context().spmm_device_exec(expr.x(), *
this, Scalar_(1), Scalar_(0), GpuOp::NoTrans);
1087template <
typename Scalar_>
1091 eigen_assert(
view.generation() ==
view.context().uploadGeneration() &&
1092 "DeviceSparseView is stale: its SparseContext has since uploaded another sparse matrix");
1093 eigen_assert(addend.rows() ==
view.rows() && addend.cols() == expr.x().cols() &&
1094 "SpMVAffineExpr: the addend must have the shape of the product");
1095 eigen_assert(&expr.x() !=
this &&
"SpMVAffineExpr: the destination aliases the dense operand");
1100 if (
view.nonZeros() == 0 ||
view.rows() == 0 ||
view.cols() == 0) {
1104 if (expr.x().cols() <= 1) {
1105 view.context().spmv_device_exec(expr.x(), *
this, expr.alpha(), expr.beta(), GpuOp::NoTrans);
1107 view.context().spmm_device_exec(expr.x(), *
this, expr.alpha(), expr.beta(), GpuOp::NoTrans);