Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GpuSparseContext.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// GPU sparse matrix-vector (SpMV) and sparse matrix-dense matrix (SpMM) multiply
12// via cuSPARSE.
13//
14// SparseContext owns the cuSPARSE descriptors and device buffers. It takes
15// SparseMatrix<Scalar, ColMajor> (CSC), implicitly converting RowMajor input, and
16// can borrow a gpu::Context so that sparse products share a stream with BLAS-1
17// operations — which removes the cross-stream event waits in solvers like CG.
18// It also takes BlockSparseMatrix: as BSR (internal::BsrBinding) where
19// internal::use_cusparse_bsr holds, as the scalar-level toSparse() copy otherwise.
20//
21// Caching: host-input calls re-upload the values *and* index arrays on every
22// call. Host pointer identity cannot detect a sparsity pattern rewritten in
23// place or assigned into the same allocations (SparseMatrix reuses them for
24// same-shape assignments), so the structure is never assumed unchanged. What
25// is cached are the cuSPARSE descriptors (sparse descriptor keyed on
26// dimensions + nonzero count + block size, dense descriptors keyed on shape)
27// and the SpMV/SpMM workspace-size queries. For repeated products with no
28// re-upload at all, use deviceView().
29//
30// Not thread-safe: concurrent multiply* calls on one instance race on the
31// cuSPARSE handle, the bound stream, and the cached buffers. Use one per thread.
32
33#ifndef EIGEN_GPU_SPARSE_CONTEXT_H
34#define EIGEN_GPU_SPARSE_CONTEXT_H
35
36// IWYU pragma: private
37#include "./InternalHeaderCheck.h"
38
39#include <cstdint>
40
41#include "./CuSparseSupport.h"
42#include "./FwdDecl.h"
43
44namespace Eigen {
45namespace gpu {
46
47namespace internal {
53template <int BlockRows, int BlockCols>
55 : Eigen::internal::bool_constant<(EIGEN_HAS_CUSPARSE_BSR != 0 && BlockRows == BlockCols && BlockRows >= 2)> {};
56} // namespace internal
57
58#if EIGEN_HAS_CUSPARSE_BSR
59namespace internal {
60
64template <typename Scalar, typename StorageIndex>
65struct BsrArrays {
66 Index brows;
67 Index bcols;
68 Index bnnz;
69 Index blockSize;
70 const StorageIndex* outer;
71 const StorageIndex* inner;
72 const Scalar* values;
73
74 Index rows() const { return brows * blockSize; }
75 Index cols() const { return bcols * blockSize; }
76 Index nonZeros() const { return bnnz * blockSize * blockSize; }
77};
78
85template <typename Bsm>
86BsrArrays<typename Bsm::Scalar, typename Bsm::StorageIndex> bsr_of(const Bsm& M) {
87 return BsrArrays<typename Bsm::Scalar, typename Bsm::StorageIndex>{
88 /*brows=*/M.blockOuterSize(), /*bcols=*/M.blockInnerSize(),
89 /*bnnz=*/M.nonZeroBlocks(), /*blockSize=*/Index(Bsm::BlockRows),
90 /*outer=*/M.outerIndexPtr(), /*inner=*/M.innerIndexPtr(),
91 /*values=*/M.valuePtr()};
92}
93
99template <typename Bsm>
100class BsrBinding {
101 EIGEN_STATIC_ASSERT((use_cusparse_bsr<int(Bsm::BlockRows), int(Bsm::BlockCols)>::value),
102 CUSPARSE_BSR_REQUIRES_SQUARE_BLOCKS_OF_SIZE_AT_LEAST_2)
103
104 public:
105 using Scalar = typename Bsm::Scalar;
106 using Arrays = BsrArrays<Scalar, typename Bsm::StorageIndex>;
107
108 BsrBinding(const Bsm& A, GpuOp op) {
109 if (op == GpuOp::ConjTrans && !NumTraits<Scalar>::IsComplex) op = GpuOp::Trans;
110 const bool direct = Bsm::IsRowMajor ? (op == GpuOp::NoTrans) : (op == GpuOp::Trans);
111 if (direct) {
112 arrays_ = bsr_of(A);
113 return;
114 }
115 if (op == GpuOp::ConjTrans && !Bsm::IsRowMajor) {
116 // ColMajor conj(A) reads as BSR of conj(A)^T = A^H.
117 copy_ = A.unaryExpr(Eigen::internal::scalar_conjugate_op<Scalar>());
118 } else if (op == GpuOp::ConjTrans) {
119 copy_ = A.adjoint();
120 } else {
121 copy_ = A.transpose();
122 }
123 arrays_ = bsr_of(copy_);
124 }
125
126 BsrBinding(const BsrBinding&) = delete;
127 BsrBinding& operator=(const BsrBinding&) = delete;
128
129 const Arrays& arrays() const { return arrays_; }
130
131 private:
132 // transpose()/adjoint() swap the block dimensions, which are equal here, so
133 // the copy has A's own type in either storage order.
134 Bsm copy_;
135 Arrays arrays_{};
136};
137
138} // namespace internal
139#endif // EIGEN_HAS_CUSPARSE_BSR
140
145template <typename Scalar_>
146class SpMVExpr {
147 public:
148 using Scalar = Scalar_;
149 SpMVExpr(const DeviceSparseView<Scalar>& view, const DeviceMatrix<Scalar>& x) : view_(view), x_(x) {}
150 const DeviceSparseView<Scalar>& view() const { return view_; }
151 const DeviceMatrix<Scalar>& x() const { return x_; }
152
153 private:
154 const DeviceSparseView<Scalar>& view_;
155 const DeviceMatrix<Scalar>& x_;
156};
157
165template <typename Scalar_>
166class SpMVAffineExpr {
167 public:
168 using Scalar = Scalar_;
169 SpMVAffineExpr(const SpMVExpr<Scalar>& product, Scalar alpha, Scalar beta, const DeviceMatrix<Scalar>& addend)
170 : view_(product.view()), x_(product.x()), alpha_(alpha), beta_(beta), addend_(addend) {}
171 const DeviceSparseView<Scalar>& view() const { return view_; }
172 const DeviceMatrix<Scalar>& x() const { return x_; }
173 Scalar alpha() const { return alpha_; }
174 Scalar beta() const { return beta_; }
175 const DeviceMatrix<Scalar>& addend() const { return addend_; }
176
177 private:
178 const DeviceSparseView<Scalar>& view_;
179 const DeviceMatrix<Scalar>& x_;
180 Scalar alpha_;
181 Scalar beta_;
182 const DeviceMatrix<Scalar>& addend_;
183};
184
185template <typename S>
186SpMVAffineExpr<S> operator-(const DeviceMatrix<S>& b, const SpMVExpr<S>& p) {
187 return SpMVAffineExpr<S>(p, S(-1), S(1), b);
188}
189
190template <typename S>
191SpMVAffineExpr<S> operator+(const DeviceMatrix<S>& b, const SpMVExpr<S>& p) {
192 return SpMVAffineExpr<S>(p, S(1), S(1), b);
193}
194
195template <typename S>
196SpMVAffineExpr<S> operator+(const SpMVExpr<S>& p, const DeviceMatrix<S>& b) {
197 return b + p;
198}
199
200template <typename S>
201SpMVAffineExpr<S> operator-(const SpMVExpr<S>& p, const DeviceMatrix<S>& b) {
202 return SpMVAffineExpr<S>(p, S(1), S(-1), b);
203}
204
205} // namespace gpu
206
207namespace internal {
208// DeviceSparseView is a matrix-free matrix type for Eigen's iterative solvers: it looks like a
209// SparseMatrix to the traits machinery, and IterativeSolverBase stores it by pointer because
210// Ref<> cannot bind to it (the same mechanism as the matrix-free example in the documentation).
211template <typename Scalar_>
212struct traits<gpu::DeviceSparseView<Scalar_>> : traits<SparseMatrix<Scalar_, ColMajor, int>> {};
213} // namespace internal
214
215namespace gpu {
216
233template <typename Scalar_>
234class DeviceSparseView : public EigenBase<DeviceSparseView<Scalar_>> {
235 public:
236 using Scalar = Scalar_;
237 using RealScalar = typename NumTraits<Scalar>::Real;
238 using StorageIndex = int;
240 // What IterativeSolverBase and ConjugateGradient read from a matrix type.
241 static constexpr int ColsAtCompileTime = Dynamic;
242 static constexpr int MaxColsAtCompileTime = Dynamic;
243 static constexpr bool IsRowMajor = false;
244
245 DeviceSparseView(SparseContext<Scalar>& ctx, Index rows, Index cols, uint64_t generation)
246 : ctx_(ctx), rows_(rows), cols_(cols), generation_(generation) {}
247
250
251 Index rows() const { return rows_; }
252 Index cols() const { return cols_; }
254 Index nonZeros() const { return ctx_.nonZeros(); }
255 const SparseContext<Scalar>& context() const { return ctx_; }
256
259 uint64_t generation() const { return generation_; }
260
261 private:
262 SparseContext<Scalar>& ctx_;
263 Index rows_;
264 Index cols_;
265 uint64_t generation_;
266};
267
268template <typename Scalar_>
269class SparseContext {
270 public:
271 using Scalar = Scalar_;
272 using RealScalar = typename NumTraits<Scalar>::Real;
273 using StorageIndex = int;
275 using DenseVector = Matrix<Scalar, Dynamic, 1>;
282 template <int Options, int BlockRows, int BlockCols>
284
286 SparseContext() : owns_handle_(true) {
287 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamCreate(&stream_));
288 owns_stream_ = true;
289 EIGEN_CUSPARSE_CHECK(cusparseCreate(&handle_));
290 EIGEN_CUSPARSE_CHECK(cusparseSetStream(handle_, stream_));
291 }
292
295 explicit SparseContext(Context& ctx)
296 : stream_(ctx.stream()), handle_(ctx.cusparseHandle()), owns_stream_(false), owns_handle_(false) {}
297
298 ~SparseContext() {
299 destroy_spmat_descriptor(/*checked=*/false);
300 destroy_dense_descriptors();
301 if (owns_handle_ && handle_) (void)cusparseDestroy(handle_);
302 if (owns_stream_ && stream_) (void)cudaStreamDestroy(stream_);
303 }
304
305 SparseContext(const SparseContext&) = delete;
306 SparseContext& operator=(const SparseContext&) = delete;
307
320 DeviceSparseView<Scalar> deviceView(const SpMat& A) {
321 eigen_assert(A.isCompressed());
322 upload_sparse(A);
323 return DeviceSparseView<Scalar>(*this, A.rows(), A.cols(), generation_);
324 }
325
328 uint64_t uploadGeneration() const { return generation_; }
329
331 Index nonZeros() const { return cached_nnz_; }
332
334 template <typename InputType, typename Rhs>
335 DenseVector multiply(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x) {
336 return multiply_host_return(A, x, GpuOp::NoTrans);
337 }
338
340 template <typename InputType, typename Rhs, typename Dest>
341 void multiply(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x, MatrixBase<Dest>& y,
342 Scalar alpha = Scalar(1), Scalar beta = Scalar(0), GpuOp op = GpuOp::NoTrans) {
343 const InputType& input = A.derived();
344 internal::check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
345 SpMat storage;
346 const SpMat& mat = internal::bind_sparse<SpMat>(input, storage);
347 multiply_host_impl(mat, x.derived(), y.derived(), alpha, beta, internal::to_cusparse_op<Scalar>(op));
348 }
349
353 template <typename InputType>
354 void multiply(const SparseMatrixBase<InputType>& A, const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y) {
355 multiply(A, d_x, d_y, Scalar(1), Scalar(0), GpuOp::NoTrans);
356 }
357
359 template <typename InputType>
360 void multiply(const SparseMatrixBase<InputType>& A, const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y,
361 Scalar alpha, Scalar beta, GpuOp op = GpuOp::NoTrans) {
362 const InputType& input = A.derived();
363 internal::check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
364 SpMat storage;
365 const SpMat& mat = internal::bind_sparse<SpMat>(input, storage);
366 upload_sparse(mat);
367 spmv_device_exec(d_x, d_y, alpha, beta, op);
368 }
369
371 template <typename InputType, typename Rhs>
372 DenseVector multiplyT(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x) {
373 return multiply_host_return(A, x, GpuOp::Trans);
374 }
375
377 template <typename InputType, typename Rhs>
378 DenseVector multiplyAdjoint(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x) {
379 return multiply_host_return(A, x, GpuOp::ConjTrans);
380 }
381
383 template <typename InputType, typename Rhs>
384 DenseMatrix multiplyMat(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& X, GpuOp op = GpuOp::NoTrans) {
385 const InputType& input = A.derived();
386 internal::check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
387 SpMat storage;
388 const SpMat& mat = internal::bind_sparse<SpMat>(input, storage);
389 const DenseMatrix rhs(X.derived());
390
391 const cusparseOperation_t cu_op = internal::to_cusparse_op<Scalar>(op);
392 const Index m = (op == GpuOp::NoTrans) ? mat.rows() : mat.cols();
393 const Index k = (op == GpuOp::NoTrans) ? mat.cols() : mat.rows();
394 eigen_assert(k == rhs.rows());
395 EIGEN_UNUSED_VARIABLE(k);
396
397 const Index n = rhs.cols();
398 if (m == 0 || n == 0 || mat.nonZeros() == 0) return DenseMatrix::Zero(m, n);
399
400 DenseMatrix Y = DenseMatrix::Zero(m, n);
401 spmm_impl(mat, rhs, Y, Scalar(1), Scalar(0), cu_op);
402 return Y;
403 }
404
405 // BlockSparseMatrix overloads: same contracts as the SparseMatrix ones,
406 // dispatched on internal::use_cusparse_bsr<BlockRows, BlockCols> to a BSR
407 // upload of op(A) or to the scalar-level CSC path.
408
412 template <int Options, int BlockRows, int BlockCols>
413 DeviceSparseView<Scalar> deviceView(const BlockSpMat<Options, BlockRows, BlockCols>& A) {
414 return device_view_block(A, internal::use_cusparse_bsr<BlockRows, BlockCols>());
415 }
416
418 template <int Options, int BlockRows, int BlockCols, typename Rhs>
419 DenseVector multiply(const BlockSpMat<Options, BlockRows, BlockCols>& A, const MatrixBase<Rhs>& x) {
420 return multiply_host_return_block(A, x, GpuOp::NoTrans);
421 }
422
424 template <int Options, int BlockRows, int BlockCols, typename Rhs, typename Dest>
425 void multiply(const BlockSpMat<Options, BlockRows, BlockCols>& A, const MatrixBase<Rhs>& x, MatrixBase<Dest>& y,
426 Scalar alpha = Scalar(1), Scalar beta = Scalar(0), GpuOp op = GpuOp::NoTrans) {
427 multiply_host_block(A, x.derived(), y.derived(), alpha, beta, op,
428 internal::use_cusparse_bsr<BlockRows, BlockCols>());
429 }
430
433 template <int Options, int BlockRows, int BlockCols>
434 void multiply(const BlockSpMat<Options, BlockRows, BlockCols>& A, const DeviceMatrix<Scalar>& d_x,
435 DeviceMatrix<Scalar>& d_y) {
436 multiply(A, d_x, d_y, Scalar(1), Scalar(0), GpuOp::NoTrans);
437 }
438
440 template <int Options, int BlockRows, int BlockCols>
441 void multiply(const BlockSpMat<Options, BlockRows, BlockCols>& A, const DeviceMatrix<Scalar>& d_x,
442 DeviceMatrix<Scalar>& d_y, Scalar alpha, Scalar beta, GpuOp op = GpuOp::NoTrans) {
443 multiply_device_block(A, d_x, d_y, alpha, beta, op, internal::use_cusparse_bsr<BlockRows, BlockCols>());
444 }
445
447 template <int Options, int BlockRows, int BlockCols, typename Rhs>
448 DenseVector multiplyT(const BlockSpMat<Options, BlockRows, BlockCols>& A, const MatrixBase<Rhs>& x) {
449 return multiply_host_return_block(A, x, GpuOp::Trans);
450 }
451
453 template <int Options, int BlockRows, int BlockCols, typename Rhs>
454 DenseVector multiplyAdjoint(const BlockSpMat<Options, BlockRows, BlockCols>& A, const MatrixBase<Rhs>& x) {
455 return multiply_host_return_block(A, x, GpuOp::ConjTrans);
456 }
457
459 template <int Options, int BlockRows, int BlockCols, typename Rhs>
460 DenseMatrix multiplyMat(const BlockSpMat<Options, BlockRows, BlockCols>& A, const MatrixBase<Rhs>& X,
461 GpuOp op = GpuOp::NoTrans) {
462 return multiply_mat_block(A, X, op, internal::use_cusparse_bsr<BlockRows, BlockCols>());
463 }
464
465 cudaStream_t stream() const { return stream_; }
466
467 private:
468 cudaStream_t stream_ = nullptr;
469 cusparseHandle_t handle_ = nullptr;
470 bool owns_stream_ = false;
471 bool owns_handle_ = false;
472
473 // Cached device buffers for sparse matrix (grow-only).
474 internal::DeviceBuffer d_outerPtr_;
475 internal::DeviceBuffer d_innerIdx_;
476 internal::DeviceBuffer d_values_;
477
478 // Cached device buffers for host-API dense vectors (grow-only).
479 internal::DeviceBuffer d_x_;
480 internal::DeviceBuffer d_y_;
481
482 mutable internal::DeviceBuffer d_workspace_;
483
484 // Cached cuSPARSE sparse matrix descriptor, keyed on the scalar-level shape
485 // and nonzero count plus the block size (0 for CSC/CSR, B for BSR).
486 cusparseSpMatDescr_t spmat_desc_ = nullptr;
487 Index cached_rows_ = -1;
488 Index cached_cols_ = -1;
489 Index cached_nnz_ = -1;
490 Index cached_block_size_ = 0;
491
492 // Bumped on every sparse upload; DeviceSparseViews record it at creation so
493 // a stale view (its data replaced by a later upload) asserts at evaluation.
494 uint64_t generation_ = 0;
495
496 // Cached dense-vector/matrix descriptors, re-pointed per call and recreated
497 // only when the shape changes.
498 mutable cusparseDnVecDescr_t x_vec_desc_ = nullptr;
499 mutable cusparseDnVecDescr_t y_vec_desc_ = nullptr;
500 mutable int64_t x_vec_size_ = -1;
501 mutable int64_t y_vec_size_ = -1;
502 mutable cusparseDnMatDescr_t x_mat_desc_ = nullptr;
503 mutable cusparseDnMatDescr_t y_mat_desc_ = nullptr;
504 mutable int64_t x_mat_rows_ = -1, x_mat_cols_ = -1;
505 mutable int64_t y_mat_rows_ = -1, y_mat_cols_ = -1;
506
507 // Cached workspace-size query results, indexed by cusparseOperation_t.
508 // Invalidated when the sparse descriptor or a dense-descriptor shape changes.
509 static constexpr size_t kWsUnknown = static_cast<size_t>(-1);
510 mutable size_t spmv_ws_size_[3] = {kWsUnknown, kWsUnknown, kWsUnknown};
511 mutable size_t spmm_ws_size_[3] = {kWsUnknown, kWsUnknown, kWsUnknown};
512
513 static constexpr cusparseIndexType_t kIndexType =
514 (sizeof(StorageIndex) == 4) ? CUSPARSE_INDEX_32I : CUSPARSE_INDEX_64I;
515 static constexpr cudaDataType_t kValueType = internal::cuda_data_type<Scalar>::value;
516
517 // Empty-operand result on the host: y <- beta * y, no device work.
518 template <typename Dest>
519 static void scale_host(Dest& y, Scalar beta) {
520 if (beta == Scalar(0))
521 y.setZero();
522 else
523 y *= beta;
524 }
525
526 // Shared host-input SpMV entry: y = op(A) * x into a fresh vector.
527 template <typename InputType, typename Rhs>
528 DenseVector multiply_host_return(const SparseMatrixBase<InputType>& A, const MatrixBase<Rhs>& x, GpuOp op) {
529 const InputType& input = A.derived();
530 internal::check_storage_index_bounds<StorageIndex>(input.rows(), input.cols(), input.nonZeros());
531 SpMat storage;
532 const SpMat& mat = internal::bind_sparse<SpMat>(input, storage);
533 DenseVector y((op == GpuOp::NoTrans) ? mat.rows() : mat.cols());
534 y.setZero();
535 multiply_host_impl(mat, x.derived(), y, Scalar(1), Scalar(0), internal::to_cusparse_op<Scalar>(op));
536 return y;
537 }
538
539 template <typename RhsDerived, typename DestDerived>
540 void multiply_host_impl(const SpMat& A, const RhsDerived& x, DestDerived& y, Scalar alpha, Scalar beta,
541 cusparseOperation_t op) {
542 eigen_assert(A.isCompressed());
543
544 const Index m = A.rows();
545 const Index n = A.cols();
546 const Index nnz = A.nonZeros();
547 const Index x_size = (op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? n : m;
548 const Index y_size = (op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? m : n;
549
550 eigen_assert(x.size() == x_size);
551 eigen_assert(y.size() == y_size);
552
553 if (m == 0 || n == 0 || nnz == 0) {
554 scale_host(y, beta);
555 return;
556 }
557
558 upload_sparse(A);
559 spmv_host_cached(x, y, x_size, y_size, alpha, beta, op);
560 }
561
562 // BlockSparseMatrix primitives, dispatched on use_cusparse_bsr: the true_type
563 // overloads bind op(A) as BSR (internal::BsrBinding) and run cuSPARSE with
564 // NON_TRANSPOSE, the false_type ones forward the scalar-level toSparse() copy
565 // to the CSC overloads.
566 template <typename Bsm, typename Rhs>
567 DenseVector multiply_host_return_block(const Bsm& A, const MatrixBase<Rhs>& x, GpuOp op) {
568 DenseVector y((op == GpuOp::NoTrans) ? A.rows() : A.cols());
569 y.setZero();
570 multiply_host_block(A, x.derived(), y, Scalar(1), Scalar(0), op,
571 internal::use_cusparse_bsr<int(Bsm::BlockRows), int(Bsm::BlockCols)>());
572 return y;
573 }
574
575 template <typename Bsm>
576 DeviceSparseView<Scalar> device_view_block(const Bsm& A, std::false_type) {
577 return deviceView(SpMat(A.toSparse()));
578 }
579
580 template <typename Bsm, typename RhsDerived, typename DestDerived>
581 void multiply_host_block(const Bsm& A, const RhsDerived& x, DestDerived& y, Scalar alpha, Scalar beta, GpuOp op,
582 std::false_type) {
583 multiply(A.toSparse(), x, y, alpha, beta, op);
584 }
585
586 template <typename Bsm>
587 void multiply_device_block(const Bsm& A, const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y, Scalar alpha,
588 Scalar beta, GpuOp op, std::false_type) {
589 multiply(A.toSparse(), d_x, d_y, alpha, beta, op);
590 }
591
592 template <typename Bsm, typename Rhs>
593 DenseMatrix multiply_mat_block(const Bsm& A, const MatrixBase<Rhs>& X, GpuOp op, std::false_type) {
594 return multiplyMat(A.toSparse(), X, op);
595 }
596
597#if EIGEN_HAS_CUSPARSE_BSR
598 template <typename Bsm>
599 DeviceSparseView<Scalar> device_view_block(const Bsm& A, std::true_type) {
600 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
601 const internal::BsrBinding<Bsm> bound(A, GpuOp::NoTrans);
602 upload_bsr(bound.arrays());
603 return DeviceSparseView<Scalar>(*this, A.rows(), A.cols(), generation_);
604 }
605
606 template <typename Bsm, typename RhsDerived, typename DestDerived>
607 void multiply_host_block(const Bsm& A, const RhsDerived& x, DestDerived& y, Scalar alpha, Scalar beta, GpuOp op,
608 std::true_type) {
609 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
610 const internal::BsrBinding<Bsm> bound(A, op);
611 const internal::BsrArrays<Scalar, StorageIndex>& opA = bound.arrays();
612
613 eigen_assert(x.size() == opA.cols());
614 eigen_assert(y.size() == opA.rows());
615
616 if (opA.rows() == 0 || opA.cols() == 0 || opA.bnnz == 0) {
617 scale_host(y, beta);
618 return;
619 }
620
621 upload_bsr(opA);
622 spmv_host_cached(x, y, opA.cols(), opA.rows(), alpha, beta, CUSPARSE_OPERATION_NON_TRANSPOSE);
623 }
624
625 template <typename Bsm>
626 void multiply_device_block(const Bsm& A, const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y, Scalar alpha,
627 Scalar beta, GpuOp op, std::true_type) {
628 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
629 const internal::BsrBinding<Bsm> bound(A, op);
630 upload_bsr(bound.arrays());
631 spmv_device_exec(d_x, d_y, alpha, beta, GpuOp::NoTrans);
632 }
633
634 template <typename Bsm, typename Rhs>
635 DenseMatrix multiply_mat_block(const Bsm& A, const MatrixBase<Rhs>& X, GpuOp op, std::true_type) {
636 internal::check_storage_index_bounds<StorageIndex>(A.rows(), A.cols(), A.nonZeros());
637 const internal::BsrBinding<Bsm> bound(A, op);
638 const internal::BsrArrays<Scalar, StorageIndex>& opA = bound.arrays();
639 const DenseMatrix rhs(X.derived());
640 eigen_assert(opA.cols() == rhs.rows());
641
642 const Index n = rhs.cols();
643 if (opA.rows() == 0 || opA.cols() == 0 || n == 0 || opA.bnnz == 0) return DenseMatrix::Zero(opA.rows(), n);
644
645 DenseMatrix Y = DenseMatrix::Zero(opA.rows(), n);
646 upload_bsr(opA);
647 spmm_host_cached(rhs, Y, opA.rows(), opA.cols(), Scalar(1), Scalar(0), CUSPARSE_OPERATION_NON_TRANSPOSE);
648 return Y;
649 }
650#endif // EIGEN_HAS_CUSPARSE_BSR
651
652 // y = alpha * op(D) * x + beta * y against the cached upload D, staging the
653 // host vectors through d_x_ / d_y_. x_size and y_size are the column and row
654 // counts of op(D).
655 template <typename RhsDerived, typename DestDerived>
656 void spmv_host_cached(const RhsDerived& x, DestDerived& y, Index x_size, Index y_size, Scalar alpha, Scalar beta,
657 cusparseOperation_t op) {
658 ensure_buffer(d_x_, static_cast<size_t>(x_size) * sizeof(Scalar));
659 // Ref binds in place when x is already a contiguous vector; only genuine
660 // expressions are evaluated into the Ref's internal temporary.
661 const Ref<const DenseVector> x_ref(x);
662 EIGEN_CUDA_RUNTIME_CHECK(
663 cudaMemcpyAsync(d_x_.get(), x_ref.data(), x_size * sizeof(Scalar), cudaMemcpyHostToDevice, stream_));
664
665 ensure_buffer(d_y_, static_cast<size_t>(y_size) * sizeof(Scalar));
666 if (beta != Scalar(0)) {
667 const Ref<const DenseVector> y_ref(y);
668 EIGEN_CUDA_RUNTIME_CHECK(
669 cudaMemcpyAsync(d_y_.get(), y_ref.data(), y_size * sizeof(Scalar), cudaMemcpyHostToDevice, stream_));
670 }
671
672 exec_spmv(x_size, y_size, d_x_.get(), d_y_.get(), alpha, beta, op);
673
674 // The spacing of y's elements. innerStride() is not it for a vector shaped only at run time: a 1 x n block of a
675 // column-major matrix has innerStride() == 1 with its elements outerStride() apart.
676 const Index incy = y.cols() == 1 ? y.rowStride() : y.colStride();
677 if (incy == 1) {
678 EIGEN_CUDA_RUNTIME_CHECK(
679 cudaMemcpyAsync(y.data(), d_y_.get(), y_size * sizeof(Scalar), cudaMemcpyDeviceToHost, stream_));
680 } else {
681 // y is a strided block, e.g. a row of a column-major matrix.
682 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy2DAsync(y.data(), static_cast<size_t>(incy) * sizeof(Scalar), d_y_.get(),
683 sizeof(Scalar), sizeof(Scalar), static_cast<size_t>(y_size),
684 cudaMemcpyDeviceToHost, stream_));
685 }
686 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
687 }
688
689 public:
695 void spmv_device_exec(const DeviceMatrix<Scalar>& d_x, DeviceMatrix<Scalar>& d_y, Scalar alpha = Scalar(1),
696 Scalar beta = Scalar(0), GpuOp op = GpuOp::NoTrans) const {
697 eigen_assert(spmat_desc_ && "sparse matrix not uploaded — call deviceView() or multiply() first");
698 check_op_against_upload(op);
699 // cuSPARSE SpMV: y must not alias x (undefined behavior).
700 eigen_assert(d_x.data() != d_y.data() && "SpMV: output aliases input vector");
701
702 const cusparseOperation_t cu_op = internal::to_cusparse_op<Scalar>(op);
703 const Index m = cached_rows_;
704 const Index n = cached_cols_;
705 const Index x_size = (cu_op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? n : m;
706 const Index y_size = (cu_op == CUSPARSE_OPERATION_NON_TRANSPOSE) ? m : n;
707
708 eigen_assert(d_x.rows() * d_x.cols() == x_size);
709
710 if (m == 0 || n == 0 || cached_nnz_ == 0) {
711 // Empty A reduces SpMV to y <- beta*y. SparseContext owns no cuBLAS handle, so the scale runs on the
712 // thread-local Context; the events on d_y order it against this context's stream. A d_y of the wrong
713 // size is new storage with no value to scale, like cusparse with beta == 0.
714 if (d_y.rows() * d_y.cols() != y_size) {
715 d_y.resize(y_size, 1);
716 d_y.setZero(stream_);
717 } else if (beta == Scalar(0)) {
718 d_y.setZero(stream_);
719 } else if (beta != Scalar(1)) {
720 d_y.scale(Context::threadLocal(), beta);
721 }
722 return;
723 }
724
725 // Ensure d_y is allocated.
726 if (d_y.rows() * d_y.cols() != y_size) {
727 d_y.resize(y_size, 1);
728 }
729
730 // Wait for input data to be ready on this stream.
731 d_x.waitReady(stream_);
732 d_y.waitReady(stream_);
733
734 exec_spmv(x_size, y_size, const_cast<void*>(static_cast<const void*>(d_x.data())), static_cast<void*>(d_y.data()),
735 alpha, beta, cu_op);
736
737 d_y.recordReady(stream_);
738 }
739
744 void spmm_device_exec(const DeviceMatrix<Scalar>& d_X, DeviceMatrix<Scalar>& d_Y, Scalar alpha = Scalar(1),
745 Scalar beta = Scalar(0), GpuOp op = GpuOp::NoTrans) const {
746 eigen_assert(spmat_desc_ && "sparse matrix not uploaded — call deviceView() or multiply() first");
747 check_op_against_upload(op);
748 eigen_assert(d_X.data() != d_Y.data() && "SpMM: output aliases input matrix");
749
750 const cusparseOperation_t cu_op = internal::to_cusparse_op<Scalar>(op);
751 const bool transposed = (cu_op != CUSPARSE_OPERATION_NON_TRANSPOSE);
752 const Index m_op = transposed ? cached_cols_ : cached_rows_;
753 const Index k_op = transposed ? cached_rows_ : cached_cols_;
754 const Index n = d_X.cols();
755
756 eigen_assert(d_X.rows() == k_op);
757
758 if (m_op == 0 || n == 0 || cached_nnz_ == 0) {
759 if (d_Y.rows() != m_op || d_Y.cols() != n) {
760 d_Y.resize(m_op, n);
761 d_Y.setZero(stream_);
762 } else if (beta == Scalar(0)) {
763 d_Y.setZero(stream_);
764 } else if (beta != Scalar(1)) {
765 d_Y.scale(Context::threadLocal(), beta);
766 }
767 return;
768 }
769
770 if (d_Y.rows() != m_op || d_Y.cols() != n) {
771 d_Y.resize(m_op, n);
772 }
773
774 d_X.waitReady(stream_);
775 d_Y.waitReady(stream_);
776
777 exec_spmm(m_op, k_op, n, const_cast<void*>(static_cast<const void*>(d_X.data())), static_cast<void*>(d_Y.data()),
778 alpha, beta, cu_op);
779
780 d_Y.recordReady(stream_);
781 }
782
783 private:
784 // cuSPARSE 11.x's cusparseSpMM rejects CSC for matA (CSC support landed in
785 // CUDA 12.0). On 11.x we register the same buffers as CSR-of-A^T (dims
786 // swapped) and invert the user-facing op before each cuSPARSE call. On 12+
787 // we keep the natural CSC path so users pay no extra cost.
788#if !defined(CUSPARSE_VERSION) || CUSPARSE_VERSION < 12000
789 static constexpr bool kUseCsrOfTranspose = true;
790 static constexpr cusparseSpMMAlg_t kSpMMAlg = CUSPARSE_SPMM_CSR_ALG2;
791#else
792 static constexpr bool kUseCsrOfTranspose = false;
793 static constexpr cusparseSpMMAlg_t kSpMMAlg = CUSPARSE_SPMM_ALG_DEFAULT;
794#endif
795
796 // Map a user-facing op on A to the cuSPARSE op on the cached descriptor.
797 // Identity on cuSPARSE 12+ (descriptor is CSC of A, or BSR of op(A));
798 // inverted on 11.x (descriptor is CSR of A^T).
799 static cusparseOperation_t descriptor_op(cusparseOperation_t user_op) {
800 EIGEN_IF_CONSTEXPR (!kUseCsrOfTranspose) return user_op;
801 switch (user_op) {
802 case CUSPARSE_OPERATION_NON_TRANSPOSE:
803 return CUSPARSE_OPERATION_TRANSPOSE;
804 case CUSPARSE_OPERATION_TRANSPOSE:
805 return CUSPARSE_OPERATION_NON_TRANSPOSE;
806 default:
807 // CONJUGATE_TRANSPOSE on the CSR-of-A^T descriptor would compute
808 // conj(A) * x, not A^H * x — not supported via this representation.
809 eigen_assert(false && "CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE not supported on cuSPARSE < 12.0");
810 return user_op;
811 }
812 }
813
814 static int op_index(cusparseOperation_t op) {
815 switch (op) {
816 case CUSPARSE_OPERATION_TRANSPOSE:
817 return 1;
818 case CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE:
819 return 2;
820 default:
821 return 0;
822 }
823 }
824
825 void invalidate_ws_caches() const {
826 for (int i = 0; i < 3; ++i) {
827 spmv_ws_size_[i] = kWsUnknown;
828 spmm_ws_size_[i] = kWsUnknown;
829 }
830 }
831
832 // Recreate the descriptor when the size changes; otherwise just re-point it
833 // at the new device buffer (cusparseDnVecSetValues is a host-side pointer
834 // update, no GPU work).
835 void update_dnvec(cusparseDnVecDescr_t& desc, int64_t& cur_size, int64_t size, void* ptr) const {
836 if (!desc || cur_size != size) {
837 if (desc) {
838 // Clear the descriptor first so a throwing check cannot leave a handle that is destroyed again.
839 cusparseDnVecDescr_t old = desc;
840 desc = nullptr;
841 EIGEN_CUSPARSE_CHECK(cusparseDestroyDnVec(old));
842 }
843 EIGEN_CUSPARSE_CHECK(cusparseCreateDnVec(&desc, size, ptr, kValueType));
844 cur_size = size;
845 invalidate_ws_caches();
846 } else {
847 EIGEN_CUSPARSE_CHECK(cusparseDnVecSetValues(desc, ptr));
848 }
849 }
850
851 void update_dnmat(cusparseDnMatDescr_t& desc, int64_t& cur_rows, int64_t& cur_cols, int64_t rows, int64_t cols,
852 void* ptr) const {
853 if (!desc || cur_rows != rows || cur_cols != cols) {
854 if (desc) {
855 cusparseDnMatDescr_t old = desc;
856 desc = nullptr;
857 EIGEN_CUSPARSE_CHECK(cusparseDestroyDnMat(old));
858 }
859 // Column-major with ld = rows.
860 EIGEN_CUSPARSE_CHECK(cusparseCreateDnMat(&desc, rows, cols, rows, ptr, kValueType, CUSPARSE_ORDER_COL));
861 cur_rows = rows;
862 cur_cols = cols;
863 invalidate_ws_caches();
864 } else {
865 EIGEN_CUSPARSE_CHECK(cusparseDnMatSetValues(desc, ptr));
866 }
867 }
868
869 void destroy_dense_descriptors() {
870 if (x_vec_desc_) (void)cusparseDestroyDnVec(x_vec_desc_);
871 if (y_vec_desc_) (void)cusparseDestroyDnVec(y_vec_desc_);
872 if (x_mat_desc_) (void)cusparseDestroyDnMat(x_mat_desc_);
873 if (y_mat_desc_) (void)cusparseDestroyDnMat(y_mat_desc_);
874 x_vec_desc_ = y_vec_desc_ = nullptr;
875 x_mat_desc_ = y_mat_desc_ = nullptr;
876 }
877
878 void exec_spmv(Index x_size, Index y_size, void* d_x_ptr, void* d_y_ptr, Scalar alpha, Scalar beta,
879 cusparseOperation_t op) const {
880 const cusparseOperation_t cu_op = descriptor_op(op);
881 update_dnvec(x_vec_desc_, x_vec_size_, x_size, d_x_ptr);
882 update_dnvec(y_vec_desc_, y_vec_size_, y_size, d_y_ptr);
883
884 size_t& ws_size = spmv_ws_size_[op_index(cu_op)];
885 if (ws_size == kWsUnknown) {
886 EIGEN_CUSPARSE_CHECK(cusparseSpMV_bufferSize(handle_, cu_op, &alpha, spmat_desc_, x_vec_desc_, &beta, y_vec_desc_,
887 kValueType, CUSPARSE_SPMV_ALG_DEFAULT, &ws_size));
888 }
889 ensure_buffer(d_workspace_, ws_size);
890
891 EIGEN_CUSPARSE_CHECK(cusparseSpMV(handle_, cu_op, &alpha, spmat_desc_, x_vec_desc_, &beta, y_vec_desc_, kValueType,
892 CUSPARSE_SPMV_ALG_DEFAULT, d_workspace_.get()));
893 }
894
895 void exec_spmm(Index m_op, Index k_op, Index n, void* d_x_ptr, void* d_y_ptr, Scalar alpha, Scalar beta,
896 cusparseOperation_t op) const {
897 const cusparseOperation_t cu_op = descriptor_op(op);
898 // X is k_op x n, Y is m_op x n (column-major, post-op shapes).
899 update_dnmat(x_mat_desc_, x_mat_rows_, x_mat_cols_, k_op, n, d_x_ptr);
900 update_dnmat(y_mat_desc_, y_mat_rows_, y_mat_cols_, m_op, n, d_y_ptr);
901
902 size_t& ws_size = spmm_ws_size_[op_index(cu_op)];
903 if (ws_size == kWsUnknown) {
904 EIGEN_CUSPARSE_CHECK(cusparseSpMM_bufferSize(handle_, cu_op, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha,
905 spmat_desc_, x_mat_desc_, &beta, y_mat_desc_, kValueType, kSpMMAlg,
906 &ws_size));
907 }
908 ensure_buffer(d_workspace_, ws_size);
909
910 EIGEN_CUSPARSE_CHECK(cusparseSpMM(handle_, cu_op, CUSPARSE_OPERATION_NON_TRANSPOSE, &alpha, spmat_desc_,
911 x_mat_desc_, &beta, y_mat_desc_, kValueType, kSpMMAlg, d_workspace_.get()));
912 }
913
914 void spmm_impl(const SpMat& A, const DenseMatrix& X, DenseMatrix& Y, Scalar alpha, Scalar beta,
915 cusparseOperation_t op) {
916 eigen_assert(A.isCompressed());
917
918 // For op != NON_TRANSPOSE, Y = op(A) * X. The dense X / Y descriptors must
919 // describe the *post-op* shapes: X has k_op rows (= input dim of op(A)),
920 // Y has m_op rows (= output dim of op(A)).
921 const bool transposed = (op != CUSPARSE_OPERATION_NON_TRANSPOSE);
922 const Index m_op = transposed ? A.cols() : A.rows();
923 const Index k_op = transposed ? A.rows() : A.cols();
924 const Index n = X.cols();
925 const Index nnz = A.nonZeros();
926
927 if (m_op == 0 || n == 0 || k_op == 0 || nnz == 0) {
928 scale_host(Y, beta);
929 return;
930 }
931
932 upload_sparse(A);
933 spmm_host_cached(X, Y, m_op, k_op, alpha, beta, op);
934 }
935
936 // Y = alpha * op(D) * X + beta * Y against the cached upload D, staging the
937 // host matrices through d_x_ / d_y_. X is k_op x n, Y is m_op x n.
938 void spmm_host_cached(const DenseMatrix& X, DenseMatrix& Y, Index m_op, Index k_op, Scalar alpha, Scalar beta,
939 cusparseOperation_t op) {
940 const Index n = X.cols();
941 const size_t x_bytes = static_cast<size_t>(k_op) * static_cast<size_t>(n) * sizeof(Scalar);
942 const size_t y_bytes = static_cast<size_t>(m_op) * static_cast<size_t>(n) * sizeof(Scalar);
943 ensure_buffer(d_x_, x_bytes);
944 ensure_buffer(d_y_, y_bytes);
945 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_x_.get(), X.data(), x_bytes, cudaMemcpyHostToDevice, stream_));
946 if (beta != Scalar(0)) {
947 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_y_.get(), Y.data(), y_bytes, cudaMemcpyHostToDevice, stream_));
948 }
949
950 exec_spmm(m_op, k_op, n, d_x_.get(), d_y_.get(), alpha, beta, op);
951
952 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(Y.data(), d_y_.get(), y_bytes, cudaMemcpyDeviceToHost, stream_));
953 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
954 }
955
956 void upload_sparse(const SpMat& A) {
957 // cuSPARSE 12.0+ accepts CSC directly. On cuSPARSE 11.x, cusparseSpMM
958 // rejects CSC and CONJUGATE_TRANSPOSE on CSC+complex SpMV silently
959 // demotes to TRANSPOSE. We register the same CSC buffers as CSR-of-A^T
960 // (dims swapped) on 11.x and invert the op at exec time via
961 // descriptor_op() — no transpose-copy required.
962 const Index m = A.rows();
963 const Index n = A.cols();
964 const Index nnz = A.nonZeros();
965 upload_arrays(/*outer_count=*/n + 1, /*inner_count=*/nnz, /*value_count=*/nnz, A.outerIndexPtr(), A.innerIndexPtr(),
966 A.valuePtr());
967 if (descriptor_key_matches(m, n, nnz, /*block_size=*/0)) return;
968
969 destroy_spmat_descriptor(/*checked=*/true);
970 EIGEN_IF_CONSTEXPR (kUseCsrOfTranspose) {
971 // cuSPARSE 11.x: cusparseSpMM rejects CSC for matA. CSC of A and CSR of
972 // A^T share the same buffers, so register the data as CSR-of-A^T (dims
973 // swapped) and invert the op in exec_spmv / spmm_impl via descriptor_op.
974 EIGEN_CUSPARSE_CHECK(cusparseCreateCsr(&spmat_desc_, n, m, nnz, d_outerPtr_.get(), d_innerIdx_.get(),
975 d_values_.get(), kIndexType, kIndexType, CUSPARSE_INDEX_BASE_ZERO,
976 kValueType));
977 } else {
978 EIGEN_CUSPARSE_CHECK(cusparseCreateCsc(&spmat_desc_, m, n, nnz, d_outerPtr_.get(), d_innerIdx_.get(),
979 d_values_.get(), kIndexType, kIndexType, CUSPARSE_INDEX_BASE_ZERO,
980 kValueType));
981 }
982 set_descriptor_key(m, n, nnz, /*block_size=*/0);
983 }
984
985#if EIGEN_HAS_CUSPARSE_BSR
986 void upload_bsr(const internal::BsrArrays<Scalar, StorageIndex>& opA) {
987 upload_arrays(/*outer_count=*/opA.brows + 1, /*inner_count=*/opA.bnnz, /*value_count=*/opA.nonZeros(), opA.outer,
988 opA.inner, opA.values);
989 if (descriptor_key_matches(opA.rows(), opA.cols(), opA.nonZeros(), opA.blockSize)) return;
990
991 destroy_spmat_descriptor(/*checked=*/true);
992 // Row-major blocks: cusparseSpMM accepts no other block layout for BSR.
993 EIGEN_CUSPARSE_CHECK(cusparseCreateBsr(&spmat_desc_, opA.brows, opA.bcols, opA.bnnz, opA.blockSize, opA.blockSize,
994 d_outerPtr_.get(), d_innerIdx_.get(), d_values_.get(), kIndexType,
995 kIndexType, CUSPARSE_INDEX_BASE_ZERO, kValueType, CUSPARSE_ORDER_ROW));
996 set_descriptor_key(opA.rows(), opA.cols(), opA.nonZeros(), opA.blockSize);
997 }
998#endif // EIGEN_HAS_CUSPARSE_BSR
999
1000 void upload_arrays(Index outer_count, Index inner_count, Index value_count, const StorageIndex* host_outer,
1001 const StorageIndex* host_inner, const Scalar* host_values) {
1002 const size_t outer_bytes = static_cast<size_t>(outer_count) * sizeof(StorageIndex);
1003 const size_t inner_bytes = static_cast<size_t>(inner_count) * sizeof(StorageIndex);
1004 const size_t val_bytes = static_cast<size_t>(value_count) * sizeof(Scalar);
1005
1006 // Values *and* index arrays are re-uploaded unconditionally: host pointer
1007 // identity cannot detect a same-shape/same-nnz pattern rewritten in place
1008 // or assigned into the same allocations (SparseMatrix reuses them), so a
1009 // structure cache keyed on pointers would silently serve stale indices.
1010 // Only the cuSPARSE descriptor and the workspace-size queries are cached,
1011 // keyed on (rows, cols, nnz, block size). Every upload invalidates
1012 // outstanding DeviceSparseViews via the generation counter.
1013 ++generation_;
1014 ensure_buffer(d_values_, val_bytes);
1015 ensure_buffer(d_outerPtr_, outer_bytes);
1016 ensure_buffer(d_innerIdx_, inner_bytes);
1017 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(d_values_.get(), host_values, val_bytes, cudaMemcpyHostToDevice, stream_));
1018 EIGEN_CUDA_RUNTIME_CHECK(
1019 cudaMemcpyAsync(d_outerPtr_.get(), host_outer, outer_bytes, cudaMemcpyHostToDevice, stream_));
1020 EIGEN_CUDA_RUNTIME_CHECK(
1021 cudaMemcpyAsync(d_innerIdx_.get(), host_inner, inner_bytes, cudaMemcpyHostToDevice, stream_));
1022 }
1023
1024 // Same shape, nnz and format: the grow-only device buffers cannot have been
1025 // reallocated, so the existing descriptor still points at the freshly
1026 // written data.
1027 bool descriptor_key_matches(Index m, Index n, Index nnz, Index block_size) const {
1028 return m == cached_rows_ && n == cached_cols_ && nnz == cached_nnz_ && block_size == cached_block_size_;
1029 }
1030
1031 void set_descriptor_key(Index m, Index n, Index nnz, Index block_size) {
1032 cached_rows_ = m;
1033 cached_cols_ = n;
1034 cached_nnz_ = nnz;
1035 cached_block_size_ = block_size;
1036 }
1037
1038 // A BSR descriptor holds op(A) as bound at upload time, and cuSPARSE runs no
1039 // transposed BSR product.
1040 void check_op_against_upload(GpuOp op) const {
1041 eigen_assert((cached_block_size_ == 0 || op == GpuOp::NoTrans) &&
1042 "cuSPARSE runs BSR products only with op == NoTrans; pass op to multiply(A, d_x, d_y, ...) instead");
1043 EIGEN_UNUSED_VARIABLE(op);
1044 }
1045
1046 // Destroy the sparse-matrix descriptor and reset the cache identity.
1047 // `checked` selects checking (mid-lifetime rebuilds) vs swallowing failure
1048 // (noexcept destructor).
1049 void destroy_spmat_descriptor(bool checked) {
1050 if (spmat_desc_) {
1051 cusparseSpMatDescr_t old = spmat_desc_;
1052 spmat_desc_ = nullptr; // before the destroy: a throwing check must not leave a handle that is destroyed again
1053 if (checked)
1054 EIGEN_CUSPARSE_CHECK(cusparseDestroySpMat(old));
1055 else
1056 (void)cusparseDestroySpMat(old);
1057 }
1058 set_descriptor_key(-1, -1, -1, 0);
1059 invalidate_ws_caches();
1060 }
1061
1062 void ensure_buffer(internal::DeviceBuffer& buf, size_t needed) const {
1063 if (needed > buf.size()) {
1064 if (buf) EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream_));
1065 buf = internal::DeviceBuffer(needed);
1066 }
1067 }
1068};
1069
1070// Defined here because it needs the full SparseContext definition.
1071
1072template <typename Scalar_>
1073DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const SpMVExpr<Scalar_>& expr) {
1074 // Uses the sparse matrix already uploaded by deviceView() — no re-upload on
1075 // repeated products with the same view. A stale view (the context has since
1076 // uploaded again, replacing the cached data) is caught here.
1077 eigen_assert(expr.view().generation() == expr.view().context().uploadGeneration() &&
1078 "DeviceSparseView is stale: its SparseContext has since uploaded another sparse matrix");
1079 if (expr.x().cols() <= 1) {
1080 expr.view().context().spmv_device_exec(expr.x(), *this, Scalar_(1), Scalar_(0), GpuOp::NoTrans);
1081 } else {
1082 expr.view().context().spmm_device_exec(expr.x(), *this, Scalar_(1), Scalar_(0), GpuOp::NoTrans);
1083 }
1084 return *this;
1085}
1086
1087template <typename Scalar_>
1088DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const SpMVAffineExpr<Scalar_>& expr) {
1089 const DeviceSparseView<Scalar_>& view = expr.view();
1090 const DeviceMatrix& addend = expr.addend();
1091 eigen_assert(view.generation() == view.context().uploadGeneration() &&
1092 "DeviceSparseView is stale: its SparseContext has since uploaded another sparse matrix");
1093 eigen_assert(addend.rows() == view.rows() && addend.cols() == expr.x().cols() &&
1094 "SpMVAffineExpr: the addend must have the shape of the product");
1095 eigen_assert(&expr.x() != this && "SpMVAffineExpr: the destination aliases the dense operand");
1096 // The product accumulates into a copy of the addend; when the addend is the destination
1097 // itself (d_r = d_r - d_A * d_x) cuSPARSE's beta accumulates in place.
1098 if (&addend != this) copyFrom(Context::threadLocal(), addend);
1099 // With no stored entries the expression is beta * addend: scale the copy, no cuSPARSE call needed.
1100 if (view.nonZeros() == 0 || view.rows() == 0 || view.cols() == 0) {
1101 if (expr.beta() != Scalar_(1)) scale(Context::threadLocal(), expr.beta());
1102 return *this;
1103 }
1104 if (expr.x().cols() <= 1) {
1105 view.context().spmv_device_exec(expr.x(), *this, expr.alpha(), expr.beta(), GpuOp::NoTrans);
1106 } else {
1107 view.context().spmm_device_exec(expr.x(), *this, expr.alpha(), expr.beta(), GpuOp::NoTrans);
1108 }
1109 return *this;
1110}
1111
1112template <typename Scalar_>
1114 *this = expr;
1115}
1116
1117template <typename Scalar_>
1118DeviceMatrix<Scalar_>::DeviceMatrix(const SpMVAffineExpr<Scalar_>& expr) : DeviceMatrix() {
1119 *this = expr;
1120}
1121} // namespace gpu
1122} // namespace Eigen
1123
1124#endif // EIGEN_GPU_SPARSE_CONTEXT_H
static Context & threadLocal()
Definition GpuContext.h:121
RAII wrapper for a dense column-major matrix in GPU device memory.
Definition DeviceMatrix.h:122
void scale(Context &ctx, Scalar alpha)
Definition DeviceDispatch.h:690
void copyFrom(Context &ctx, const DeviceMatrix &other)
Definition DeviceDispatch.h:700
static DeviceMatrix view(Scalar *device_ptr, Index rows, Index cols)
Definition DeviceMatrix.h:578
Definition GpuSparseContext.h:234
SpMVExpr< Scalar > operator*(const DeviceMatrix< Scalar > &x) const
Definition GpuSparseContext.h:249
Index nonZeros() const
Definition GpuSparseContext.h:254
uint64_t generation() const
Definition GpuSparseContext.h:259
Definition GpuSparseContext.h:166
Definition GpuSparseContext.h:146
Namespace containing all symbols from the Eigen library.
Definition GpuSparseContext.h:55