Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
DeviceMatrix.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// Typed RAII wrapper for a dense column-major matrix in GPU device memory.
12//
13// Cross-stream safety is automatic: an internal CUDA event records when the last
14// write completed, and consumers on a different stream wait on it before reading.
15
16#ifndef EIGEN_GPU_DEVICE_MATRIX_H
17#define EIGEN_GPU_DEVICE_MATRIX_H
18
19// IWYU pragma: private
20#include "./InternalHeaderCheck.h"
21
22#include <cstring>
23
24#include "./FwdDecl.h"
25#include "./GpuSupport.h"
26
27namespace Eigen {
28namespace gpu {
29
38template <typename Scalar_>
39class HostTransfer {
40 public:
41 using Scalar = Scalar_;
43
46 PlainMatrix& get() {
47 if (!synced_) {
48 EIGEN_CUDA_RUNTIME_CHECK(cudaEventSynchronize(event_));
49 if (pinned_buf_ && host_buf_.size() > 0) {
50 std::memcpy(host_buf_.data(), pinned_buf_.get(), static_cast<size_t>(host_buf_.size()) * sizeof(Scalar));
51 }
52 pinned_buf_ = internal::PinnedHostBuffer(); // free pinned memory early
53 synced_ = true;
54 }
55 return host_buf_;
56 }
57
59 bool ready() const {
60 if (synced_) return true;
61 const cudaError_t err = cudaEventQuery(event_);
62 if (err == cudaSuccess) return true;
63 if (err != cudaErrorNotReady)
64 EIGEN_GPU_CHECK_FAILED(cudaGetErrorName(err), "cudaEventQuery(event_)", __FILE__, __LINE__);
65 return false;
66 }
67
69 if (event_) (void)cudaEventDestroy(event_);
70 }
71
72 HostTransfer(HostTransfer&& o) noexcept
73 : host_buf_(std::move(o.host_buf_)), pinned_buf_(std::move(o.pinned_buf_)), event_(o.event_), synced_(o.synced_) {
74 o.event_ = nullptr;
75 o.synced_ = true;
76 }
77
78 HostTransfer& operator=(HostTransfer&& o) noexcept {
79 if (this != &o) {
80 // Unchecked like the destructor: eigen_assert may throw, and this operator is noexcept.
81 if (event_) (void)cudaEventDestroy(event_);
82 host_buf_ = std::move(o.host_buf_);
83 pinned_buf_ = std::move(o.pinned_buf_);
84 event_ = o.event_;
85 synced_ = o.synced_;
86 o.event_ = nullptr;
87 o.synced_ = true;
88 }
89 return *this;
90 }
91
92 HostTransfer(const HostTransfer&) = delete;
93 HostTransfer& operator=(const HostTransfer&) = delete;
94
95 private:
96 template <typename>
97 friend class DeviceMatrix;
98
99 HostTransfer(PlainMatrix&& buf, internal::PinnedHostBuffer&& pinned, cudaEvent_t event)
100 : host_buf_(std::move(buf)), pinned_buf_(std::move(pinned)), event_(event), synced_(false) {}
101
102 PlainMatrix host_buf_; // final destination (pageable)
103 internal::PinnedHostBuffer pinned_buf_; // staging buffer for async DMA
104 cudaEvent_t event_ = nullptr;
105 bool synced_ = false;
106};
107
121template <typename Scalar_>
123 public:
124 using Scalar = Scalar_;
125 using RealScalar = typename NumTraits<Scalar>::Real;
126 using PlainObject = DeviceMatrix; // owning type, as generic solver code expects
128
130 DeviceMatrix() = default;
131
134 explicit DeviceMatrix(Index n) : rows_(n), cols_(1) {
135 eigen_assert(n >= 0);
136 allocate(sizeInBytes());
137 }
138
140 DeviceMatrix(Index rows, Index cols) : rows_(rows), cols_(cols) {
141 eigen_assert(rows >= 0 && cols >= 0);
142 allocate(sizeInBytes());
143 }
144
145 // Copy-initialization from a device expression, mirroring the Eigen CPU idiom
146 // `DeviceMatrix<double> d_C = d_A * d_B;`. Each delegates to the corresponding
147 // operator= on the thread-local Context, and is defined out-of-line in
148 // DeviceDispatch.h — GpuSparseContext.h for SpMV — where Context is complete.
149
150 template <typename Lhs, typename Rhs>
151 DeviceMatrix(const GemmExpr<Lhs, Rhs>& expr);
154 template <int UpLo>
157 template <int UpLo>
159 template <int UpLo>
161 DeviceMatrix(const SpMVExpr<Scalar>& expr);
163
164 ~DeviceMatrix() {
165 // cudaEventDestroy on a pending event is non-blocking: the runtime defers
166 // teardown until the event completes. The trailing cudaFree() (via
167 // data_.reset()) is itself synchronous, so the buffer outlives any
168 // in-flight kernel that may still be touching it.
169 if (ready_event_) (void)cudaEventDestroy(ready_event_);
170 }
171
172 DeviceMatrix(DeviceMatrix&& o) noexcept
173 : data_(std::move(o.data_)),
174 rows_(o.rows_),
175 cols_(o.cols_),
176 capacity_bytes_(o.capacity_bytes_),
177 ready_event_(o.ready_event_),
178 ready_stream_(o.ready_stream_),
179 retained_buffer_(std::move(o.retained_buffer_)) {
180 o.rows_ = 0;
181 o.cols_ = 0;
182 o.capacity_bytes_ = 0;
183 o.ready_event_ = nullptr;
184 o.ready_stream_ = nullptr;
185 }
186
187 DeviceMatrix& operator=(DeviceMatrix&& o) noexcept {
188 if (this != &o) {
189 // Unchecked like the destructor: eigen_assert may throw, and this operator is noexcept.
190 if (ready_event_) (void)cudaEventDestroy(ready_event_);
191 data_ = std::move(o.data_);
192 rows_ = o.rows_;
193 cols_ = o.cols_;
194 capacity_bytes_ = o.capacity_bytes_;
195 ready_event_ = o.ready_event_;
196 ready_stream_ = o.ready_stream_;
197 retained_buffer_ = std::move(o.retained_buffer_);
198 o.rows_ = 0;
199 o.cols_ = 0;
200 o.capacity_bytes_ = 0;
201 o.ready_event_ = nullptr;
202 o.ready_stream_ = nullptr;
203 }
204 return *this;
205 }
206
213 DeviceMatrix(const DeviceMatrix& other);
214 DeviceMatrix& operator=(const DeviceMatrix& other);
215
225 template <typename Derived>
226 static DeviceMatrix fromHost(const DenseBase<Derived>& host, cudaStream_t stream = nullptr) {
227 // Ref binds any column-major direct-access input in place (no host copy);
228 // row-major layouts and expressions evaluate into its temporary. A bound
229 // block keeps its parent's outer stride, so the upload must honour
230 // outerStride() rather than assume rows() -- see upload_host_matrix.
231 const Ref<const PlainMatrix> mat(host.derived());
232 DeviceMatrix dm(mat.rows(), mat.cols());
233 if (dm.sizeInBytes() > 0) {
234 internal::upload_host_matrix(dm.data_.get(), mat.rows(), mat.data(), mat.outerStride(), mat.rows(), mat.cols(),
235 stream);
236 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream));
237 }
238 return dm;
239 }
240
251 static DeviceMatrix fromHostAsync(const Scalar* host_data, Index rows, Index cols, cudaStream_t stream) {
252 eigen_assert(rows >= 0 && cols >= 0);
253 eigen_assert(host_data != nullptr || (rows == 0 || cols == 0));
254 DeviceMatrix dm(rows, cols);
255 if (dm.sizeInBytes() > 0) {
256 EIGEN_CUDA_RUNTIME_CHECK(
257 cudaMemcpyAsync(dm.data_.get(), host_data, dm.sizeInBytes(), cudaMemcpyHostToDevice, stream));
258 dm.recordReady(stream);
259 }
260 return dm;
261 }
262
270 PlainMatrix toHost(cudaStream_t stream = nullptr) const {
271 PlainMatrix host_buf(rows_, cols_);
272 if (sizeInBytes() > 0) {
273 waitReady(stream);
274 EIGEN_CUDA_RUNTIME_CHECK(
275 cudaMemcpyAsync(host_buf.data(), data_.get(), sizeInBytes(), cudaMemcpyDeviceToHost, stream));
276 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(stream));
277 }
278 return host_buf;
279 }
280
289 HostTransfer<Scalar> toHostAsync(cudaStream_t stream = nullptr) const {
290 PlainMatrix host_buf(rows_, cols_);
291 internal::PinnedHostBuffer pinned_buf(sizeInBytes());
292 if (sizeInBytes() > 0) {
293 waitReady(stream);
294 EIGEN_CUDA_RUNTIME_CHECK(
295 cudaMemcpyAsync(pinned_buf.get(), data_.get(), sizeInBytes(), cudaMemcpyDeviceToHost, stream));
296 }
297 cudaEvent_t transfer_event;
298 EIGEN_CUDA_RUNTIME_CHECK(cudaEventCreateWithFlags(&transfer_event, cudaEventDisableTiming));
299 EIGEN_CUDA_RUNTIME_CHECK(cudaEventRecord(transfer_event, stream));
300 return HostTransfer<Scalar>(std::move(host_buf), std::move(pinned_buf), transfer_event);
301 }
302
307 DeviceMatrix clone(cudaStream_t stream = nullptr) const {
308 DeviceMatrix result(rows_, cols_);
309 if (sizeInBytes() > 0) {
310 waitReady(stream);
311 EIGEN_CUDA_RUNTIME_CHECK(
312 cudaMemcpyAsync(result.data_.get(), data_.get(), sizeInBytes(), cudaMemcpyDeviceToDevice, stream));
313 result.recordReady(stream);
314 }
315 return result;
316 }
317
323 void resize(Index rows, Index cols) {
324 eigen_assert(rows >= 0 && cols >= 0);
325 if (rows == rows_ && cols == cols_) return;
326 const size_t bytes = static_cast<size_t>(rows) * static_cast<size_t>(cols) * sizeof(Scalar);
327 if (bytes > 0 && bytes <= capacity_bytes_ && data_) {
328 // Reuse the allocation; the ready event still orders any in-flight
329 // writes to this buffer ahead of its next producer.
330 rows_ = rows;
331 cols_ = cols;
332 return;
333 }
334 data_.reset();
335 capacity_bytes_ = 0;
336 if (ready_event_) {
337 // Clear the member first so a throwing check cannot leave a handle for the destructor to destroy again.
338 cudaEvent_t old = ready_event_;
339 ready_event_ = nullptr;
340 EIGEN_CUDA_RUNTIME_CHECK(cudaEventDestroy(old));
341 }
342 ready_stream_ = nullptr;
343 retained_buffer_ = internal::DeviceBuffer();
344 rows_ = rows;
345 cols_ = cols;
346 allocate(bytes);
347 }
348
349 Scalar* data() { return data_.get(); }
350 const Scalar* data() const { return data_.get(); }
351 Index rows() const { return rows_; }
352 Index cols() const { return cols_; }
353 bool empty() const { return rows_ == 0 || cols_ == 0; }
354
359 bool isView() const { return data_ != nullptr && data_.get_deleter().borrow; }
360
362 size_t sizeInBytes() const { return static_cast<size_t>(rows_) * static_cast<size_t>(cols_) * sizeof(Scalar); }
363
365 void recordReady(cudaStream_t stream) {
366 ensureEvent();
367 EIGEN_CUDA_RUNTIME_CHECK(cudaEventRecord(ready_event_, stream));
368 ready_stream_ = stream;
369 }
370
374 void waitReady(cudaStream_t stream) const {
375 if (ready_event_ && stream != ready_stream_) {
376 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamWaitEvent(stream, ready_event_, 0));
377 }
378 }
379
382
385
388 Assignment<Scalar> device(Context& ctx) { return Assignment<Scalar>(*this, ctx); }
389
390 template <typename Lhs, typename Rhs>
391 DeviceMatrix& operator=(const GemmExpr<Lhs, Rhs>& expr);
392
393 template <typename Lhs, typename Rhs>
394 DeviceMatrix& operator+=(const GemmExpr<Lhs, Rhs>& expr);
395
397 template <typename Lhs, typename Rhs>
399
402
404 template <int UpLo>
406 return LLTView<Scalar, UpLo>(*this);
407 }
408
410 LUView<Scalar> lu() const { return LUView<Scalar>(*this); }
411
412 template <int UpLo>
413 DeviceMatrix& operator=(const LltSolveExpr<Scalar, UpLo>& expr);
414
415 DeviceMatrix& operator=(const LuSolveExpr<Scalar>& expr);
416
418 template <int UpLo>
422
424 template <int UpLo>
428
430 template <int UpLo>
434
435 template <int UpLo>
436 DeviceMatrix& operator=(const TrsmExpr<Scalar, UpLo>& expr);
437
438 template <int UpLo>
439 DeviceMatrix& operator=(const SymmExpr<Scalar, UpLo>& expr);
440
441 // A DeviceMatrix is always dense (lda == rows) and a vector is one with
442 // cols == 1, so the BLAS-1 methods below simply run over the flat rows*cols
443 // array and serve both. Passing an explicit Context& lets callers keep every
444 // operation on one stream, which elides the cross-stream event waits.
445
448 DeviceScalar<Scalar> dot(Context& ctx, const DeviceMatrix& other) const;
449
452
457
461
466 void dot(Context& ctx, const DeviceMatrix& other, DeviceScalar<Scalar>& result) const;
467 void squaredNorm(Context& ctx, DeviceScalar<RealScalar>& result) const;
468 void norm(Context& ctx, DeviceScalar<RealScalar>& result) const;
469 void stableNorm(Context& ctx, DeviceScalar<RealScalar>& result) const;
470
472 void setZero(Context& ctx);
473 void setZero(cudaStream_t stream);
474
476 void addScaled(Context& ctx, Scalar alpha, const DeviceMatrix& x);
477
479 void scale(Context& ctx, Scalar alpha);
480
484 void divide(Context& ctx, Scalar alpha);
485
487 void copyFrom(Context& ctx, const DeviceMatrix& other);
488
489 DeviceScalar<Scalar> dot(const DeviceMatrix& other) const;
493 void setZero();
494
495 // The operators below let iterative-solver code written against Matrix — say
496 // `x += alpha * p` — compile unchanged against DeviceMatrix, dispatching to
497 // cuBLAS axpy/scal. `alpha * DeviceMatrix` yields Scaled<DeviceMatrix<Scalar>>
498 // from DeviceExpr.h.
499
501 DeviceMatrix& operator+=(const Scaled<DeviceMatrix>& expr);
502
505
507 DeviceMatrix& operator+=(const DeviceMatrix& other);
508
510 DeviceMatrix& operator-=(const DeviceMatrix& other);
511
513 DeviceMatrix& operator*=(Scalar alpha);
514
516 DeviceMatrix& operator/=(Scalar alpha);
517
520
522 DeviceMatrix cwiseProduct(Context& ctx, const DeviceMatrix& other) const;
523
526 void cwiseProduct(Context& ctx, const DeviceMatrix& a, const DeviceMatrix& b);
527
529 DeviceMatrix& operator+=(const DeviceScaledDevice<Scalar>& expr);
530
533
535 DeviceMatrix& operator=(const SpMVExpr<Scalar>& expr);
536
541 DeviceMatrix& operator=(const SpMVAffineExpr<Scalar>& expr);
542
544 DeviceMatrix& operator=(const DeviceAddExpr<Scalar>& expr);
545
548 DeviceMatrix& operator=(const Scaled<DeviceMatrix>& expr);
549
559 DeviceMatrix& noalias() { return *this; }
560
562 static DeviceMatrix adopt(Scalar* device_ptr, Index rows, Index cols) {
563 DeviceMatrix dm;
564 dm.data_.reset(device_ptr);
565 dm.rows_ = rows;
566 dm.cols_ = cols;
567 dm.capacity_bytes_ = dm.sizeInBytes();
568 return dm;
569 }
570
578 static DeviceMatrix view(Scalar* device_ptr, Index rows, Index cols) {
579 DeviceMatrix dm;
580 dm.data_ =
581 std::unique_ptr<Scalar, internal::CudaFreeDeleter>(device_ptr, internal::CudaFreeDeleter{/*borrow=*/true});
582 dm.rows_ = rows;
583 dm.cols_ = cols;
584 return dm;
585 }
586
590 Scalar* release() {
591 Scalar* p = data_.release();
592 rows_ = 0;
593 cols_ = 0;
594 capacity_bytes_ = 0;
595 if (ready_event_) {
596 // Clear the member first so a throwing check cannot leave a handle for the destructor to destroy again.
597 cudaEvent_t old = ready_event_;
598 ready_event_ = nullptr;
599 EIGEN_CUDA_RUNTIME_CHECK(cudaEventDestroy(old));
600 }
601 ready_stream_ = nullptr;
602 return p;
603 }
604
605 private:
606 // Fresh owning allocation of `bytes` (no-op for empty). Also resets the
607 // deleter so a previously borrowed (view) deleter cannot leak the new
608 // owned pointer.
609 void allocate(size_t bytes) {
610 if (bytes > 0) {
611 data_ = std::unique_ptr<Scalar, internal::CudaFreeDeleter>(static_cast<Scalar*>(internal::device_malloc(bytes)),
612 internal::CudaFreeDeleter{});
613 capacity_bytes_ = bytes;
614 }
615 }
616
617 void ensureEvent() {
618 if (!ready_event_) {
619 EIGEN_CUDA_RUNTIME_CHECK(cudaEventCreateWithFlags(&ready_event_, cudaEventDisableTiming));
620 }
621 }
622
623 void retainBuffer(internal::DeviceBuffer&& buffer) { retained_buffer_ = std::move(buffer); }
624
625 std::unique_ptr<Scalar, internal::CudaFreeDeleter> data_;
626 Index rows_ = 0;
627 Index cols_ = 0;
628 size_t capacity_bytes_ = 0; // owned allocation size (0 for borrowed views)
629 cudaEvent_t ready_event_ = nullptr; // internal: tracks last write completion
630 cudaStream_t ready_stream_ = nullptr; // stream that recorded ready_event_ (for same-stream skip)
631 internal::DeviceBuffer retained_buffer_; // internal: keeps async aux buffers alive
632};
633} // namespace gpu
634} // namespace Eigen
635
636#endif // EIGEN_GPU_DEVICE_MATRIX_H
constexpr Scalar * data()
View returned by DeviceMatrix::adjoint(); maps to the cuBLAS conjugate-transpose operand flag.
Definition DeviceExpr.h:46
Definition DeviceBlasExpr.h:81
Unified GPU execution context owning a CUDA stream and library handles.
Definition GpuContext.h:81
Linear combination of two device matrices.
Definition DeviceExpr.h:260
RAII wrapper for a dense column-major matrix in GPU device memory.
Definition DeviceMatrix.h:122
static DeviceMatrix fromHost(const DenseBase< Derived > &host, cudaStream_t stream=nullptr)
Definition DeviceMatrix.h:226
DeviceMatrix & operator*=(Scalar alpha)
Definition DeviceDispatch.h:745
LLTView< Scalar, Lower > llt() const
Definition DeviceMatrix.h:401
HostTransfer< Scalar > toHostAsync(cudaStream_t stream=nullptr) const
Definition DeviceMatrix.h:289
static DeviceMatrix fromHostAsync(const Scalar *host_data, Index rows, Index cols, cudaStream_t stream)
Definition DeviceMatrix.h:251
Scalar * release()
Definition DeviceMatrix.h:590
DeviceMatrix(Index n)
Definition DeviceMatrix.h:134
void addScaled(Context &ctx, Scalar alpha, const DeviceMatrix &x)
Definition DeviceDispatch.h:678
DeviceMatrix(Index rows, Index cols)
Definition DeviceMatrix.h:140
Assignment< Scalar > device(Context &ctx)
Definition DeviceMatrix.h:388
DeviceMatrix cwiseProduct(Context &ctx, const DeviceMatrix &other) const
Definition DeviceDispatch.h:857
TransposeView< Scalar > transpose() const
Definition DeviceMatrix.h:384
void scale(Context &ctx, Scalar alpha)
Definition DeviceDispatch.h:690
void copyFrom(Context &ctx, const DeviceMatrix &other)
Definition DeviceDispatch.h:700
size_t sizeInBytes() const
Definition DeviceMatrix.h:362
DeviceMatrix & operator/=(Scalar alpha)
Definition DeviceDispatch.h:777
LLTView< Scalar, UpLo > llt() const
Definition DeviceMatrix.h:405
bool isView() const
Definition DeviceMatrix.h:359
DeviceMatrix & operator-=(const GemmExpr< Lhs, Rhs > &expr)
void divide(Context &ctx, Scalar alpha)
Definition DeviceDispatch.h:766
ConstSelfAdjointView< Scalar, UpLo > selfadjointView() const
Definition DeviceMatrix.h:431
static DeviceMatrix view(Scalar *device_ptr, Index rows, Index cols)
Definition DeviceMatrix.h:578
static DeviceMatrix adopt(Scalar *device_ptr, Index rows, Index cols)
Definition DeviceMatrix.h:562
void recordReady(cudaStream_t stream)
Definition DeviceMatrix.h:365
DeviceMatrix clone(cudaStream_t stream=nullptr) const
Definition DeviceMatrix.h:307
AdjointView< Scalar > adjoint() const
Definition DeviceMatrix.h:381
PlainMatrix toHost(cudaStream_t stream=nullptr) const
Definition DeviceMatrix.h:270
DeviceScalar< Scalar > dot(Context &ctx, const DeviceMatrix &other) const
Definition DeviceDispatch.h:600
SelfAdjointView< Scalar, UpLo > selfadjointView()
Definition DeviceMatrix.h:425
LUView< Scalar > lu() const
Definition DeviceMatrix.h:410
void resize(Index rows, Index cols)
Definition DeviceMatrix.h:323
DeviceMatrix & noalias()
Definition DeviceMatrix.h:559
TriangularView< Scalar, UpLo > triangularView() const
Definition DeviceMatrix.h:419
void waitReady(cudaStream_t stream) const
Definition DeviceMatrix.h:374
RAII wrapper for a scalar in GPU device memory.
Definition DeviceScalar.h:33
Expression that scales a device matrix by a DeviceScalar.
Definition DeviceExpr.h:231
Expression returned by operator*(lhs_expr, rhs_expr), dispatched to cuBLAS GEMM.
Definition DeviceExpr.h:92
Future for an asynchronous device-to-host matrix transfer.
Definition DeviceMatrix.h:39
PlainMatrix & get()
Definition DeviceMatrix.h:46
bool ready() const
Definition DeviceMatrix.h:59
Definition DeviceSolverExpr.h:61
Definition DeviceSolverExpr.h:76
Definition DeviceSolverExpr.h:30
Definition DeviceSolverExpr.h:46
Expression returned by operator*(Scalar, DeviceMatrix/View), carrying the scalar factor.
Definition DeviceExpr.h:77
Definition DeviceBlasExpr.h:61
Definition GpuSparseContext.h:166
Definition GpuSparseContext.h:146
Definition DeviceBlasExpr.h:96
View returned by DeviceMatrix::transpose(); maps to the cuBLAS transpose operand flag.
Definition DeviceExpr.h:58
Definition DeviceBlasExpr.h:29
Definition DeviceBlasExpr.h:45
Internal RAII owner for an untyped GPU device allocation.
Definition GpuSupport.h:296
Namespace containing all symbols from the Eigen library.