Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
GpuSupport.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// Generic CUDA runtime support shared by all GPU library integrations.
12// Depends only on <cuda_runtime.h>; no NVIDIA library headers.
13
14#ifndef EIGEN_GPU_SUPPORT_H
15#define EIGEN_GPU_SUPPORT_H
16
17// IWYU pragma: private
18#include "./InternalHeaderCheck.h"
19
20#include <cuda_runtime.h>
21#include <cstdio>
22#include <cstdlib>
23#include <vector>
24
25#include <limits>
26#include <memory>
27#include <type_traits>
28
29namespace Eigen {
30namespace gpu {
31// Transpose/adjoint flag for BLAS-, solver-, and sparse-style calls. Each
32// library's support header maps it to its own enum (cublasOperation_t,
33// cusparseOperation_t, ...) via a to_<lib>_op() helper.
34enum class GpuOp { NoTrans, Trans, ConjTrans };
35
36namespace internal {
37// Prints `file:line: call: error` to stderr and stops: std::abort() where
38// assertions are compiled out, a failed eigen_assert otherwise. No build goes
39// on past a failed call: its work was not done, and a sticky error leaves the
40// context unusable for every later call.
41inline void gpu_check_failed(const char* error, const char* expression, const char* file, int line) {
42 std::fprintf(stderr, "%s:%d: %s: %s\n", file, line, expression, error);
43#if defined(EIGEN_NO_DEBUG)
44 std::abort();
45#else
46 eigen_assert(false && "GPU runtime or library call failed");
47#endif
48}
49
50// Every check macro of the module reports failures here, in every build. It may
51// be defined to throw, so no destructor or noexcept function uses the checks.
52#ifndef EIGEN_GPU_CHECK_FAILED
53#define EIGEN_GPU_CHECK_FAILED(error, expression, file, line) \
54 ::Eigen::gpu::internal::gpu_check_failed(error, expression, file, line)
55#endif
56
57#define EIGEN_CUDA_RUNTIME_CHECK(expr) \
58 do { \
59 const cudaError_t _e = (expr); \
60 if (_e != cudaSuccess) EIGEN_GPU_CHECK_FAILED(cudaGetErrorName(_e), #expr, __FILE__, __LINE__); \
61 } while (0)
62
63// For the libraries without a status-to-string function: "<library> status <code>".
64inline void gpu_check_failed_code(const char* library, int status, const char* expression, const char* file, int line) {
65 char error[64];
66 std::snprintf(error, sizeof(error), "%s status %d", library, status);
67 // A user-defined EIGEN_GPU_CHECK_FAILED need not use every argument.
68 EIGEN_UNUSED_VARIABLE(expression);
69 EIGEN_UNUSED_VARIABLE(file);
70 EIGEN_UNUSED_VARIABLE(line);
71 EIGEN_GPU_CHECK_FAILED(error, expression, file, line);
72}
73
74// cuBLAS and the legacy cuSOLVER APIs take dimensions and leading dimensions as
75// 32-bit `int`, while Eigen's Index is 64-bit by default and GPU allocations can
76// exceed INT_MAX in one dimension. Narrow through this helper at every such call
77// site so an out-of-range value is reported, in every build, instead of silently
78// overflowing.
79inline int to_blas_int(int64_t v) {
80 if (v < 0 || v > static_cast<int64_t>((std::numeric_limits<int>::max)())) {
81 EIGEN_GPU_CHECK_FAILED("dimension exceeds the int range supported by cuBLAS / cuSOLVER", "to_blas_int", __FILE__,
82 __LINE__);
83 }
84 return static_cast<int>(v);
85}
86
87// cudaMallocAsync / cudaFreeAsync (CUDA 11.2+) allocate from a stream-ordered
88// memory pool: both are cheap enqueues instead of the device-wide
89// synchronization performed by cudaMalloc / cudaFree. All module allocations
90// go through device_malloc / device_free on the *legacy default stream*:
91// legacy-stream ordering guarantees that work enqueued later on any blocking
92// stream observes the allocation, and that a free waits for all previously
93// enqueued work on blocking streams — the same lifetime guarantees callers
94// got from cudaMalloc / cudaFree, minus the host stalls.
95//
96// Caveat: streams created with cudaStreamNonBlocking do not synchronize with
97// the legacy stream. When borrowing such a stream (gpu::Context(stream)),
98// define EIGEN_GPU_NO_STREAM_ORDERED_ALLOC to fall back to cudaMalloc/cudaFree.
99//
100// Support is detected once per process, from the device current at first use.
101
102inline bool device_supports_memory_pools() {
103#ifdef EIGEN_GPU_NO_STREAM_ORDERED_ALLOC
104 return false;
105#else
106 static const bool supported = [] {
107 int device = 0;
108 if (cudaGetDevice(&device) != cudaSuccess) return false;
109 int v = 0;
110 if (cudaDeviceGetAttribute(&v, cudaDevAttrMemoryPoolsSupported, device) != cudaSuccess) return false;
111 if (v == 0) return false;
112 // Keep freed memory in the pool instead of trimming at every stream
113 // synchronize — repeated alloc/free cycles (temporaries in loops) then
114 // recycle at user-space speed.
115 cudaMemPool_t pool = nullptr;
116 if (cudaDeviceGetDefaultMemPool(&pool, device) == cudaSuccess) {
117 // The attribute value type is cuuint64_t; use a same-size stand-in to
118 // avoid requiring the driver-API header.
119 unsigned long long threshold = ~0ULL;
120 (void)cudaMemPoolSetAttribute(pool, cudaMemPoolAttrReleaseThreshold, &threshold);
121 }
122 return true;
123 }();
124 return supported;
125#endif
126}
127
128inline void* device_malloc(size_t bytes) {
129 void* p = nullptr;
130 if (device_supports_memory_pools()) {
131 EIGEN_CUDA_RUNTIME_CHECK(cudaMallocAsync(&p, bytes, /*legacy default stream*/ nullptr));
132 } else {
133 EIGEN_CUDA_RUNTIME_CHECK(cudaMalloc(&p, bytes));
134 }
135 return p;
136}
137
138inline void device_free(void* p) noexcept {
139 if (!p) return;
140 if (device_supports_memory_pools()) {
141 (void)cudaFreeAsync(p, /*legacy default stream*/ nullptr);
142 } else {
143 (void)cudaFree(p);
144 }
145}
146
147struct CudaFreeDeleter {
148 // When `borrow == true`, the unique_ptr does not free the pointer. Used by
149 // DeviceMatrix::view() to wrap a non-owning device pointer with the same
150 // smart-pointer machinery as owning storage, without changing the type.
151 bool borrow = false;
152 void operator()(void* p) const noexcept {
153 if (p && !borrow) device_free(p);
154 }
155};
156
157struct CudaFreeHostDeleter {
158 void operator()(void* p) const noexcept {
159 if (p) (void)cudaFreeHost(p);
160 }
161};
162
163// RAII CUDA stream; the ownership flag supports borrowed, caller-owned streams.
164struct CudaStreamDeleter {
165 bool owns = true;
166 void operator()(cudaStream_t s) const noexcept {
167 if (owns && s) (void)cudaStreamDestroy(s);
168 }
169};
170using UniqueStream = std::unique_ptr<std::remove_pointer_t<cudaStream_t>, CudaStreamDeleter>;
171
172// Recycles allocations up to kSmallBufferThreshold bytes (e.g. DeviceScalar) to
173// avoid cudaMalloc/cudaFree overhead on devices without memory pools; where
174// cudaMallocAsync exists it is both stream-ordered and cheaper than this pool's
175// release event, so DeviceBuffer bypasses the pool there (bench_overhead:
176// CudaMallocAsyncFree vs PoolAllocFree). Larger allocations always bypass it.
177// Invariant: a block is recycled only after the device has retired every
178// operation enqueued before its release on any blocking stream: deallocate()
179// records an event on the legacy default stream (the ordering device_free
180// relies on), the free list stays in release order, and events on one stream
181// retire in order, so allocate() scans until the first pending entry.
182template <size_t SmallBufferThreshold = 256, size_t MaxPoolSize = 64>
183struct DeviceBufferPool {
184 static constexpr size_t kSmallBufferThreshold = SmallBufferThreshold;
185 static constexpr size_t kMaxPoolSize = MaxPoolSize;
186
187 struct Entry {
188 void* ptr;
189 size_t bytes;
190 cudaEvent_t release_event;
191 };
192
193 // Lifetime marker for the thread-local pool. thread_local destruction runs
194 // in reverse construction order, so a long-lived object holding pooled
195 // buffers (e.g. the thread-local gpu::Context, or a static) can be
196 // destroyed *after* the pool. The marker is trivially destructible — it
197 // stays readable during TLS teardown — letting the deleter fall back to a
198 // direct device_free once the pool is gone instead of touching a destroyed
199 // vector.
200 enum class State : signed char { kNotConstructed = 0, kAlive = 1, kDestroyed = 2 };
201
202 static State& threadState() {
203 thread_local State state = State::kNotConstructed;
204 return state;
205 }
206
207 DeviceBufferPool() { threadState() = State::kAlive; }
208
209 ~DeviceBufferPool() {
210 for (const Entry& entry : free_list_) freeBlock(entry.ptr, entry.release_event);
211 for (cudaEvent_t event : spare_events_) (void)cudaEventDestroy(event);
212 threadState() = State::kDestroyed;
213 }
214
215 // First fit among the retired blocks, oldest release first; the first pending
216 // entry ends the scan because every later one is pending too.
217 void* allocate(size_t bytes) {
218 for (auto it = free_list_.begin(); it != free_list_.end(); ++it) {
219 if (cudaEventQuery(it->release_event) != cudaSuccess) break;
220 if (it->bytes >= bytes) {
221 void* p = it->ptr;
222 spare_events_.push_back(it->release_event);
223 free_list_.erase(it);
224 return p;
225 }
226 }
227 return device_malloc(bytes);
228 }
229
230 // Called from a noexcept deleter: every failure falls back to device_free.
231 void deallocate(void* p, size_t bytes) noexcept {
232 if (free_list_.size() >= kMaxPoolSize) {
233 device_free(p);
234 return;
235 }
236 cudaEvent_t release_event = acquireEvent();
237 if (release_event == nullptr) {
238 device_free(p);
239 return;
240 }
241 if (cudaEventRecord(release_event, /*legacy default stream*/ nullptr) != cudaSuccess) {
242 freeBlock(p, release_event);
243 return;
244 }
245 free_list_.push_back({p, bytes, release_event});
246 }
247
248 static DeviceBufferPool& threadLocal() {
249 thread_local DeviceBufferPool pool;
250 return pool;
251 }
252
253 private:
254 // Returns a spare event, or a newly created one; nullptr if creation fails.
255 cudaEvent_t acquireEvent() noexcept {
256 if (!spare_events_.empty()) {
257 cudaEvent_t event = spare_events_.back();
258 spare_events_.pop_back();
259 return event;
260 }
261 cudaEvent_t event = nullptr;
262 if (cudaEventCreateWithFlags(&event, cudaEventDisableTiming) != cudaSuccess) return nullptr;
263 return event;
264 }
265
266 // Gives up a block and the event tracking its release. Destroying a pending
267 // event is non-blocking; the runtime defers it until the event completes.
268 static void freeBlock(void* p, cudaEvent_t release_event) noexcept {
269 (void)cudaEventDestroy(release_event);
270 device_free(p);
271 }
272
273 std::vector<Entry> free_list_;
274 // Events of recycled entries; free_list_.size() + spare_events_.size() <= kMaxPoolSize.
275 std::vector<cudaEvent_t> spare_events_;
276};
277
278// Stateful deleter that returns pooled buffers to the thread-local pool and
279// device_free's the rest. size==0 means "always device_free" (adopted pointers
280// and allocations that went straight to device_malloc).
281struct PooledCudaFreeDeleter {
282 size_t size = 0;
283
284 void operator()(void* p) const noexcept {
285 if (!p) return;
286 if (size > 0 && size <= DeviceBufferPool<>::kSmallBufferThreshold &&
287 DeviceBufferPool<>::threadState() == DeviceBufferPool<>::State::kAlive) {
288 DeviceBufferPool<>::threadLocal().deallocate(p, size);
289 } else {
290 device_free(p);
291 }
292 }
293};
294
296class DeviceBuffer {
297 public:
298 DeviceBuffer() = default;
299
300 explicit DeviceBuffer(size_t bytes) : bytes_(bytes) {
301 if (bytes > 0) {
302 // The pool serves small blocks only on the cudaMalloc fallback path, and
303 // not once its thread_local has been destroyed (allocation from a
304 // static/TLS destructor). A deleter size of 0 keeps the other
305 // allocations on the direct device_free path.
306 const bool pooled = bytes <= DeviceBufferPool<>::kSmallBufferThreshold && !device_supports_memory_pools() &&
307 DeviceBufferPool<>::threadState() != DeviceBufferPool<>::State::kDestroyed;
308 void* p = pooled ? DeviceBufferPool<>::threadLocal().allocate(bytes) : device_malloc(bytes);
309 ptr_ = std::unique_ptr<void, PooledCudaFreeDeleter>(p, PooledCudaFreeDeleter{pooled ? bytes : 0});
310 }
311 }
312
313 // Explicit moves so a moved-from buffer reports size() == 0 (callers use
314 // size() for grow-only reuse decisions; a stale size on a null buffer would
315 // suppress the reallocation).
316 DeviceBuffer(DeviceBuffer&& o) noexcept : ptr_(std::move(o.ptr_)), bytes_(o.bytes_) { o.bytes_ = 0; }
317 DeviceBuffer& operator=(DeviceBuffer&& o) noexcept {
318 if (this != &o) {
319 ptr_ = std::move(o.ptr_);
320 bytes_ = o.bytes_;
321 o.bytes_ = 0;
322 }
323 return *this;
324 }
325
326 void* get() const noexcept { return ptr_.get(); }
327 void* release() noexcept {
328 bytes_ = 0;
329 return ptr_.release();
330 }
331 explicit operator bool() const noexcept { return static_cast<bool>(ptr_); }
332
334 size_t size() const noexcept { return bytes_; }
335
336 // Adopt an existing device pointer of `bytes` usable bytes. Caller
337 // relinquishes ownership. Adopted buffers bypass the pool on destruction
338 // (deleter size == 0).
339 static DeviceBuffer adopt(void* p, size_t bytes) noexcept {
340 DeviceBuffer b;
341 b.ptr_ = std::unique_ptr<void, PooledCudaFreeDeleter>(p, PooledCudaFreeDeleter{});
342 b.bytes_ = p ? bytes : 0;
343 return b;
344 }
345
346 private:
347 std::unique_ptr<void, PooledCudaFreeDeleter> ptr_;
348 size_t bytes_ = 0;
349};
350
351// cudaMemcpyAsync only overlaps with compute when the host side is pinned, so
352// async D2H staging goes through this buffer.
353class PinnedHostBuffer {
354 public:
355 PinnedHostBuffer() = default;
356
357 explicit PinnedHostBuffer(size_t bytes) {
358 if (bytes > 0) {
359 void* p = nullptr;
360 EIGEN_CUDA_RUNTIME_CHECK(cudaMallocHost(&p, bytes));
361 ptr_.reset(p);
362 }
363 }
364
365 void* get() const noexcept { return ptr_.get(); }
366 explicit operator bool() const noexcept { return static_cast<bool>(ptr_); }
367
368 private:
369 std::unique_ptr<void, CudaFreeHostDeleter> ptr_;
370};
371
372// Upload a column-major host matrix whose strides are in elements. Ref<const
373// PlainMatrix> can bind any outer stride in place. Use a 2D DMA for ordinary
374// padded layouts; copy legal negative or overlapping Eigen strides one
375// contiguous column at a time because CUDA cannot express them as a pitch.
376template <typename Scalar>
377void upload_host_matrix(Scalar* dst, Index dst_outer_stride, const Scalar* src, Index src_outer_stride, Index rows,
378 Index cols, cudaStream_t stream) {
379 if (rows <= 0 || cols <= 0) return;
380 eigen_assert(dst_outer_stride >= rows);
381 const size_t column_bytes = static_cast<size_t>(rows) * sizeof(Scalar);
382 if (src_outer_stride >= rows) {
383 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy2DAsync(dst, static_cast<size_t>(dst_outer_stride) * sizeof(Scalar), src,
384 static_cast<size_t>(src_outer_stride) * sizeof(Scalar), column_bytes,
385 static_cast<size_t>(cols), cudaMemcpyHostToDevice, stream));
386 } else {
387 for (Index col = 0; col < cols; ++col) {
388 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst + col * dst_outer_stride, src + col * src_outer_stride, column_bytes,
389 cudaMemcpyHostToDevice, stream));
390 }
391 }
392}
393
394// cudaDataType_t lives in library_types.h, pulled in transitively by
395// cuda_runtime.h, so this trait needs no NVIDIA library header of its own.
396template <typename Scalar>
397struct cuda_data_type;
398
399template <>
400struct cuda_data_type<float> {
401 static constexpr cudaDataType_t value = CUDA_R_32F;
402};
403template <>
404struct cuda_data_type<double> {
405 static constexpr cudaDataType_t value = CUDA_R_64F;
406};
407template <>
408struct cuda_data_type<std::complex<float>> {
409 static constexpr cudaDataType_t value = CUDA_C_32F;
410};
411template <>
412struct cuda_data_type<std::complex<double>> {
413 static constexpr cudaDataType_t value = CUDA_C_64F;
414};
415} // namespace internal
416} // namespace gpu
417} // namespace Eigen
418
419#endif // EIGEN_GPU_SUPPORT_H
Internal RAII owner for an untyped GPU device allocation.
Definition GpuSupport.h:296
size_t size() const noexcept
Definition GpuSupport.h:334
Namespace containing all symbols from the Eigen library.