14#ifndef EIGEN_GPU_SUPPORT_H
15#define EIGEN_GPU_SUPPORT_H
18#include "./InternalHeaderCheck.h"
20#include <cuda_runtime.h>
34enum class GpuOp { NoTrans, Trans, ConjTrans };
41inline void gpu_check_failed(
const char* error,
const char* expression,
const char* file,
int line) {
42 std::fprintf(stderr,
"%s:%d: %s: %s\n", file, line, expression, error);
43#if defined(EIGEN_NO_DEBUG)
46 eigen_assert(
false &&
"GPU runtime or library call failed");
52#ifndef EIGEN_GPU_CHECK_FAILED
53#define EIGEN_GPU_CHECK_FAILED(error, expression, file, line) \
54 ::Eigen::gpu::internal::gpu_check_failed(error, expression, file, line)
57#define EIGEN_CUDA_RUNTIME_CHECK(expr) \
59 const cudaError_t _e = (expr); \
60 if (_e != cudaSuccess) EIGEN_GPU_CHECK_FAILED(cudaGetErrorName(_e), #expr, __FILE__, __LINE__); \
64inline void gpu_check_failed_code(
const char* library,
int status,
const char* expression,
const char* file,
int line) {
66 std::snprintf(error,
sizeof(error),
"%s status %d", library, status);
68 EIGEN_UNUSED_VARIABLE(expression);
69 EIGEN_UNUSED_VARIABLE(file);
70 EIGEN_UNUSED_VARIABLE(line);
71 EIGEN_GPU_CHECK_FAILED(error, expression, file, line);
79inline int to_blas_int(int64_t v) {
80 if (v < 0 || v >
static_cast<int64_t
>((std::numeric_limits<int>::max)())) {
81 EIGEN_GPU_CHECK_FAILED(
"dimension exceeds the int range supported by cuBLAS / cuSOLVER",
"to_blas_int", __FILE__,
84 return static_cast<int>(v);
102inline bool device_supports_memory_pools() {
103#ifdef EIGEN_GPU_NO_STREAM_ORDERED_ALLOC
106 static const bool supported = [] {
108 if (cudaGetDevice(&device) != cudaSuccess)
return false;
110 if (cudaDeviceGetAttribute(&v, cudaDevAttrMemoryPoolsSupported, device) != cudaSuccess)
return false;
111 if (v == 0)
return false;
115 cudaMemPool_t pool =
nullptr;
116 if (cudaDeviceGetDefaultMemPool(&pool, device) == cudaSuccess) {
119 unsigned long long threshold = ~0ULL;
120 (void)cudaMemPoolSetAttribute(pool, cudaMemPoolAttrReleaseThreshold, &threshold);
128inline void* device_malloc(
size_t bytes) {
130 if (device_supports_memory_pools()) {
131 EIGEN_CUDA_RUNTIME_CHECK(cudaMallocAsync(&p, bytes,
nullptr));
133 EIGEN_CUDA_RUNTIME_CHECK(cudaMalloc(&p, bytes));
138inline void device_free(
void* p)
noexcept {
140 if (device_supports_memory_pools()) {
141 (void)cudaFreeAsync(p,
nullptr);
147struct CudaFreeDeleter {
152 void operator()(
void* p)
const noexcept {
153 if (p && !borrow) device_free(p);
157struct CudaFreeHostDeleter {
158 void operator()(
void* p)
const noexcept {
159 if (p) (void)cudaFreeHost(p);
164struct CudaStreamDeleter {
166 void operator()(cudaStream_t s)
const noexcept {
167 if (owns && s) (void)cudaStreamDestroy(s);
170using UniqueStream = std::unique_ptr<std::remove_pointer_t<cudaStream_t>, CudaStreamDeleter>;
182template <
size_t SmallBufferThreshold = 256,
size_t MaxPoolSize = 64>
183struct DeviceBufferPool {
184 static constexpr size_t kSmallBufferThreshold = SmallBufferThreshold;
185 static constexpr size_t kMaxPoolSize = MaxPoolSize;
190 cudaEvent_t release_event;
200 enum class State :
signed char { kNotConstructed = 0, kAlive = 1, kDestroyed = 2 };
202 static State& threadState() {
203 thread_local State state = State::kNotConstructed;
207 DeviceBufferPool() { threadState() = State::kAlive; }
209 ~DeviceBufferPool() {
210 for (
const Entry& entry : free_list_) freeBlock(entry.ptr, entry.release_event);
211 for (cudaEvent_t event : spare_events_) (void)cudaEventDestroy(event);
212 threadState() = State::kDestroyed;
217 void* allocate(
size_t bytes) {
218 for (
auto it = free_list_.begin(); it != free_list_.end(); ++it) {
219 if (cudaEventQuery(it->release_event) != cudaSuccess)
break;
220 if (it->bytes >= bytes) {
222 spare_events_.push_back(it->release_event);
223 free_list_.erase(it);
227 return device_malloc(bytes);
231 void deallocate(
void* p,
size_t bytes)
noexcept {
232 if (free_list_.size() >= kMaxPoolSize) {
236 cudaEvent_t release_event = acquireEvent();
237 if (release_event ==
nullptr) {
241 if (cudaEventRecord(release_event,
nullptr) != cudaSuccess) {
242 freeBlock(p, release_event);
245 free_list_.push_back({p, bytes, release_event});
248 static DeviceBufferPool& threadLocal() {
249 thread_local DeviceBufferPool pool;
255 cudaEvent_t acquireEvent() noexcept {
256 if (!spare_events_.empty()) {
257 cudaEvent_t
event = spare_events_.back();
258 spare_events_.pop_back();
261 cudaEvent_t
event =
nullptr;
262 if (cudaEventCreateWithFlags(&event, cudaEventDisableTiming) != cudaSuccess)
return nullptr;
268 static void freeBlock(
void* p, cudaEvent_t release_event)
noexcept {
269 (void)cudaEventDestroy(release_event);
273 std::vector<Entry> free_list_;
275 std::vector<cudaEvent_t> spare_events_;
281struct PooledCudaFreeDeleter {
284 void operator()(
void* p)
const noexcept {
286 if (size > 0 && size <= DeviceBufferPool<>::kSmallBufferThreshold &&
287 DeviceBufferPool<>::threadState() == DeviceBufferPool<>::State::kAlive) {
288 DeviceBufferPool<>::threadLocal().deallocate(p, size);
298 DeviceBuffer() =
default;
300 explicit DeviceBuffer(
size_t bytes) : bytes_(bytes) {
306 const bool pooled = bytes <= DeviceBufferPool<>::kSmallBufferThreshold && !device_supports_memory_pools() &&
307 DeviceBufferPool<>::threadState() != DeviceBufferPool<>::State::kDestroyed;
308 void* p = pooled ? DeviceBufferPool<>::threadLocal().allocate(bytes) : device_malloc(bytes);
309 ptr_ = std::unique_ptr<void, PooledCudaFreeDeleter>(p, PooledCudaFreeDeleter{pooled ? bytes : 0});
316 DeviceBuffer(DeviceBuffer&& o) noexcept : ptr_(std::move(o.ptr_)), bytes_(o.bytes_) { o.bytes_ = 0; }
317 DeviceBuffer& operator=(DeviceBuffer&& o)
noexcept {
319 ptr_ = std::move(o.ptr_);
326 void* get()
const noexcept {
return ptr_.get(); }
327 void* release()
noexcept {
329 return ptr_.release();
331 explicit operator bool()
const noexcept {
return static_cast<bool>(ptr_); }
334 size_t size() const noexcept {
return bytes_; }
339 static DeviceBuffer adopt(
void* p,
size_t bytes)
noexcept {
341 b.ptr_ = std::unique_ptr<void, PooledCudaFreeDeleter>(p, PooledCudaFreeDeleter{});
342 b.bytes_ = p ? bytes : 0;
347 std::unique_ptr<void, PooledCudaFreeDeleter> ptr_;
353class PinnedHostBuffer {
355 PinnedHostBuffer() =
default;
357 explicit PinnedHostBuffer(
size_t bytes) {
360 EIGEN_CUDA_RUNTIME_CHECK(cudaMallocHost(&p, bytes));
365 void* get() const noexcept {
return ptr_.get(); }
366 explicit operator bool() const noexcept {
return static_cast<bool>(ptr_); }
369 std::unique_ptr<void, CudaFreeHostDeleter> ptr_;
376template <
typename Scalar>
377void upload_host_matrix(Scalar* dst, Index dst_outer_stride,
const Scalar* src, Index src_outer_stride, Index rows,
378 Index cols, cudaStream_t stream) {
379 if (rows <= 0 || cols <= 0)
return;
380 eigen_assert(dst_outer_stride >= rows);
381 const size_t column_bytes =
static_cast<size_t>(rows) *
sizeof(Scalar);
382 if (src_outer_stride >= rows) {
383 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpy2DAsync(dst,
static_cast<size_t>(dst_outer_stride) *
sizeof(Scalar), src,
384 static_cast<size_t>(src_outer_stride) *
sizeof(Scalar), column_bytes,
385 static_cast<size_t>(cols), cudaMemcpyHostToDevice, stream));
387 for (Index col = 0; col < cols; ++col) {
388 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst + col * dst_outer_stride, src + col * src_outer_stride, column_bytes,
389 cudaMemcpyHostToDevice, stream));
396template <
typename Scalar>
397struct cuda_data_type;
400struct cuda_data_type<float> {
401 static constexpr cudaDataType_t value = CUDA_R_32F;
404struct cuda_data_type<double> {
405 static constexpr cudaDataType_t value = CUDA_R_64F;
408struct cuda_data_type<std::complex<float>> {
409 static constexpr cudaDataType_t value = CUDA_C_32F;
412struct cuda_data_type<std::complex<double>> {
413 static constexpr cudaDataType_t value = CUDA_C_64F;
Internal RAII owner for an untyped GPU device allocation.
Definition GpuSupport.h:296
size_t size() const noexcept
Definition GpuSupport.h:334
Namespace containing all symbols from the Eigen library.