15#ifndef EIGEN_GPU_DEVICE_SCALAR_OPS_H
16#define EIGEN_GPU_DEVICE_SCALAR_OPS_H
18#include <cuda_runtime.h>
19#include <npps_arithmetic_and_logical_operations.h>
23#include "./GpuSupport.h"
30#define EIGEN_NPP_CHECK(expr) \
32 const NppStatus _s = (expr); \
33 if (_s < NPP_SUCCESS) \
34 ::Eigen::gpu::internal::gpu_check_failed_code("NPP", static_cast<int>(_s), #expr, __FILE__, __LINE__); \
40inline NppStreamContext make_npp_stream_ctx(cudaStream_t stream) {
41 NppStreamContext ctx = {};
43#if CUDART_VERSION >= 12080
46 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamGetDevice(stream, &ctx.nCudaDeviceId));
50 EIGEN_CUDA_RUNTIME_CHECK(cudaGetDevice(&ctx.nCudaDeviceId));
52 EIGEN_CUDA_RUNTIME_CHECK(cudaDeviceGetAttribute(&ctx.nCudaDevAttrComputeCapabilityMajor,
53 cudaDevAttrComputeCapabilityMajor, ctx.nCudaDeviceId));
54 EIGEN_CUDA_RUNTIME_CHECK(cudaDeviceGetAttribute(&ctx.nCudaDevAttrComputeCapabilityMinor,
55 cudaDevAttrComputeCapabilityMinor, ctx.nCudaDeviceId));
56 EIGEN_CUDA_RUNTIME_CHECK(
57 cudaDeviceGetAttribute(&ctx.nMultiProcessorCount, cudaDevAttrMultiProcessorCount, ctx.nCudaDeviceId));
58 EIGEN_CUDA_RUNTIME_CHECK(cudaDeviceGetAttribute(&ctx.nMaxThreadsPerMultiProcessor,
59 cudaDevAttrMaxThreadsPerMultiProcessor, ctx.nCudaDeviceId));
60 EIGEN_CUDA_RUNTIME_CHECK(
61 cudaDeviceGetAttribute(&ctx.nMaxThreadsPerBlock, cudaDevAttrMaxThreadsPerBlock, ctx.nCudaDeviceId));
62 int shared_mem_per_block = 0;
63 EIGEN_CUDA_RUNTIME_CHECK(
64 cudaDeviceGetAttribute(&shared_mem_per_block, cudaDevAttrMaxSharedMemoryPerBlock, ctx.nCudaDeviceId));
65 ctx.nSharedMemPerBlock =
static_cast<size_t>(shared_mem_per_block);
66 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamGetFlags(stream, &ctx.nStreamFlags));
72inline void device_scalar_div(
const float* a,
const float* b,
float* c, cudaStream_t stream) {
73 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
74 EIGEN_NPP_CHECK(nppsDiv_32f_Ctx(b, a, c, 1, npp_ctx));
77inline void device_scalar_div(
const double* a,
const double* b,
double* c, cudaStream_t stream) {
78 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
79 EIGEN_NPP_CHECK(nppsDiv_64f_Ctx(b, a, c, 1, npp_ctx));
83inline void device_scalar_sqrt(
float* a,
const NppStreamContext& npp_ctx) {
84 EIGEN_NPP_CHECK(nppsSqrt_32f_I_Ctx(a, 1, npp_ctx));
87inline void device_scalar_sqrt(
double* a,
const NppStreamContext& npp_ctx) {
88 EIGEN_NPP_CHECK(nppsSqrt_64f_I_Ctx(a, 1, npp_ctx));
92inline void device_scalar_neg(
const float* a,
float* c, cudaStream_t stream) {
93 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
94 EIGEN_NPP_CHECK(nppsMulC_32f_Ctx(a, -1.0f, c, 1, npp_ctx));
97inline void device_scalar_neg(
const double* a,
double* c, cudaStream_t stream) {
98 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
99 EIGEN_NPP_CHECK(nppsMulC_64f_Ctx(a, -1.0, c, 1, npp_ctx));
105template <
typename Fn>
106void for_each_npp_chunk(int64_t n, Fn fn) {
107 const int64_t max_length = int64_t(1) << 30;
108 for (int64_t offset = 0; offset < n; offset += max_length) {
109 fn(offset,
static_cast<int>((std::min)(max_length, n - offset)));
114inline void device_cwiseProduct(
const float* a,
const float* b,
float* c, int64_t n, cudaStream_t stream) {
115 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
117 n, [&](int64_t i,
int length) { EIGEN_NPP_CHECK(nppsMul_32f_Ctx(a + i, b + i, c + i, length, npp_ctx)); });
120inline void device_cwiseProduct(
const double* a,
const double* b,
double* c, int64_t n, cudaStream_t stream) {
121 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
123 n, [&](int64_t i,
int length) { EIGEN_NPP_CHECK(nppsMul_64f_Ctx(a + i, b + i, c + i, length, npp_ctx)); });
127inline void device_cwiseQuotient(
const float* a,
const float* b,
float* c, int64_t n, cudaStream_t stream) {
128 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
130 n, [&](int64_t i,
int length) { EIGEN_NPP_CHECK(nppsDiv_32f_Ctx(b + i, a + i, c + i, length, npp_ctx)); });
133inline void device_cwiseQuotient(
const double* a,
const double* b,
double* c, int64_t n, cudaStream_t stream) {
134 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
136 n, [&](int64_t i,
int length) { EIGEN_NPP_CHECK(nppsDiv_64f_Ctx(b + i, a + i, c + i, length, npp_ctx)); });
140inline void device_divC(
float alpha,
float* x, int64_t n, cudaStream_t stream) {
141 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
143 n, [&](int64_t i,
int length) { EIGEN_NPP_CHECK(nppsDivC_32f_I_Ctx(alpha, x + i, length, npp_ctx)); });
146inline void device_divC(
double alpha,
double* x, int64_t n, cudaStream_t stream) {
147 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
149 n, [&](int64_t i,
int length) { EIGEN_NPP_CHECK(nppsDivC_64f_I_Ctx(alpha, x + i, length, npp_ctx)); });
Namespace containing all symbols from the Eigen library.