Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
DeviceScalarOps.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// Device-resident scalar and element-wise operations built on the NPP signal
12// primitives (CUDA::npps), which keeps the module header-only: no custom CUDA
13// kernels to compile.
14
15#ifndef EIGEN_GPU_DEVICE_SCALAR_OPS_H
16#define EIGEN_GPU_DEVICE_SCALAR_OPS_H
17
18#include <cuda_runtime.h>
19#include <npps_arithmetic_and_logical_operations.h>
20
21#include <algorithm>
22
23#include "./GpuSupport.h"
24
25namespace Eigen {
26namespace gpu {
27namespace internal {
28
29// NPP statuses below NPP_SUCCESS are errors; positive ones are warnings.
30#define EIGEN_NPP_CHECK(expr) \
31 do { \
32 const NppStatus _s = (expr); \
33 if (_s < NPP_SUCCESS) \
34 ::Eigen::gpu::internal::gpu_check_failed_code("NPP", static_cast<int>(_s), #expr, __FILE__, __LINE__); \
35 } while (0)
36
37// NPP requires nCudaDeviceId and the device attributes to match the device that
38// owns `stream`. Re-querying per call is cheap next to the NPP launch itself and
39// keeps multi-device and borrowed-stream callers correct.
40inline NppStreamContext make_npp_stream_ctx(cudaStream_t stream) {
41 NppStreamContext ctx = {};
42 ctx.hStream = stream;
43#if CUDART_VERSION >= 12080
44 // cudaStreamGetDevice reports the stream's owning device irrespective of the
45 // calling thread's current device.
46 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamGetDevice(stream, &ctx.nCudaDeviceId));
47#else
48 // Without cudaStreamGetDevice (pre-CUDA 12.8), a caller borrowing a stream from
49 // another device must cudaSetDevice() first.
50 EIGEN_CUDA_RUNTIME_CHECK(cudaGetDevice(&ctx.nCudaDeviceId));
51#endif
52 EIGEN_CUDA_RUNTIME_CHECK(cudaDeviceGetAttribute(&ctx.nCudaDevAttrComputeCapabilityMajor,
53 cudaDevAttrComputeCapabilityMajor, ctx.nCudaDeviceId));
54 EIGEN_CUDA_RUNTIME_CHECK(cudaDeviceGetAttribute(&ctx.nCudaDevAttrComputeCapabilityMinor,
55 cudaDevAttrComputeCapabilityMinor, ctx.nCudaDeviceId));
56 EIGEN_CUDA_RUNTIME_CHECK(
57 cudaDeviceGetAttribute(&ctx.nMultiProcessorCount, cudaDevAttrMultiProcessorCount, ctx.nCudaDeviceId));
58 EIGEN_CUDA_RUNTIME_CHECK(cudaDeviceGetAttribute(&ctx.nMaxThreadsPerMultiProcessor,
59 cudaDevAttrMaxThreadsPerMultiProcessor, ctx.nCudaDeviceId));
60 EIGEN_CUDA_RUNTIME_CHECK(
61 cudaDeviceGetAttribute(&ctx.nMaxThreadsPerBlock, cudaDevAttrMaxThreadsPerBlock, ctx.nCudaDeviceId));
62 int shared_mem_per_block = 0;
63 EIGEN_CUDA_RUNTIME_CHECK(
64 cudaDeviceGetAttribute(&shared_mem_per_block, cudaDevAttrMaxSharedMemoryPerBlock, ctx.nCudaDeviceId));
65 ctx.nSharedMemPerBlock = static_cast<size_t>(shared_mem_per_block);
66 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamGetFlags(stream, &ctx.nStreamFlags));
67 return ctx;
68}
69
70// c = a / b. The operands are swapped because NPP computes
71// pDst[i] = pSrc2[i] / pSrc1[i].
72inline void device_scalar_div(const float* a, const float* b, float* c, cudaStream_t stream) {
73 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
74 EIGEN_NPP_CHECK(nppsDiv_32f_Ctx(b, a, c, 1, npp_ctx));
75}
76
77inline void device_scalar_div(const double* a, const double* b, double* c, cudaStream_t stream) {
78 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
79 EIGEN_NPP_CHECK(nppsDiv_64f_Ctx(b, a, c, 1, npp_ctx));
80}
81
82// a = sqrt(a), in place, with a context filled in before any capture began.
83inline void device_scalar_sqrt(float* a, const NppStreamContext& npp_ctx) {
84 EIGEN_NPP_CHECK(nppsSqrt_32f_I_Ctx(a, 1, npp_ctx));
85}
86
87inline void device_scalar_sqrt(double* a, const NppStreamContext& npp_ctx) {
88 EIGEN_NPP_CHECK(nppsSqrt_64f_I_Ctx(a, 1, npp_ctx));
89}
90
91// c = -a.
92inline void device_scalar_neg(const float* a, float* c, cudaStream_t stream) {
93 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
94 EIGEN_NPP_CHECK(nppsMulC_32f_Ctx(a, -1.0f, c, 1, npp_ctx));
95}
96
97inline void device_scalar_neg(const double* a, double* c, cudaStream_t stream) {
98 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
99 EIGEN_NPP_CHECK(nppsMulC_64f_Ctx(a, -1.0, c, 1, npp_ctx));
100}
101
102// Calls fn(offset, length) on consecutive pieces of [0, n) of at most 2^30 elements: older NPP releases take the
103// length of a signal as a 32-bit int (newer ones as size_t), and a longer vector would wrap to another length. A
104// power of two keeps every piece's start a multiple of 2^30 elements, so the pieces stay aligned.
105template <typename Fn>
106void for_each_npp_chunk(int64_t n, Fn fn) {
107 const int64_t max_length = int64_t(1) << 30;
108 for (int64_t offset = 0; offset < n; offset += max_length) {
109 fn(offset, static_cast<int>((std::min)(max_length, n - offset)));
110 }
111}
112
113// c[i] = a[i] * b[i].
114inline void device_cwiseProduct(const float* a, const float* b, float* c, int64_t n, cudaStream_t stream) {
115 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
116 for_each_npp_chunk(
117 n, [&](int64_t i, int length) { EIGEN_NPP_CHECK(nppsMul_32f_Ctx(a + i, b + i, c + i, length, npp_ctx)); });
118}
119
120inline void device_cwiseProduct(const double* a, const double* b, double* c, int64_t n, cudaStream_t stream) {
121 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
122 for_each_npp_chunk(
123 n, [&](int64_t i, int length) { EIGEN_NPP_CHECK(nppsMul_64f_Ctx(a + i, b + i, c + i, length, npp_ctx)); });
124}
125
126// c[i] = a[i] / b[i], with operands swapped as in device_scalar_div.
127inline void device_cwiseQuotient(const float* a, const float* b, float* c, int64_t n, cudaStream_t stream) {
128 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
129 for_each_npp_chunk(
130 n, [&](int64_t i, int length) { EIGEN_NPP_CHECK(nppsDiv_32f_Ctx(b + i, a + i, c + i, length, npp_ctx)); });
131}
132
133inline void device_cwiseQuotient(const double* a, const double* b, double* c, int64_t n, cudaStream_t stream) {
134 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
135 for_each_npp_chunk(
136 n, [&](int64_t i, int length) { EIGEN_NPP_CHECK(nppsDiv_64f_Ctx(b + i, a + i, c + i, length, npp_ctx)); });
137}
138
139// x[i] /= alpha, a true division (NPP divide-by-constant, in place).
140inline void device_divC(float alpha, float* x, int64_t n, cudaStream_t stream) {
141 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
142 for_each_npp_chunk(
143 n, [&](int64_t i, int length) { EIGEN_NPP_CHECK(nppsDivC_32f_I_Ctx(alpha, x + i, length, npp_ctx)); });
144}
145
146inline void device_divC(double alpha, double* x, int64_t n, cudaStream_t stream) {
147 NppStreamContext npp_ctx = make_npp_stream_ctx(stream);
148 for_each_npp_chunk(
149 n, [&](int64_t i, int length) { EIGEN_NPP_CHECK(nppsDivC_64f_I_Ctx(alpha, x + i, length, npp_ctx)); });
150}
151
152} // namespace internal
153} // namespace gpu
154} // namespace Eigen
155
156#endif // EIGEN_GPU_DEVICE_SCALAR_OPS_H
Namespace containing all symbols from the Eigen library.