Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
DeviceDispatch.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2026 Rasmus Munk Larsen <rmlarsen@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11// Dispatch functions mapping DeviceMatrix expressions to NVIDIA library calls,
12// plus the DeviceMatrix members that need a complete gpu::Context. The
13// expression argument selects the dispatch() overload.
14
15#ifndef EIGEN_GPU_DEVICE_DISPATCH_H
16#define EIGEN_GPU_DEVICE_DISPATCH_H
17
18// IWYU pragma: private
19#include "./InternalHeaderCheck.h"
20
21#include <cstdint>
22
23#include "./DeviceExpr.h"
24#include "./DeviceBlasExpr.h"
25#include "./DeviceSolverExpr.h"
26#include "./GpuContext.h"
27#include "./CuSolverSupport.h"
28
29namespace Eigen {
30namespace gpu {
31namespace internal {
32template <typename Scalar>
33bool aliases_device_memory(const DeviceMatrix<Scalar>& a, const DeviceMatrix<Scalar>& b) {
34 return a.data() != nullptr && a.data() == b.data();
35}
36
37template <typename Lhs, typename Rhs>
38void dispatch(Context& ctx, DeviceMatrix<scalar_type_t<Lhs>>& dst, const GemmExpr<Lhs, Rhs>& expr,
39 scalar_type_t<Lhs> beta_val, scalar_type_t<Lhs> alpha_scale = scalar_type_t<Lhs>(1)) {
40 using Scalar = scalar_type_t<Lhs>;
41 using traits_lhs = device_expr_traits<Lhs>;
42 using traits_rhs = device_expr_traits<Rhs>;
43
44 const DeviceMatrix<Scalar>& A = traits_lhs::matrix(expr.lhs());
45 const DeviceMatrix<Scalar>& B = traits_rhs::matrix(expr.rhs());
46
47 // cuBLAS leaves C aliasing A or B undefined.
48 eigen_assert(!aliases_device_memory(dst, A) && "GEMM: output aliases left operand (use a temporary)");
49 eigen_assert(!aliases_device_memory(dst, B) && "GEMM: output aliases right operand (use a temporary)");
50
51 constexpr cublasOperation_t transA = to_cublas_op(traits_lhs::op);
52 constexpr cublasOperation_t transB = to_cublas_op(traits_rhs::op);
53
54 const int64_t m = (traits_lhs::op == GpuOp::NoTrans) ? A.rows() : A.cols();
55 const int64_t k = (traits_lhs::op == GpuOp::NoTrans) ? A.cols() : A.rows();
56 const int64_t n = (traits_rhs::op == GpuOp::NoTrans) ? B.cols() : B.rows();
57 const int64_t rhs_k = (traits_rhs::op == GpuOp::NoTrans) ? B.rows() : B.cols();
58
59 eigen_assert(k == rhs_k && "DeviceMatrix GEMM dimension mismatch");
60 EIGEN_UNUSED_VARIABLE(rhs_k);
61
62 const int64_t lda = A.rows();
63 const int64_t ldb = B.rows();
64
65 if (!dst.empty()) {
66 dst.waitReady(ctx.stream());
67 }
68
69 const bool resized = dst.empty() || dst.rows() != m || dst.cols() != n;
70 if (resized) {
71 dst.resize(m, n);
72 }
73
74 // cuBLAS rejects ld = 0 (ld >= max(1, rows)) and the cublasLt heuristic
75 // faults on n == 0, so empty products stop here. For k == 0, C = beta * C
76 // with beta in {0, 1}; a resized C starts from zero.
77 if (m == 0 || n == 0 || k == 0) {
78 if ((resized || beta_val == Scalar(0)) && dst.sizeInBytes() > 0) {
79 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream()));
80 dst.recordReady(ctx.stream());
81 }
82 return;
83 }
84 const int64_t ldc = dst.rows();
85
86 Scalar alpha_local = alpha_scale * traits_lhs::alpha(expr.lhs()) * traits_rhs::alpha(expr.rhs());
87
88 A.waitReady(ctx.stream());
89 B.waitReady(ctx.stream());
90
91 if (resized && beta_val != Scalar(0) && dst.sizeInBytes() > 0) {
92 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream()));
93 }
94
95 cublaslt_gemm(ctx.cublasLtHandle(), ctx.cublasHandle(), transA, transB, m, n, k, &alpha_local, A.data(), lda,
96 B.data(), ldb, &beta_val, dst.data(), ldc, ctx.gemmWorkspace(), ctx.gemmPlanCache(),
97 ctx.cublasLtMaxWorkspaceBytes(), ctx.stream());
98
99 dst.recordReady(ctx.stream());
100}
101
102// Debug-build status check shared by the one-shot solver dispatches: syncs
103// the stream and asserts on the two info words (factorize, solve). Release
104// builds skip both the check and the sync — one-shot expressions are then
105// fully async with no failure detection; use gpu::LLT / gpu::LU + info()
106// when failures must be detected.
107inline void oneshot_check_info(Context& ctx, OneShotSolverScratch& scratch, const char* what) {
108#ifdef EIGEN_NO_DEBUG
109 EIGEN_UNUSED_VARIABLE(ctx);
110 EIGEN_UNUSED_VARIABLE(scratch);
111 EIGEN_UNUSED_VARIABLE(what);
112#else
113 int* info_words = static_cast<int*>(scratch.h_info.get());
114 EIGEN_CUDA_RUNTIME_CHECK(
115 cudaMemcpyAsync(info_words, scratch.d_info.get(), kOneShotInfoBytes, cudaMemcpyDeviceToHost, ctx.stream()));
116 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(ctx.stream()));
117 eigen_assert(info_words[0] == 0 && "cuSOLVER one-shot factorization failed" && what);
118 eigen_assert(info_words[1] == 0 && "cuSOLVER one-shot solve failed" && what);
119 EIGEN_UNUSED_VARIABLE(what);
120#endif
121}
122
123template <typename Scalar, int UpLo>
124void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst, const LltSolveExpr<Scalar, UpLo>& expr) {
125 const DeviceMatrix<Scalar>& A = expr.matrix();
126 const DeviceMatrix<Scalar>& B = expr.rhs();
127
128 eigen_assert(A.rows() == A.cols() && "LLT requires a square matrix");
129 eigen_assert(B.rows() == A.rows() && "LLT solve: RHS rows must match matrix size");
130
131 if (A.rows() == 0 || B.cols() == 0) {
132 if (!dst.empty()) dst.waitReady(ctx.stream());
133 dst.resize(A.rows(), B.cols());
134 return;
135 }
136
137 A.waitReady(ctx.stream());
138 B.waitReady(ctx.stream());
139 if (!dst.empty()) dst.waitReady(ctx.stream());
140
141 // thread_local: must outlive the async kernels (no end-of-call sync), and
142 // only TUs that instantiate the one-shot path pull in cuSOLVER symbols.
143 static thread_local CusolverParams params;
144 constexpr cublasFillMode_t uplo = cusolver_fill_mode<UpLo>::value;
145 const int64_t n = static_cast<int64_t>(A.rows());
146 constexpr cudaDataType_t dtype = cuda_data_type<Scalar>::value;
147 OneShotSolverScratch& scratch = ctx.oneshotSolverScratch();
148 {
149 const size_t mat_bytes = A.sizeInBytes();
150 // Context-owned grow-only scratch: no per-call allocation, no end-of-call sync.
151 ensure_sized(scratch.d_factor, mat_bytes);
152 EIGEN_CUDA_RUNTIME_CHECK(
153 cudaMemcpyAsync(scratch.d_factor.get(), A.data(), mat_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
154 }
155 const int64_t lda = static_cast<int64_t>(A.rows());
156 size_t dev_ws = 0;
157 size_t host_ws = 0;
158 EIGEN_CUSOLVER_CHECK(cusolverDnXpotrf_bufferSize(ctx.cusolverHandle(), params.p, uplo, n, dtype,
159 scratch.d_factor.get(), lda, dtype, &dev_ws, &host_ws));
160 ensure_sized(scratch.d_workspace, dev_ws);
161 if (scratch.h_workspace.size() < host_ws) scratch.h_workspace.resize(host_ws);
162 // Two info slots (potrf, potrs) so both kernels queue back-to-back. If potrf
163 // fails, potrs runs on garbage but the debug check catches both at once.
164 int* d_info_potrf = static_cast<int*>(scratch.d_info.get());
165 int* d_info_potrs = d_info_potrf + 1;
166 EIGEN_CUSOLVER_CHECK(cusolverDnXpotrf(ctx.cusolverHandle(), params.p, uplo, n, dtype, scratch.d_factor.get(), lda,
167 dtype, scratch.d_workspace.get(), dev_ws,
168 host_ws > 0 ? scratch.h_workspace.data() : nullptr, host_ws, d_info_potrf));
169
170 dst.resize(n, B.cols());
171 const size_t rhs_bytes = B.sizeInBytes();
172 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst.data(), B.data(), rhs_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
173
174 const int64_t nrhs = static_cast<int64_t>(B.cols());
175 EIGEN_CUSOLVER_CHECK(cusolverDnXpotrs(ctx.cusolverHandle(), params.p, uplo, n, nrhs, dtype, scratch.d_factor.get(),
176 lda, dtype, dst.data(), static_cast<int64_t>(dst.rows()), d_info_potrs));
177 oneshot_check_info(ctx, scratch, "llt");
178 dst.recordReady(ctx.stream());
179}
180
181template <typename Scalar>
182void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst, const LuSolveExpr<Scalar>& expr) {
183 const DeviceMatrix<Scalar>& A = expr.matrix();
184 const DeviceMatrix<Scalar>& B = expr.rhs();
185
186 eigen_assert(A.rows() == A.cols() && "LU requires a square matrix");
187 eigen_assert(B.rows() == A.rows() && "LU solve: RHS rows must match matrix size");
188
189 if (A.rows() == 0 || B.cols() == 0) {
190 if (!dst.empty()) dst.waitReady(ctx.stream());
191 dst.resize(A.rows(), B.cols());
192 return;
193 }
194
195 A.waitReady(ctx.stream());
196 B.waitReady(ctx.stream());
197 if (!dst.empty()) dst.waitReady(ctx.stream());
198
199 // thread_local: must outlive the async kernels (no end-of-call sync), and
200 // only TUs that instantiate the one-shot path pull in cuSOLVER symbols.
201 static thread_local CusolverParams params;
202 const int64_t n = static_cast<int64_t>(A.rows());
203 constexpr cudaDataType_t dtype = cuda_data_type<Scalar>::value;
204 OneShotSolverScratch& scratch = ctx.oneshotSolverScratch();
205 {
206 const size_t mat_bytes = A.sizeInBytes();
207 // Context-owned grow-only scratch: no per-call allocation, no end-of-call sync.
208 ensure_sized(scratch.d_factor, mat_bytes);
209 EIGEN_CUDA_RUNTIME_CHECK(
210 cudaMemcpyAsync(scratch.d_factor.get(), A.data(), mat_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
211 }
212 ensure_sized(scratch.d_ipiv, static_cast<size_t>(n) * sizeof(int64_t));
213 const int64_t lda = static_cast<int64_t>(A.rows());
214 size_t dev_ws = 0;
215 size_t host_ws = 0;
216 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrf_bufferSize(ctx.cusolverHandle(), params.p, n, n, dtype, scratch.d_factor.get(),
217 lda, dtype, &dev_ws, &host_ws));
218 ensure_sized(scratch.d_workspace, dev_ws);
219 if (scratch.h_workspace.size() < host_ws) scratch.h_workspace.resize(host_ws);
220 int* d_info_getrf = static_cast<int*>(scratch.d_info.get());
221 int* d_info_getrs = d_info_getrf + 1;
222 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrf(ctx.cusolverHandle(), params.p, n, n, dtype, scratch.d_factor.get(), lda,
223 static_cast<int64_t*>(scratch.d_ipiv.get()), dtype, scratch.d_workspace.get(),
224 dev_ws, host_ws > 0 ? scratch.h_workspace.data() : nullptr, host_ws,
225 d_info_getrf));
226
227 dst.resize(n, B.cols());
228 const size_t rhs_bytes = B.sizeInBytes();
229 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst.data(), B.data(), rhs_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
230
231 const int64_t nrhs = static_cast<int64_t>(B.cols());
232 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrs(ctx.cusolverHandle(), params.p, CUBLAS_OP_N, n, nrhs, dtype,
233 scratch.d_factor.get(), lda, static_cast<const int64_t*>(scratch.d_ipiv.get()),
234 dtype, dst.data(), static_cast<int64_t>(dst.rows()), d_info_getrs));
235 oneshot_check_info(ctx, scratch, "lu");
236 dst.recordReady(ctx.stream());
237}
238
239template <typename Scalar, int UpLo>
240void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst, const TrsmExpr<Scalar, UpLo>& expr) {
241 const DeviceMatrix<Scalar>& A = expr.matrix();
242 const DeviceMatrix<Scalar>& B = expr.rhs();
243
244 eigen_assert(A.rows() == A.cols() && "TRSM requires a square triangular matrix");
245 eigen_assert(B.rows() == A.rows() && "TRSM: RHS rows must match matrix size");
246
247 const int64_t n = A.rows();
248 const int64_t nrhs = B.cols();
249
250 if (n == 0 || nrhs == 0) {
251 if (!dst.empty()) dst.waitReady(ctx.stream());
252 dst.resize(n, B.cols());
253 return;
254 }
255
256 A.waitReady(ctx.stream());
257 B.waitReady(ctx.stream());
258 eigen_assert(!aliases_device_memory(dst, A) && "DeviceMatrix TRSM destination aliases triangular operand");
259 eigen_assert(!aliases_device_memory(dst, B) && "DeviceMatrix TRSM destination aliases RHS operand");
260 if (!dst.empty()) dst.waitReady(ctx.stream());
261
262 dst.resize(n, B.cols());
263 const size_t rhs_bytes = static_cast<size_t>(dst.rows()) * static_cast<size_t>(nrhs) * sizeof(Scalar);
264 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst.data(), B.data(), rhs_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
265
266 constexpr cublasFillMode_t uplo = (UpLo == Lower) ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER;
267 Scalar alpha(1);
268
269 EIGEN_CUBLAS_CHECK(cublasXtrsm(ctx.cublasHandle(), CUBLAS_SIDE_LEFT, uplo, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, n, nrhs,
270 &alpha, A.data(), A.rows(), dst.data(), dst.rows()));
271
272 dst.recordReady(ctx.stream());
273}
274
275template <typename Scalar, int UpLo>
276void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst, const SymmExpr<Scalar, UpLo>& expr) {
277 const DeviceMatrix<Scalar>& A = expr.matrix();
278 const DeviceMatrix<Scalar>& B = expr.rhs();
279
280 eigen_assert(A.rows() == A.cols() && "SYMM requires a square matrix");
281 eigen_assert(B.rows() == A.rows() && "SYMM: RHS rows must match matrix size");
282
283 const int64_t m = A.rows();
284 const int64_t n = B.cols();
285
286 if (m == 0 || n == 0) {
287 if (!dst.empty()) dst.waitReady(ctx.stream());
288 dst.resize(m, B.cols());
289 return;
290 }
291
292 A.waitReady(ctx.stream());
293 B.waitReady(ctx.stream());
294 eigen_assert(!aliases_device_memory(dst, A) && "DeviceMatrix SYMM destination aliases self-adjoint operand");
295 eigen_assert(!aliases_device_memory(dst, B) && "DeviceMatrix SYMM destination aliases RHS operand");
296 if (!dst.empty()) dst.waitReady(ctx.stream());
297
298 dst.resize(m, n);
299
300 constexpr cublasFillMode_t uplo = (UpLo == Lower) ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER;
301 const Scalar one(1), zero(0);
302
303 EIGEN_CUBLAS_CHECK(cublasXsymm(ctx.cublasHandle(), CUBLAS_SIDE_LEFT, uplo, m, n, &one, A.data(), A.rows(), B.data(),
304 B.rows(), &zero, dst.data(), dst.rows()));
305
306 dst.recordReady(ctx.stream());
307}
308
309template <typename Scalar, int UpLo>
310void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst, const SyrkExpr<Scalar, UpLo>& expr,
311 typename NumTraits<Scalar>::Real alpha_val, typename NumTraits<Scalar>::Real beta_val) {
312 using RealScalar = typename NumTraits<Scalar>::Real;
313 const DeviceMatrix<Scalar>& A = expr.matrix();
314
315 const int64_t n = A.rows();
316 const int64_t k = A.cols();
317
318 if (n == 0) {
319 if (!dst.empty()) dst.waitReady(ctx.stream());
320 dst.resize(0, 0);
321 return;
322 }
323
324 A.waitReady(ctx.stream());
325 eigen_assert(!aliases_device_memory(dst, A) && "DeviceMatrix SYRK destination aliases input operand");
326 if (!dst.empty()) dst.waitReady(ctx.stream());
327
328 if (dst.empty() || dst.rows() != n || dst.cols() != n) {
329 dst.resize(n, n);
330 if (beta_val != RealScalar(0)) {
331 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream()));
332 }
333 }
334
335 constexpr cublasFillMode_t uplo = (UpLo == Lower) ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER;
336
337 EIGEN_CUBLAS_CHECK(cublasXsyrk(ctx.cublasHandle(), uplo, CUBLAS_OP_N, n, k, &alpha_val, A.data(), A.rows(), &beta_val,
338 dst.data(), dst.rows()));
339
340 dst.recordReady(ctx.stream());
341}
342
343// DeviceAddExpr → cublasXgeam: dst = alpha * A + beta * B. Safe when dst
344// aliases A and/or B (geam supports in-place operation with equal leading
345// dimensions, which always holds here since DeviceMatrix is fully dense).
346
347template <typename Scalar>
348void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst, const DeviceAddExpr<Scalar>& expr) {
349 const DeviceMatrix<Scalar>& A = expr.A();
350 const DeviceMatrix<Scalar>& B = expr.B();
351 eigen_assert(A.rows() == B.rows() && A.cols() == B.cols());
352 const int64_t m = A.rows();
353 const int64_t n = A.cols();
354 // Wait on dst before resize — resize may free the old buffer while another
355 // stream is still reading it.
356 if (!dst.empty()) dst.waitReady(ctx.stream());
357 dst.resize(A.rows(), A.cols());
358 if (m > 0 && n > 0) {
359 A.waitReady(ctx.stream());
360 B.waitReady(ctx.stream());
361 const Scalar alpha_val = expr.alpha(), beta_val = expr.beta();
362 EIGEN_CUBLAS_CHECK(cublasXgeam(ctx.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N, m, n, &alpha_val, A.data(), m,
363 &beta_val, B.data(), m, dst.data(), m));
364 dst.recordReady(ctx.stream());
365 }
366}
367} // namespace internal
368
369template <typename Scalar_>
370class Assignment {
371 public:
372 using Scalar = Scalar_;
373
374 Assignment(DeviceMatrix<Scalar>& dst, Context& ctx) : dst_(dst), ctx_(ctx) {}
375
376 template <typename Lhs, typename Rhs>
377 DeviceMatrix<Scalar>& operator=(const GemmExpr<Lhs, Rhs>& expr) {
378 internal::dispatch(ctx_, dst_, expr, Scalar(0));
379 return dst_;
380 }
381
382 template <typename Lhs, typename Rhs>
383 DeviceMatrix<Scalar>& operator+=(const GemmExpr<Lhs, Rhs>& expr) {
384 internal::dispatch(ctx_, dst_, expr, Scalar(1));
385 return dst_;
386 }
387
388 template <typename Lhs, typename Rhs>
389 DeviceMatrix<Scalar>& operator-=(const GemmExpr<Lhs, Rhs>& expr) {
390 internal::dispatch(ctx_, dst_, expr, Scalar(1), Scalar(-1));
391 return dst_;
392 }
393
394 template <int UpLo>
395 DeviceMatrix<Scalar>& operator=(const LltSolveExpr<Scalar, UpLo>& expr) {
396 internal::dispatch(ctx_, dst_, expr);
397 return dst_;
398 }
399
400 DeviceMatrix<Scalar>& operator=(const LuSolveExpr<Scalar>& expr) {
401 internal::dispatch(ctx_, dst_, expr);
402 return dst_;
403 }
404
405 template <int UpLo>
406 DeviceMatrix<Scalar>& operator=(const TrsmExpr<Scalar, UpLo>& expr) {
407 internal::dispatch(ctx_, dst_, expr);
408 return dst_;
409 }
410
411 template <int UpLo>
412 DeviceMatrix<Scalar>& operator=(const SymmExpr<Scalar, UpLo>& expr) {
413 internal::dispatch(ctx_, dst_, expr);
414 return dst_;
415 }
416
417 DeviceMatrix<Scalar>& operator=(const DeviceAddExpr<Scalar>& expr) {
418 internal::dispatch(ctx_, dst_, expr);
419 return dst_;
420 }
421
422 DeviceMatrix<Scalar>& operator=(const Scaled<DeviceMatrix<Scalar>>& expr) {
423 // geam with beta == 0: cuBLAS documents B as unread, so pass A twice.
424 internal::dispatch(ctx_, dst_, DeviceAddExpr<Scalar>(expr.scalar(), expr.inner(), Scalar(0), expr.inner()));
425 return dst_;
426 }
427
428 template <typename Expr>
429 DeviceMatrix<Scalar>& operator=(const Expr&) {
430 static_assert(sizeof(Expr) == 0,
431 "DeviceMatrix expression not supported: no cuBLAS/cuSOLVER mapping. "
432 "Supported: GEMM (A*B), geam (A + alpha*B, alpha*A), "
433 "TRSM (.triangularView().solve()), SYMM (.selfadjointView()*B), "
434 "LLT (.llt().solve()), LU (.lu().solve()).");
435 return dst_;
436 }
437
438 private:
439 DeviceMatrix<Scalar>& dst_;
440 Context& ctx_;
441};
442
443// The definitions below call Context::threadLocal(), so they cannot live in
444// DeviceMatrix.h, where Context is still incomplete.
445
446template <typename Scalar_>
447template <typename Lhs, typename Rhs>
448DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const GemmExpr<Lhs, Rhs>& expr) {
450 return *this;
452
453template <typename Scalar_>
454template <typename Lhs, typename Rhs>
455DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator+=(const GemmExpr<Lhs, Rhs>& expr) {
457 return *this;
458}
459
460template <typename Scalar_>
461template <typename Lhs, typename Rhs>
463 device(Context::threadLocal()) -= expr;
464 return *this;
465}
467template <typename Scalar_>
468template <int UpLo>
469DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const LltSolveExpr<Scalar_, UpLo>& expr) {
471 return *this;
473
474template <typename Scalar_>
475DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const LuSolveExpr<Scalar_>& expr) {
477 return *this;
478}
480template <typename Scalar_>
481template <int UpLo>
482DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const TrsmExpr<Scalar_, UpLo>& expr) {
484 return *this;
485}
486
487template <typename Scalar_>
488template <int UpLo>
489DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const SymmExpr<Scalar_, UpLo>& expr) {
491 return *this;
492}
493
494template <typename Scalar_>
495DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const Scaled<DeviceMatrix>& expr) {
497 return *this;
498}
499
500// Enable copy-initialization straight from an expression, e.g.
501// DeviceMatrix<double> d_C = d_A * d_B;
502// Each default-constructs and delegates to the matching operator=.
503
504template <typename Scalar_>
505template <typename Lhs, typename Rhs>
507 *this = expr;
508}
509
510template <typename Scalar_>
512 *this = expr;
514
515template <typename Scalar_>
517 *this = expr;
518}
520template <typename Scalar_>
521template <int UpLo>
523 *this = expr;
524}
525
526template <typename Scalar_>
528 *this = expr;
530
531template <typename Scalar_>
532template <int UpLo>
534 *this = expr;
535}
536
537template <typename Scalar_>
538template <int UpLo>
540 *this = expr;
541}
542
543template <typename Scalar_, int UpLo_>
546 RealScalar beta = matrix().empty() ? RealScalar(0) : RealScalar(1);
547 internal::dispatch(Context::threadLocal(), matrix(), expr, alpha, beta);
549
550namespace internal {
551// Runs `f` with the handle temporarily in CUBLAS_POINTER_MODE_DEVICE, restoring
552// the caller's mode afterwards, also when a failure in `f` throws: the handle
553// would otherwise read host scalars of later calls as device pointers.
554template <typename F>
555void with_device_pointer_mode(cublasHandle_t h, F&& f) {
556 struct RestoreOnThrow {
557 cublasHandle_t handle;
558 cublasPointerMode_t mode;
559 bool armed;
560 ~RestoreOnThrow() {
561 if (armed) (void)cublasSetPointerMode(handle, mode); // unchecked: may run during unwinding
562 }
563 };
564 cublasPointerMode_t prev;
565 EIGEN_CUBLAS_CHECK(cublasGetPointerMode(h, &prev));
566 EIGEN_CUBLAS_CHECK(cublasSetPointerMode(h, CUBLAS_POINTER_MODE_DEVICE));
567 RestoreOnThrow restore{h, prev, true};
568 f();
569 restore.armed = false;
570 EIGEN_CUBLAS_CHECK(cublasSetPointerMode(h, prev));
571}
572} // namespace internal
573
574// The reductions below (dot, norm, squaredNorm) run under
575// CUBLAS_POINTER_MODE_DEVICE: the scalar result is written to device memory and
576// stays there until DeviceScalar's conversion to Scalar syncs and reads it.
577
578namespace internal {
579inline int64_t blas1_size(Index rows, Index cols) { return static_cast<int64_t>(rows) * static_cast<int64_t>(cols); }
580} // namespace internal
581
582template <typename Scalar_>
584 const int64_t n = internal::blas1_size(rows_, cols_);
585 eigen_assert(n == internal::blas1_size(other.rows_, other.cols_));
586 eigen_assert(result.stream() == ctx.stream() && "DeviceMatrix::dot: result must live on ctx's stream");
587 if (n > 0) {
588 waitReady(ctx.stream());
589 other.waitReady(ctx.stream());
590 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
591 EIGEN_CUBLAS_CHECK(
592 internal::cublasXdot(ctx.cublasHandle(), n, data_.get(), 1, other.data_.get(), 1, result.devicePtr()));
593 });
594 } else {
595 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0, sizeof(Scalar), ctx.stream()));
596 }
597}
598
599template <typename Scalar_>
601 const DeviceMatrix& other) const {
602 // Allocated uninitialized: the reduction overwrites the slot.
603 DeviceScalar<Scalar> result(ctx.stream());
604 dot(ctx, other, result);
605 return result;
606}
607
608template <typename Scalar_>
610 const int64_t n = internal::blas1_size(rows_, cols_);
611 eigen_assert(result.stream() == ctx.stream() && "DeviceMatrix::squaredNorm: result must live on ctx's stream");
612 if (n > 0) {
613 // ||x||^2 = x^T x over the 2n real and imaginary parts of a complex x
614 // (std::complex<T> has the layout of T[2]): real, so no host sync. Unscaled,
615 // and ~4.5x faster than nrm2^2; stableNorm() is the scaled form.
616 const int64_t reals = NumTraits<Scalar>::IsComplex ? 2 * n : n;
617 const RealScalar* x = reinterpret_cast<const RealScalar*>(data_.get());
618 waitReady(ctx.stream());
619 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
620 EIGEN_CUBLAS_CHECK(internal::cublasXdot(ctx.cublasHandle(), reals, x, 1, x, 1, result.devicePtr()));
621 });
622 } else {
623 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0, sizeof(RealScalar), ctx.stream()));
624 }
625}
626
627template <typename Scalar_>
629 DeviceScalar<RealScalar> result(ctx.stream());
630 squaredNorm(ctx, result);
631 return result;
632}
633
634template <typename Scalar_>
636 // sqrt of the dot product: a dot and a one-element NPP sqrt cost less than
637 // cuBLAS nrm2's scaled accumulation, see stableNorm().
638 squaredNorm(ctx, result);
639 internal::device_scalar_sqrt(result.devicePtr(), ctx.nppStreamContext());
640}
641
642template <typename Scalar_>
644 DeviceScalar<RealScalar> result(ctx.stream());
645 norm(ctx, result);
646 return result;
647}
648
649template <typename Scalar_>
651 const int64_t n = internal::blas1_size(rows_, cols_);
652 eigen_assert(result.stream() == ctx.stream() && "DeviceMatrix::stableNorm: result must live on ctx's stream");
653 if (n > 0) {
654 waitReady(ctx.stream());
655 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
656 EIGEN_CUBLAS_CHECK(internal::cublasXnrm2(ctx.cublasHandle(), n, data_.get(), 1, result.devicePtr()));
657 });
658 } else {
659 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0, sizeof(RealScalar), ctx.stream()));
660 }
661}
662
663template <typename Scalar_>
664void DeviceMatrix<Scalar_>::setZero(cudaStream_t stream) {
665 if (sizeInBytes() > 0) {
666 waitReady(stream);
667 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(data_.get(), 0, sizeInBytes(), stream));
668 recordReady(stream);
669 }
670}
671
672template <typename Scalar_>
674 setZero(ctx.stream());
675}
676
677template <typename Scalar_>
678void DeviceMatrix<Scalar_>::addScaled(Context& ctx, Scalar alpha, const DeviceMatrix& x) {
679 const int64_t n = internal::blas1_size(rows_, cols_);
680 eigen_assert(n == internal::blas1_size(x.rows_, x.cols_));
681 if (n > 0) {
682 waitReady(ctx.stream());
683 x.waitReady(ctx.stream());
684 EIGEN_CUBLAS_CHECK(internal::cublasXaxpy(ctx.cublasHandle(), n, &alpha, x.data_.get(), 1, data_.get(), 1));
685 recordReady(ctx.stream());
686 }
687}
688
689template <typename Scalar_>
690void DeviceMatrix<Scalar_>::scale(Context& ctx, Scalar alpha) {
691 const int64_t n = internal::blas1_size(rows_, cols_);
692 if (n > 0) {
693 waitReady(ctx.stream());
694 EIGEN_CUBLAS_CHECK(internal::cublasXscal(ctx.cublasHandle(), n, &alpha, data_.get(), 1));
695 recordReady(ctx.stream());
696 }
697}
698
699template <typename Scalar_>
701 // Wait on *this before resize — resize may free the old buffer while another
702 // stream is still reading it.
703 if (!empty()) waitReady(ctx.stream());
704 resize(other.rows_, other.cols_);
705 const int64_t n = internal::blas1_size(rows_, cols_);
706 if (n > 0) {
707 other.waitReady(ctx.stream());
708 EIGEN_CUBLAS_CHECK(internal::cublasXcopy(ctx.cublasHandle(), n, other.data_.get(), 1, data_.get(), 1));
709 recordReady(ctx.stream());
710 }
711}
712
713// this += alpha * x (axpy)
714template <typename Scalar_>
715DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator+=(const Scaled<DeviceMatrix>& expr) {
717 return *this;
718}
719
720// this -= alpha * x (axpy with negated alpha)
721template <typename Scalar_>
726
727// this += x (axpy with alpha=1)
728template <typename Scalar_>
729DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator+=(const DeviceMatrix& other) {
730 Scalar one(1);
731 addScaled(Context::threadLocal(), one, other);
732 return *this;
733}
734
735// this -= x (axpy with alpha=-1)
736template <typename Scalar_>
738 Scalar neg_one(-1);
739 addScaled(Context::threadLocal(), neg_one, other);
740 return *this;
741}
742
743// this *= alpha (scal, host pointer)
744template <typename Scalar_>
746 scale(Context::threadLocal(), alpha);
747 return *this;
748}
749
750namespace internal {
751// x[i] /= alpha. Real Scalar: NPP divides in place. Complex Scalar: cuBLAS scal by the
752// host reciprocal, which std::complex computes with scaling, so |alpha| > sqrt(max)
753// does not overflow; one extra rounding per element.
754inline void divide_in_place(Context& ctx, float* x, int64_t n, float alpha) { device_divC(alpha, x, n, ctx.stream()); }
755inline void divide_in_place(Context& ctx, double* x, int64_t n, double alpha) {
756 device_divC(alpha, x, n, ctx.stream());
757}
758template <typename Real>
759void divide_in_place(Context& ctx, std::complex<Real>* x, int64_t n, std::complex<Real> alpha) {
760 const std::complex<Real> inv = std::complex<Real>(1) / alpha;
761 EIGEN_CUBLAS_CHECK(cublasXscal(ctx.cublasHandle(), n, &inv, x, 1));
762}
763} // namespace internal
764
765template <typename Scalar_>
766void DeviceMatrix<Scalar_>::divide(Context& ctx, Scalar alpha) {
767 const int64_t n = internal::blas1_size(rows_, cols_);
768 if (n > 0) {
769 waitReady(ctx.stream());
770 internal::divide_in_place(ctx, data_.get(), n, alpha);
771 recordReady(ctx.stream());
772 }
773}
774
775// this /= alpha
776template <typename Scalar_>
779 return *this;
780}
781
782// Deep copies: device-to-device cuBLAS copy on the thread-local Context.
783template <typename Scalar_>
787
788template <typename Scalar_>
789DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const DeviceMatrix& other) {
790 if (this != &other) copyFrom(Context::threadLocal(), other);
791 return *this;
792}
793
794template <typename Scalar_>
796 DeviceScalar<RealScalar> result(ctx.stream());
797 stableNorm(ctx, result);
798 return result;
799}
800
801template <typename Scalar_>
803 return stableNorm(Context::threadLocal());
804}
805
806// this *= alpha (scal, device pointer — avoids host sync)
807template <typename Scalar_>
809 const int64_t n = internal::blas1_size(rows_, cols_);
810 if (n > 0) {
811 auto& ctx = Context::threadLocal();
812 waitReady(ctx.stream());
813 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
814 EIGEN_CUBLAS_CHECK(internal::cublasXscal(ctx.cublasHandle(), n, alpha.devicePtr(), data_.get(), 1));
815 });
816 recordReady(ctx.stream());
817 }
818 return *this;
819}
820
821// this += DeviceScalar * x (axpy with CUBLAS_POINTER_MODE_DEVICE)
822template <typename Scalar_>
823DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator+=(const DeviceScaledDevice<Scalar_>& expr) {
824 const int64_t n = internal::blas1_size(rows_, cols_);
825 const auto& x = expr.matrix();
826 eigen_assert(n == internal::blas1_size(x.rows_, x.cols_));
827 if (n > 0) {
828 auto& ctx = Context::threadLocal();
829 waitReady(ctx.stream());
830 x.waitReady(ctx.stream());
831 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
832 EIGEN_CUBLAS_CHECK(
833 internal::cublasXaxpy(ctx.cublasHandle(), n, expr.alpha().devicePtr(), x.data_.get(), 1, data_.get(), 1));
834 });
835 recordReady(ctx.stream());
836 }
837 return *this;
838}
839
840// this -= DeviceScalar * x (axpy with negated device scalar)
841template <typename Scalar_>
843 auto neg_alpha = -expr.alpha();
844 DeviceScaledDevice<Scalar_> neg_expr(neg_alpha, expr.matrix());
845 return operator+=(neg_expr);
846}
847
848// this = alpha * A + beta * B (cuBLAS geam)
849template <typename Scalar_>
850DeviceMatrix<Scalar_>& DeviceMatrix<Scalar_>::operator=(const DeviceAddExpr<Scalar_>& expr) {
851 internal::dispatch(Context::threadLocal(), *this, expr);
852 return *this;
853}
854
855// cwiseProduct (allocating).
856template <typename Scalar_>
858 const int64_t n = internal::blas1_size(rows_, cols_);
859 eigen_assert(n == internal::blas1_size(other.rows_, other.cols_));
860 DeviceMatrix result(rows_, cols_);
861 if (n > 0) {
862 waitReady(ctx.stream());
863 other.waitReady(ctx.stream());
864 internal::device_cwiseProduct(data_.get(), other.data_.get(), result.data_.get(), n, ctx.stream());
865 result.recordReady(ctx.stream());
866 }
867 return result;
868}
869
870// In-place cwiseProduct: this = a .* b (reuses this buffer, no allocation).
871template <typename Scalar_>
873 const int64_t n = internal::blas1_size(a.rows_, a.cols_);
874 eigen_assert(n == internal::blas1_size(b.rows_, b.cols_));
875 if (!empty()) waitReady(ctx.stream());
876 resize(a.rows_, a.cols_);
877 if (n > 0) {
878 a.waitReady(ctx.stream());
879 b.waitReady(ctx.stream());
880 internal::device_cwiseProduct(a.data_.get(), b.data_.get(), data_.get(), n, ctx.stream());
881 recordReady(ctx.stream());
882 }
883}
884
885// Convenience overloads using thread-local default Context.
886template <typename Scalar_>
888 return dot(Context::threadLocal(), other);
889}
890
891template <typename Scalar_>
892DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::squaredNorm() const {
893 return squaredNorm(Context::threadLocal());
894}
895
896template <typename Scalar_>
897DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::norm() const {
898 return norm(Context::threadLocal());
899}
900
901template <typename Scalar_>
903 setZero(Context::threadLocal());
904}
905} // namespace gpu
906} // namespace Eigen
907
908#endif // EIGEN_GPU_DEVICE_DISPATCH_H
Unified GPU execution context owning a CUDA stream and library handles.
Definition GpuContext.h:81
const NppStreamContext & nppStreamContext() const
Definition GpuContext.h:139
static Context & threadLocal()
Definition GpuContext.h:121
Linear combination of two device matrices.
Definition DeviceExpr.h:260
RAII wrapper for a dense column-major matrix in GPU device memory.
Definition DeviceMatrix.h:122
DeviceMatrix & operator*=(Scalar alpha)
Definition DeviceDispatch.h:745
void addScaled(Context &ctx, Scalar alpha, const DeviceMatrix &x)
Definition DeviceDispatch.h:678
Assignment< Scalar > device(Context &ctx)
Definition DeviceMatrix.h:388
DeviceMatrix cwiseProduct(Context &ctx, const DeviceMatrix &other) const
Definition DeviceDispatch.h:857
void scale(Context &ctx, Scalar alpha)
Definition DeviceDispatch.h:690
void copyFrom(Context &ctx, const DeviceMatrix &other)
Definition DeviceDispatch.h:700
DeviceScalar< typename NumTraits< Scalar >::Real > squaredNorm(Context &ctx) const
Definition DeviceDispatch.h:628
DeviceMatrix & operator/=(Scalar alpha)
Definition DeviceDispatch.h:777
DeviceScalar< typename NumTraits< Scalar >::Real > stableNorm(Context &ctx) const
Definition DeviceDispatch.h:795
DeviceMatrix & operator-=(const GemmExpr< Lhs, Rhs > &expr)
void divide(Context &ctx, Scalar alpha)
Definition DeviceDispatch.h:766
void recordReady(cudaStream_t stream)
Definition DeviceMatrix.h:365
DeviceScalar< typename NumTraits< Scalar >::Real > norm(Context &ctx) const
Definition DeviceDispatch.h:643
void setZero(Context &ctx)
Definition DeviceDispatch.h:673
DeviceScalar< Scalar > dot(Context &ctx, const DeviceMatrix &other) const
Definition DeviceDispatch.h:600
void resize(Index rows, Index cols)
Definition DeviceMatrix.h:323
void waitReady(cudaStream_t stream) const
Definition DeviceMatrix.h:374
RAII wrapper for a scalar in GPU device memory.
Definition DeviceScalar.h:33
Expression that scales a device matrix by a DeviceScalar.
Definition DeviceExpr.h:231
Expression returned by operator*(lhs_expr, rhs_expr), dispatched to cuBLAS GEMM.
Definition DeviceExpr.h:92
Definition DeviceSolverExpr.h:30
Definition DeviceSolverExpr.h:46
Expression returned by operator*(Scalar, DeviceMatrix/View), carrying the scalar factor.
Definition DeviceExpr.h:77
void rankUpdate(const DeviceMatrix< Scalar > &A, RealScalar alpha=RealScalar(1))
Definition DeviceDispatch.h:544
Definition DeviceBlasExpr.h:96
Definition DeviceBlasExpr.h:122
Definition DeviceBlasExpr.h:45
Namespace containing all symbols from the Eigen library.
Describes GPU device expression types.
Definition DeviceExpr.h:173