15#ifndef EIGEN_GPU_DEVICE_DISPATCH_H
16#define EIGEN_GPU_DEVICE_DISPATCH_H
19#include "./InternalHeaderCheck.h"
23#include "./DeviceExpr.h"
24#include "./DeviceBlasExpr.h"
25#include "./DeviceSolverExpr.h"
26#include "./GpuContext.h"
27#include "./CuSolverSupport.h"
32template <
typename Scalar>
33bool aliases_device_memory(
const DeviceMatrix<Scalar>& a,
const DeviceMatrix<Scalar>& b) {
34 return a.data() !=
nullptr && a.data() == b.data();
37template <
typename Lhs,
typename Rhs>
38void dispatch(Context& ctx, DeviceMatrix<scalar_type_t<Lhs>>& dst,
const GemmExpr<Lhs, Rhs>& expr,
39 scalar_type_t<Lhs> beta_val, scalar_type_t<Lhs> alpha_scale = scalar_type_t<Lhs>(1)) {
40 using Scalar = scalar_type_t<Lhs>;
44 const DeviceMatrix<Scalar>& A = traits_lhs::matrix(expr.lhs());
45 const DeviceMatrix<Scalar>& B = traits_rhs::matrix(expr.rhs());
48 eigen_assert(!aliases_device_memory(dst, A) &&
"GEMM: output aliases left operand (use a temporary)");
49 eigen_assert(!aliases_device_memory(dst, B) &&
"GEMM: output aliases right operand (use a temporary)");
51 constexpr cublasOperation_t transA = to_cublas_op(traits_lhs::op);
52 constexpr cublasOperation_t transB = to_cublas_op(traits_rhs::op);
54 const int64_t m = (traits_lhs::op == GpuOp::NoTrans) ? A.rows() : A.cols();
55 const int64_t k = (traits_lhs::op == GpuOp::NoTrans) ? A.cols() : A.rows();
56 const int64_t n = (traits_rhs::op == GpuOp::NoTrans) ? B.cols() : B.rows();
57 const int64_t rhs_k = (traits_rhs::op == GpuOp::NoTrans) ? B.rows() : B.cols();
59 eigen_assert(k == rhs_k &&
"DeviceMatrix GEMM dimension mismatch");
60 EIGEN_UNUSED_VARIABLE(rhs_k);
62 const int64_t lda = A.rows();
63 const int64_t ldb = B.rows();
66 dst.waitReady(ctx.stream());
69 const bool resized = dst.empty() || dst.rows() != m || dst.cols() != n;
77 if (m == 0 || n == 0 || k == 0) {
78 if ((resized || beta_val == Scalar(0)) && dst.sizeInBytes() > 0) {
79 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream()));
80 dst.recordReady(ctx.stream());
84 const int64_t ldc = dst.rows();
86 Scalar alpha_local = alpha_scale * traits_lhs::alpha(expr.lhs()) * traits_rhs::alpha(expr.rhs());
88 A.waitReady(ctx.stream());
89 B.waitReady(ctx.stream());
91 if (resized && beta_val != Scalar(0) && dst.sizeInBytes() > 0) {
92 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream()));
95 cublaslt_gemm(ctx.cublasLtHandle(), ctx.cublasHandle(), transA, transB, m, n, k, &alpha_local, A.data(), lda,
96 B.data(), ldb, &beta_val, dst.data(), ldc, ctx.gemmWorkspace(), ctx.gemmPlanCache(),
97 ctx.cublasLtMaxWorkspaceBytes(), ctx.stream());
99 dst.recordReady(ctx.stream());
107inline void oneshot_check_info(Context& ctx, OneShotSolverScratch& scratch,
const char* what) {
109 EIGEN_UNUSED_VARIABLE(ctx);
110 EIGEN_UNUSED_VARIABLE(scratch);
111 EIGEN_UNUSED_VARIABLE(what);
113 int* info_words =
static_cast<int*
>(scratch.h_info.get());
114 EIGEN_CUDA_RUNTIME_CHECK(
115 cudaMemcpyAsync(info_words, scratch.d_info.get(), kOneShotInfoBytes, cudaMemcpyDeviceToHost, ctx.stream()));
116 EIGEN_CUDA_RUNTIME_CHECK(cudaStreamSynchronize(ctx.stream()));
117 eigen_assert(info_words[0] == 0 &&
"cuSOLVER one-shot factorization failed" && what);
118 eigen_assert(info_words[1] == 0 &&
"cuSOLVER one-shot solve failed" && what);
119 EIGEN_UNUSED_VARIABLE(what);
123template <
typename Scalar,
int UpLo>
124void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst,
const LltSolveExpr<Scalar, UpLo>& expr) {
125 const DeviceMatrix<Scalar>& A = expr.matrix();
126 const DeviceMatrix<Scalar>& B = expr.rhs();
128 eigen_assert(A.rows() == A.cols() &&
"LLT requires a square matrix");
129 eigen_assert(B.rows() == A.rows() &&
"LLT solve: RHS rows must match matrix size");
131 if (A.rows() == 0 || B.cols() == 0) {
132 if (!dst.empty()) dst.waitReady(ctx.stream());
133 dst.resize(A.rows(), B.cols());
137 A.waitReady(ctx.stream());
138 B.waitReady(ctx.stream());
139 if (!dst.empty()) dst.waitReady(ctx.stream());
143 static thread_local CusolverParams params;
144 constexpr cublasFillMode_t uplo = cusolver_fill_mode<UpLo>::value;
145 const int64_t n =
static_cast<int64_t
>(A.rows());
146 constexpr cudaDataType_t dtype = cuda_data_type<Scalar>::value;
147 OneShotSolverScratch& scratch = ctx.oneshotSolverScratch();
149 const size_t mat_bytes = A.sizeInBytes();
151 ensure_sized(scratch.d_factor, mat_bytes);
152 EIGEN_CUDA_RUNTIME_CHECK(
153 cudaMemcpyAsync(scratch.d_factor.get(), A.data(), mat_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
155 const int64_t lda =
static_cast<int64_t
>(A.rows());
158 EIGEN_CUSOLVER_CHECK(cusolverDnXpotrf_bufferSize(ctx.cusolverHandle(), params.p, uplo, n, dtype,
159 scratch.d_factor.get(), lda, dtype, &dev_ws, &host_ws));
160 ensure_sized(scratch.d_workspace, dev_ws);
161 if (scratch.h_workspace.size() < host_ws) scratch.h_workspace.resize(host_ws);
164 int* d_info_potrf =
static_cast<int*
>(scratch.d_info.get());
165 int* d_info_potrs = d_info_potrf + 1;
166 EIGEN_CUSOLVER_CHECK(cusolverDnXpotrf(ctx.cusolverHandle(), params.p, uplo, n, dtype, scratch.d_factor.get(), lda,
167 dtype, scratch.d_workspace.get(), dev_ws,
168 host_ws > 0 ? scratch.h_workspace.data() :
nullptr, host_ws, d_info_potrf));
170 dst.resize(n, B.cols());
171 const size_t rhs_bytes = B.sizeInBytes();
172 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst.data(), B.data(), rhs_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
174 const int64_t nrhs =
static_cast<int64_t
>(B.cols());
175 EIGEN_CUSOLVER_CHECK(cusolverDnXpotrs(ctx.cusolverHandle(), params.p, uplo, n, nrhs, dtype, scratch.d_factor.get(),
176 lda, dtype, dst.data(),
static_cast<int64_t
>(dst.rows()), d_info_potrs));
177 oneshot_check_info(ctx, scratch,
"llt");
178 dst.recordReady(ctx.stream());
181template <
typename Scalar>
182void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst,
const LuSolveExpr<Scalar>& expr) {
183 const DeviceMatrix<Scalar>& A = expr.matrix();
184 const DeviceMatrix<Scalar>& B = expr.rhs();
186 eigen_assert(A.rows() == A.cols() &&
"LU requires a square matrix");
187 eigen_assert(B.rows() == A.rows() &&
"LU solve: RHS rows must match matrix size");
189 if (A.rows() == 0 || B.cols() == 0) {
190 if (!dst.empty()) dst.waitReady(ctx.stream());
191 dst.resize(A.rows(), B.cols());
195 A.waitReady(ctx.stream());
196 B.waitReady(ctx.stream());
197 if (!dst.empty()) dst.waitReady(ctx.stream());
201 static thread_local CusolverParams params;
202 const int64_t n =
static_cast<int64_t
>(A.rows());
203 constexpr cudaDataType_t dtype = cuda_data_type<Scalar>::value;
204 OneShotSolverScratch& scratch = ctx.oneshotSolverScratch();
206 const size_t mat_bytes = A.sizeInBytes();
208 ensure_sized(scratch.d_factor, mat_bytes);
209 EIGEN_CUDA_RUNTIME_CHECK(
210 cudaMemcpyAsync(scratch.d_factor.get(), A.data(), mat_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
212 ensure_sized(scratch.d_ipiv,
static_cast<size_t>(n) *
sizeof(int64_t));
213 const int64_t lda =
static_cast<int64_t
>(A.rows());
216 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrf_bufferSize(ctx.cusolverHandle(), params.p, n, n, dtype, scratch.d_factor.get(),
217 lda, dtype, &dev_ws, &host_ws));
218 ensure_sized(scratch.d_workspace, dev_ws);
219 if (scratch.h_workspace.size() < host_ws) scratch.h_workspace.resize(host_ws);
220 int* d_info_getrf =
static_cast<int*
>(scratch.d_info.get());
221 int* d_info_getrs = d_info_getrf + 1;
222 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrf(ctx.cusolverHandle(), params.p, n, n, dtype, scratch.d_factor.get(), lda,
223 static_cast<int64_t*
>(scratch.d_ipiv.get()), dtype, scratch.d_workspace.get(),
224 dev_ws, host_ws > 0 ? scratch.h_workspace.data() :
nullptr, host_ws,
227 dst.resize(n, B.cols());
228 const size_t rhs_bytes = B.sizeInBytes();
229 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst.data(), B.data(), rhs_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
231 const int64_t nrhs =
static_cast<int64_t
>(B.cols());
232 EIGEN_CUSOLVER_CHECK(cusolverDnXgetrs(ctx.cusolverHandle(), params.p, CUBLAS_OP_N, n, nrhs, dtype,
233 scratch.d_factor.get(), lda,
static_cast<const int64_t*
>(scratch.d_ipiv.get()),
234 dtype, dst.data(),
static_cast<int64_t
>(dst.rows()), d_info_getrs));
235 oneshot_check_info(ctx, scratch,
"lu");
236 dst.recordReady(ctx.stream());
239template <
typename Scalar,
int UpLo>
244 eigen_assert(A.rows() == A.cols() &&
"TRSM requires a square triangular matrix");
245 eigen_assert(B.rows() == A.rows() &&
"TRSM: RHS rows must match matrix size");
247 const int64_t n = A.rows();
248 const int64_t nrhs = B.cols();
250 if (n == 0 || nrhs == 0) {
251 if (!dst.empty()) dst.
waitReady(ctx.stream());
258 eigen_assert(!aliases_device_memory(dst, A) &&
"DeviceMatrix TRSM destination aliases triangular operand");
259 eigen_assert(!aliases_device_memory(dst, B) &&
"DeviceMatrix TRSM destination aliases RHS operand");
260 if (!dst.empty()) dst.
waitReady(ctx.stream());
263 const size_t rhs_bytes =
static_cast<size_t>(dst.rows()) *
static_cast<size_t>(nrhs) *
sizeof(Scalar);
264 EIGEN_CUDA_RUNTIME_CHECK(cudaMemcpyAsync(dst.data(), B.data(), rhs_bytes, cudaMemcpyDeviceToDevice, ctx.stream()));
266 constexpr cublasFillMode_t uplo = (UpLo ==
Lower) ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER;
269 EIGEN_CUBLAS_CHECK(cublasXtrsm(ctx.cublasHandle(), CUBLAS_SIDE_LEFT, uplo, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT, n, nrhs,
270 &alpha, A.data(), A.rows(), dst.data(), dst.rows()));
275template <
typename Scalar,
int UpLo>
276void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst,
const SymmExpr<Scalar, UpLo>& expr) {
277 const DeviceMatrix<Scalar>& A = expr.matrix();
278 const DeviceMatrix<Scalar>& B = expr.rhs();
280 eigen_assert(A.rows() == A.cols() &&
"SYMM requires a square matrix");
281 eigen_assert(B.rows() == A.rows() &&
"SYMM: RHS rows must match matrix size");
283 const int64_t m = A.rows();
284 const int64_t n = B.cols();
286 if (m == 0 || n == 0) {
287 if (!dst.empty()) dst.waitReady(ctx.stream());
288 dst.resize(m, B.cols());
292 A.waitReady(ctx.stream());
293 B.waitReady(ctx.stream());
294 eigen_assert(!aliases_device_memory(dst, A) &&
"DeviceMatrix SYMM destination aliases self-adjoint operand");
295 eigen_assert(!aliases_device_memory(dst, B) &&
"DeviceMatrix SYMM destination aliases RHS operand");
296 if (!dst.empty()) dst.waitReady(ctx.stream());
300 constexpr cublasFillMode_t uplo = (UpLo ==
Lower) ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER;
301 const Scalar one(1), zero(0);
303 EIGEN_CUBLAS_CHECK(cublasXsymm(ctx.cublasHandle(), CUBLAS_SIDE_LEFT, uplo, m, n, &one, A.data(), A.rows(), B.data(),
304 B.rows(), &zero, dst.data(), dst.rows()));
306 dst.recordReady(ctx.stream());
309template <
typename Scalar,
int UpLo>
310void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst,
const SyrkExpr<Scalar, UpLo>& expr,
311 typename NumTraits<Scalar>::Real alpha_val,
typename NumTraits<Scalar>::Real beta_val) {
312 using RealScalar =
typename NumTraits<Scalar>::Real;
313 const DeviceMatrix<Scalar>& A = expr.matrix();
315 const int64_t n = A.rows();
316 const int64_t k = A.cols();
319 if (!dst.empty()) dst.waitReady(ctx.stream());
324 A.waitReady(ctx.stream());
325 eigen_assert(!aliases_device_memory(dst, A) &&
"DeviceMatrix SYRK destination aliases input operand");
326 if (!dst.empty()) dst.waitReady(ctx.stream());
328 if (dst.empty() || dst.rows() != n || dst.cols() != n) {
330 if (beta_val != RealScalar(0)) {
331 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(dst.data(), 0, dst.sizeInBytes(), ctx.stream()));
335 constexpr cublasFillMode_t uplo = (UpLo ==
Lower) ? CUBLAS_FILL_MODE_LOWER : CUBLAS_FILL_MODE_UPPER;
337 EIGEN_CUBLAS_CHECK(cublasXsyrk(ctx.cublasHandle(), uplo, CUBLAS_OP_N, n, k, &alpha_val, A.data(), A.rows(), &beta_val,
338 dst.data(), dst.rows()));
340 dst.recordReady(ctx.stream());
347template <
typename Scalar>
348void dispatch(Context& ctx, DeviceMatrix<Scalar>& dst,
const DeviceAddExpr<Scalar>& expr) {
349 const DeviceMatrix<Scalar>& A = expr.A();
350 const DeviceMatrix<Scalar>& B = expr.B();
351 eigen_assert(A.rows() == B.rows() && A.cols() == B.cols());
352 const int64_t m = A.rows();
353 const int64_t n = A.cols();
356 if (!dst.empty()) dst.waitReady(ctx.stream());
357 dst.resize(A.rows(), A.cols());
358 if (m > 0 && n > 0) {
359 A.waitReady(ctx.stream());
360 B.waitReady(ctx.stream());
361 const Scalar alpha_val = expr.alpha(), beta_val = expr.beta();
362 EIGEN_CUBLAS_CHECK(cublasXgeam(ctx.cublasHandle(), CUBLAS_OP_N, CUBLAS_OP_N, m, n, &alpha_val, A.data(), m,
363 &beta_val, B.data(), m, dst.data(), m));
364 dst.recordReady(ctx.stream());
369template <
typename Scalar_>
372 using Scalar = Scalar_;
374 Assignment(DeviceMatrix<Scalar>& dst, Context& ctx) : dst_(dst), ctx_(ctx) {}
376 template <
typename Lhs,
typename Rhs>
377 DeviceMatrix<Scalar>& operator=(
const GemmExpr<Lhs, Rhs>& expr) {
378 internal::dispatch(ctx_, dst_, expr, Scalar(0));
382 template <
typename Lhs,
typename Rhs>
383 DeviceMatrix<Scalar>& operator+=(
const GemmExpr<Lhs, Rhs>& expr) {
384 internal::dispatch(ctx_, dst_, expr, Scalar(1));
388 template <
typename Lhs,
typename Rhs>
389 DeviceMatrix<Scalar>& operator-=(
const GemmExpr<Lhs, Rhs>& expr) {
390 internal::dispatch(ctx_, dst_, expr, Scalar(1), Scalar(-1));
395 DeviceMatrix<Scalar>& operator=(
const LltSolveExpr<Scalar, UpLo>& expr) {
396 internal::dispatch(ctx_, dst_, expr);
400 DeviceMatrix<Scalar>& operator=(
const LuSolveExpr<Scalar>& expr) {
401 internal::dispatch(ctx_, dst_, expr);
406 DeviceMatrix<Scalar>& operator=(
const TrsmExpr<Scalar, UpLo>& expr) {
407 internal::dispatch(ctx_, dst_, expr);
412 DeviceMatrix<Scalar>& operator=(
const SymmExpr<Scalar, UpLo>& expr) {
413 internal::dispatch(ctx_, dst_, expr);
417 DeviceMatrix<Scalar>& operator=(
const DeviceAddExpr<Scalar>& expr) {
418 internal::dispatch(ctx_, dst_, expr);
422 DeviceMatrix<Scalar>& operator=(
const Scaled<DeviceMatrix<Scalar>>& expr) {
424 internal::dispatch(ctx_, dst_, DeviceAddExpr<Scalar>(expr.scalar(), expr.inner(), Scalar(0), expr.inner()));
428 template <
typename Expr>
429 DeviceMatrix<Scalar>& operator=(
const Expr&) {
430 static_assert(
sizeof(Expr) == 0,
431 "DeviceMatrix expression not supported: no cuBLAS/cuSOLVER mapping. "
432 "Supported: GEMM (A*B), geam (A + alpha*B, alpha*A), "
433 "TRSM (.triangularView().solve()), SYMM (.selfadjointView()*B), "
434 "LLT (.llt().solve()), LU (.lu().solve()).");
439 DeviceMatrix<Scalar>& dst_;
446template <
typename Scalar_>
447template <
typename Lhs,
typename Rhs>
453template <
typename Scalar_>
454template <
typename Lhs,
typename Rhs>
460template <
typename Scalar_>
461template <
typename Lhs,
typename Rhs>
467template <
typename Scalar_>
474template <
typename Scalar_>
480template <
typename Scalar_>
487template <
typename Scalar_>
494template <
typename Scalar_>
504template <
typename Scalar_>
505template <
typename Lhs,
typename Rhs>
510template <
typename Scalar_>
515template <
typename Scalar_>
520template <
typename Scalar_>
526template <
typename Scalar_>
531template <
typename Scalar_>
537template <
typename Scalar_>
543template <
typename Scalar_,
int UpLo_>
546 RealScalar beta = matrix().empty() ? RealScalar(0) : RealScalar(1);
555void with_device_pointer_mode(cublasHandle_t h, F&& f) {
556 struct RestoreOnThrow {
557 cublasHandle_t handle;
558 cublasPointerMode_t mode;
561 if (armed) (void)cublasSetPointerMode(handle, mode);
564 cublasPointerMode_t prev;
565 EIGEN_CUBLAS_CHECK(cublasGetPointerMode(h, &prev));
566 EIGEN_CUBLAS_CHECK(cublasSetPointerMode(h, CUBLAS_POINTER_MODE_DEVICE));
567 RestoreOnThrow restore{h, prev,
true};
569 restore.armed =
false;
570 EIGEN_CUBLAS_CHECK(cublasSetPointerMode(h, prev));
579inline int64_t blas1_size(Index rows, Index cols) {
return static_cast<int64_t
>(rows) *
static_cast<int64_t
>(cols); }
582template <
typename Scalar_>
584 const int64_t n = internal::blas1_size(rows_, cols_);
585 eigen_assert(n == internal::blas1_size(other.rows_, other.cols_));
586 eigen_assert(result.stream() == ctx.stream() &&
"DeviceMatrix::dot: result must live on ctx's stream");
590 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
592 internal::cublasXdot(ctx.cublasHandle(), n, data_.get(), 1, other.data_.get(), 1, result.devicePtr()));
595 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0,
sizeof(Scalar), ctx.stream()));
599template <
typename Scalar_>
604 dot(ctx, other, result);
608template <
typename Scalar_>
610 const int64_t n = internal::blas1_size(rows_, cols_);
611 eigen_assert(result.stream() == ctx.stream() &&
"DeviceMatrix::squaredNorm: result must live on ctx's stream");
617 const RealScalar* x =
reinterpret_cast<const RealScalar*
>(data_.get());
618 waitReady(ctx.stream());
619 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
620 EIGEN_CUBLAS_CHECK(internal::cublasXdot(ctx.cublasHandle(), reals, x, 1, x, 1, result.devicePtr()));
623 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0,
sizeof(RealScalar), ctx.stream()));
627template <
typename Scalar_>
634template <
typename Scalar_>
638 squaredNorm(ctx, result);
642template <
typename Scalar_>
649template <
typename Scalar_>
651 const int64_t n = internal::blas1_size(rows_, cols_);
652 eigen_assert(result.stream() == ctx.stream() &&
"DeviceMatrix::stableNorm: result must live on ctx's stream");
654 waitReady(ctx.stream());
655 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
656 EIGEN_CUBLAS_CHECK(internal::cublasXnrm2(ctx.cublasHandle(), n, data_.get(), 1, result.devicePtr()));
659 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(result.devicePtr(), 0,
sizeof(RealScalar), ctx.stream()));
663template <
typename Scalar_>
665 if (sizeInBytes() > 0) {
667 EIGEN_CUDA_RUNTIME_CHECK(cudaMemsetAsync(data_.get(), 0, sizeInBytes(), stream));
672template <
typename Scalar_>
677template <
typename Scalar_>
679 const int64_t n = internal::blas1_size(rows_, cols_);
680 eigen_assert(n == internal::blas1_size(x.rows_, x.cols_));
684 EIGEN_CUBLAS_CHECK(internal::cublasXaxpy(ctx.cublasHandle(), n, &alpha, x.data_.get(), 1, data_.get(), 1));
689template <
typename Scalar_>
691 const int64_t n = internal::blas1_size(rows_, cols_);
694 EIGEN_CUBLAS_CHECK(internal::cublasXscal(ctx.cublasHandle(), n, &alpha, data_.get(), 1));
699template <
typename Scalar_>
704 resize(other.rows_, other.cols_);
705 const int64_t n = internal::blas1_size(rows_, cols_);
708 EIGEN_CUBLAS_CHECK(internal::cublasXcopy(ctx.cublasHandle(), n, other.data_.get(), 1, data_.get(), 1));
714template <
typename Scalar_>
721template <
typename Scalar_>
728template <
typename Scalar_>
736template <
typename Scalar_>
744template <
typename Scalar_>
754inline void divide_in_place(
Context& ctx,
float* x, int64_t n,
float alpha) { device_divC(alpha, x, n, ctx.stream()); }
755inline void divide_in_place(Context& ctx,
double* x, int64_t n,
double alpha) {
756 device_divC(alpha, x, n, ctx.stream());
758template <
typename Real>
759void divide_in_place(Context& ctx, std::complex<Real>* x, int64_t n, std::complex<Real> alpha) {
760 const std::complex<Real> inv = std::complex<Real>(1) / alpha;
761 EIGEN_CUBLAS_CHECK(cublasXscal(ctx.cublasHandle(), n, &inv, x, 1));
765template <
typename Scalar_>
767 const int64_t n = internal::blas1_size(rows_, cols_);
770 internal::divide_in_place(ctx, data_.get(), n, alpha);
776template <
typename Scalar_>
783template <
typename Scalar_>
788template <
typename Scalar_>
794template <
typename Scalar_>
801template <
typename Scalar_>
807template <
typename Scalar_>
809 const int64_t n = internal::blas1_size(rows_, cols_);
813 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
814 EIGEN_CUBLAS_CHECK(internal::cublasXscal(ctx.cublasHandle(), n, alpha.devicePtr(), data_.get(), 1));
822template <
typename Scalar_>
824 const int64_t n = internal::blas1_size(rows_, cols_);
825 const auto& x = expr.matrix();
826 eigen_assert(n == internal::blas1_size(x.rows_, x.cols_));
830 x.waitReady(ctx.stream());
831 internal::with_device_pointer_mode(ctx.cublasHandle(), [&] {
833 internal::cublasXaxpy(ctx.cublasHandle(), n, expr.alpha().devicePtr(), x.data_.get(), 1, data_.get(), 1));
841template <
typename Scalar_>
843 auto neg_alpha = -expr.alpha();
845 return operator+=(neg_expr);
849template <
typename Scalar_>
856template <
typename Scalar_>
858 const int64_t n = internal::blas1_size(rows_, cols_);
859 eigen_assert(n == internal::blas1_size(other.rows_, other.cols_));
864 internal::device_cwiseProduct(data_.get(), other.data_.get(), result.data_.get(), n, ctx.stream());
871template <
typename Scalar_>
873 const int64_t n = internal::blas1_size(a.rows_, a.cols_);
874 eigen_assert(n == internal::blas1_size(b.rows_, b.cols_));
880 internal::device_cwiseProduct(a.data_.get(), b.data_.get(), data_.get(), n, ctx.stream());
886template <
typename Scalar_>
891template <
typename Scalar_>
892DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::squaredNorm()
const {
896template <
typename Scalar_>
897DeviceScalar<typename NumTraits<Scalar_>::Real> DeviceMatrix<Scalar_>::norm()
const {
901template <
typename Scalar_>
Unified GPU execution context owning a CUDA stream and library handles.
Definition GpuContext.h:81
const NppStreamContext & nppStreamContext() const
Definition GpuContext.h:139
static Context & threadLocal()
Definition GpuContext.h:121
Linear combination of two device matrices.
Definition DeviceExpr.h:260
RAII wrapper for a dense column-major matrix in GPU device memory.
Definition DeviceMatrix.h:122
DeviceMatrix & operator*=(Scalar alpha)
Definition DeviceDispatch.h:745
void addScaled(Context &ctx, Scalar alpha, const DeviceMatrix &x)
Definition DeviceDispatch.h:678
Assignment< Scalar > device(Context &ctx)
Definition DeviceMatrix.h:388
DeviceMatrix cwiseProduct(Context &ctx, const DeviceMatrix &other) const
Definition DeviceDispatch.h:857
void scale(Context &ctx, Scalar alpha)
Definition DeviceDispatch.h:690
void copyFrom(Context &ctx, const DeviceMatrix &other)
Definition DeviceDispatch.h:700
DeviceScalar< typename NumTraits< Scalar >::Real > squaredNorm(Context &ctx) const
Definition DeviceDispatch.h:628
DeviceMatrix & operator/=(Scalar alpha)
Definition DeviceDispatch.h:777
DeviceScalar< typename NumTraits< Scalar >::Real > stableNorm(Context &ctx) const
Definition DeviceDispatch.h:795
DeviceMatrix & operator-=(const GemmExpr< Lhs, Rhs > &expr)
void divide(Context &ctx, Scalar alpha)
Definition DeviceDispatch.h:766
void recordReady(cudaStream_t stream)
Definition DeviceMatrix.h:365
DeviceScalar< typename NumTraits< Scalar >::Real > norm(Context &ctx) const
Definition DeviceDispatch.h:643
void setZero(Context &ctx)
Definition DeviceDispatch.h:673
DeviceScalar< Scalar > dot(Context &ctx, const DeviceMatrix &other) const
Definition DeviceDispatch.h:600
void resize(Index rows, Index cols)
Definition DeviceMatrix.h:323
void waitReady(cudaStream_t stream) const
Definition DeviceMatrix.h:374
RAII wrapper for a scalar in GPU device memory.
Definition DeviceScalar.h:33
Expression that scales a device matrix by a DeviceScalar.
Definition DeviceExpr.h:231
Expression returned by operator*(lhs_expr, rhs_expr), dispatched to cuBLAS GEMM.
Definition DeviceExpr.h:92
Definition DeviceSolverExpr.h:30
Definition DeviceSolverExpr.h:46
Expression returned by operator*(Scalar, DeviceMatrix/View), carrying the scalar factor.
Definition DeviceExpr.h:77
void rankUpdate(const DeviceMatrix< Scalar > &A, RealScalar alpha=RealScalar(1))
Definition DeviceDispatch.h:544
Definition DeviceBlasExpr.h:96
Definition DeviceBlasExpr.h:122
Definition DeviceBlasExpr.h:45
Namespace containing all symbols from the Eigen library.
Describes GPU device expression types.
Definition DeviceExpr.h:173