11#ifndef EIGEN_TENSOR_TENSOR_CONVOLUTION_H
12#define EIGEN_TENSOR_TENSOR_CONVOLUTION_H
15#include "./InternalHeaderCheck.h"
21template <
typename Index,
typename InputDims,
int NumKernelDims,
int Layout>
24 IndexMapper(
const InputDims& input_dims,
const array<Index, NumKernelDims>& kernel_dims,
25 const array<Index, NumKernelDims>& indices) {
26 array<Index, NumDims> dimensions = input_dims;
27 for (
int i = 0; i < NumKernelDims; ++i) {
28 const Index index = indices[i];
29 const Index input_dim = input_dims[index];
30 const Index kernel_dim = kernel_dims[i];
31 const Index result_dim = input_dim - kernel_dim + 1;
32 dimensions[index] = result_dim;
35 array<Index, NumDims> inputStrides;
36 array<Index, NumDims> outputStrides;
37 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
40 for (
int i = 1; i < NumDims; ++i) {
41 inputStrides[i] = inputStrides[i - 1] * input_dims[i - 1];
42 outputStrides[i] = outputStrides[i - 1] * dimensions[i - 1];
45 inputStrides[NumDims - 1] = 1;
46 outputStrides[NumDims - 1] = 1;
47 for (
int i =
static_cast<int>(NumDims) - 2; i >= 0; --i) {
48 inputStrides[i] = inputStrides[i + 1] * input_dims[i + 1];
49 outputStrides[i] = outputStrides[i + 1] * dimensions[i + 1];
53 array<Index, NumDims> gpuInputDimensions;
54 array<Index, NumDims> gpuOutputDimensions;
55 array<Index, NumDims> tmp = dimensions;
56 array<Index, NumDims> ordering;
57 constexpr size_t offset =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - NumKernelDims;
58 for (
int i = 0; i < NumKernelDims; ++i) {
59 const Index index = i + offset;
60 ordering[index] = indices[i];
62 gpuInputDimensions[index] = input_dims[indices[i]];
63 gpuOutputDimensions[index] = dimensions[indices[i]];
66 int written =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? NumKernelDims : 0;
67 for (
int i = 0; i < NumDims; ++i) {
69 ordering[written] = i;
70 gpuInputDimensions[written] = input_dims[i];
71 gpuOutputDimensions[written] = dimensions[i];
76 for (
int i = 0; i < NumDims; ++i) {
77 m_inputStrides[i] = inputStrides[ordering[i]];
78 m_outputStrides[i] = outputStrides[ordering[i]];
81 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
82 for (
int i = 0; i < NumDims; ++i) {
83 if (i > NumKernelDims) {
84 m_gpuInputStrides[i] = m_gpuInputStrides[i - 1] * gpuInputDimensions[i - 1];
85 m_gpuOutputStrides[i] = m_gpuOutputStrides[i - 1] * gpuOutputDimensions[i - 1];
87 m_gpuInputStrides[i] = 1;
88 m_gpuOutputStrides[i] = 1;
92 for (
int i = NumDims - 1; i >= 0; --i) {
93 if (i + 1 <
static_cast<int>(offset)) {
94 m_gpuInputStrides[i] = m_gpuInputStrides[i + 1] * gpuInputDimensions[i + 1];
95 m_gpuOutputStrides[i] = m_gpuOutputStrides[i + 1] * gpuOutputDimensions[i + 1];
97 m_gpuInputStrides[i] = 1;
98 m_gpuOutputStrides[i] = 1;
104 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputPlaneToTensorInputOffset(Index p)
const {
105 Index inputIndex = 0;
106 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
107 for (
int d = NumDims - 1; d > NumKernelDims; --d) {
108 const Index idx = p / m_gpuInputStrides[d];
109 inputIndex += idx * m_inputStrides[d];
110 p -= idx * m_gpuInputStrides[d];
112 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
113 inputIndex += p * m_inputStrides[NumKernelDims];
116 std::ptrdiff_t limit = 0;
117 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
118 limit = NumDims - NumKernelDims - 1;
120 for (
int d = 0; d < limit; ++d) {
121 const Index idx = p / m_gpuInputStrides[d];
122 inputIndex += idx * m_inputStrides[d];
123 p -= idx * m_gpuInputStrides[d];
125 inputIndex += p * m_inputStrides[limit];
130 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputPlaneToTensorOutputOffset(Index p)
const {
131 Index outputIndex = 0;
132 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
133 for (
int d = NumDims - 1; d > NumKernelDims; --d) {
134 const Index idx = p / m_gpuOutputStrides[d];
135 outputIndex += idx * m_outputStrides[d];
136 p -= idx * m_gpuOutputStrides[d];
138 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
139 outputIndex += p * m_outputStrides[NumKernelDims];
142 std::ptrdiff_t limit = 0;
143 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
144 limit = NumDims - NumKernelDims - 1;
146 for (
int d = 0; d < limit; ++d) {
147 const Index idx = p / m_gpuOutputStrides[d];
148 outputIndex += idx * m_outputStrides[d];
149 p -= idx * m_gpuOutputStrides[d];
151 outputIndex += p * m_outputStrides[limit];
156 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i)
const {
157 constexpr size_t offset =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - NumKernelDims;
158 return i * m_inputStrides[offset];
161 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i)
const {
162 constexpr size_t offset =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - NumKernelDims;
163 return i * m_outputStrides[offset];
166 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j)
const {
167 constexpr size_t offset =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - NumKernelDims;
168 return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1];
171 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j)
const {
172 constexpr size_t offset =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - NumKernelDims;
173 return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1];
176 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j, Index k)
const {
177 constexpr size_t offset =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - NumKernelDims;
178 return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1] + k * m_inputStrides[offset + 2];
181 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j, Index k)
const {
182 constexpr size_t offset =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - NumKernelDims;
183 return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1] + k * m_outputStrides[offset + 2];
187 static constexpr int NumDims = internal::array_size<InputDims>::value;
188 array<Index, NumDims> m_inputStrides;
189 array<Index, NumDims> m_outputStrides;
190 array<Index, NumDims> m_gpuInputStrides;
191 array<Index, NumDims> m_gpuOutputStrides;
194template <
typename Dimensions,
typename InputXprType,
typename KernelXprType>
195struct traits<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType> > {
197 using Scalar =
typename promote_storage_type<typename InputXprType::Scalar, typename KernelXprType::Scalar>::ret;
198 using StorageKind =
typename promote_storage_type<typename traits<InputXprType>::StorageKind,
199 typename traits<KernelXprType>::StorageKind>::ret;
201 typename promote_index_type<typename traits<InputXprType>::Index,
typename traits<KernelXprType>::Index>::type;
202 static constexpr int NumDimensions = traits<InputXprType>::NumDimensions;
203 static constexpr int Layout = traits<InputXprType>::Layout;
205 std::conditional_t<Pointer_type_promotion<typename InputXprType::Scalar, Scalar>::val,
206 typename traits<InputXprType>::PointerType,
typename traits<KernelXprType>::PointerType>;
211template <
typename Dimensions,
typename InputXprType,
typename KernelXprType>
212struct eval<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType>, Eigen::Dense> {
213 using type =
const TensorConvolutionOp<Dimensions, InputXprType, KernelXprType>&;
221template <
typename Indices,
typename InputXprType,
typename KernelXprType>
222class TensorConvolutionOp
223 :
public TensorBase<TensorConvolutionOp<Indices, InputXprType, KernelXprType>, ReadOnlyAccessors> {
225 using Scalar =
typename Eigen::internal::traits<TensorConvolutionOp>::Scalar;
227 using CoeffReturnType =
typename internal::promote_storage_type<
typename InputXprType::CoeffReturnType,
228 typename KernelXprType::CoeffReturnType>::ret;
229 using Nested =
typename Eigen::internal::ref_selector<TensorConvolutionOp>::type;
230 using StorageKind =
typename Eigen::internal::traits<TensorConvolutionOp>::StorageKind;
231 using Index =
typename Eigen::internal::traits<TensorConvolutionOp>::Index;
233 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorConvolutionOp(
const InputXprType& input,
const KernelXprType& kernel,
235 : m_input_xpr(input), m_kernel_xpr(kernel), m_indices(dims) {}
237 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Indices& indices()
const {
return m_indices; }
240 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const internal::remove_all_t<typename InputXprType::Nested>&
inputExpression()
245 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const internal::remove_all_t<typename KernelXprType::Nested>& kernelExpression()
251 typename InputXprType::Nested m_input_xpr;
252 typename KernelXprType::Nested m_kernel_xpr;
253 const Indices m_indices;
256template <
typename Indices,
typename InputArgType,
typename KernelArgType,
typename Device>
258 using XprType = TensorConvolutionOp<Indices, InputArgType, KernelArgType>;
260 static constexpr int NumDims =
261 internal::array_size<typename TensorEvaluator<InputArgType, Device>::Dimensions>::value;
262 static constexpr int NumKernelDims = internal::array_size<Indices>::value;
263 using Index =
typename XprType::Index;
264 using Dimensions = DSizes<Index, NumDims>;
266 using Scalar =
typename XprType::Scalar;
267 using CoeffReturnType =
typename XprType::CoeffReturnType;
268 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
269 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
270 using Storage = StorageMemory<Scalar, Device>;
271 using EvaluatorPointerType =
typename Storage::Type;
273 static constexpr int Layout = TensorEvaluator<InputArgType, Device>::Layout;
276 int(TensorEvaluator<InputArgType, Device>::IsAligned) & int(TensorEvaluator<KernelArgType, Device>::IsAligned),
277 PacketAccess = int(TensorEvaluator<InputArgType, Device>::PacketAccess) &
278 int(TensorEvaluator<KernelArgType, Device>::PacketAccess),
280 PreferBlockAccess =
false,
286 using TensorBlock = internal::TensorBlockNotImplemented;
289 EIGEN_STRONG_INLINE TensorEvaluator(
const XprType& op,
const Device& device)
290 : m_inputImpl(op.inputExpression(), device),
291 m_kernelImpl(op.kernelExpression(), device),
292 m_kernelArg(op.kernelExpression()),
294 m_local_kernel(false),
296 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<InputArgType, Device>::Layout) ==
297 static_cast<int>(TensorEvaluator<KernelArgType, Device>::Layout)),
298 YOU_MADE_A_PROGRAMMING_MISTAKE);
300 const typename TensorEvaluator<InputArgType, Device>::Dimensions& input_dims = m_inputImpl.dimensions();
301 const typename TensorEvaluator<KernelArgType, Device>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
303 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
304 m_inputStride[0] = 1;
305 for (
int i = 1; i < NumDims; ++i) {
306 m_inputStride[i] = m_inputStride[i - 1] * input_dims[i - 1];
309 m_inputStride[NumDims - 1] = 1;
310 for (
int i = NumDims - 2; i >= 0; --i) {
311 m_inputStride[i] = m_inputStride[i + 1] * input_dims[i + 1];
315 m_dimensions = m_inputImpl.dimensions();
316 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
317 for (
int i = 0; i < NumKernelDims; ++i) {
318 const Index index = op.indices()[i];
319 const Index input_dim = input_dims[index];
320 const Index kernel_dim = kernel_dims[i];
321 const Index result_dim = input_dim - kernel_dim + 1;
322 m_dimensions[index] = result_dim;
324 m_kernelStride[i] = m_kernelStride[i - 1] * kernel_dims[i - 1];
326 m_kernelStride[0] = 1;
328 m_indexStride[i] = m_inputStride[index];
331 m_outputStride[0] = 1;
332 for (
int i = 1; i < NumDims; ++i) {
333 m_outputStride[i] = m_outputStride[i - 1] * m_dimensions[i - 1];
336 for (
int i = NumKernelDims - 1; i >= 0; --i) {
337 const Index index = op.indices()[i];
338 const Index input_dim = input_dims[index];
339 const Index kernel_dim = kernel_dims[i];
340 const Index result_dim = input_dim - kernel_dim + 1;
341 m_dimensions[index] = result_dim;
342 if (i < NumKernelDims - 1) {
343 m_kernelStride[i] = m_kernelStride[i + 1] * kernel_dims[i + 1];
345 m_kernelStride[NumKernelDims - 1] = 1;
347 m_indexStride[i] = m_inputStride[index];
350 m_outputStride[NumDims - 1] = 1;
351 for (
int i = NumDims - 2; i >= 0; --i) {
352 m_outputStride[i] = m_outputStride[i + 1] * m_dimensions[i + 1];
357 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dimensions& dimensions()
const {
return m_dimensions; }
359 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(Scalar*) {
360 m_inputImpl.evalSubExprsIfNeeded(
nullptr);
364 EIGEN_STRONG_INLINE
void cleanup() {
365 m_inputImpl.cleanup();
366 if (m_local_kernel) {
367 m_device.deallocate((
void*)m_kernel);
368 m_local_kernel =
false;
373 void evalTo(
typename XprType::Scalar* buffer) {
374 evalSubExprsIfNeeded(
nullptr);
375 for (
int i = 0; i < dimensions().TotalSize(); ++i) {
376 buffer[i] += coeff(i);
381 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
382 CoeffReturnType result = CoeffReturnType(0);
383 convolve(firstInput(index), 0, NumKernelDims - 1, result);
387 template <
int LoadMode>
388 EIGEN_DEVICE_FUNC PacketReturnType packet(
const Index index)
const {
389 Index indices[2] = {index, index + PacketSize - 1};
390 Index startInputs[2] = {0, 0};
391 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
392 for (
int i = NumDims - 1; i > 0; --i) {
393 const Index idx0 = indices[0] / m_outputStride[i];
394 const Index idx1 = indices[1] / m_outputStride[i];
395 startInputs[0] += idx0 * m_inputStride[i];
396 startInputs[1] += idx1 * m_inputStride[i];
397 indices[0] -= idx0 * m_outputStride[i];
398 indices[1] -= idx1 * m_outputStride[i];
401 for (
int i = 0; i < NumDims - 1; ++i) {
402 const Index idx0 = indices[0] / m_outputStride[i];
403 const Index idx1 = indices[1] / m_outputStride[i];
404 startInputs[0] += idx0 * m_inputStride[i];
405 startInputs[1] += idx1 * m_inputStride[i];
406 indices[0] -= idx0 * m_outputStride[i];
407 indices[1] -= idx1 * m_outputStride[i];
410 startInputs[0] += indices[0];
411 startInputs[1] += indices[1];
413 if (startInputs[1] - startInputs[0] == PacketSize - 1) {
414 PacketReturnType result = internal::pset1<PacketReturnType>(0);
415 convolvePacket(startInputs[0], 0, NumKernelDims - 1, result);
418 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) Scalar data[PacketSize];
420 convolve(startInputs[0], 0, NumKernelDims - 1, data[0]);
421 for (
int i = 1; i < PacketSize - 1; ++i) {
423 convolve(firstInput(index + i), 0, NumKernelDims - 1, data[i]);
425 data[PacketSize - 1] = Scalar(0);
426 convolve(startInputs[1], 0, NumKernelDims - 1, data[PacketSize - 1]);
427 return internal::pload<PacketReturnType>(data);
431 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
432 const double kernel_size = m_kernelImpl.dimensions().TotalSize();
434 const double convolve_compute_cost = TensorOpCost::AddCost<Scalar>() + TensorOpCost::MulCost<Scalar>();
435 const double firstIndex_compute_cost =
437 (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() + TensorOpCost::DivCost<Index>());
438 return TensorOpCost(0, 0, firstIndex_compute_cost, vectorized, PacketSize) +
439 kernel_size * (m_inputImpl.costPerCoeff(vectorized) + m_kernelImpl.costPerCoeff(vectorized) +
440 TensorOpCost(0, 0, convolve_compute_cost, vectorized, PacketSize));
443 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return nullptr; }
446 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index)
const {
447 Index startInput = 0;
448 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
449 for (
int i = NumDims - 1; i > 0; --i) {
450 const Index idx = index / m_outputStride[i];
451 startInput += idx * m_inputStride[i];
452 index -= idx * m_outputStride[i];
455 for (
int i = 0; i < NumDims - 1; ++i) {
456 const Index idx = index / m_outputStride[i];
457 startInput += idx * m_inputStride[i];
458 index -= idx * m_outputStride[i];
465 EIGEN_DEVICE_FUNC
void convolve(Index firstIndex, Index firstKernel,
int DimIndex, CoeffReturnType& accum)
const {
466 for (
int j = 0; j < m_kernelImpl.dimensions()[DimIndex]; ++j) {
467 const Index input = firstIndex + j * m_indexStride[DimIndex];
468 const Index kernel = firstKernel + j * m_kernelStride[DimIndex];
470 convolve(input, kernel, DimIndex - 1, accum);
472 accum += m_inputImpl.coeff(input) * m_kernel[kernel];
477 template <
typename Packet>
478 EIGEN_DEVICE_FUNC
void convolvePacket(Index firstIndex, Index firstKernel,
int DimIndex, Packet& accum)
const {
479 for (
int j = 0; j < m_kernelImpl.dimensions()[DimIndex]; ++j) {
480 const Index input = firstIndex + j * m_indexStride[DimIndex];
481 const Index kernel = firstKernel + j * m_kernelStride[DimIndex];
483 convolvePacket(input, kernel, DimIndex - 1, accum);
485 accum = internal::pmadd<Packet>(m_inputImpl.template packet<Unaligned>(input),
486 internal::pset1<Packet>(m_kernel[kernel]), accum);
491 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void preloadKernel() {
494 const Scalar* in_place = m_kernelImpl.data();
497 m_local_kernel =
false;
499 size_t kernel_sz = m_kernelImpl.dimensions().TotalSize() *
sizeof(Scalar);
500 Scalar* local = (Scalar*)m_device.allocate_temp(kernel_sz);
501 using EvalTo = TensorEvalToOp<const KernelArgType>;
502 EvalTo evalToTmp(local, m_kernelArg);
503 const bool Vectorize = internal::IsVectorizable<Device, KernelArgType>::value;
504 internal::TensorExecutor<const EvalTo, Device, Vectorize>::run(evalToTmp, m_device);
507 m_local_kernel =
true;
511 array<Index, NumDims> m_inputStride;
512 array<Index, NumDims> m_outputStride;
514 array<Index, NumKernelDims> m_indexStride;
515 array<Index, NumKernelDims> m_kernelStride;
516 TensorEvaluator<InputArgType, Device> m_inputImpl;
517 TensorEvaluator<KernelArgType, Device> m_kernelImpl;
518 Dimensions m_dimensions;
520 KernelArgType m_kernelArg;
521 const Scalar* m_kernel;
523 const Device EIGEN_DEVICE_REF m_device;
527#if defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC)
529template <
int StaticKernelSize>
530struct GetKernelSize {
531 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
int operator()(
const int )
const {
return StaticKernelSize; }
534struct GetKernelSize<Dynamic> {
535 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
int operator()(
const int kernelSize)
const {
return kernelSize; }
539#pragma clang diagnostic push
541#pragma clang diagnostic ignored "-Wpass-failed"
544template <
typename InputEvaluator,
typename Index,
typename InputDims,
int StaticKernelSize>
545__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void EigenConvolutionKernel1D(
546 InputEvaluator eval,
const internal::IndexMapper<Index, InputDims, 1, InputEvaluator::Layout> indexMapper,
547 const float* __restrict kernel,
const int numPlanes,
const int numX,
const int maxX,
const int kernelSize,
549#if defined(EIGEN_HIPCC)
550 HIP_DYNAMIC_SHARED(
float, s)
552 extern __shared__
float s[];
555 const int first_x = blockIdx.x * maxX;
556 const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1;
557 const int num_x_input = last_x - first_x + GetKernelSize<StaticKernelSize>()(kernelSize);
558 const int num_x_output = last_x - first_x + 1;
560 const int first_plane = blockIdx.y * blockDim.y;
561 const int plane_stride = blockDim.y * gridDim.y;
563 for (
int p = first_plane + threadIdx.y; p < numPlanes; p += plane_stride) {
565 const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p);
566 const int plane_kernel_offset = threadIdx.y * num_x_input;
568 for (
int i = threadIdx.x; i < num_x_input; i += blockDim.x) {
569 const int tensor_index = plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(i + first_x);
570 s[i + plane_kernel_offset] = eval.coeff(tensor_index);
576 const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p);
579 for (
int i = threadIdx.x; i < num_x_output; i += blockDim.x) {
580 const int kernel_offset = plane_kernel_offset + i;
583 for (
int k = 0; k < GetKernelSize<StaticKernelSize>()(kernelSize); ++k) {
584 result += s[k + kernel_offset] * kernel[k];
586 const int tensor_index = plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(i + first_x);
587 buffer[tensor_index] = result;
593template <
typename InputEvaluator,
typename Index,
typename InputDims,
int StaticKernelSizeX,
int StaticKernelSizeY>
594__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void EigenConvolutionKernel2D(
595 InputEvaluator eval,
const internal::IndexMapper<Index, InputDims, 2, InputEvaluator::Layout> indexMapper,
596 const float* __restrict kernel,
const int numPlanes,
const int numX,
const int maxX,
const int numY,
const int maxY,
597 const int kernelSizeX,
const int kernelSizeY,
float* buffer) {
598#if defined(EIGEN_HIPCC)
599 HIP_DYNAMIC_SHARED(
float, s)
601 extern __shared__
float s[];
604 const int first_x = blockIdx.x * maxX;
605 const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1;
606 const int num_x_input = last_x - first_x + GetKernelSize<StaticKernelSizeX>()(kernelSizeX);
607 const int num_x_output = last_x - first_x + 1;
609 const int first_y = blockIdx.y * maxY;
610 const int last_y = (first_y + maxY < numY ? first_y + maxY : numY) - 1;
611 const int num_y_input = last_y - first_y + GetKernelSize<StaticKernelSizeY>()(kernelSizeY);
612 const int num_y_output = last_y - first_y + 1;
614 const int first_plane = blockIdx.z * blockDim.z;
615 const int plane_stride = blockDim.z * gridDim.z;
617 for (
int p = first_plane + threadIdx.z; p < numPlanes; p += plane_stride) {
618 const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p);
619 const int plane_kernel_offset = threadIdx.z * num_y_input;
623 for (
int j = threadIdx.y; j < num_y_input; j += blockDim.y) {
624 const int input_offset = num_x_input * (j + plane_kernel_offset);
626 for (
int i = threadIdx.x; i < num_x_input; i += blockDim.x) {
627 const int tensor_index =
628 plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(i + first_x, j + first_y);
629 s[i + input_offset] = eval.coeff(tensor_index);
636 const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p);
639 for (
int j = threadIdx.y; j < num_y_output; j += blockDim.y) {
641 for (
int i = threadIdx.x; i < num_x_output; i += blockDim.x) {
644 for (
int l = 0; l < GetKernelSize<StaticKernelSizeY>()(kernelSizeY); ++l) {
645 const int kernel_offset = kernelSizeX * l;
646 const int input_offset = i + num_x_input * (j + l + plane_kernel_offset);
648 for (
int k = 0; k < GetKernelSize<StaticKernelSizeX>()(kernelSizeX); ++k) {
649 result += s[k + input_offset] * kernel[k + kernel_offset];
652 const int tensor_index =
653 plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(i + first_x, j + first_y);
654 buffer[tensor_index] = result;
663#pragma clang diagnostic pop
667template <
typename InputEvaluator,
typename Index,
typename InputDims>
668__global__
void EigenConvolutionKernel3DGlobal(
669 InputEvaluator eval,
const internal::IndexMapper<Index, InputDims, 3, InputEvaluator::Layout> indexMapper,
670 const float* kernel, Index total, Index numX, Index numY, Index numZ, Index kernelSizeX, Index kernelSizeY,
671 Index kernelSizeZ,
float* buffer) {
672 const Index step = Index(blockDim.x) * gridDim.x;
673 for (Index output = Index(blockIdx.x) * blockDim.x + threadIdx.x; output < total;) {
674 Index remaining = output;
675 const Index x = remaining % numX;
677 const Index y = remaining % numY;
679 const Index z = remaining % numZ;
680 const Index plane = remaining / numZ;
681 const Index inputOffset = indexMapper.mapGpuInputPlaneToTensorInputOffset(plane);
683 Index kernelIndex = 0;
684 for (Index k = 0; k < kernelSizeZ; ++k) {
685 for (Index j = 0; j < kernelSizeY; ++j) {
686 for (Index i = 0; i < kernelSizeX; ++i) {
687 result += eval.coeff(inputOffset + indexMapper.mapGpuInputKernelToTensorInputOffset(x + i, y + j, z + k)) *
688 kernel[kernelIndex++];
692 buffer[indexMapper.mapGpuOutputPlaneToTensorOutputOffset(plane) +
693 indexMapper.mapGpuOutputKernelToTensorOutputOffset(x, y, z)] = result;
695 if (step >= total - output)
break;
700template <
typename InputEvaluator,
typename Index,
typename InputDims>
701__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024
void EigenConvolutionKernel3D(
702 InputEvaluator eval,
const internal::IndexMapper<Index, InputDims, 3, InputEvaluator::Layout> indexMapper,
703 const float* __restrict kernel,
const size_t numPlanes,
const size_t numX,
const size_t maxX,
const size_t numY,
704 const size_t maxY,
const size_t numZ,
const size_t maxZ,
const int kernelSizeX,
const int kernelSizeY,
705 const int kernelSizeZ,
float* buffer) {
706#if defined(EIGEN_HIPCC)
707 HIP_DYNAMIC_SHARED(
float, s)
709 extern __shared__
float s[];
713 const int first_x = blockIdx.x *
static_cast<int>(maxX);
714 const int last_x = (first_x +
static_cast<int>(maxX) <
static_cast<int>(numX) ? first_x +
static_cast<int>(maxX)
715 :
static_cast<int>(numX)) -
717 const int num_x_input = last_x - first_x + kernelSizeX;
719 const int first_y = blockIdx.y *
static_cast<int>(maxY);
720 const int last_y = (first_y +
static_cast<int>(maxY) <
static_cast<int>(numY) ? first_y +
static_cast<int>(maxY)
721 :
static_cast<int>(numY)) -
723 const int num_y_input = last_y - first_y + kernelSizeY;
725 const int first_z = blockIdx.z *
static_cast<int>(maxZ);
726 const int last_z = (first_z +
static_cast<int>(maxZ) <
static_cast<int>(numZ) ? first_z +
static_cast<int>(maxZ)
727 :
static_cast<int>(numZ)) -
729 const int num_z_input = last_z - first_z + kernelSizeZ;
731 for (
int p = 0; p < static_cast<int>(numPlanes); ++p) {
732 const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p);
733 const int plane_kernel_offset = 0;
735 for (
int k = threadIdx.z; k < num_z_input; k += blockDim.z) {
736 for (
int j = threadIdx.y; j < num_y_input; j += blockDim.y) {
737 for (
int i = threadIdx.x; i < num_x_input; i += blockDim.x) {
738 const int tensor_index = plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(
739 i + first_x, j + first_y, k + first_z);
740 s[i + num_x_input * (j + num_y_input * (k + plane_kernel_offset))] = eval.coeff(tensor_index);
748 const int num_z_output = last_z - first_z + 1;
749 const int num_y_output = last_y - first_y + 1;
750 const int num_x_output = last_x - first_x + 1;
751 const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p);
753 for (
int k = threadIdx.z; k < num_z_output; k += blockDim.z) {
754 for (
int j = threadIdx.y; j < num_y_output; j += blockDim.y) {
755 for (
int i = threadIdx.x; i < num_x_output; i += blockDim.x) {
757 for (
int n = 0; n < kernelSizeZ; ++n) {
758 for (
int m = 0; m < kernelSizeY; ++m) {
759 for (
int l = 0; l < kernelSizeX; ++l) {
760 result += s[i + l + num_x_input * (j + m + num_y_input * (k + n + plane_kernel_offset))] *
761 kernel[l + kernelSizeX * (m + kernelSizeY * n)];
765 const int tensor_index = plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(
766 i + first_x, j + first_y, k + first_z);
767 buffer[tensor_index] = result;
775template <
typename Indices,
typename InputArgType,
typename KernelArgType>
777 using XprType = TensorConvolutionOp<Indices, InputArgType, KernelArgType>;
779 static constexpr int NumDims =
780 internal::array_size<typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions>::value;
781 static constexpr int NumKernelDims = internal::array_size<Indices>::value;
782 using Index =
typename XprType::Index;
783 using Dimensions = DSizes<Index, NumDims>;
784 using KernelDimensions =
typename TensorEvaluator<KernelArgType, GpuDevice>::Dimensions;
786 static constexpr int Layout = TensorEvaluator<InputArgType, GpuDevice>::Layout;
788 IsAligned = int(TensorEvaluator<InputArgType, GpuDevice>::IsAligned) &
789 int(TensorEvaluator<KernelArgType, GpuDevice>::IsAligned),
790 PacketAccess =
false,
792 PreferBlockAccess =
false,
798 using TensorBlock = internal::TensorBlockNotImplemented;
801 TensorEvaluator(
const XprType& op,
const GpuDevice& device)
802 : m_inputImpl(op.inputExpression(), device),
803 m_kernelImpl(op.kernelExpression(), device),
804 m_kernelArg(op.kernelExpression()),
805 m_indices(op.indices()),
808 m_local_kernel(false),
810 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<InputArgType, GpuDevice>::Layout) ==
811 static_cast<int>(TensorEvaluator<KernelArgType, GpuDevice>::Layout)),
812 YOU_MADE_A_PROGRAMMING_MISTAKE);
814 const typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions& input_dims = m_inputImpl.dimensions();
815 const typename TensorEvaluator<KernelArgType, GpuDevice>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
817 m_dimensions = m_inputImpl.dimensions();
818 for (
int i = 0; i < NumKernelDims; ++i) {
819 const Index index = op.indices()[i];
820 const Index input_dim = input_dims[index];
821 const Index kernel_dim = kernel_dims[i];
822 const Index result_dim = input_dim - kernel_dim + 1;
823 m_dimensions[index] = result_dim;
827 using CoeffReturnType =
typename XprType::CoeffReturnType;
828 using PacketReturnType =
typename PacketType<CoeffReturnType, GpuDevice>::type;
829 using Scalar =
typename InputArgType::Scalar;
830 static constexpr int PacketSize = internal::unpacket_traits<PacketReturnType>::size;
832 EIGEN_DEVICE_FUNC
const Dimensions& dimensions()
const {
return m_dimensions; }
834 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(Scalar* data) {
836 m_inputImpl.evalSubExprsIfNeeded(
nullptr);
841 m_buf = (Scalar*)m_device.allocate(dimensions().TotalSize() *
sizeof(Scalar));
847 EIGEN_STRONG_INLINE
void cleanup() {
848 m_inputImpl.cleanup();
850 m_device.deallocate(m_buf);
853 if (m_local_kernel) {
854 m_device.deallocate((
void*)m_kernel);
855 m_local_kernel =
false;
860 EIGEN_STRONG_INLINE
void preloadKernel() {
863 const Scalar* in_place = m_kernelImpl.data();
866 m_local_kernel =
false;
868 size_t kernel_sz = m_kernelImpl.dimensions().TotalSize() *
sizeof(Scalar);
869 Scalar* local = (Scalar*)m_device.allocate(kernel_sz);
870 using EvalTo = TensorEvalToOp<const KernelArgType>;
871 EvalTo evalToTmp(local, m_kernelArg);
872 const bool PacketAccess = internal::IsVectorizable<GpuDevice, KernelArgType>::value;
873 internal::TensorExecutor<const EvalTo, GpuDevice, PacketAccess>::run(evalToTmp, m_device);
876 m_local_kernel =
true;
880 static unsigned int ceil(
unsigned int num,
unsigned int denom) {
881 const unsigned int rounded_toward_zero = num / denom;
882 if (num > rounded_toward_zero * denom) {
883 return rounded_toward_zero + 1;
885 return rounded_toward_zero;
888 void executeEval(Scalar* data)
const {
889 using InputDims =
typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions;
891 const int maxSharedMem = m_device.sharedMemPerBlock();
892 const int maxThreadsPerBlock = m_device.maxGpuThreadsPerBlock();
893 const int maxBlocksPerProcessor = m_device.maxGpuThreadsPerMultiProcessor() / maxThreadsPerBlock;
894 const int numMultiProcessors = m_device.getNumGpuMultiProcessors();
895 const int warpSize = 32;
897 switch (NumKernelDims) {
899 const int kernel_size = m_kernelImpl.dimensions().TotalSize();
901 const int numX = dimensions()[m_indices[0]];
902 const int numP = dimensions().TotalSize() / numX;
906 const int single_stride_dim =
907 static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : m_inputImpl.dimensions().rank() - 1;
908 if (m_indices[0] == single_stride_dim) {
910 const int inner_dim = ((maxSharedMem / (
sizeof(Scalar)) - kernel_size + 1 + 31) / 32) * 32;
911 maxX = numext::mini<int>(inner_dim, numX);
912 const int maxP = numext::mini<int>(maxSharedMem / ((kernel_size - 1 + maxX) *
sizeof(Scalar)), numP);
913 block_size.x = numext::mini(maxThreadsPerBlock, maxX);
914 block_size.y = numext::mini<int>(maxThreadsPerBlock / block_size.x, maxP);
917 const int inner_dim = maxSharedMem / ((warpSize + kernel_size) *
sizeof(Scalar));
918 const int maxP = numext::mini<int>(inner_dim, numP);
919 maxX = numext::mini<int>(maxSharedMem / (inner_dim *
sizeof(Scalar)) - kernel_size + 1, numX);
921 block_size.x = numext::mini(warpSize, maxX);
922 block_size.y = numext::mini<int>(maxThreadsPerBlock / block_size.x, maxP);
925 const int shared_mem = block_size.y * (maxX + kernel_size - 1) *
sizeof(Scalar);
926 gpu_assert(shared_mem <= maxSharedMem);
928 const int num_x_blocks = ceil(numX, maxX);
929 const int blocksPerProcessor = numext::mini(maxBlocksPerProcessor, maxSharedMem / shared_mem);
930 const int num_y_blocks = ceil(numMultiProcessors * blocksPerProcessor, num_x_blocks);
932 dim3 num_blocks(num_x_blocks, numext::mini<int>(num_y_blocks, ceil(numP, block_size.y)));
934 const array<Index, 1> indices{m_indices[0]};
935 const array<Index, 1> kernel_dims{m_kernelImpl.dimensions()[0]};
936 internal::IndexMapper<Index, InputDims, 1, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices);
937 switch (kernel_size) {
939 LAUNCH_GPU_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4>),
940 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP,
941 numX, maxX, 4, data);
945 LAUNCH_GPU_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7>),
946 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP,
947 numX, maxX, 7, data);
952 (EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic>),
953 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
961 constexpr int idxX =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : 1;
962 constexpr int idxY =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 1 : 0;
963 const int kernel_size_x = m_kernelImpl.dimensions()[idxX];
964 const int kernel_size_y = m_kernelImpl.dimensions()[idxY];
966 const int numX = dimensions()[m_indices[idxX]];
967 const int numY = dimensions()[m_indices[idxY]];
968 const int numP = dimensions().TotalSize() / (numX * numY);
970 const float scaling_factor =
971 sqrtf(
static_cast<float>(maxSharedMem) / (
sizeof(Scalar) * kernel_size_y * kernel_size_x));
974 int inner_dim = ((
static_cast<int>(scaling_factor * kernel_size_x) - kernel_size_x + 1 + 32) / 32) * 32;
975 const int maxX = numext::mini<int>(inner_dim, numX);
977 numext::mini<int>(maxSharedMem / (
sizeof(Scalar) * (maxX + kernel_size_x - 1)) - kernel_size_y + 1, numY);
978 const int maxP = numext::mini<int>(
979 maxSharedMem / ((kernel_size_x - 1 + maxX) * (kernel_size_y - 1 + maxY) *
sizeof(Scalar)), numP);
982 block_size.x = numext::mini(1024, maxX);
983 block_size.y = numext::mini<int>(1024 / block_size.x, maxY);
984 block_size.z = numext::mini<int>(1024 / (block_size.x * block_size.y), maxP);
986 const int shared_mem = block_size.z * (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) *
sizeof(Scalar);
987 gpu_assert(shared_mem <= maxSharedMem);
989 const int num_x_blocks = ceil(numX, maxX);
990 const int num_y_blocks = ceil(numY, maxY);
991 const int blocksPerProcessor = numext::mini(maxBlocksPerProcessor, maxSharedMem / shared_mem);
992 const int num_z_blocks = ceil(numMultiProcessors * blocksPerProcessor, num_x_blocks * num_y_blocks);
994 dim3 num_blocks(num_x_blocks, num_y_blocks, numext::mini<int>(num_z_blocks, ceil(numP, block_size.z)));
996 const array<Index, 2> indices{m_indices[idxX], m_indices[idxY]};
997 const array<Index, 2> kernel_dims{m_kernelImpl.dimensions()[idxX], m_kernelImpl.dimensions()[idxY]};
998 internal::IndexMapper<Index, InputDims, 2, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices);
999 switch (kernel_size_x) {
1001 switch (kernel_size_y) {
1004 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, 7>),
1005 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1006 numY, maxY, 4, 7, data);
1011 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, Dynamic>),
1012 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1013 numY, maxY, 4, kernel_size_y, data);
1020 switch (kernel_size_y) {
1023 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, 4>),
1024 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1025 numY, maxY, 7, 4, data);
1030 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, Dynamic>),
1031 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1032 numY, maxY, 7, kernel_size_y, data);
1039 LAUNCH_GPU_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims,
1041 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP,
1042 numX, maxX, numY, maxY, kernel_size_x, kernel_size_y, data);
1050 constexpr int idxX =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : 2;
1051 constexpr int idxY =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 1 : 1;
1052 constexpr int idxZ =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 2 : 0;
1054 const int kernel_size_x = m_kernelImpl.dimensions()[idxX];
1055 const int kernel_size_y = m_kernelImpl.dimensions()[idxY];
1056 const int kernel_size_z = m_kernelImpl.dimensions()[idxZ];
1058 const int numX = dimensions()[m_indices[idxX]];
1059 const int numY = dimensions()[m_indices[idxY]];
1060 const int numZ = dimensions()[m_indices[idxZ]];
1061 const Index total = dimensions().TotalSize();
1062 if (total == 0)
return;
1064 const array<Index, 3> indices{m_indices[idxX], m_indices[idxY], m_indices[idxZ]};
1065 const array<Index, 3> kernel_dims{m_kernelImpl.dimensions()[idxX], m_kernelImpl.dimensions()[idxY],
1066 m_kernelImpl.dimensions()[idxZ]};
1067 internal::IndexMapper<Index, InputDims, 3, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices);
1069 if (m_kernelImpl.dimensions().TotalSize() > Index(maxSharedMem /
sizeof(Scalar))) {
1070 constexpr int preferredBlockSize = 256;
1071 const int threads = numext::mini(preferredBlockSize, maxThreadsPerBlock);
1072 const int maxBlocks =
static_cast<int>(
1073 numext::mini<int64_t>(int64_t(numMultiProcessors) * 32, NumTraits<Index>::highest() / threads));
1074 const int blocks =
static_cast<int>(numext::mini<Index>((total - 1) / threads + 1, Index(maxBlocks)));
1076 (EigenConvolutionKernel3DGlobal<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims>), blocks,
1077 threads, 0, m_device, m_inputImpl, indexMapper, m_kernel, total, Index(numX), Index(numY), Index(numZ),
1078 Index(kernel_size_x), Index(kernel_size_y), Index(kernel_size_z), data);
1082 const int numP = total / (Index(numX) * numY * numZ);
1083 const int maxX = numext::mini<int>(
1084 128, numext::mini<int>(maxSharedMem / (
sizeof(Scalar) * kernel_size_y * kernel_size_z) - kernel_size_x + 1,
1086 const int maxY = numext::mini<int>(
1087 128, numext::mini<int>(
1088 maxSharedMem / (
sizeof(Scalar) * (maxX + kernel_size_x - 1) * kernel_size_z) - kernel_size_y + 1,
1090 const int maxZ = numext::mini<int>(
1091 128, numext::mini<int>(
1092 maxSharedMem / (
sizeof(Scalar) * (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1)) -
1097 block_size.x = numext::mini(32, maxX);
1098 block_size.y = numext::mini(32, maxY);
1099 block_size.z = numext::mini<int>(1024 / (block_size.x * block_size.y), maxZ);
1100 dim3 num_blocks(ceil(numX, maxX), ceil(numY, maxY), ceil(numZ, maxZ));
1102 const int shared_mem =
1103 (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) * (maxZ + kernel_size_z - 1) *
sizeof(Scalar);
1104 gpu_assert(shared_mem <= maxSharedMem);
1106 LAUNCH_GPU_KERNEL((EigenConvolutionKernel3D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims>),
1107 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX,
1108 maxX, numY, maxY, numZ, maxZ, kernel_size_x, kernel_size_y, kernel_size_z, data);
1113 EIGEN_STATIC_ASSERT((NumKernelDims >= 1 && NumKernelDims <= 3),
1114 THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE);
1119 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
1120 eigen_assert(m_buf);
1121 eigen_assert(index < m_dimensions.TotalSize());
1122 return m_buf[index];
1125 template <
int LoadMode>
1126 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(
const Index index)
const {
1127 eigen_assert(m_buf);
1128 eigen_assert(index < m_dimensions.TotalSize());
1129 return internal::ploadt<PacketReturnType, LoadMode>(m_buf + index);
1132 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
1135 const double kernel_size = m_kernelImpl.dimensions().TotalSize();
1137 const double convolve_compute_cost = TensorOpCost::AddCost<Scalar>() + TensorOpCost::MulCost<Scalar>();
1138 const double firstIndex_compute_cost =
1140 (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() + TensorOpCost::DivCost<Index>());
1141 return TensorOpCost(0, 0, firstIndex_compute_cost, vectorized, PacketSize) +
1142 kernel_size * (m_inputImpl.costPerCoeff(vectorized) + m_kernelImpl.costPerCoeff(vectorized) +
1143 TensorOpCost(0, 0, convolve_compute_cost, vectorized, PacketSize));
1147 TensorEvaluator<InputArgType, GpuDevice> m_inputImpl;
1148 TensorEvaluator<KernelArgType, GpuDevice> m_kernelImpl;
1149 KernelArgType m_kernelArg;
1151 Dimensions m_dimensions;
1153 const Scalar* m_kernel;
1154 bool m_local_kernel;
1156 const GpuDevice& m_device;
The tensor base class.
Definition TensorForwardDeclarations.h:69
Definition TensorConvolution.h:223
const internal::remove_all_t< typename InputXprType::Nested > & inputExpression() const
Definition TensorConvolution.h:240
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47