Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorConvolution.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TENSOR_TENSOR_CONVOLUTION_H
12#define EIGEN_TENSOR_TENSOR_CONVOLUTION_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20
21template <typename Index, typename InputDims, int NumKernelDims, int Layout>
22class IndexMapper {
23 public:
24 IndexMapper(const InputDims& input_dims, const array<Index, NumKernelDims>& kernel_dims,
25 const array<Index, NumKernelDims>& indices) {
26 array<Index, NumDims> dimensions = input_dims;
27 for (int i = 0; i < NumKernelDims; ++i) {
28 const Index index = indices[i];
29 const Index input_dim = input_dims[index];
30 const Index kernel_dim = kernel_dims[i];
31 const Index result_dim = input_dim - kernel_dim + 1;
32 dimensions[index] = result_dim;
33 }
34
35 array<Index, NumDims> inputStrides;
36 array<Index, NumDims> outputStrides;
37 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
38 inputStrides[0] = 1;
39 outputStrides[0] = 1;
40 for (int i = 1; i < NumDims; ++i) {
41 inputStrides[i] = inputStrides[i - 1] * input_dims[i - 1];
42 outputStrides[i] = outputStrides[i - 1] * dimensions[i - 1];
43 }
44 } else {
45 inputStrides[NumDims - 1] = 1;
46 outputStrides[NumDims - 1] = 1;
47 for (int i = static_cast<int>(NumDims) - 2; i >= 0; --i) {
48 inputStrides[i] = inputStrides[i + 1] * input_dims[i + 1];
49 outputStrides[i] = outputStrides[i + 1] * dimensions[i + 1];
50 }
51 }
52
53 array<Index, NumDims> gpuInputDimensions;
54 array<Index, NumDims> gpuOutputDimensions;
55 array<Index, NumDims> tmp = dimensions;
56 array<Index, NumDims> ordering;
57 constexpr size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims;
58 for (int i = 0; i < NumKernelDims; ++i) {
59 const Index index = i + offset;
60 ordering[index] = indices[i];
61 tmp[indices[i]] = -1;
62 gpuInputDimensions[index] = input_dims[indices[i]];
63 gpuOutputDimensions[index] = dimensions[indices[i]];
64 }
65
66 int written = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? NumKernelDims : 0;
67 for (int i = 0; i < NumDims; ++i) {
68 if (tmp[i] >= 0) {
69 ordering[written] = i;
70 gpuInputDimensions[written] = input_dims[i];
71 gpuOutputDimensions[written] = dimensions[i];
72 ++written;
73 }
74 }
75
76 for (int i = 0; i < NumDims; ++i) {
77 m_inputStrides[i] = inputStrides[ordering[i]];
78 m_outputStrides[i] = outputStrides[ordering[i]];
79 }
80
81 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
82 for (int i = 0; i < NumDims; ++i) {
83 if (i > NumKernelDims) {
84 m_gpuInputStrides[i] = m_gpuInputStrides[i - 1] * gpuInputDimensions[i - 1];
85 m_gpuOutputStrides[i] = m_gpuOutputStrides[i - 1] * gpuOutputDimensions[i - 1];
86 } else {
87 m_gpuInputStrides[i] = 1;
88 m_gpuOutputStrides[i] = 1;
89 }
90 }
91 } else {
92 for (int i = NumDims - 1; i >= 0; --i) {
93 if (i + 1 < static_cast<int>(offset)) {
94 m_gpuInputStrides[i] = m_gpuInputStrides[i + 1] * gpuInputDimensions[i + 1];
95 m_gpuOutputStrides[i] = m_gpuOutputStrides[i + 1] * gpuOutputDimensions[i + 1];
96 } else {
97 m_gpuInputStrides[i] = 1;
98 m_gpuOutputStrides[i] = 1;
99 }
100 }
101 }
102 }
103
104 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputPlaneToTensorInputOffset(Index p) const {
105 Index inputIndex = 0;
106 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
107 for (int d = NumDims - 1; d > NumKernelDims; --d) {
108 const Index idx = p / m_gpuInputStrides[d];
109 inputIndex += idx * m_inputStrides[d];
110 p -= idx * m_gpuInputStrides[d];
111 }
112 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
113 inputIndex += p * m_inputStrides[NumKernelDims];
114 }
115 } else {
116 std::ptrdiff_t limit = 0;
117 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
118 limit = NumDims - NumKernelDims - 1;
119 }
120 for (int d = 0; d < limit; ++d) {
121 const Index idx = p / m_gpuInputStrides[d];
122 inputIndex += idx * m_inputStrides[d];
123 p -= idx * m_gpuInputStrides[d];
124 }
125 inputIndex += p * m_inputStrides[limit];
126 }
127 return inputIndex;
128 }
129
130 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputPlaneToTensorOutputOffset(Index p) const {
131 Index outputIndex = 0;
132 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
133 for (int d = NumDims - 1; d > NumKernelDims; --d) {
134 const Index idx = p / m_gpuOutputStrides[d];
135 outputIndex += idx * m_outputStrides[d];
136 p -= idx * m_gpuOutputStrides[d];
137 }
138 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
139 outputIndex += p * m_outputStrides[NumKernelDims];
140 }
141 } else {
142 std::ptrdiff_t limit = 0;
143 EIGEN_IF_CONSTEXPR (NumKernelDims < NumDims) {
144 limit = NumDims - NumKernelDims - 1;
145 }
146 for (int d = 0; d < limit; ++d) {
147 const Index idx = p / m_gpuOutputStrides[d];
148 outputIndex += idx * m_outputStrides[d];
149 p -= idx * m_gpuOutputStrides[d];
150 }
151 outputIndex += p * m_outputStrides[limit];
152 }
153 return outputIndex;
154 }
155
156 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i) const {
157 constexpr size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims;
158 return i * m_inputStrides[offset];
159 }
160
161 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i) const {
162 constexpr size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims;
163 return i * m_outputStrides[offset];
164 }
165
166 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j) const {
167 constexpr size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims;
168 return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1];
169 }
170
171 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j) const {
172 constexpr size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims;
173 return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1];
174 }
175
176 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuInputKernelToTensorInputOffset(Index i, Index j, Index k) const {
177 constexpr size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims;
178 return i * m_inputStrides[offset] + j * m_inputStrides[offset + 1] + k * m_inputStrides[offset + 2];
179 }
180
181 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Index mapGpuOutputKernelToTensorOutputOffset(Index i, Index j, Index k) const {
182 constexpr size_t offset = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : NumDims - NumKernelDims;
183 return i * m_outputStrides[offset] + j * m_outputStrides[offset + 1] + k * m_outputStrides[offset + 2];
184 }
185
186 private:
187 static constexpr int NumDims = internal::array_size<InputDims>::value;
188 array<Index, NumDims> m_inputStrides;
189 array<Index, NumDims> m_outputStrides;
190 array<Index, NumDims> m_gpuInputStrides;
191 array<Index, NumDims> m_gpuOutputStrides;
192};
193
194template <typename Dimensions, typename InputXprType, typename KernelXprType>
195struct traits<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType> > {
196 // Type promotion to handle the case where the types of the lhs and the rhs are different.
197 using Scalar = typename promote_storage_type<typename InputXprType::Scalar, typename KernelXprType::Scalar>::ret;
198 using StorageKind = typename promote_storage_type<typename traits<InputXprType>::StorageKind,
199 typename traits<KernelXprType>::StorageKind>::ret;
200 using Index =
201 typename promote_index_type<typename traits<InputXprType>::Index, typename traits<KernelXprType>::Index>::type;
202 static constexpr int NumDimensions = traits<InputXprType>::NumDimensions;
203 static constexpr int Layout = traits<InputXprType>::Layout;
204 using PointerType =
205 std::conditional_t<Pointer_type_promotion<typename InputXprType::Scalar, Scalar>::val,
206 typename traits<InputXprType>::PointerType, typename traits<KernelXprType>::PointerType>;
207
208 enum { Flags = 0 };
209};
210
211template <typename Dimensions, typename InputXprType, typename KernelXprType>
212struct eval<TensorConvolutionOp<Dimensions, InputXprType, KernelXprType>, Eigen::Dense> {
213 using type = const TensorConvolutionOp<Dimensions, InputXprType, KernelXprType>&;
214};
215
216} // end namespace internal
217
221template <typename Indices, typename InputXprType, typename KernelXprType>
222class TensorConvolutionOp
223 : public TensorBase<TensorConvolutionOp<Indices, InputXprType, KernelXprType>, ReadOnlyAccessors> {
224 public:
225 using Scalar = typename Eigen::internal::traits<TensorConvolutionOp>::Scalar;
226 using RealScalar = typename Eigen::NumTraits<Scalar>::Real;
227 using CoeffReturnType = typename internal::promote_storage_type<typename InputXprType::CoeffReturnType,
228 typename KernelXprType::CoeffReturnType>::ret;
229 using Nested = typename Eigen::internal::ref_selector<TensorConvolutionOp>::type;
230 using StorageKind = typename Eigen::internal::traits<TensorConvolutionOp>::StorageKind;
231 using Index = typename Eigen::internal::traits<TensorConvolutionOp>::Index;
232
233 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorConvolutionOp(const InputXprType& input, const KernelXprType& kernel,
234 const Indices& dims)
235 : m_input_xpr(input), m_kernel_xpr(kernel), m_indices(dims) {}
236
237 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Indices& indices() const { return m_indices; }
238
240 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const internal::remove_all_t<typename InputXprType::Nested>& inputExpression()
241 const {
242 return m_input_xpr;
243 }
244
245 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const internal::remove_all_t<typename KernelXprType::Nested>& kernelExpression()
246 const {
247 return m_kernel_xpr;
248 }
249
250 protected:
251 typename InputXprType::Nested m_input_xpr;
252 typename KernelXprType::Nested m_kernel_xpr;
253 const Indices m_indices;
254};
255
256template <typename Indices, typename InputArgType, typename KernelArgType, typename Device>
257struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelArgType>, Device> {
258 using XprType = TensorConvolutionOp<Indices, InputArgType, KernelArgType>;
259
260 static constexpr int NumDims =
261 internal::array_size<typename TensorEvaluator<InputArgType, Device>::Dimensions>::value;
262 static constexpr int NumKernelDims = internal::array_size<Indices>::value;
263 using Index = typename XprType::Index;
264 using Dimensions = DSizes<Index, NumDims>;
265
266 using Scalar = typename XprType::Scalar;
267 using CoeffReturnType = typename XprType::CoeffReturnType;
268 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
269 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
270 using Storage = StorageMemory<Scalar, Device>;
271 using EvaluatorPointerType = typename Storage::Type;
272
273 static constexpr int Layout = TensorEvaluator<InputArgType, Device>::Layout;
274 enum {
275 IsAligned =
276 int(TensorEvaluator<InputArgType, Device>::IsAligned) & int(TensorEvaluator<KernelArgType, Device>::IsAligned),
277 PacketAccess = int(TensorEvaluator<InputArgType, Device>::PacketAccess) &
278 int(TensorEvaluator<KernelArgType, Device>::PacketAccess),
279 BlockAccess = false,
280 PreferBlockAccess = false,
281 CoordAccess = false, // to be implemented
282 RawAccess = false
283 };
284
285 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
286 using TensorBlock = internal::TensorBlockNotImplemented;
287 //===--------------------------------------------------------------------===//
288
289 EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
290 : m_inputImpl(op.inputExpression(), device),
291 m_kernelImpl(op.kernelExpression(), device),
292 m_kernelArg(op.kernelExpression()),
293 m_kernel(nullptr),
294 m_local_kernel(false),
295 m_device(device) {
296 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<InputArgType, Device>::Layout) ==
297 static_cast<int>(TensorEvaluator<KernelArgType, Device>::Layout)),
298 YOU_MADE_A_PROGRAMMING_MISTAKE);
299
300 const typename TensorEvaluator<InputArgType, Device>::Dimensions& input_dims = m_inputImpl.dimensions();
301 const typename TensorEvaluator<KernelArgType, Device>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
302
303 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
304 m_inputStride[0] = 1;
305 for (int i = 1; i < NumDims; ++i) {
306 m_inputStride[i] = m_inputStride[i - 1] * input_dims[i - 1];
307 }
308 } else {
309 m_inputStride[NumDims - 1] = 1;
310 for (int i = NumDims - 2; i >= 0; --i) {
311 m_inputStride[i] = m_inputStride[i + 1] * input_dims[i + 1];
312 }
313 }
314
315 m_dimensions = m_inputImpl.dimensions();
316 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
317 for (int i = 0; i < NumKernelDims; ++i) {
318 const Index index = op.indices()[i];
319 const Index input_dim = input_dims[index];
320 const Index kernel_dim = kernel_dims[i];
321 const Index result_dim = input_dim - kernel_dim + 1;
322 m_dimensions[index] = result_dim;
323 if (i > 0) {
324 m_kernelStride[i] = m_kernelStride[i - 1] * kernel_dims[i - 1];
325 } else {
326 m_kernelStride[0] = 1;
327 }
328 m_indexStride[i] = m_inputStride[index];
329 }
330
331 m_outputStride[0] = 1;
332 for (int i = 1; i < NumDims; ++i) {
333 m_outputStride[i] = m_outputStride[i - 1] * m_dimensions[i - 1];
334 }
335 } else {
336 for (int i = NumKernelDims - 1; i >= 0; --i) {
337 const Index index = op.indices()[i];
338 const Index input_dim = input_dims[index];
339 const Index kernel_dim = kernel_dims[i];
340 const Index result_dim = input_dim - kernel_dim + 1;
341 m_dimensions[index] = result_dim;
342 if (i < NumKernelDims - 1) {
343 m_kernelStride[i] = m_kernelStride[i + 1] * kernel_dims[i + 1];
344 } else {
345 m_kernelStride[NumKernelDims - 1] = 1;
346 }
347 m_indexStride[i] = m_inputStride[index];
348 }
349
350 m_outputStride[NumDims - 1] = 1;
351 for (int i = NumDims - 2; i >= 0; --i) {
352 m_outputStride[i] = m_outputStride[i + 1] * m_dimensions[i + 1];
353 }
354 }
355 }
356
357 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }
358
359 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar*) {
360 m_inputImpl.evalSubExprsIfNeeded(nullptr);
361 preloadKernel();
362 return true;
363 }
364 EIGEN_STRONG_INLINE void cleanup() {
365 m_inputImpl.cleanup();
366 if (m_local_kernel) {
367 m_device.deallocate((void*)m_kernel);
368 m_local_kernel = false;
369 }
370 m_kernel = nullptr;
371 }
372
373 void evalTo(typename XprType::Scalar* buffer) {
374 evalSubExprsIfNeeded(nullptr);
375 for (int i = 0; i < dimensions().TotalSize(); ++i) {
376 buffer[i] += coeff(i);
377 }
378 cleanup();
379 }
380
381 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
382 CoeffReturnType result = CoeffReturnType(0);
383 convolve(firstInput(index), 0, NumKernelDims - 1, result);
384 return result;
385 }
386
387 template <int LoadMode>
388 EIGEN_DEVICE_FUNC PacketReturnType packet(const Index index) const {
389 Index indices[2] = {index, index + PacketSize - 1};
390 Index startInputs[2] = {0, 0};
391 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
392 for (int i = NumDims - 1; i > 0; --i) {
393 const Index idx0 = indices[0] / m_outputStride[i];
394 const Index idx1 = indices[1] / m_outputStride[i];
395 startInputs[0] += idx0 * m_inputStride[i];
396 startInputs[1] += idx1 * m_inputStride[i];
397 indices[0] -= idx0 * m_outputStride[i];
398 indices[1] -= idx1 * m_outputStride[i];
399 }
400 } else {
401 for (int i = 0; i < NumDims - 1; ++i) {
402 const Index idx0 = indices[0] / m_outputStride[i];
403 const Index idx1 = indices[1] / m_outputStride[i];
404 startInputs[0] += idx0 * m_inputStride[i];
405 startInputs[1] += idx1 * m_inputStride[i];
406 indices[0] -= idx0 * m_outputStride[i];
407 indices[1] -= idx1 * m_outputStride[i];
408 }
409 }
410 startInputs[0] += indices[0];
411 startInputs[1] += indices[1];
412
413 if (startInputs[1] - startInputs[0] == PacketSize - 1) {
414 PacketReturnType result = internal::pset1<PacketReturnType>(0);
415 convolvePacket(startInputs[0], 0, NumKernelDims - 1, result);
416 return result;
417 } else {
418 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) Scalar data[PacketSize];
419 data[0] = Scalar(0);
420 convolve(startInputs[0], 0, NumKernelDims - 1, data[0]);
421 for (int i = 1; i < PacketSize - 1; ++i) {
422 data[i] = Scalar(0);
423 convolve(firstInput(index + i), 0, NumKernelDims - 1, data[i]);
424 }
425 data[PacketSize - 1] = Scalar(0);
426 convolve(startInputs[1], 0, NumKernelDims - 1, data[PacketSize - 1]);
427 return internal::pload<PacketReturnType>(data);
428 }
429 }
430
431 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
432 const double kernel_size = m_kernelImpl.dimensions().TotalSize();
433 // We ignore the use of fused multiply-add.
434 const double convolve_compute_cost = TensorOpCost::AddCost<Scalar>() + TensorOpCost::MulCost<Scalar>();
435 const double firstIndex_compute_cost =
436 NumDims *
437 (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() + TensorOpCost::DivCost<Index>());
438 return TensorOpCost(0, 0, firstIndex_compute_cost, vectorized, PacketSize) +
439 kernel_size * (m_inputImpl.costPerCoeff(vectorized) + m_kernelImpl.costPerCoeff(vectorized) +
440 TensorOpCost(0, 0, convolve_compute_cost, vectorized, PacketSize));
441 }
442
443 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return nullptr; }
444
445 private:
446 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index firstInput(Index index) const {
447 Index startInput = 0;
448 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
449 for (int i = NumDims - 1; i > 0; --i) {
450 const Index idx = index / m_outputStride[i];
451 startInput += idx * m_inputStride[i];
452 index -= idx * m_outputStride[i];
453 }
454 } else {
455 for (int i = 0; i < NumDims - 1; ++i) {
456 const Index idx = index / m_outputStride[i];
457 startInput += idx * m_inputStride[i];
458 index -= idx * m_outputStride[i];
459 }
460 }
461 startInput += index;
462 return startInput;
463 }
464
465 EIGEN_DEVICE_FUNC void convolve(Index firstIndex, Index firstKernel, int DimIndex, CoeffReturnType& accum) const {
466 for (int j = 0; j < m_kernelImpl.dimensions()[DimIndex]; ++j) {
467 const Index input = firstIndex + j * m_indexStride[DimIndex];
468 const Index kernel = firstKernel + j * m_kernelStride[DimIndex];
469 if (DimIndex > 0) {
470 convolve(input, kernel, DimIndex - 1, accum);
471 } else {
472 accum += m_inputImpl.coeff(input) * m_kernel[kernel];
473 }
474 }
475 }
476
477 template <typename Packet>
478 EIGEN_DEVICE_FUNC void convolvePacket(Index firstIndex, Index firstKernel, int DimIndex, Packet& accum) const {
479 for (int j = 0; j < m_kernelImpl.dimensions()[DimIndex]; ++j) {
480 const Index input = firstIndex + j * m_indexStride[DimIndex];
481 const Index kernel = firstKernel + j * m_kernelStride[DimIndex];
482 if (DimIndex > 0) {
483 convolvePacket(input, kernel, DimIndex - 1, accum);
484 } else {
485 accum = internal::pmadd<Packet>(m_inputImpl.template packet<Unaligned>(input),
486 internal::pset1<Packet>(m_kernel[kernel]), accum);
487 }
488 }
489 }
490
491 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void preloadKernel() {
492 // Don't make a local copy of the kernel unless we have to (i.e. it's an
493 // expression that needs to be evaluated)
494 const Scalar* in_place = m_kernelImpl.data();
495 if (in_place) {
496 m_kernel = in_place;
497 m_local_kernel = false;
498 } else {
499 size_t kernel_sz = m_kernelImpl.dimensions().TotalSize() * sizeof(Scalar);
500 Scalar* local = (Scalar*)m_device.allocate_temp(kernel_sz);
501 using EvalTo = TensorEvalToOp<const KernelArgType>;
502 EvalTo evalToTmp(local, m_kernelArg);
503 const bool Vectorize = internal::IsVectorizable<Device, KernelArgType>::value;
504 internal::TensorExecutor<const EvalTo, Device, Vectorize>::run(evalToTmp, m_device);
505
506 m_kernel = local;
507 m_local_kernel = true;
508 }
509 }
510
511 array<Index, NumDims> m_inputStride;
512 array<Index, NumDims> m_outputStride;
513
514 array<Index, NumKernelDims> m_indexStride;
515 array<Index, NumKernelDims> m_kernelStride;
516 TensorEvaluator<InputArgType, Device> m_inputImpl;
517 TensorEvaluator<KernelArgType, Device> m_kernelImpl;
518 Dimensions m_dimensions;
519
520 KernelArgType m_kernelArg;
521 const Scalar* m_kernel;
522 bool m_local_kernel;
523 const Device EIGEN_DEVICE_REF m_device;
524};
525
526// Use an optimized implementation of the evaluation code for GPUs whenever possible.
527#if defined(EIGEN_USE_GPU) && defined(EIGEN_GPUCC)
528
529template <int StaticKernelSize>
530struct GetKernelSize {
531 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int operator()(const int /*kernelSize*/) const { return StaticKernelSize; }
532};
533template <>
534struct GetKernelSize<Dynamic> {
535 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE int operator()(const int kernelSize) const { return kernelSize; }
536};
537
538#if EIGEN_COMP_CLANG
539#pragma clang diagnostic push
540// Runtime tile bounds can prevent full unrolling; the hints are best-effort.
541#pragma clang diagnostic ignored "-Wpass-failed"
542#endif
543
544template <typename InputEvaluator, typename Index, typename InputDims, int StaticKernelSize>
545__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void EigenConvolutionKernel1D(
546 InputEvaluator eval, const internal::IndexMapper<Index, InputDims, 1, InputEvaluator::Layout> indexMapper,
547 const float* __restrict kernel, const int numPlanes, const int numX, const int maxX, const int kernelSize,
548 float* buffer) {
549#if defined(EIGEN_HIPCC)
550 HIP_DYNAMIC_SHARED(float, s)
551#else
552 extern __shared__ float s[];
553#endif
554
555 const int first_x = blockIdx.x * maxX;
556 const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1;
557 const int num_x_input = last_x - first_x + GetKernelSize<StaticKernelSize>()(kernelSize);
558 const int num_x_output = last_x - first_x + 1;
559
560 const int first_plane = blockIdx.y * blockDim.y;
561 const int plane_stride = blockDim.y * gridDim.y;
562
563 for (int p = first_plane + threadIdx.y; p < numPlanes; p += plane_stride) {
564 // Load inputs to shared memory
565 const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p);
566 const int plane_kernel_offset = threadIdx.y * num_x_input;
567#pragma unroll
568 for (int i = threadIdx.x; i < num_x_input; i += blockDim.x) {
569 const int tensor_index = plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(i + first_x);
570 s[i + plane_kernel_offset] = eval.coeff(tensor_index);
571 }
572
573 __syncthreads();
574
575 // Compute the convolution
576 const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p);
577
578#pragma unroll
579 for (int i = threadIdx.x; i < num_x_output; i += blockDim.x) {
580 const int kernel_offset = plane_kernel_offset + i;
581 float result = 0.0f;
582#pragma unroll
583 for (int k = 0; k < GetKernelSize<StaticKernelSize>()(kernelSize); ++k) {
584 result += s[k + kernel_offset] * kernel[k];
585 }
586 const int tensor_index = plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(i + first_x);
587 buffer[tensor_index] = result;
588 }
589 __syncthreads();
590 }
591}
592
593template <typename InputEvaluator, typename Index, typename InputDims, int StaticKernelSizeX, int StaticKernelSizeY>
594__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void EigenConvolutionKernel2D(
595 InputEvaluator eval, const internal::IndexMapper<Index, InputDims, 2, InputEvaluator::Layout> indexMapper,
596 const float* __restrict kernel, const int numPlanes, const int numX, const int maxX, const int numY, const int maxY,
597 const int kernelSizeX, const int kernelSizeY, float* buffer) {
598#if defined(EIGEN_HIPCC)
599 HIP_DYNAMIC_SHARED(float, s)
600#else
601 extern __shared__ float s[];
602#endif
603
604 const int first_x = blockIdx.x * maxX;
605 const int last_x = (first_x + maxX < numX ? first_x + maxX : numX) - 1;
606 const int num_x_input = last_x - first_x + GetKernelSize<StaticKernelSizeX>()(kernelSizeX);
607 const int num_x_output = last_x - first_x + 1;
608
609 const int first_y = blockIdx.y * maxY;
610 const int last_y = (first_y + maxY < numY ? first_y + maxY : numY) - 1;
611 const int num_y_input = last_y - first_y + GetKernelSize<StaticKernelSizeY>()(kernelSizeY);
612 const int num_y_output = last_y - first_y + 1;
613
614 const int first_plane = blockIdx.z * blockDim.z;
615 const int plane_stride = blockDim.z * gridDim.z;
616
617 for (int p = first_plane + threadIdx.z; p < numPlanes; p += plane_stride) {
618 const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p);
619 const int plane_kernel_offset = threadIdx.z * num_y_input;
620
621// Load inputs to shared memory
622#pragma unroll
623 for (int j = threadIdx.y; j < num_y_input; j += blockDim.y) {
624 const int input_offset = num_x_input * (j + plane_kernel_offset);
625#pragma unroll
626 for (int i = threadIdx.x; i < num_x_input; i += blockDim.x) {
627 const int tensor_index =
628 plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(i + first_x, j + first_y);
629 s[i + input_offset] = eval.coeff(tensor_index);
630 }
631 }
632
633 __syncthreads();
634
635 // Convolution
636 const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p);
637
638#pragma unroll
639 for (int j = threadIdx.y; j < num_y_output; j += blockDim.y) {
640#pragma unroll
641 for (int i = threadIdx.x; i < num_x_output; i += blockDim.x) {
642 float result = 0.0f;
643#pragma unroll
644 for (int l = 0; l < GetKernelSize<StaticKernelSizeY>()(kernelSizeY); ++l) {
645 const int kernel_offset = kernelSizeX * l;
646 const int input_offset = i + num_x_input * (j + l + plane_kernel_offset);
647#pragma unroll
648 for (int k = 0; k < GetKernelSize<StaticKernelSizeX>()(kernelSizeX); ++k) {
649 result += s[k + input_offset] * kernel[k + kernel_offset];
650 }
651 }
652 const int tensor_index =
653 plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(i + first_x, j + first_y);
654 buffer[tensor_index] = result;
655 }
656 }
657
658 __syncthreads();
659 }
660}
661
662#if EIGEN_COMP_CLANG
663#pragma clang diagnostic pop
664#endif
665
666// Oversized kernels cannot stage even one output's input footprint in shared memory.
667template <typename InputEvaluator, typename Index, typename InputDims>
668__global__ void EigenConvolutionKernel3DGlobal(
669 InputEvaluator eval, const internal::IndexMapper<Index, InputDims, 3, InputEvaluator::Layout> indexMapper,
670 const float* kernel, Index total, Index numX, Index numY, Index numZ, Index kernelSizeX, Index kernelSizeY,
671 Index kernelSizeZ, float* buffer) {
672 const Index step = Index(blockDim.x) * gridDim.x;
673 for (Index output = Index(blockIdx.x) * blockDim.x + threadIdx.x; output < total;) {
674 Index remaining = output;
675 const Index x = remaining % numX;
676 remaining /= numX;
677 const Index y = remaining % numY;
678 remaining /= numY;
679 const Index z = remaining % numZ;
680 const Index plane = remaining / numZ;
681 const Index inputOffset = indexMapper.mapGpuInputPlaneToTensorInputOffset(plane);
682 float result = 0;
683 Index kernelIndex = 0;
684 for (Index k = 0; k < kernelSizeZ; ++k) {
685 for (Index j = 0; j < kernelSizeY; ++j) {
686 for (Index i = 0; i < kernelSizeX; ++i) {
687 result += eval.coeff(inputOffset + indexMapper.mapGpuInputKernelToTensorInputOffset(x + i, y + j, z + k)) *
688 kernel[kernelIndex++];
689 }
690 }
691 }
692 buffer[indexMapper.mapGpuOutputPlaneToTensorOutputOffset(plane) +
693 indexMapper.mapGpuOutputKernelToTensorOutputOffset(x, y, z)] = result;
694 // Test the remaining distance before adding: even the final increment must not overflow Index.
695 if (step >= total - output) break;
696 output += step;
697 }
698}
699
700template <typename InputEvaluator, typename Index, typename InputDims>
701__global__ EIGEN_HIP_LAUNCH_BOUNDS_1024 void EigenConvolutionKernel3D(
702 InputEvaluator eval, const internal::IndexMapper<Index, InputDims, 3, InputEvaluator::Layout> indexMapper,
703 const float* __restrict kernel, const size_t numPlanes, const size_t numX, const size_t maxX, const size_t numY,
704 const size_t maxY, const size_t numZ, const size_t maxZ, const int kernelSizeX, const int kernelSizeY,
705 const int kernelSizeZ, float* buffer) {
706#if defined(EIGEN_HIPCC)
707 HIP_DYNAMIC_SHARED(float, s)
708#else
709 extern __shared__ float s[];
710#endif
711
712 // Load inputs to shared memory
713 const int first_x = blockIdx.x * static_cast<int>(maxX);
714 const int last_x = (first_x + static_cast<int>(maxX) < static_cast<int>(numX) ? first_x + static_cast<int>(maxX)
715 : static_cast<int>(numX)) -
716 1;
717 const int num_x_input = last_x - first_x + kernelSizeX;
718
719 const int first_y = blockIdx.y * static_cast<int>(maxY);
720 const int last_y = (first_y + static_cast<int>(maxY) < static_cast<int>(numY) ? first_y + static_cast<int>(maxY)
721 : static_cast<int>(numY)) -
722 1;
723 const int num_y_input = last_y - first_y + kernelSizeY;
724
725 const int first_z = blockIdx.z * static_cast<int>(maxZ);
726 const int last_z = (first_z + static_cast<int>(maxZ) < static_cast<int>(numZ) ? first_z + static_cast<int>(maxZ)
727 : static_cast<int>(numZ)) -
728 1;
729 const int num_z_input = last_z - first_z + kernelSizeZ;
730
731 for (int p = 0; p < static_cast<int>(numPlanes); ++p) {
732 const int plane_input_offset = indexMapper.mapGpuInputPlaneToTensorInputOffset(p);
733 const int plane_kernel_offset = 0;
734
735 for (int k = threadIdx.z; k < num_z_input; k += blockDim.z) {
736 for (int j = threadIdx.y; j < num_y_input; j += blockDim.y) {
737 for (int i = threadIdx.x; i < num_x_input; i += blockDim.x) {
738 const int tensor_index = plane_input_offset + indexMapper.mapGpuInputKernelToTensorInputOffset(
739 i + first_x, j + first_y, k + first_z);
740 s[i + num_x_input * (j + num_y_input * (k + plane_kernel_offset))] = eval.coeff(tensor_index);
741 }
742 }
743 }
744
745 __syncthreads();
746
747 // Convolution
748 const int num_z_output = last_z - first_z + 1;
749 const int num_y_output = last_y - first_y + 1;
750 const int num_x_output = last_x - first_x + 1;
751 const int plane_output_offset = indexMapper.mapGpuOutputPlaneToTensorOutputOffset(p);
752
753 for (int k = threadIdx.z; k < num_z_output; k += blockDim.z) {
754 for (int j = threadIdx.y; j < num_y_output; j += blockDim.y) {
755 for (int i = threadIdx.x; i < num_x_output; i += blockDim.x) {
756 float result = 0.0f;
757 for (int n = 0; n < kernelSizeZ; ++n) {
758 for (int m = 0; m < kernelSizeY; ++m) {
759 for (int l = 0; l < kernelSizeX; ++l) {
760 result += s[i + l + num_x_input * (j + m + num_y_input * (k + n + plane_kernel_offset))] *
761 kernel[l + kernelSizeX * (m + kernelSizeY * n)];
762 }
763 }
764 }
765 const int tensor_index = plane_output_offset + indexMapper.mapGpuOutputKernelToTensorOutputOffset(
766 i + first_x, j + first_y, k + first_z);
767 buffer[tensor_index] = result;
768 }
769 }
770 }
771 __syncthreads();
772 }
773}
774
775template <typename Indices, typename InputArgType, typename KernelArgType>
776struct TensorEvaluator<const TensorConvolutionOp<Indices, InputArgType, KernelArgType>, GpuDevice> {
777 using XprType = TensorConvolutionOp<Indices, InputArgType, KernelArgType>;
778
779 static constexpr int NumDims =
780 internal::array_size<typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions>::value;
781 static constexpr int NumKernelDims = internal::array_size<Indices>::value;
782 using Index = typename XprType::Index;
783 using Dimensions = DSizes<Index, NumDims>;
784 using KernelDimensions = typename TensorEvaluator<KernelArgType, GpuDevice>::Dimensions;
785
786 static constexpr int Layout = TensorEvaluator<InputArgType, GpuDevice>::Layout;
787 enum {
788 IsAligned = int(TensorEvaluator<InputArgType, GpuDevice>::IsAligned) &
789 int(TensorEvaluator<KernelArgType, GpuDevice>::IsAligned),
790 PacketAccess = false,
791 BlockAccess = false,
792 PreferBlockAccess = false,
793 CoordAccess = false, // to be implemented
794 RawAccess = false
795 };
796
797 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
798 using TensorBlock = internal::TensorBlockNotImplemented;
799 //===--------------------------------------------------------------------===//
800
801 TensorEvaluator(const XprType& op, const GpuDevice& device)
802 : m_inputImpl(op.inputExpression(), device),
803 m_kernelImpl(op.kernelExpression(), device),
804 m_kernelArg(op.kernelExpression()),
805 m_indices(op.indices()),
806 m_buf(nullptr),
807 m_kernel(nullptr),
808 m_local_kernel(false),
809 m_device(device) {
810 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<InputArgType, GpuDevice>::Layout) ==
811 static_cast<int>(TensorEvaluator<KernelArgType, GpuDevice>::Layout)),
812 YOU_MADE_A_PROGRAMMING_MISTAKE);
813
814 const typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions& input_dims = m_inputImpl.dimensions();
815 const typename TensorEvaluator<KernelArgType, GpuDevice>::Dimensions& kernel_dims = m_kernelImpl.dimensions();
816
817 m_dimensions = m_inputImpl.dimensions();
818 for (int i = 0; i < NumKernelDims; ++i) {
819 const Index index = op.indices()[i];
820 const Index input_dim = input_dims[index];
821 const Index kernel_dim = kernel_dims[i];
822 const Index result_dim = input_dim - kernel_dim + 1;
823 m_dimensions[index] = result_dim;
824 }
825 }
826
827 using CoeffReturnType = typename XprType::CoeffReturnType;
828 using PacketReturnType = typename PacketType<CoeffReturnType, GpuDevice>::type;
829 using Scalar = typename InputArgType::Scalar;
830 static constexpr int PacketSize = internal::unpacket_traits<PacketReturnType>::size;
831
832 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_dimensions; }
833
834 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(Scalar* data) {
835 preloadKernel();
836 m_inputImpl.evalSubExprsIfNeeded(nullptr);
837 if (data) {
838 executeEval(data);
839 return false;
840 } else {
841 m_buf = (Scalar*)m_device.allocate(dimensions().TotalSize() * sizeof(Scalar));
842 executeEval(m_buf);
843 return true;
844 }
845 }
846
847 EIGEN_STRONG_INLINE void cleanup() {
848 m_inputImpl.cleanup();
849 if (m_buf) {
850 m_device.deallocate(m_buf);
851 m_buf = nullptr;
852 }
853 if (m_local_kernel) {
854 m_device.deallocate((void*)m_kernel);
855 m_local_kernel = false;
856 }
857 m_kernel = nullptr;
858 }
859
860 EIGEN_STRONG_INLINE void preloadKernel() {
861 // Don't make a local copy of the kernel unless we have to (i.e. it's an
862 // expression that needs to be evaluated)
863 const Scalar* in_place = m_kernelImpl.data();
864 if (in_place) {
865 m_kernel = in_place;
866 m_local_kernel = false;
867 } else {
868 size_t kernel_sz = m_kernelImpl.dimensions().TotalSize() * sizeof(Scalar);
869 Scalar* local = (Scalar*)m_device.allocate(kernel_sz);
870 using EvalTo = TensorEvalToOp<const KernelArgType>;
871 EvalTo evalToTmp(local, m_kernelArg);
872 const bool PacketAccess = internal::IsVectorizable<GpuDevice, KernelArgType>::value;
873 internal::TensorExecutor<const EvalTo, GpuDevice, PacketAccess>::run(evalToTmp, m_device);
874
875 m_kernel = local;
876 m_local_kernel = true;
877 }
878 }
879
880 static unsigned int ceil(unsigned int num, unsigned int denom) {
881 const unsigned int rounded_toward_zero = num / denom;
882 if (num > rounded_toward_zero * denom) {
883 return rounded_toward_zero + 1;
884 }
885 return rounded_toward_zero;
886 }
887
888 void executeEval(Scalar* data) const {
889 using InputDims = typename TensorEvaluator<InputArgType, GpuDevice>::Dimensions;
890
891 const int maxSharedMem = m_device.sharedMemPerBlock();
892 const int maxThreadsPerBlock = m_device.maxGpuThreadsPerBlock();
893 const int maxBlocksPerProcessor = m_device.maxGpuThreadsPerMultiProcessor() / maxThreadsPerBlock;
894 const int numMultiProcessors = m_device.getNumGpuMultiProcessors();
895 const int warpSize = 32;
896
897 switch (NumKernelDims) {
898 case 1: {
899 const int kernel_size = m_kernelImpl.dimensions().TotalSize();
900
901 const int numX = dimensions()[m_indices[0]];
902 const int numP = dimensions().TotalSize() / numX;
903 int maxX;
904 dim3 block_size;
905
906 const int single_stride_dim =
907 static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : m_inputImpl.dimensions().rank() - 1;
908 if (m_indices[0] == single_stride_dim) {
909 // Maximize the reuse
910 const int inner_dim = ((maxSharedMem / (sizeof(Scalar)) - kernel_size + 1 + 31) / 32) * 32;
911 maxX = numext::mini<int>(inner_dim, numX);
912 const int maxP = numext::mini<int>(maxSharedMem / ((kernel_size - 1 + maxX) * sizeof(Scalar)), numP);
913 block_size.x = numext::mini(maxThreadsPerBlock, maxX);
914 block_size.y = numext::mini<int>(maxThreadsPerBlock / block_size.x, maxP);
915 } else {
916 // Read as much as possible alongside the inner most dimension, that is the plane
917 const int inner_dim = maxSharedMem / ((warpSize + kernel_size) * sizeof(Scalar));
918 const int maxP = numext::mini<int>(inner_dim, numP);
919 maxX = numext::mini<int>(maxSharedMem / (inner_dim * sizeof(Scalar)) - kernel_size + 1, numX);
920
921 block_size.x = numext::mini(warpSize, maxX);
922 block_size.y = numext::mini<int>(maxThreadsPerBlock / block_size.x, maxP);
923 }
924
925 const int shared_mem = block_size.y * (maxX + kernel_size - 1) * sizeof(Scalar);
926 gpu_assert(shared_mem <= maxSharedMem);
927
928 const int num_x_blocks = ceil(numX, maxX);
929 const int blocksPerProcessor = numext::mini(maxBlocksPerProcessor, maxSharedMem / shared_mem);
930 const int num_y_blocks = ceil(numMultiProcessors * blocksPerProcessor, num_x_blocks);
931
932 dim3 num_blocks(num_x_blocks, numext::mini<int>(num_y_blocks, ceil(numP, block_size.y)));
933
934 const array<Index, 1> indices{m_indices[0]};
935 const array<Index, 1> kernel_dims{m_kernelImpl.dimensions()[0]};
936 internal::IndexMapper<Index, InputDims, 1, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices);
937 switch (kernel_size) {
938 case 4: {
939 LAUNCH_GPU_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4>),
940 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP,
941 numX, maxX, 4, data);
942 break;
943 }
944 case 7: {
945 LAUNCH_GPU_KERNEL((EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7>),
946 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP,
947 numX, maxX, 7, data);
948 break;
949 }
950 default: {
951 LAUNCH_GPU_KERNEL(
952 (EigenConvolutionKernel1D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, Dynamic>),
953 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
954 kernel_size, data);
955 }
956 }
957 break;
958 }
959
960 case 2: {
961 constexpr int idxX = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : 1;
962 constexpr int idxY = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 0;
963 const int kernel_size_x = m_kernelImpl.dimensions()[idxX];
964 const int kernel_size_y = m_kernelImpl.dimensions()[idxY];
965
966 const int numX = dimensions()[m_indices[idxX]];
967 const int numY = dimensions()[m_indices[idxY]];
968 const int numP = dimensions().TotalSize() / (numX * numY);
969
970 const float scaling_factor =
971 sqrtf(static_cast<float>(maxSharedMem) / (sizeof(Scalar) * kernel_size_y * kernel_size_x));
972
973 // Snap maxX to warp size
974 int inner_dim = ((static_cast<int>(scaling_factor * kernel_size_x) - kernel_size_x + 1 + 32) / 32) * 32;
975 const int maxX = numext::mini<int>(inner_dim, numX);
976 const int maxY =
977 numext::mini<int>(maxSharedMem / (sizeof(Scalar) * (maxX + kernel_size_x - 1)) - kernel_size_y + 1, numY);
978 const int maxP = numext::mini<int>(
979 maxSharedMem / ((kernel_size_x - 1 + maxX) * (kernel_size_y - 1 + maxY) * sizeof(Scalar)), numP);
980
981 dim3 block_size;
982 block_size.x = numext::mini(1024, maxX);
983 block_size.y = numext::mini<int>(1024 / block_size.x, maxY);
984 block_size.z = numext::mini<int>(1024 / (block_size.x * block_size.y), maxP);
985
986 const int shared_mem = block_size.z * (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) * sizeof(Scalar);
987 gpu_assert(shared_mem <= maxSharedMem);
988
989 const int num_x_blocks = ceil(numX, maxX);
990 const int num_y_blocks = ceil(numY, maxY);
991 const int blocksPerProcessor = numext::mini(maxBlocksPerProcessor, maxSharedMem / shared_mem);
992 const int num_z_blocks = ceil(numMultiProcessors * blocksPerProcessor, num_x_blocks * num_y_blocks);
993
994 dim3 num_blocks(num_x_blocks, num_y_blocks, numext::mini<int>(num_z_blocks, ceil(numP, block_size.z)));
995
996 const array<Index, 2> indices{m_indices[idxX], m_indices[idxY]};
997 const array<Index, 2> kernel_dims{m_kernelImpl.dimensions()[idxX], m_kernelImpl.dimensions()[idxY]};
998 internal::IndexMapper<Index, InputDims, 2, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices);
999 switch (kernel_size_x) {
1000 case 4: {
1001 switch (kernel_size_y) {
1002 case 7: {
1003 LAUNCH_GPU_KERNEL(
1004 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, 7>),
1005 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1006 numY, maxY, 4, 7, data);
1007 break;
1008 }
1009 default: {
1010 LAUNCH_GPU_KERNEL(
1011 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 4, Dynamic>),
1012 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1013 numY, maxY, 4, kernel_size_y, data);
1014 break;
1015 }
1016 }
1017 break;
1018 }
1019 case 7: {
1020 switch (kernel_size_y) {
1021 case 4: {
1022 LAUNCH_GPU_KERNEL(
1023 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, 4>),
1024 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1025 numY, maxY, 7, 4, data);
1026 break;
1027 }
1028 default: {
1029 LAUNCH_GPU_KERNEL(
1030 (EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims, 7, Dynamic>),
1031 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX, maxX,
1032 numY, maxY, 7, kernel_size_y, data);
1033 break;
1034 }
1035 }
1036 break;
1037 }
1038 default: {
1039 LAUNCH_GPU_KERNEL((EigenConvolutionKernel2D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims,
1040 Dynamic, Dynamic>),
1041 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP,
1042 numX, maxX, numY, maxY, kernel_size_x, kernel_size_y, data);
1043 break;
1044 }
1045 }
1046 break;
1047 }
1048
1049 case 3: {
1050 constexpr int idxX = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 0 : 2;
1051 constexpr int idxY = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 1 : 1;
1052 constexpr int idxZ = static_cast<int>(Layout) == static_cast<int>(ColMajor) ? 2 : 0;
1053
1054 const int kernel_size_x = m_kernelImpl.dimensions()[idxX];
1055 const int kernel_size_y = m_kernelImpl.dimensions()[idxY];
1056 const int kernel_size_z = m_kernelImpl.dimensions()[idxZ];
1057
1058 const int numX = dimensions()[m_indices[idxX]];
1059 const int numY = dimensions()[m_indices[idxY]];
1060 const int numZ = dimensions()[m_indices[idxZ]];
1061 const Index total = dimensions().TotalSize();
1062 if (total == 0) return;
1063
1064 const array<Index, 3> indices{m_indices[idxX], m_indices[idxY], m_indices[idxZ]};
1065 const array<Index, 3> kernel_dims{m_kernelImpl.dimensions()[idxX], m_kernelImpl.dimensions()[idxY],
1066 m_kernelImpl.dimensions()[idxZ]};
1067 internal::IndexMapper<Index, InputDims, 3, Layout> indexMapper(m_inputImpl.dimensions(), kernel_dims, indices);
1068
1069 if (m_kernelImpl.dimensions().TotalSize() > Index(maxSharedMem / sizeof(Scalar))) {
1070 constexpr int preferredBlockSize = 256;
1071 const int threads = numext::mini(preferredBlockSize, maxThreadsPerBlock);
1072 const int maxBlocks = static_cast<int>(
1073 numext::mini<int64_t>(int64_t(numMultiProcessors) * 32, NumTraits<Index>::highest() / threads));
1074 const int blocks = static_cast<int>(numext::mini<Index>((total - 1) / threads + 1, Index(maxBlocks)));
1075 LAUNCH_GPU_KERNEL(
1076 (EigenConvolutionKernel3DGlobal<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims>), blocks,
1077 threads, 0, m_device, m_inputImpl, indexMapper, m_kernel, total, Index(numX), Index(numY), Index(numZ),
1078 Index(kernel_size_x), Index(kernel_size_y), Index(kernel_size_z), data);
1079 break;
1080 }
1081
1082 const int numP = total / (Index(numX) * numY * numZ);
1083 const int maxX = numext::mini<int>(
1084 128, numext::mini<int>(maxSharedMem / (sizeof(Scalar) * kernel_size_y * kernel_size_z) - kernel_size_x + 1,
1085 numX));
1086 const int maxY = numext::mini<int>(
1087 128, numext::mini<int>(
1088 maxSharedMem / (sizeof(Scalar) * (maxX + kernel_size_x - 1) * kernel_size_z) - kernel_size_y + 1,
1089 numY));
1090 const int maxZ = numext::mini<int>(
1091 128, numext::mini<int>(
1092 maxSharedMem / (sizeof(Scalar) * (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1)) -
1093 kernel_size_z + 1,
1094 numZ));
1095
1096 dim3 block_size;
1097 block_size.x = numext::mini(32, maxX);
1098 block_size.y = numext::mini(32, maxY);
1099 block_size.z = numext::mini<int>(1024 / (block_size.x * block_size.y), maxZ);
1100 dim3 num_blocks(ceil(numX, maxX), ceil(numY, maxY), ceil(numZ, maxZ));
1101
1102 const int shared_mem =
1103 (maxX + kernel_size_x - 1) * (maxY + kernel_size_y - 1) * (maxZ + kernel_size_z - 1) * sizeof(Scalar);
1104 gpu_assert(shared_mem <= maxSharedMem);
1105
1106 LAUNCH_GPU_KERNEL((EigenConvolutionKernel3D<TensorEvaluator<InputArgType, GpuDevice>, Index, InputDims>),
1107 num_blocks, block_size, shared_mem, m_device, m_inputImpl, indexMapper, m_kernel, numP, numX,
1108 maxX, numY, maxY, numZ, maxZ, kernel_size_x, kernel_size_y, kernel_size_z, data);
1109 break;
1110 }
1111
1112 default: {
1113 EIGEN_STATIC_ASSERT((NumKernelDims >= 1 && NumKernelDims <= 3),
1114 THIS_METHOD_IS_ONLY_FOR_OBJECTS_OF_A_SPECIFIC_SIZE);
1115 }
1116 }
1117 }
1118
1119 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
1120 eigen_assert(m_buf);
1121 eigen_assert(index < m_dimensions.TotalSize());
1122 return m_buf[index];
1123 }
1124
1125 template <int LoadMode>
1126 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(const Index index) const {
1127 eigen_assert(m_buf);
1128 eigen_assert(index < m_dimensions.TotalSize());
1129 return internal::ploadt<PacketReturnType, LoadMode>(m_buf + index);
1130 }
1131
1132 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
1133 // TODO(rmlarsen): For now, this is just a copy of the CPU cost
1134 // model.
1135 const double kernel_size = m_kernelImpl.dimensions().TotalSize();
1136 // We ignore the use of fused multiply-add.
1137 const double convolve_compute_cost = TensorOpCost::AddCost<Scalar>() + TensorOpCost::MulCost<Scalar>();
1138 const double firstIndex_compute_cost =
1139 NumDims *
1140 (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() + TensorOpCost::DivCost<Index>());
1141 return TensorOpCost(0, 0, firstIndex_compute_cost, vectorized, PacketSize) +
1142 kernel_size * (m_inputImpl.costPerCoeff(vectorized) + m_kernelImpl.costPerCoeff(vectorized) +
1143 TensorOpCost(0, 0, convolve_compute_cost, vectorized, PacketSize));
1144 }
1145
1146 private:
1147 TensorEvaluator<InputArgType, GpuDevice> m_inputImpl;
1148 TensorEvaluator<KernelArgType, GpuDevice> m_kernelImpl;
1149 KernelArgType m_kernelArg;
1150 Indices m_indices;
1151 Dimensions m_dimensions;
1152 Scalar* m_buf;
1153 const Scalar* m_kernel;
1154 bool m_local_kernel;
1155
1156 const GpuDevice& m_device;
1157};
1158#endif
1159
1160} // end namespace Eigen
1161
1162#endif // EIGEN_TENSOR_TENSOR_CONVOLUTION_H
The tensor base class.
Definition TensorForwardDeclarations.h:69
Definition TensorConvolution.h:223
const internal::remove_all_t< typename InputXprType::Nested > & inputExpression() const
Definition TensorConvolution.h:240
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47