6#ifndef EIGEN_TENSOR_TENSOR_VOLUME_PATCH_H
7#define EIGEN_TENSOR_TENSOR_VOLUME_PATCH_H
10#include "./InternalHeaderCheck.h"
16template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols,
typename XprType>
17struct traits<TensorVolumePatchOp<Planes, Rows, Cols, XprType>> : traits<XprType> {
18 using Scalar = std::remove_const_t<typename XprType::Scalar>;
19 using XprTraits = traits<XprType>;
20 using StorageKind =
typename XprTraits::StorageKind;
21 using Index =
typename XprTraits::Index;
22 static constexpr int NumDimensions = XprTraits::NumDimensions + 1;
23 static constexpr int Layout = XprTraits::Layout;
24 using PointerType =
typename XprTraits::PointerType;
27template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols,
typename XprType>
28struct eval<TensorVolumePatchOp<Planes, Rows, Cols, XprType>, Eigen::Dense> {
29 using type =
const TensorVolumePatchOp<Planes, Rows, Cols, XprType>&;
49template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols,
typename XprType>
50class TensorVolumePatchOp :
public TensorBase<TensorVolumePatchOp<Planes, Rows, Cols, XprType>, ReadOnlyAccessors> {
52 using Scalar =
typename Eigen::internal::traits<TensorVolumePatchOp>::Scalar;
54 using CoeffReturnType =
typename XprType::CoeffReturnType;
55 using Nested =
typename Eigen::internal::ref_selector<TensorVolumePatchOp>::type;
56 using StorageKind =
typename Eigen::internal::traits<TensorVolumePatchOp>::StorageKind;
57 using Index =
typename Eigen::internal::traits<TensorVolumePatchOp>::Index;
59 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorVolumePatchOp(
60 const XprType& expr, DenseIndex patch_planes, DenseIndex patch_rows, DenseIndex patch_cols,
61 DenseIndex plane_strides, DenseIndex row_strides, DenseIndex col_strides, DenseIndex in_plane_strides,
62 DenseIndex in_row_strides, DenseIndex in_col_strides, DenseIndex plane_inflate_strides,
63 DenseIndex row_inflate_strides, DenseIndex col_inflate_strides, PaddingType padding_type, Scalar padding_value)
65 m_patch_planes(patch_planes),
66 m_patch_rows(patch_rows),
67 m_patch_cols(patch_cols),
68 m_plane_strides(plane_strides),
69 m_row_strides(row_strides),
70 m_col_strides(col_strides),
71 m_in_plane_strides(in_plane_strides),
72 m_in_row_strides(in_row_strides),
73 m_in_col_strides(in_col_strides),
74 m_plane_inflate_strides(plane_inflate_strides),
75 m_row_inflate_strides(row_inflate_strides),
76 m_col_inflate_strides(col_inflate_strides),
77 m_padding_explicit(
false),
79 m_padding_bottom_z(0),
84 m_padding_type(padding_type),
85 m_padding_value(padding_value) {}
87 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorVolumePatchOp(
88 const XprType& expr, DenseIndex patch_planes, DenseIndex patch_rows, DenseIndex patch_cols,
89 DenseIndex plane_strides, DenseIndex row_strides, DenseIndex col_strides, DenseIndex in_plane_strides,
90 DenseIndex in_row_strides, DenseIndex in_col_strides, DenseIndex plane_inflate_strides,
91 DenseIndex row_inflate_strides, DenseIndex col_inflate_strides, DenseIndex padding_top_z,
92 DenseIndex padding_bottom_z, DenseIndex padding_top, DenseIndex padding_bottom, DenseIndex padding_left,
93 DenseIndex padding_right, Scalar padding_value)
95 m_patch_planes(patch_planes),
96 m_patch_rows(patch_rows),
97 m_patch_cols(patch_cols),
98 m_plane_strides(plane_strides),
99 m_row_strides(row_strides),
100 m_col_strides(col_strides),
101 m_in_plane_strides(in_plane_strides),
102 m_in_row_strides(in_row_strides),
103 m_in_col_strides(in_col_strides),
104 m_plane_inflate_strides(plane_inflate_strides),
105 m_row_inflate_strides(row_inflate_strides),
106 m_col_inflate_strides(col_inflate_strides),
107 m_padding_explicit(
true),
108 m_padding_top_z(padding_top_z),
109 m_padding_bottom_z(padding_bottom_z),
110 m_padding_top(padding_top),
111 m_padding_bottom(padding_bottom),
112 m_padding_left(padding_left),
113 m_padding_right(padding_right),
114 m_padding_type(PADDING_VALID),
115 m_padding_value(padding_value) {}
117 EIGEN_DEVICE_FUNC DenseIndex patch_planes()
const {
return m_patch_planes; }
118 EIGEN_DEVICE_FUNC DenseIndex patch_rows()
const {
return m_patch_rows; }
119 EIGEN_DEVICE_FUNC DenseIndex patch_cols()
const {
return m_patch_cols; }
120 EIGEN_DEVICE_FUNC DenseIndex plane_strides()
const {
return m_plane_strides; }
121 EIGEN_DEVICE_FUNC DenseIndex row_strides()
const {
return m_row_strides; }
122 EIGEN_DEVICE_FUNC DenseIndex col_strides()
const {
return m_col_strides; }
123 EIGEN_DEVICE_FUNC DenseIndex in_plane_strides()
const {
return m_in_plane_strides; }
124 EIGEN_DEVICE_FUNC DenseIndex in_row_strides()
const {
return m_in_row_strides; }
125 EIGEN_DEVICE_FUNC DenseIndex in_col_strides()
const {
return m_in_col_strides; }
126 EIGEN_DEVICE_FUNC DenseIndex plane_inflate_strides()
const {
return m_plane_inflate_strides; }
127 EIGEN_DEVICE_FUNC DenseIndex row_inflate_strides()
const {
return m_row_inflate_strides; }
128 EIGEN_DEVICE_FUNC DenseIndex col_inflate_strides()
const {
return m_col_inflate_strides; }
129 EIGEN_DEVICE_FUNC
bool padding_explicit()
const {
return m_padding_explicit; }
130 EIGEN_DEVICE_FUNC DenseIndex padding_top_z()
const {
return m_padding_top_z; }
131 EIGEN_DEVICE_FUNC DenseIndex padding_bottom_z()
const {
return m_padding_bottom_z; }
132 EIGEN_DEVICE_FUNC DenseIndex padding_top()
const {
return m_padding_top; }
133 EIGEN_DEVICE_FUNC DenseIndex padding_bottom()
const {
return m_padding_bottom; }
134 EIGEN_DEVICE_FUNC DenseIndex padding_left()
const {
return m_padding_left; }
135 EIGEN_DEVICE_FUNC DenseIndex padding_right()
const {
return m_padding_right; }
136 EIGEN_DEVICE_FUNC PaddingType padding_type()
const {
return m_padding_type; }
137 EIGEN_DEVICE_FUNC Scalar padding_value()
const {
return m_padding_value; }
139 EIGEN_DEVICE_FUNC
const internal::remove_all_t<typename XprType::Nested>& expression()
const {
return m_xpr; }
142 typename XprType::Nested m_xpr;
143 const DenseIndex m_patch_planes;
144 const DenseIndex m_patch_rows;
145 const DenseIndex m_patch_cols;
146 const DenseIndex m_plane_strides;
147 const DenseIndex m_row_strides;
148 const DenseIndex m_col_strides;
149 const DenseIndex m_in_plane_strides;
150 const DenseIndex m_in_row_strides;
151 const DenseIndex m_in_col_strides;
152 const DenseIndex m_plane_inflate_strides;
153 const DenseIndex m_row_inflate_strides;
154 const DenseIndex m_col_inflate_strides;
155 const bool m_padding_explicit;
156 const DenseIndex m_padding_top_z;
157 const DenseIndex m_padding_bottom_z;
158 const DenseIndex m_padding_top;
159 const DenseIndex m_padding_bottom;
160 const DenseIndex m_padding_left;
161 const DenseIndex m_padding_right;
162 const PaddingType m_padding_type;
163 const Scalar m_padding_value;
167template <DenseIndex Planes, DenseIndex Rows, DenseIndex Cols,
typename ArgType,
typename Device>
170 using Index =
typename XprType::Index;
171 static constexpr int NumInputDims =
172 internal::array_size<typename TensorEvaluator<ArgType, Device>::Dimensions>::value;
173 static constexpr int NumDims = NumInputDims + 1;
174 using Dimensions = DSizes<Index, NumDims>;
175 using Scalar = std::remove_const_t<typename XprType::Scalar>;
176 using CoeffReturnType =
typename XprType::CoeffReturnType;
177 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
178 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
179 using Storage = StorageMemory<CoeffReturnType, Device>;
180 using EvaluatorPointerType =
typename Storage::Type;
182 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
185 PacketAccess = TensorEvaluator<ArgType, Device>::PacketAccess,
193 PreferBlockAccess =
true,
199 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
200 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
201 using TensorBlock =
typename internal::TensorMaterializedBlock<Scalar, NumDims, Layout, Index>;
204 EIGEN_STRONG_INLINE TensorEvaluator(
const XprType& op,
const Device& device)
205 : m_impl(op.expression(), device), m_device(device) {
206 EIGEN_STATIC_ASSERT((NumDims >= 5), YOU_MADE_A_PROGRAMMING_MISTAKE);
208 m_paddingValue = op.padding_value();
210 const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
213 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
214 m_inputDepth = input_dims[0];
215 m_inputPlanes = input_dims[1];
216 m_inputRows = input_dims[2];
217 m_inputCols = input_dims[3];
219 m_inputDepth = input_dims[NumInputDims - 1];
220 m_inputPlanes = input_dims[NumInputDims - 2];
221 m_inputRows = input_dims[NumInputDims - 3];
222 m_inputCols = input_dims[NumInputDims - 4];
225 m_plane_strides = op.plane_strides();
226 m_row_strides = op.row_strides();
227 m_col_strides = op.col_strides();
230 m_in_plane_strides = op.in_plane_strides();
231 m_in_row_strides = op.in_row_strides();
232 m_in_col_strides = op.in_col_strides();
233 m_plane_inflate_strides = op.plane_inflate_strides();
234 m_row_inflate_strides = op.row_inflate_strides();
235 m_col_inflate_strides = op.col_inflate_strides();
238 m_input_planes_eff = (m_inputPlanes - 1) * m_plane_inflate_strides + 1;
239 m_input_rows_eff = (m_inputRows - 1) * m_row_inflate_strides + 1;
240 m_input_cols_eff = (m_inputCols - 1) * m_col_inflate_strides + 1;
241 m_patch_planes_eff = op.patch_planes() + (op.patch_planes() - 1) * (m_in_plane_strides - 1);
242 m_patch_rows_eff = op.patch_rows() + (op.patch_rows() - 1) * (m_in_row_strides - 1);
243 m_patch_cols_eff = op.patch_cols() + (op.patch_cols() - 1) * (m_in_col_strides - 1);
245 if (op.padding_explicit()) {
247 numext::ceil((m_input_planes_eff + op.padding_top_z() + op.padding_bottom_z() - m_patch_planes_eff + 1.0) /
248 static_cast<double>(m_plane_strides));
249 m_outputRows = numext::ceil((m_input_rows_eff + op.padding_top() + op.padding_bottom() - m_patch_rows_eff + 1.0) /
250 static_cast<double>(m_row_strides));
251 m_outputCols = numext::ceil((m_input_cols_eff + op.padding_left() + op.padding_right() - m_patch_cols_eff + 1.0) /
252 static_cast<double>(m_col_strides));
253 m_planePaddingTop = op.padding_top_z();
254 m_rowPaddingTop = op.padding_top();
255 m_colPaddingLeft = op.padding_left();
258 switch (op.padding_type()) {
261 numext::ceil((m_input_planes_eff - m_patch_planes_eff + 1.0) /
static_cast<double>(m_plane_strides));
262 m_outputRows = numext::ceil((m_input_rows_eff - m_patch_rows_eff + 1.0) /
static_cast<double>(m_row_strides));
263 m_outputCols = numext::ceil((m_input_cols_eff - m_patch_cols_eff + 1.0) /
static_cast<double>(m_col_strides));
264 m_planePaddingTop = 0;
266 m_colPaddingLeft = 0;
269 m_outputPlanes = numext::ceil(m_input_planes_eff /
static_cast<double>(m_plane_strides));
270 m_outputRows = numext::ceil(m_input_rows_eff /
static_cast<double>(m_row_strides));
271 m_outputCols = numext::ceil(m_input_cols_eff /
static_cast<double>(m_col_strides));
272 const Index dz = (m_outputPlanes - 1) * m_plane_strides + m_patch_planes_eff - m_input_planes_eff;
273 const Index dy = (m_outputRows - 1) * m_row_strides + m_patch_rows_eff - m_input_rows_eff;
274 const Index dx = (m_outputCols - 1) * m_col_strides + m_patch_cols_eff - m_input_cols_eff;
276 m_planePaddingTop = numext::maxi<Index>(0, dz) / 2;
277 m_rowPaddingTop = numext::maxi<Index>(0, dy) / 2;
278 m_colPaddingLeft = numext::maxi<Index>(0, dx) / 2;
282 eigen_assert(
false &&
"unexpected padding");
287 eigen_assert(m_outputRows > 0);
288 eigen_assert(m_outputCols > 0);
289 eigen_assert(m_outputPlanes > 0);
292 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
300 m_dimensions[0] = input_dims[0];
301 m_dimensions[1] = op.patch_planes();
302 m_dimensions[2] = op.patch_rows();
303 m_dimensions[3] = op.patch_cols();
304 m_dimensions[4] = m_outputPlanes * m_outputRows * m_outputCols;
305 for (
int i = 5; i < NumDims; ++i) {
306 m_dimensions[i] = input_dims[i - 1];
316 m_dimensions[NumDims - 1] = input_dims[NumInputDims - 1];
317 m_dimensions[NumDims - 2] = op.patch_planes();
318 m_dimensions[NumDims - 3] = op.patch_rows();
319 m_dimensions[NumDims - 4] = op.patch_cols();
320 m_dimensions[NumDims - 5] = m_outputPlanes * m_outputRows * m_outputCols;
321 for (
int i = NumDims - 6; i >= 0; --i) {
322 m_dimensions[i] = input_dims[i];
327 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
328 m_rowStride = m_dimensions[1];
329 m_colStride = m_dimensions[2] * m_rowStride;
330 m_patchStride = m_colStride * m_dimensions[3] * m_dimensions[0];
331 m_otherStride = m_patchStride * m_dimensions[4];
333 m_rowStride = m_dimensions[NumDims - 2];
334 m_colStride = m_dimensions[NumDims - 3] * m_rowStride;
335 m_patchStride = m_colStride * m_dimensions[NumDims - 4] * m_dimensions[NumDims - 1];
336 m_otherStride = m_patchStride * m_dimensions[NumDims - 5];
340 m_planeInputStride = m_inputDepth;
341 m_rowInputStride = m_inputDepth * m_inputPlanes;
342 m_colInputStride = m_inputDepth * m_inputRows * m_inputPlanes;
343 m_otherInputStride = m_inputDepth * m_inputRows * m_inputCols * m_inputPlanes;
345 m_outputPlanesRows = m_outputPlanes * m_outputRows;
348 m_fastOtherStride = internal::TensorIntDivisor<Index>(m_otherStride);
350 m_fastPatchStride = internal::TensorIntDivisor<Index>(m_patchStride);
351 m_fastColStride = internal::TensorIntDivisor<Index>(m_colStride);
352 m_fastRowStride = internal::TensorIntDivisor<Index>(m_rowStride);
353 m_fastInputRowStride = internal::TensorIntDivisor<Index>(m_row_inflate_strides);
354 m_fastInputColStride = internal::TensorIntDivisor<Index>(m_col_inflate_strides);
355 m_fastInputPlaneStride = internal::TensorIntDivisor<Index>(m_plane_inflate_strides);
356 m_fastInputColsEff = internal::TensorIntDivisor<Index>(m_input_cols_eff);
357 m_fastOutputPlanes = internal::TensorIntDivisor<Index>(m_outputPlanes);
358 m_fastOutputPlanesRows = internal::TensorIntDivisor<Index>(m_outputPlanesRows);
360 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
361 m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[0]);
363 m_fastOutputDepth = internal::TensorIntDivisor<Index>(m_dimensions[NumDims - 1]);
367 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dimensions& dimensions()
const {
return m_dimensions; }
369 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType ) {
370 m_impl.evalSubExprsIfNeeded(
nullptr);
374#ifdef EIGEN_USE_THREADS
375 template <
typename EvalSubExprsCallback>
376 EIGEN_STRONG_INLINE
void evalSubExprsIfNeededAsync(EvaluatorPointerType , EvalSubExprsCallback done) {
377 m_impl.evalSubExprsIfNeededAsync(
nullptr, [done](
bool) { done(
true); });
381 EIGEN_STRONG_INLINE
void cleanup() { m_impl.cleanup(); }
383 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
385 const Index patchIndex = index / m_fastPatchStride;
389 const Index patchOffset = (index - patchIndex * m_patchStride) / m_fastOutputDepth;
392 const Index otherIndex = (NumDims == 5) ? 0 : index / m_fastOtherStride;
393 const Index patch3DIndex = (NumDims == 5) ? patchIndex : (index - otherIndex * m_otherStride) / m_fastPatchStride;
396 const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
397 const Index colOffset = patchOffset / m_fastColStride;
398 const Index inputCol = colIndex * m_col_strides + colOffset * m_in_col_strides - m_colPaddingLeft;
399 const Index origInputCol =
400 (m_col_inflate_strides == 1) ? inputCol : ((inputCol >= 0) ? (inputCol / m_fastInputColStride) : 0);
401 if (inputCol < 0 || inputCol >= m_input_cols_eff ||
402 ((m_col_inflate_strides != 1) && (inputCol != origInputCol * m_col_inflate_strides))) {
403 return Scalar(m_paddingValue);
407 const Index rowIndex = (patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
408 const Index rowOffset = (patchOffset - colOffset * m_colStride) / m_fastRowStride;
409 const Index inputRow = rowIndex * m_row_strides + rowOffset * m_in_row_strides - m_rowPaddingTop;
410 const Index origInputRow =
411 (m_row_inflate_strides == 1) ? inputRow : ((inputRow >= 0) ? (inputRow / m_fastInputRowStride) : 0);
412 if (inputRow < 0 || inputRow >= m_input_rows_eff ||
413 ((m_row_inflate_strides != 1) && (inputRow != origInputRow * m_row_inflate_strides))) {
414 return Scalar(m_paddingValue);
418 const Index planeIndex = patch3DIndex - m_outputPlanes * (colIndex * m_outputRows + rowIndex);
419 const Index planeOffset = patchOffset - colOffset * m_colStride - rowOffset * m_rowStride;
420 const Index inputPlane = planeIndex * m_plane_strides + planeOffset * m_in_plane_strides - m_planePaddingTop;
421 const Index origInputPlane =
422 (m_plane_inflate_strides == 1) ? inputPlane : ((inputPlane >= 0) ? (inputPlane / m_fastInputPlaneStride) : 0);
423 if (inputPlane < 0 || inputPlane >= m_input_planes_eff ||
424 ((m_plane_inflate_strides != 1) && (inputPlane != origInputPlane * m_plane_inflate_strides))) {
425 return Scalar(m_paddingValue);
428 constexpr int depth_index =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - 1;
429 const Index depth = index - (index / m_fastOutputDepth) * m_dimensions[depth_index];
431 const Index inputIndex = depth + origInputRow * m_rowInputStride + origInputCol * m_colInputStride +
432 origInputPlane * m_planeInputStride + otherIndex * m_otherInputStride;
434 return m_impl.coeff(inputIndex);
437 template <
int LoadMode>
438 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
439 eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
441 if (m_in_row_strides != 1 || m_in_col_strides != 1 || m_row_inflate_strides != 1 || m_col_inflate_strides != 1 ||
442 m_in_plane_strides != 1 || m_plane_inflate_strides != 1) {
443 return packetWithPossibleZero(index);
446 const Index indices[2] = {index, index + PacketSize - 1};
447 const Index patchIndex = indices[0] / m_fastPatchStride;
448 if (patchIndex != indices[1] / m_fastPatchStride) {
449 return packetWithPossibleZero(index);
451 const Index otherIndex = (NumDims == 5) ? 0 : indices[0] / m_fastOtherStride;
452 eigen_assert(otherIndex == indices[1] / m_fastOtherStride);
455 Index first_entry = (indices[0] - patchIndex * m_patchStride) / m_fastOutputDepth;
456 Index second_entry = PacketSize == 1 ? first_entry : (indices[1] - patchIndex * m_patchStride) / m_fastOutputDepth;
458 const Index patchOffsets[2] = {first_entry, second_entry};
460 const Index patch3DIndex =
461 (NumDims == 5) ? patchIndex : (indices[0] - otherIndex * m_otherStride) / m_fastPatchStride;
462 eigen_assert(patch3DIndex == (indices[1] - otherIndex * m_otherStride) / m_fastPatchStride);
464 const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
465 const Index colOffsets[2] = {patchOffsets[0] / m_fastColStride, patchOffsets[1] / m_fastColStride};
468 const Index inputCols[2] = {colIndex * m_col_strides + colOffsets[0] - m_colPaddingLeft,
469 colIndex * m_col_strides + colOffsets[1] - m_colPaddingLeft};
470 if (inputCols[1] < 0 || inputCols[0] >= m_inputCols) {
471 return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
474 if (inputCols[0] != inputCols[1]) {
475 return packetWithPossibleZero(index);
478 const Index rowIndex = (patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
479 const Index rowOffsets[2] = {(patchOffsets[0] - colOffsets[0] * m_colStride) / m_fastRowStride,
480 (patchOffsets[1] - colOffsets[1] * m_colStride) / m_fastRowStride};
481 eigen_assert(rowOffsets[0] <= rowOffsets[1]);
483 const Index inputRows[2] = {rowIndex * m_row_strides + rowOffsets[0] - m_rowPaddingTop,
484 rowIndex * m_row_strides + rowOffsets[1] - m_rowPaddingTop};
486 if (inputRows[1] < 0 || inputRows[0] >= m_inputRows) {
487 return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
490 if (inputRows[0] != inputRows[1]) {
491 return packetWithPossibleZero(index);
494 const Index planeIndex = patch3DIndex - m_outputPlanes * (colIndex * m_outputRows + rowIndex);
495 const Index planeOffsets[2] = {patchOffsets[0] - colOffsets[0] * m_colStride - rowOffsets[0] * m_rowStride,
496 patchOffsets[1] - colOffsets[1] * m_colStride - rowOffsets[1] * m_rowStride};
497 eigen_assert(planeOffsets[0] <= planeOffsets[1]);
498 const Index inputPlanes[2] = {planeIndex * m_plane_strides + planeOffsets[0] - m_planePaddingTop,
499 planeIndex * m_plane_strides + planeOffsets[1] - m_planePaddingTop};
501 if (inputPlanes[1] < 0 || inputPlanes[0] >= m_inputPlanes) {
502 return internal::pset1<PacketReturnType>(Scalar(m_paddingValue));
505 if (inputPlanes[0] >= 0 && inputPlanes[1] < m_inputPlanes) {
507 constexpr int depth_index =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor) ? 0 : NumDims - 1;
508 const Index depth = index - (index / m_fastOutputDepth) * m_dimensions[depth_index];
509 const Index inputIndex = depth + inputRows[0] * m_rowInputStride + inputCols[0] * m_colInputStride +
510 m_planeInputStride * inputPlanes[0] + otherIndex * m_otherInputStride;
511 return m_impl.template packet<Unaligned>(inputIndex);
514 return packetWithPossibleZero(index);
517 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
518 const size_t target_size = m_device.firstLevelCacheSize();
524 const TensorOpCost cost_per_coeff = m_impl.costPerCoeff(
false) + TensorOpCost(0,
sizeof(Scalar), 0);
525 return internal::TensorBlockResourceRequirements::withShapeAndSize<Scalar>(
526 internal::TensorBlockShapeType::kSkewedInnerDims, target_size, cost_per_coeff);
533 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
534 bool =
false)
const {
535 constexpr bool is_col_major =
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor);
537 if (desc.size() == 0) {
538 return TensorBlock(internal::TensorBlockKind::kView,
nullptr, desc.dimensions());
541 typename TensorBlock::Storage block_storage = TensorBlock::prepareStorage(desc, scratch);
542 Scalar* block_buffer = block_storage.data();
545 const DSizes<Index, NumDims> output_strides = internal::strides<Layout>(m_dimensions);
546 array<Index, NumDims> coords;
547 Index remaining = desc.offset();
548 EIGEN_IF_CONSTEXPR (is_col_major) {
549 for (
int i = NumDims - 1; i > 0; --i) {
550 coords[i] = remaining / output_strides[i];
551 remaining -= coords[i] * output_strides[i];
553 coords[0] = remaining;
555 for (
int i = 0; i < NumDims - 1; ++i) {
556 coords[i] = remaining / output_strides[i];
557 remaining -= coords[i] * output_strides[i];
559 coords[NumDims - 1] = remaining;
563 const int dd = is_col_major ? 0 : NumDims - 1;
564 const int nd = is_col_major ? 1 : NumDims - 2;
565 const int rd = is_col_major ? 2 : NumDims - 3;
566 const int cd = is_col_major ? 3 : NumDims - 4;
567 const int pd = is_col_major ? 4 : NumDims - 5;
569 const Index depth_start = coords[dd];
570 const Index depth_size = desc.dimension(dd);
571 const Index plane_start = coords[nd];
572 const Index plane_size = desc.dimension(nd);
573 const Index row_start = coords[rd];
574 const Index row_size = desc.dimension(rd);
575 const Index col_start = coords[cd];
576 const Index col_size = desc.dimension(cd);
577 const Index patch_start = coords[pd];
578 const Index patch_size = desc.dimension(pd);
582 array<Index, NumDims> other_sizes;
583 array<Index, NumDims> other_src_stride;
584 array<Index, NumDims> other_count;
588 Index in_stride = m_otherInputStride;
589 for (
int k = 5; k < NumDims; ++k) {
590 const int d = is_col_major ? k : NumDims - 1 - k;
591 other_sizes[num_other] = desc.dimension(d);
592 other_src_stride[num_other] = in_stride;
593 other_count[num_other] = 0;
594 src_other += coords[d] * in_stride;
595 in_stride *= m_dimensions[d];
600 using LinCopy = internal::StridedLinearBufferCopy<Scalar, Index>;
607 for (Index p = 0; p < patch_size; ++p) {
608 const Index patch3DIndex = patch_start + p;
609 const Index colIndex = patch3DIndex / m_fastOutputPlanesRows;
610 const Index rowIndex = (patch3DIndex - colIndex * m_outputPlanesRows) / m_fastOutputPlanes;
611 const Index planeIndex = patch3DIndex - m_outputPlanes * (colIndex * m_outputRows + rowIndex);
613 for (Index c = 0; c < col_size; ++c) {
614 const Index colOffset = col_start + c;
615 const Index inputCol = colIndex * m_col_strides + colOffset * m_in_col_strides - m_colPaddingLeft;
616 Index origInputCol = inputCol;
617 bool col_valid = inputCol >= 0 && inputCol < m_input_cols_eff;
618 if (col_valid && m_col_inflate_strides != 1) {
619 origInputCol = inputCol / m_fastInputColStride;
620 col_valid = (inputCol == origInputCol * m_col_inflate_strides);
623 for (Index r = 0; r < row_size; ++r) {
624 const Index rowOffset = row_start + r;
625 bool row_valid = col_valid;
626 Index origInputRow = 0;
628 const Index inputRow = rowIndex * m_row_strides + rowOffset * m_in_row_strides - m_rowPaddingTop;
629 row_valid = inputRow >= 0 && inputRow < m_input_rows_eff;
631 origInputRow = inputRow;
632 if (m_row_inflate_strides != 1) {
633 origInputRow = inputRow / m_fastInputRowStride;
634 row_valid = (inputRow == origInputRow * m_row_inflate_strides);
639 for (Index n = 0; n < plane_size; ++n) {
640 const Index planeOffset = plane_start + n;
641 bool valid = row_valid;
642 Index origInputPlane = 0;
644 const Index inputPlane =
645 planeIndex * m_plane_strides + planeOffset * m_in_plane_strides - m_planePaddingTop;
646 valid = inputPlane >= 0 && inputPlane < m_input_planes_eff;
648 origInputPlane = inputPlane;
649 if (m_plane_inflate_strides != 1) {
650 origInputPlane = inputPlane / m_fastInputPlaneStride;
651 valid = (inputPlane == origInputPlane * m_plane_inflate_strides);
657 const Index src = depth_start + origInputPlane * m_planeInputStride + origInputRow * m_rowInputStride +
658 origInputCol * m_colInputStride + src_other;
659 for (Index d = 0; d < depth_size; ++d) {
660 block_buffer[dst + d] = m_impl.coeff(src + d);
663 LinCopy::template Run<LinCopy::Kind::FillLinear>(
typename LinCopy::Dst(dst, 1, block_buffer),
664 typename LinCopy::Src(0, 0, &m_paddingValue),
674 for (; k < num_other; ++k) {
675 if (++other_count[k] < other_sizes[k]) {
676 src_other += other_src_stride[k];
680 src_other -= other_src_stride[k] * (other_sizes[k] - 1);
682 if (k == num_other)
break;
684 eigen_assert(dst == desc.size());
686 return block_storage.AsTensorMaterializedBlock();
689 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
690 const double compute_cost =
691 10 * TensorOpCost::DivCost<Index>() + 21 * TensorOpCost::MulCost<Index>() + 8 * TensorOpCost::AddCost<Index>();
692 return TensorOpCost(0, 0, compute_cost, vectorized, PacketSize);
695 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return nullptr; }
697 const TensorEvaluator<ArgType, Device>& impl()
const {
return m_impl; }
699 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index planePaddingTop()
const {
return m_planePaddingTop; }
700 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rowPaddingTop()
const {
return m_rowPaddingTop; }
701 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index colPaddingLeft()
const {
return m_colPaddingLeft; }
702 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index outputPlanes()
const {
return m_outputPlanes; }
703 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index outputRows()
const {
return m_outputRows; }
704 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index outputCols()
const {
return m_outputCols; }
705 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index userPlaneStride()
const {
return m_plane_strides; }
706 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index userRowStride()
const {
return m_row_strides; }
707 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index userColStride()
const {
return m_col_strides; }
708 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index userInPlaneStride()
const {
return m_in_plane_strides; }
709 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index userInRowStride()
const {
return m_in_row_strides; }
710 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index userInColStride()
const {
return m_in_col_strides; }
711 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index planeInflateStride()
const {
return m_plane_inflate_strides; }
712 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index rowInflateStride()
const {
return m_row_inflate_strides; }
713 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index colInflateStride()
const {
return m_col_inflate_strides; }
716 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packetWithPossibleZero(Index index)
const {
717 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
718 std::remove_const_t<CoeffReturnType> values[PacketSize];
720 for (
int i = 0; i < PacketSize; ++i) {
721 values[i] = coeff(index + i);
723 PacketReturnType rslt = internal::pload<PacketReturnType>(values);
727 Dimensions m_dimensions;
730 Index m_plane_strides;
734 Index m_outputPlanes;
738 Index m_planePaddingTop;
739 Index m_rowPaddingTop;
740 Index m_colPaddingLeft;
742 Index m_in_plane_strides;
743 Index m_in_row_strides;
744 Index m_in_col_strides;
746 Index m_plane_inflate_strides;
747 Index m_row_inflate_strides;
748 Index m_col_inflate_strides;
757 Index m_outputPlanesRows;
760 Index m_input_planes_eff;
761 Index m_input_rows_eff;
762 Index m_input_cols_eff;
763 Index m_patch_planes_eff;
764 Index m_patch_rows_eff;
765 Index m_patch_cols_eff;
774 Index m_planeInputStride;
775 Index m_rowInputStride;
776 Index m_colInputStride;
777 Index m_otherInputStride;
779 internal::TensorIntDivisor<Index> m_fastOtherStride;
780 internal::TensorIntDivisor<Index> m_fastPatchStride;
781 internal::TensorIntDivisor<Index> m_fastColStride;
782 internal::TensorIntDivisor<Index> m_fastRowStride;
783 internal::TensorIntDivisor<Index> m_fastInputPlaneStride;
784 internal::TensorIntDivisor<Index> m_fastInputRowStride;
785 internal::TensorIntDivisor<Index> m_fastInputColStride;
786 internal::TensorIntDivisor<Index> m_fastInputColsEff;
787 internal::TensorIntDivisor<Index> m_fastOutputPlanesRows;
788 internal::TensorIntDivisor<Index> m_fastOutputPlanes;
789 internal::TensorIntDivisor<Index> m_fastOutputDepth;
791 Scalar m_paddingValue;
793 TensorEvaluator<ArgType, Device> m_impl;
794 const Device EIGEN_DEVICE_REF m_device;
The tensor base class.
Definition TensorForwardDeclarations.h:69
Patch extraction specialized for processing of volumetric data. This assumes that the input has at le...
Definition TensorVolumePatch.h:50
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47