Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorShuffling.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TENSOR_TENSOR_SHUFFLING_H
12#define EIGEN_TENSOR_TENSOR_SHUFFLING_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20template <typename Shuffle, typename XprType>
21struct traits<TensorShufflingOp<Shuffle, XprType> > : public traits<XprType> {
22 using Scalar = typename XprType::Scalar;
23 using XprTraits = traits<XprType>;
24 using StorageKind = typename XprTraits::StorageKind;
25 using Index = typename XprTraits::Index;
26 static constexpr int NumDimensions = XprTraits::NumDimensions;
27 static constexpr int Layout = XprTraits::Layout;
28 using PointerType = typename XprTraits::PointerType;
29};
30
31template <typename Shuffle, typename XprType>
32struct eval<TensorShufflingOp<Shuffle, XprType>, Eigen::Dense> {
33 using type = const TensorShufflingOp<Shuffle, XprType>&;
34};
35
36} // end namespace internal
37
43template <typename Shuffle, typename XprType>
44class TensorShufflingOp : public TensorBase<TensorShufflingOp<Shuffle, XprType> > {
45 public:
47 using Scalar = typename Eigen::internal::traits<TensorShufflingOp>::Scalar;
48 using RealScalar = typename Eigen::NumTraits<Scalar>::Real;
49 using CoeffReturnType = typename XprType::CoeffReturnType;
50 using Nested = typename Eigen::internal::ref_selector<TensorShufflingOp>::type;
51 using StorageKind = typename Eigen::internal::traits<TensorShufflingOp>::StorageKind;
52 using Index = typename Eigen::internal::traits<TensorShufflingOp>::Index;
53
54 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorShufflingOp(const XprType& expr, const Shuffle& shfl)
55 : m_xpr(expr), m_shuffle(shfl) {}
56
57 EIGEN_DEVICE_FUNC const Shuffle& shufflePermutation() const { return m_shuffle; }
58
59 EIGEN_DEVICE_FUNC const internal::remove_all_t<typename XprType::Nested>& expression() const { return m_xpr; }
60
61 EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TensorShufflingOp)
62
63 protected:
64 typename XprType::Nested m_xpr;
65 const Shuffle m_shuffle;
66};
67
68// Eval as rvalue
69template <typename Shuffle, typename ArgType, typename Device>
70struct TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
73 using Index = typename XprType::Index;
74 static constexpr int NumDims = internal::array_size<typename TensorEvaluator<ArgType, Device>::Dimensions>::value;
75 using Dimensions = DSizes<Index, NumDims>;
76 using Scalar = typename XprType::Scalar;
77 using CoeffReturnType = typename XprType::CoeffReturnType;
78 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
79 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
80 using Storage = StorageMemory<CoeffReturnType, Device>;
81 using EvaluatorPointerType = typename Storage::Type;
82
83 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
84 enum {
85 IsAligned = false,
86 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
88 PreferBlockAccess = true,
89 CoordAccess = false, // to be implemented
90 RawAccess = false
91 };
92
93 using ScalarNoConst = std::remove_const_t<Scalar>;
94
95 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
96 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
97 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
98
99 using TensorBlock = typename internal::TensorMaterializedBlock<ScalarNoConst, NumDims, Layout, Index>;
100 //===--------------------------------------------------------------------===//
101
102 EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
103 : m_device(device), m_impl(op.expression(), device) {
104 const typename TensorEvaluator<ArgType, Device>::Dimensions& input_dims = m_impl.dimensions();
105 const Shuffle& shuffle = op.shufflePermutation();
106 m_is_identity = true;
107 for (int i = 0; i < NumDims; ++i) {
108 m_shuffle[i] = static_cast<int>(shuffle[i]);
109 m_dimensions[i] = input_dims[shuffle[i]];
110 m_inverseShuffle[shuffle[i]] = i;
111 if (m_is_identity && shuffle[i] != i) {
112 m_is_identity = false;
113 }
114 }
115
116 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
117 m_unshuffledInputStrides[0] = 1;
118 m_outputStrides[0] = 1;
119
120 for (int i = 1; i < NumDims; ++i) {
121 m_unshuffledInputStrides[i] = m_unshuffledInputStrides[i - 1] * input_dims[i - 1];
122 m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
123 m_fastOutputStrides[i] =
124 internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : Index(1));
125 }
126 } else {
127 m_unshuffledInputStrides[NumDims - 1] = 1;
128 m_outputStrides[NumDims - 1] = 1;
129 for (int i = NumDims - 2; i >= 0; --i) {
130 m_unshuffledInputStrides[i] = m_unshuffledInputStrides[i + 1] * input_dims[i + 1];
131 m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1];
132 m_fastOutputStrides[i] =
133 internal::TensorIntDivisor<Index>(m_outputStrides[i] > 0 ? m_outputStrides[i] : Index(1));
134 }
135 }
136
137 for (int i = 0; i < NumDims; ++i) {
138 m_inputStrides[i] = m_unshuffledInputStrides[shuffle[i]];
139 }
140 }
141
142 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }
143
144 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType /*data*/) {
145 m_impl.evalSubExprsIfNeeded(nullptr);
146 return true;
147 }
148
149#ifdef EIGEN_USE_THREADS
150 template <typename EvalSubExprsCallback>
151 EIGEN_STRONG_INLINE void evalSubExprsIfNeededAsync(EvaluatorPointerType, EvalSubExprsCallback done) {
152 m_impl.evalSubExprsIfNeededAsync(nullptr, [done](bool) { done(true); });
153 }
154#endif // EIGEN_USE_THREADS
155
156 EIGEN_STRONG_INLINE void cleanup() { m_impl.cleanup(); }
157
158 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
159 if (m_is_identity) {
160 return m_impl.coeff(index);
161 } else {
162 return m_impl.coeff(srcCoeff(index));
163 }
164 }
165
166 // Assembles a packet whose elements all lie in one inner-most run of the
167 // output: the input indices form an arithmetic progression starting at
168 // `base` with step `inner_stride`, so the index mapping is computed once
169 // per packet instead of once per coefficient.
170 template <int LoadMode, typename Self, bool ImplPacketAccess>
171 struct InnerRunLoader {
172 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType Run(const Self& self, Index base,
173 Index inner_stride) {
174 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
175 std::remove_const_t<CoeffReturnType> values[PacketSize];
176 EIGEN_UNROLL_LOOP
177 for (int i = 0; i < PacketSize; ++i) {
178 values[i] = self.m_impl.coeff(base + i * inner_stride);
179 }
180 return internal::pload<PacketReturnType>(values);
181 }
182 };
183
184 template <int LoadMode, typename Self>
185 struct InnerRunLoader<LoadMode, Self, true> {
186 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType Run(const Self& self, Index base,
187 Index inner_stride) {
188 if (inner_stride == 1) {
189 // Inner dimension not shuffled: one contiguous load.
190 return self.m_impl.template packet<Unaligned>(base);
191 }
192 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
193 std::remove_const_t<CoeffReturnType> values[PacketSize];
194 EIGEN_UNROLL_LOOP
195 for (int i = 0; i < PacketSize; ++i) {
196 values[i] = self.m_impl.coeff(base + i * inner_stride);
197 }
198 return internal::pload<PacketReturnType>(values);
199 }
200 };
201
202 template <int LoadMode, typename Self, bool ImplPacketAccess>
203 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType LoadPacketViaInnerRun(const Self& self, Index index) {
204 constexpr int inner_dim = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? 0 : NumDims - 1;
205 Index inner_pos;
206 const Index base = self.srcCoeffInner(index, inner_pos);
207 if (inner_pos + PacketSize <= self.m_dimensions[inner_dim]) {
208 return InnerRunLoader<LoadMode, Self, ImplPacketAccess>::Run(self, base, self.m_inputStrides[inner_dim]);
209 }
210
211 // The packet crosses an inner-run boundary: assemble it scalar by scalar.
212 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
213 std::remove_const_t<CoeffReturnType> values[PacketSize];
214 EIGEN_UNROLL_LOOP
215 for (int i = 0; i < PacketSize; ++i) {
216 values[i] = self.coeff(index + i);
217 }
218 return internal::pload<PacketReturnType>(values);
219 }
220
221 template <int LoadMode, typename Self, bool ImplPacketAccess>
222 struct PacketLoader {
223 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType Run(const Self& self, Index index) {
224 if (self.m_is_identity) {
225 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
226 std::remove_const_t<CoeffReturnType> values[PacketSize];
227 EIGEN_UNROLL_LOOP
228 for (int i = 0; i < PacketSize; ++i) {
229 values[i] = self.m_impl.coeff(index + i);
230 }
231 return internal::pload<PacketReturnType>(values);
232 }
233 return LoadPacketViaInnerRun<LoadMode, Self, ImplPacketAccess>(self, index);
234 }
235 };
236
237 template <int LoadMode, typename Self>
238 struct PacketLoader<LoadMode, Self, true> {
239 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static PacketReturnType Run(const Self& self, Index index) {
240 if (self.m_is_identity) {
241 return self.m_impl.template packet<LoadMode>(index);
242 }
243 return LoadPacketViaInnerRun<LoadMode, Self, true>(self, index);
244 }
245 };
246
247 template <int LoadMode>
248 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
249 eigen_assert(index + PacketSize - 1 < dimensions().TotalSize());
250 return PacketLoader<LoadMode, Self, TensorEvaluator<ArgType, Device>::PacketAccess>::Run(*this, index);
251 }
252
253 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
254 static constexpr int inner_dim = Layout == static_cast<int>(ColMajor) ? 0 : NumDims - 1;
255
256 const size_t target_size = m_device.firstLevelCacheSize();
257 const bool inner_dim_shuffled = m_shuffle[inner_dim] != inner_dim;
258
259 // Shuffling the inner dimension requires a strided gather across source
260 // rows into the block buffer, which is unvectorized across lanes (~1 cycle
261 // per scalar element in pgather) and spans one cache line per source row
262 // in the tile. Use target_size / 8 (~4 KiB) so the gathered source cache
263 // lines stay within L1 cache capacity and set associativity.
264 using BlockRequirements = internal::TensorBlockResourceRequirements;
265 if (inner_dim_shuffled) {
266 return BlockRequirements::uniform<Scalar>(target_size / 8).addCostPerCoeff({0, 0, 1.0});
267 } else {
268 return BlockRequirements::skewed<Scalar>(target_size);
269 }
270 }
271
272 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
273 bool root_of_expr_ast = false) const {
274 eigen_assert(m_impl.data() != nullptr);
275
276 using TensorBlockIO = internal::TensorBlockIO<ScalarNoConst, Index, NumDims, Layout>;
277 using TensorBlockIODst = typename TensorBlockIO::Dst;
278 using TensorBlockIOSrc = typename TensorBlockIO::Src;
279
280 const typename TensorBlock::Storage block_storage =
281 TensorBlock::prepareStorage(desc, scratch, /*allow_strided_storage=*/root_of_expr_ast);
282
283 typename TensorBlockIO::Dimensions input_strides(m_unshuffledInputStrides);
284 TensorBlockIOSrc src(input_strides, m_impl.data(), srcCoeff(desc.offset()));
285
286 TensorBlockIODst dst(block_storage.dimensions(), block_storage.strides(), block_storage.data());
287
288 typename TensorBlockIO::DimensionsMap dst_to_src_dim_map(m_shuffle);
289 TensorBlockIO::Copy(dst, src, dst_to_src_dim_map);
290
291 return block_storage.AsTensorMaterializedBlock();
292 }
293
294 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
295 const double compute_cost = m_is_identity
296 ? TensorOpCost::AddCost<Index>()
297 : NumDims * (2 * TensorOpCost::AddCost<Index>() +
298 2 * TensorOpCost::MulCost<Index>() + TensorOpCost::DivCost<Index>());
299 return m_impl.costPerCoeff(vectorized) + TensorOpCost(0, 0, compute_cost, vectorized, PacketSize);
300 }
301
302 EIGEN_DEVICE_FUNC typename Storage::Type data() const { return nullptr; }
303
304 protected:
305 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index
306 GetBlockOutputIndex(Index input_index, const DSizes<Index, NumDims>& input_block_strides,
307 const DSizes<Index, NumDims>& output_block_strides,
308 const DSizes<internal::TensorIntDivisor<Index>, NumDims>& fast_input_block_strides) const {
309 Index output_index = 0;
310 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
311 for (int i = NumDims - 1; i > 0; --i) {
312 const Index idx = input_index / fast_input_block_strides[i];
313 output_index += idx * output_block_strides[m_inverseShuffle[i]];
314 input_index -= idx * input_block_strides[i];
315 }
316 return output_index + input_index * output_block_strides[m_inverseShuffle[0]];
317 } else {
318 for (int i = 0; i < NumDims - 1; ++i) {
319 const Index idx = input_index / fast_input_block_strides[i];
320 output_index += idx * output_block_strides[m_inverseShuffle[i]];
321 input_index -= idx * input_block_strides[i];
322 }
323 return output_index + input_index * output_block_strides[m_inverseShuffle[NumDims - 1]];
324 }
325 }
326
327 // Computes the input index of output index `index` and, as a by-product of
328 // the same fast-divisor walk, the output's inner-dimension coordinate. The
329 // packet paths use the latter to test whether a whole packet stays inside
330 // one inner-most run without spending an extra division on it.
331 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeffInner(Index index, Index& inner_pos) const {
332 Index inputIndex = 0;
333 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
334 for (int i = NumDims - 1; i > 0; --i) {
335 const Index idx = index / m_fastOutputStrides[i];
336 inputIndex += idx * m_inputStrides[i];
337 index -= idx * m_outputStrides[i];
338 }
339 inner_pos = index;
340 return inputIndex + index * m_inputStrides[0];
341 } else {
342 for (int i = 0; i < NumDims - 1; ++i) {
343 const Index idx = index / m_fastOutputStrides[i];
344 inputIndex += idx * m_inputStrides[i];
345 index -= idx * m_outputStrides[i];
346 }
347 inner_pos = index;
348 return inputIndex + index * m_inputStrides[NumDims - 1];
349 }
350 }
351
352 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
353 Index inner_pos;
354 return srcCoeffInner(index, inner_pos);
355 }
356
357 Dimensions m_dimensions;
358 bool m_is_identity;
359 array<int, NumDims> m_shuffle;
360 array<Index, NumDims> m_inverseShuffle; // TODO(ezhulenev): Make it int type.
361 array<Index, NumDims> m_outputStrides;
362 array<internal::TensorIntDivisor<Index>, NumDims> m_fastOutputStrides;
363 array<Index, NumDims> m_inputStrides;
364 array<Index, NumDims> m_unshuffledInputStrides;
365
366 const Device EIGEN_DEVICE_REF m_device;
367 TensorEvaluator<ArgType, Device> m_impl;
368};
369
370// Eval as lvalue
371template <typename Shuffle, typename ArgType, typename Device>
372struct TensorEvaluator<TensorShufflingOp<Shuffle, ArgType>, Device>
373 : public TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device> {
374 using Base = TensorEvaluator<const TensorShufflingOp<Shuffle, ArgType>, Device>;
375
376 using XprType = TensorShufflingOp<Shuffle, ArgType>;
377 using Index = typename XprType::Index;
378 static constexpr int NumDims = internal::array_size<typename TensorEvaluator<ArgType, Device>::Dimensions>::value;
379 using Dimensions = DSizes<Index, NumDims>;
380 using Scalar = typename XprType::Scalar;
381 using CoeffReturnType = typename XprType::CoeffReturnType;
382 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
383 static constexpr int PacketSize = PacketType<CoeffReturnType, Device>::size;
384 static constexpr int Layout = TensorEvaluator<ArgType, Device>::Layout;
385
386 enum {
387 IsAligned = false,
388 PacketAccess = (PacketType<CoeffReturnType, Device>::size > 1),
389 BlockAccess = TensorEvaluator<ArgType, Device>::RawAccess,
390 PreferBlockAccess = true,
391 RawAccess = false
392 };
393
394 using ScalarNoConst = std::remove_const_t<Scalar>;
395
396 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
397 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
398 //===--------------------------------------------------------------------===//
399
400 EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) {}
401
402 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) const {
403 return this->m_impl.coeffRef(this->srcCoeff(index));
404 }
405
406 // Contiguous store into an unshuffled inner run; only instantiated when the
407 // nested evaluator has packet access.
408 template <int StoreMode, typename Self, bool ImplPacketAccess>
409 struct InnerRunWriter {
410 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static bool Run(const Self&, Index, const PacketReturnType&) { return false; }
411 };
412
413 template <int StoreMode, typename Self>
414 struct InnerRunWriter<StoreMode, Self, true> {
415 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE static bool Run(const Self& self, Index base, const PacketReturnType& x) {
416 self.m_impl.template writePacket<Unaligned>(base, x);
417 return true;
418 }
419 };
420
421 template <int StoreMode>
422 EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) const {
423 using Self = TensorEvaluator<TensorShufflingOp<Shuffle, ArgType>, Device>;
424 constexpr bool ImplPacketAccess = bool(TensorEvaluator<ArgType, Device>::PacketAccess);
425
426 // Mirrors the rvalue PacketLoader: within one inner-most run the target
427 // input indices form an arithmetic progression, so the index mapping is
428 // computed once per packet; an unshuffled inner dimension becomes a
429 // single contiguous store.
430 constexpr int inner_dim = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? 0 : NumDims - 1;
431 Index inner_pos;
432 const Index base = this->srcCoeffInner(index, inner_pos);
433 if (inner_pos + PacketSize <= this->m_dimensions[inner_dim]) {
434 const Index inner_stride = this->m_inputStrides[inner_dim];
435 if (inner_stride == 1 && InnerRunWriter<StoreMode, Self, ImplPacketAccess>::Run(*this, base, x)) {
436 return;
437 }
438 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
439 std::remove_const_t<CoeffReturnType> values[PacketSize];
440 internal::pstore<CoeffReturnType, PacketReturnType>(values, x);
441 EIGEN_UNROLL_LOOP
442 for (int i = 0; i < PacketSize; ++i) {
443 this->m_impl.coeffRef(base + i * inner_stride) = values[i];
444 }
445 return;
446 }
447
448 // The packet crosses an inner-run boundary: scatter scalar by scalar.
449 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment)
450 std::remove_const_t<CoeffReturnType> values[PacketSize];
451 internal::pstore<CoeffReturnType, PacketReturnType>(values, x);
452 EIGEN_UNROLL_LOOP
453 for (int i = 0; i < PacketSize; ++i) {
454 this->coeffRef(index + i) = values[i];
455 }
456 }
457
458 template <typename TensorBlock>
459 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writeBlock(const TensorBlockDesc& desc, const TensorBlock& block) {
460 eigen_assert(this->m_impl.data() != nullptr);
461
462 using TensorBlockIO = internal::TensorBlockIO<ScalarNoConst, Index, NumDims, Layout>;
463 using TensorBlockIODst = typename TensorBlockIO::Dst;
464 using TensorBlockIOSrc = typename TensorBlockIO::Src;
465
466 const Scalar* block_buffer = block.data();
467
468 // TODO(ezhulenev): TensorBlockIO should be able to read from any Eigen
469 // expression with coefficient and packet access as `src`.
470 void* mem = nullptr;
471 if (block_buffer == nullptr) {
472 mem = this->m_device.allocate(desc.size() * sizeof(Scalar));
473 ScalarNoConst* buf = static_cast<ScalarNoConst*>(mem);
474
475 using TensorBlockAssignment =
476 internal::TensorBlockAssignment<ScalarNoConst, NumDims, typename TensorBlock::XprType, Index>;
477
478 TensorBlockAssignment::Run(
479 TensorBlockAssignment::target(desc.dimensions(), internal::strides<Layout>(desc.dimensions()), buf),
480 block.expr());
481
482 block_buffer = buf;
483 }
484
485 // Read from block.
486 TensorBlockIOSrc src(internal::strides<Layout>(desc.dimensions()), block_buffer);
487
488 // Write to the output buffer.
489 typename TensorBlockIO::Dimensions output_strides(this->m_unshuffledInputStrides);
490 typename TensorBlockIO::Dimensions output_dimensions;
491 for (int i = 0; i < NumDims; ++i) {
492 output_dimensions[this->m_shuffle[i]] = desc.dimension(i);
493 }
494 TensorBlockIODst dst(output_dimensions, output_strides, this->m_impl.data(), this->srcCoeff(desc.offset()));
495
496 // Reorder dimensions according to the shuffle.
497 typename TensorBlockIO::DimensionsMap dst_to_src_dim_map;
498 for (int i = 0; i < NumDims; ++i) {
499 dst_to_src_dim_map[i] = static_cast<int>(this->m_inverseShuffle[i]);
500 }
501 TensorBlockIO::Copy(dst, src, dst_to_src_dim_map);
502
503 // Deallocate temporary buffer used for the block materialization.
504 if (mem != nullptr) this->m_device.deallocate(mem);
505 }
506};
507
508} // end namespace Eigen
509
510#endif // EIGEN_TENSOR_TENSOR_SHUFFLING_H
The tensor base class.
Definition TensorForwardDeclarations.h:69
Tensor shuffling class.
Definition TensorShuffling.h:44
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47