Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorBlock.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// This Source Code Form is subject to the terms of the Mozilla
5// Public License v. 2.0. If a copy of the MPL was not distributed
6// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
7// SPDX-FileCopyrightText: The Eigen Authors
8// SPDX-License-Identifier: MPL-2.0
9
10#ifndef EIGEN_TENSOR_TENSOR_BLOCK_H
11#define EIGEN_TENSOR_TENSOR_BLOCK_H
12
13// IWYU pragma: private
14#include "./InternalHeaderCheck.h"
15
16namespace Eigen {
17namespace internal {
18
19// -------------------------------------------------------------------------- //
20// Forward declarations for templates defined below.
21template <typename Scalar, typename IndexType, int NumDims, int Layout>
22class TensorBlockIO;
23
24// -------------------------------------------------------------------------- //
25// Helper function to compute strides for densely stored buffer of given
26// dimensions.
27
28// TODO(ezhulenev): We compute strides many times in different evaluators, use
29// this function instead everywhere.
30template <int Layout, typename IndexType, int NumDims>
31EIGEN_ALWAYS_INLINE std::enable_if_t<NumDims == 0, DSizes<IndexType, NumDims> > strides_impl(
32 const DSizes<IndexType, NumDims>& /*dimensions*/) {
33 DSizes<IndexType, NumDims> strides;
34 return strides;
35}
36
37template <int Layout, typename IndexType, int NumDims>
38EIGEN_ALWAYS_INLINE std::enable_if_t<(NumDims > 0), DSizes<IndexType, NumDims> > strides_impl(
39 const DSizes<IndexType, NumDims>& dimensions) {
40 DSizes<IndexType, NumDims> strides;
41 // TODO(ezhulenev): Benchmark whether template-unrolling this loop is beneficial.
42 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
43 strides[0] = 1;
44 for (int i = 1; i < NumDims; ++i) {
45 strides[i] = strides[i - 1] * dimensions[i - 1];
46 }
47 } else {
48 strides[NumDims - 1] = 1;
49 for (int i = NumDims - 2; i >= 0; --i) {
50 strides[i] = strides[i + 1] * dimensions[i + 1];
51 }
52 }
53
54 return strides;
55}
56
57template <int Layout, typename IndexType, int NumDims>
58EIGEN_ALWAYS_INLINE DSizes<IndexType, NumDims> strides(const DSizes<IndexType, NumDims>& dimensions) {
59 return strides_impl<Layout>(dimensions);
60}
61
62template <int Layout, typename IndexType, size_t NumDims>
63EIGEN_ALWAYS_INLINE DSizes<IndexType, NumDims> strides(const Eigen::array<IndexType, NumDims>& dimensions) {
64 return strides<Layout>(DSizes<IndexType, NumDims>(dimensions));
65}
66
67template <int Layout, std::ptrdiff_t... Indices>
68EIGEN_STRONG_INLINE DSizes<std::ptrdiff_t, sizeof...(Indices)> strides(const Sizes<Indices...>& sizes) {
69 return strides<Layout>(DSizes<std::ptrdiff_t, sizeof...(Indices)>(sizes));
70}
71
72// -------------------------------------------------------------------------- //
73
74// Tensor block shape type defines what are the shape preference for the blocks
75// extracted from the larger tensor.
76//
77// Example: blocks of 100 elements from the large 100x100 tensor:
78// - tensor: 100x100
79// - target_block_size: 100
80//
81// TensorBlockShapeType:
82// - kUniformAllDims: 100 blocks of size 10x10
83// - kSkewedInnerDims: 100 blocks of size 100x1 (or 1x100 depending on a column
84// or row major layout)
85enum class TensorBlockShapeType { kUniformAllDims, kSkewedInnerDims };
86
87struct TensorBlockResourceRequirements {
88 TensorBlockShapeType shape_type; // target block shape
89 size_t size; // target block size
90 TensorOpCost cost_per_coeff; // cost of computing a single block element
91
92#ifdef EIGEN_HIPCC
93 // For HIPCC, we need to explicitly declare as a "device fun", the constructor
94 // which is implicitly invoked in the "merge" / "any" routines. else HIPCC
95 // errors out complaining about the lack of a matching constructor
96 EIGEN_DEVICE_FUNC TensorBlockResourceRequirements(TensorBlockShapeType shape_type_, size_t size_, TensorOpCost cost_)
97 : shape_type(shape_type_), size(size_), cost_per_coeff(cost_) {}
98#endif
99
100 template <typename Scalar>
101 EIGEN_DEVICE_FUNC static TensorBlockResourceRequirements withShapeAndSize(TensorBlockShapeType shape_type,
102 size_t size_in_bytes, TensorOpCost cost) {
103 const size_t size = numext::maxi(size_t(1), size_in_bytes / sizeof(Scalar));
104 return {shape_type, size, cost};
105 }
106
107 template <typename Scalar>
108 EIGEN_DEVICE_FUNC static TensorBlockResourceRequirements withShapeAndSize(TensorBlockShapeType shape_type,
109 size_t size_in_bytes) {
110 // This default cost per coefficient is valid for most materialized tensor
111 // block evaluation implementations, because they typically just read
112 // coefficients from the underlying tensor storage, and write to the tensor
113 // block buffer (scratch or destination memory, reads and writes have linear
114 // access pattern). We ignore the fixed cost of block evaluation, because in
115 // practice it should be negligible.
116 //
117 // Lazy block evaluation adds the cost of calling a functor for each
118 // coefficient.
119 //
120 // All non-trivial block evaluation implementations must provide their own
121 // cost approximation (e.g. shuffling inner dimension has a much higher cost
122 // because it reads memory randomly, although the total number of moved
123 // bytes is the same).
124 return withShapeAndSize<Scalar>(shape_type, size_in_bytes,
125 {/*bytes_loaded=*/sizeof(Scalar),
126 /*bytes_stored=*/sizeof(Scalar),
127 /*compute_cycles=*/0});
128 }
129
130 template <typename Scalar>
131 EIGEN_DEVICE_FUNC static TensorBlockResourceRequirements skewed(size_t size_in_bytes) {
132 return withShapeAndSize<Scalar>(TensorBlockShapeType::kSkewedInnerDims, size_in_bytes);
133 }
134
135 template <typename Scalar>
136 EIGEN_DEVICE_FUNC static TensorBlockResourceRequirements uniform(size_t size_in_bytes) {
137 return withShapeAndSize<Scalar>(TensorBlockShapeType::kUniformAllDims, size_in_bytes);
138 }
139
140 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE TensorBlockResourceRequirements
141 merge(const TensorBlockResourceRequirements& lhs, const TensorBlockResourceRequirements& rhs) {
142 return {merge(lhs.shape_type, rhs.shape_type), // shape_type
143 merge(lhs.size, rhs.size), // size
144 merge(lhs.cost_per_coeff, rhs.cost_per_coeff)}; // cost_per_coeff
145 }
146
147 EIGEN_DEVICE_FUNC TensorBlockResourceRequirements& addCostPerCoeff(TensorOpCost cost) {
148 cost_per_coeff += cost;
149 return *this;
150 }
151
152 // This is a resource requirement that should be returned from expressions
153 // that do not have any block evaluation preference (e.g. default tensor
154 // expression with raw buffer access).
155 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE TensorBlockResourceRequirements any() {
156 return {TensorBlockShapeType::kUniformAllDims, 1, {0, 0, 0}};
157 }
158
159 private:
160 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE size_t merge(size_t lhs_size, size_t rhs_size) {
161 return numext::maxi(lhs_size, rhs_size);
162 }
163
164 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE TensorBlockShapeType merge(TensorBlockShapeType lhs,
165 TensorBlockShapeType rhs) {
166 return (lhs == TensorBlockShapeType::kSkewedInnerDims || rhs == TensorBlockShapeType::kSkewedInnerDims)
167 ? TensorBlockShapeType::kSkewedInnerDims
168 : TensorBlockShapeType::kUniformAllDims;
169 }
170
171 EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE TensorOpCost merge(TensorOpCost lhs_cost, TensorOpCost rhs_cost) {
172 return lhs_cost + rhs_cost;
173 }
174};
175
176// -------------------------------------------------------------------------- //
177// TensorBlockDescriptor specifies a block offset within a tensor and the block
178// sizes along each of the tensor dimensions.
179
180template <int NumDims, typename IndexType = Eigen::Index>
181class TensorBlockDescriptor {
182 public:
183 using Dimensions = DSizes<IndexType, NumDims>;
184
185 // If we evaluate a Tensor assignment, and expression on the left, already has
186 // a memory buffer, then we might do performance optimization, and evaluate
187 // the root expression directly into the final output memory. Some time it's
188 // possible to reuse it for materializing subexpressions inside an expression
189 // tree, to avoid dynamic memory allocation.
190 //
191 // The pointer type of the underlying storage is erased, because passing
192 // Scalar type through all the expression evaluation layers is way too many
193 // templates. In practice destination buffer type should always match the
194 // evaluated expression scalar type.
195 class DestinationBuffer {
196 public:
197 enum DestinationBufferKind : int {
198 // The above explicit specification of "int" as the enum basetype is
199 // needed to get around a HIPCC link error ("the field type is not
200 // amp-compatible")
201 // which is issued for class members with the enum type.
202 // TODO(rocm):
203 // remove the "int" basetype once HIPCC has been fixed to not error out
204 // in the above scenario.
205
206 // Destination buffer is not defined (`m_data` == nullptr).
207 kEmpty,
208
209 // Tensor block defined by an owning tensor block descriptor can fit
210 // contiguously into the destination buffer. In this case it's safe to
211 // materialize a tensor block in the destination buffer and build an
212 // expression over a dense view of it.
213 kContiguous,
214
215 // Destination buffer strides do not match strides of the contiguously
216 // stored block, and it's impossible to define a TensorMap over this
217 // buffer. However if we are evaluating a root of an expression tree, we
218 // still can materialize an output into this destination, because we can
219 // guarantee that no one will ever access it through block API.
220 //
221 // Strided input views are represented by TensorBlockView. Output
222 // materialization still reserves this destination for the root so that
223 // a child cannot overwrite another operand before it has been consumed.
224 kStrided
225 };
226
227 template <typename Scalar>
228 Scalar* data() const {
229 eigen_assert(m_data_type_size == sizeof(Scalar));
230 return static_cast<Scalar*>(m_data);
231 }
232
233 const Dimensions& strides() const { return m_strides; }
234 const DestinationBufferKind& kind() const { return m_kind; }
235
236 private:
237 friend class TensorBlockDescriptor<NumDims, IndexType>;
238
239 DestinationBuffer() = default;
240
241 template <typename Scalar>
242 DestinationBuffer(Scalar* data, const Dimensions& strides, DestinationBufferKind kind)
243 : m_data(static_cast<void*>(data)), m_data_type_size(sizeof(Scalar)), m_strides(strides), m_kind(kind) {}
244
245 template <int Layout, typename Scalar>
246 static DestinationBuffer make(const TensorBlockDescriptor& desc, Scalar* data, const Dimensions& strides) {
247 return DestinationBuffer(data, strides, kind<Layout>(desc, strides));
248 }
249
250 template <int Layout>
251 static DestinationBufferKind kind(const TensorBlockDescriptor& desc, const Dimensions& strides) {
252 const Dimensions& desc_dims = desc.dimensions();
253 const Dimensions& desc_strides = internal::strides<Layout>(desc_dims);
254 for (int i = 0; i < NumDims; ++i) {
255 if (desc_dims[i] == 1) continue;
256 if (desc_strides[i] != strides[i]) return kStrided;
257 }
258 return kContiguous;
259 }
260
261 // Storage pointer is type erased, to reduce template bloat, but we still
262 // keep the size of the underlying element type for error checking.
263 void* m_data = nullptr;
264 size_t m_data_type_size = 0;
265
266 // Destination buffer dimensions always match the dimensions of a tensor
267 // block descriptor it belongs to, however strides might be different.
268 Dimensions m_strides;
269
270 DestinationBufferKind m_kind = kEmpty;
271 };
272
273 TensorBlockDescriptor(const IndexType offset, const Dimensions& dimensions, const DestinationBuffer& destination)
274 : m_offset(offset), m_dimensions(dimensions), m_destination(destination) {}
275
276 TensorBlockDescriptor(const IndexType offset, const Dimensions& dimensions)
277 : m_offset(offset), m_dimensions(dimensions), m_destination(DestinationBuffer()) {}
278
279 IndexType offset() const { return m_offset; }
280 const Dimensions& dimensions() const { return m_dimensions; }
281 IndexType dimension(Index n) const {
282 eigen_internal_assert(0 <= n && n < NumDims);
283 return m_dimensions[n];
284 }
285 IndexType size() const { return array_prod<IndexType>(m_dimensions); }
286
287 const DestinationBuffer& destination() const { return m_destination; }
288
289 template <int Layout, typename Scalar>
290 void AddDestinationBuffer(Scalar* dst_base, const Dimensions& dst_strides) {
291 eigen_assert(dst_base != nullptr);
292 m_destination = DestinationBuffer::template make<Layout>(*this, dst_base, dst_strides);
293 }
294
295 template <int Layout, typename Scalar, typename DstStridesIndexType>
296 void AddDestinationBuffer(Scalar* dst_base, const DSizes<DstStridesIndexType, NumDims>& dst_strides) {
297 // DSizes constructor will do index type promotion if it's safe.
298 AddDestinationBuffer<Layout>(dst_base, Dimensions(dst_strides));
299 }
300
301 TensorBlockDescriptor& DropDestinationBuffer() {
302 m_destination.m_data = nullptr;
303 m_destination.m_kind = DestinationBuffer::kEmpty;
304 return *this;
305 }
306
307 bool HasDestinationBuffer() const { return m_destination.kind() != DestinationBuffer::kEmpty; }
308
309 // Returns a copy of `*this` with updated offset.
310 TensorBlockDescriptor WithOffset(IndexType offset) const {
311 return TensorBlockDescriptor(offset, m_dimensions, m_destination);
312 }
313
314 private:
315 // Offset and dimensions are immutable after construction. Block descriptor
316 // can only be mutated by adding or dropping destination.
317 const IndexType m_offset;
318 const Dimensions m_dimensions;
319 DestinationBuffer m_destination;
320};
321
322// -------------------------------------------------------------------------- //
323// TensorBlockMapper is responsible for iterating over the blocks of a tensor.
324
325template <int NumDims, int Layout, typename IndexType = Eigen::Index>
326class TensorBlockMapper {
327 using BlockDescriptor = TensorBlockDescriptor<NumDims, IndexType>;
328
329 public:
330 using Dimensions = DSizes<IndexType, NumDims>;
331
332 TensorBlockMapper() = default;
333 TensorBlockMapper(const DSizes<IndexType, NumDims>& dimensions, const TensorBlockResourceRequirements& requirements)
334 : m_tensor_dimensions(dimensions), m_requirements(requirements) {
335 // Compute block dimensions and the total number of blocks.
336 InitializeBlockDimensions();
337 }
338
339 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE IndexType blockCount() const { return m_total_block_count; }
340
341 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE IndexType blockTotalSize() const { return m_block_dimensions.TotalSize(); }
342
343 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const DSizes<IndexType, NumDims>& blockDimensions() const {
344 return m_block_dimensions;
345 }
346
347 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE BlockDescriptor blockDescriptor(IndexType block_index) const {
348 static constexpr bool isColMajor = Layout == static_cast<int>(ColMajor);
349
350 IndexType offset = 0;
351 DSizes<IndexType, NumDims> dimensions;
352
353 EIGEN_IF_CONSTEXPR (NumDims == 0) return BlockDescriptor(offset, dimensions);
354
355 // Iterate outer -> inner dimensions.
356 for (int i = NumDims - 1; i >= 0; --i) {
357 const int dim = isColMajor ? i : NumDims - i - 1;
358
359 const IndexType idx = block_index / m_block_strides[dim];
360 block_index -= idx * m_block_strides[dim];
361
362 const IndexType coord = idx * m_block_dimensions[dim];
363 dimensions[dim] = numext::mini(m_tensor_dimensions[dim] - coord, m_block_dimensions[dim]);
364 offset += coord * m_tensor_strides[dim];
365 }
366
367 return {offset, dimensions};
368 }
369
370 private:
371 void InitializeBlockDimensions() {
372 // Requested block shape and size.
373 const TensorBlockShapeType shape_type = m_requirements.shape_type;
374 IndexType target_block_size = numext::maxi<IndexType>(1, static_cast<IndexType>(m_requirements.size));
375
376 IndexType tensor_size = m_tensor_dimensions.TotalSize();
377
378 // Corner case: one of the dimensions is zero. Logic below is too complex
379 // to handle this case on a general basis, just use unit block size.
380 // Note: we must not yield blocks with zero dimensions (recipe for
381 // overflows/underflows, divisions by zero and NaNs later).
382 if (tensor_size == 0) {
383 for (int i = 0; i < NumDims; ++i) {
384 m_block_dimensions[i] = 1;
385 }
386 m_total_block_count = 0;
387 return;
388 }
389
390 // If tensor fits into a target block size, evaluate it as a single block.
391 if (tensor_size <= target_block_size) {
392 m_block_dimensions = m_tensor_dimensions;
393 m_total_block_count = 1;
394 // The only valid block index is `0`, and in this case we do not need
395 // to compute real strides for tensor or blocks (see blockDescriptor).
396 for (int i = 0; i < NumDims; ++i) {
397 m_tensor_strides[i] = 0;
398 m_block_strides[i] = 1;
399 }
400 return;
401 }
402
403 static constexpr bool isColMajor = Layout == static_cast<int>(ColMajor);
404
405 // Block shape skewed towards inner dimension.
406 if (shape_type == TensorBlockShapeType::kSkewedInnerDims) {
407 IndexType coeff_to_allocate = target_block_size;
408
409 for (int i = 0; i < NumDims; ++i) {
410 const int dim = isColMajor ? i : NumDims - i - 1;
411 m_block_dimensions[dim] = numext::mini(coeff_to_allocate, m_tensor_dimensions[dim]);
412 coeff_to_allocate =
413 numext::div_ceil(coeff_to_allocate, numext::maxi(static_cast<IndexType>(1), m_block_dimensions[dim]));
414 }
415 eigen_assert(coeff_to_allocate == 1);
416
417 } else if (shape_type == TensorBlockShapeType::kUniformAllDims) {
418 // Tensor will not fit within 'target_block_size' budget: calculate tensor
419 // block dimension sizes based on "square" dimension size target.
420 const IndexType dim_size_target = convert_index<IndexType>(
421 numext::pow(static_cast<float>(target_block_size), 1.0f / static_cast<float>(m_block_dimensions.rank())));
422
423 for (int i = 0; i < NumDims; ++i) {
424 // TODO(andydavis): Adjust the inner most 'block_dim_size' to make it
425 // a multiple of the packet size. Note that reducing
426 // 'block_dim_size' in this manner can increase the number of
427 // blocks, and so will amplify any per-block overhead.
428 m_block_dimensions[i] = numext::mini(dim_size_target, m_tensor_dimensions[i]);
429 }
430
431 // Add any un-allocated coefficients to inner dimension(s).
432 IndexType total_size = m_block_dimensions.TotalSize();
433 for (int i = 0; i < NumDims; ++i) {
434 const int dim = isColMajor ? i : NumDims - i - 1;
435
436 if (m_block_dimensions[dim] < m_tensor_dimensions[dim]) {
437 const IndexType total_size_other_dims = total_size / m_block_dimensions[dim];
438 const IndexType alloc_avail = numext::div_ceil<IndexType>(target_block_size, total_size_other_dims);
439 if (alloc_avail == m_block_dimensions[dim]) {
440 // Insufficient excess coefficients to allocate.
441 break;
442 }
443 m_block_dimensions[dim] = numext::mini(m_tensor_dimensions[dim], alloc_avail);
444 total_size = total_size_other_dims * m_block_dimensions[dim];
445 }
446 }
447
448 } else {
449 eigen_assert(false); // unknown block shape
450 }
451
452 eigen_assert(m_block_dimensions.TotalSize() >=
453 numext::mini<IndexType>(target_block_size, m_tensor_dimensions.TotalSize()));
454
455 // Calculate block counts by dimension and total block count.
456 DSizes<IndexType, NumDims> block_count;
457 for (int i = 0; i < NumDims; ++i) {
458 block_count[i] = numext::div_ceil(m_tensor_dimensions[i], m_block_dimensions[i]);
459 }
460 m_total_block_count = array_prod(block_count);
461
462 // Calculate block strides (used for enumerating blocks).
463 m_tensor_strides = strides<Layout>(m_tensor_dimensions);
464 m_block_strides = strides<Layout>(block_count);
465 }
466
467 DSizes<IndexType, NumDims> m_tensor_dimensions;
468 TensorBlockResourceRequirements m_requirements;
469
470 DSizes<IndexType, NumDims> m_block_dimensions;
471 IndexType m_total_block_count;
472
473 DSizes<IndexType, NumDims> m_tensor_strides;
474 DSizes<IndexType, NumDims> m_block_strides;
475};
476
477// -------------------------------------------------------------------------- //
478// TensorBlockScratchAllocator is responsible for allocating temporary buffers
479// for block evaluation (output or input block materialization). Given that
480// Eigen expression traversal order is deterministic, all temporary allocations
481// are happening in the same order, and usually have exactly the same size.
482// Scratch allocator keeps a trace of all dynamic allocations, and after the
483// first block evaluation is completed, we should be able to reuse all the
484// temporary buffers for the next block evaluation.
485
486template <typename Device>
487class TensorBlockScratchAllocator {
488 public:
489 explicit TensorBlockScratchAllocator(const Device& device) : m_device(device), m_allocation_index(0) {}
490
491 ~TensorBlockScratchAllocator() {
492 for (size_t i = 0; i < m_allocations.size(); ++i) {
493 m_device.deallocate(m_allocations[i].ptr);
494 }
495 }
496
497 void* allocate(size_t size) {
498 // TODO(ezhulenev): Remove when replaced with inlined vector.
499 if (m_allocations.capacity() == 0) m_allocations.reserve(8);
500
501 // Check if we already have an existing allocation at current index.
502 const int num_allocations = static_cast<int>(m_allocations.size());
503 const bool has_allocation = m_allocation_index < num_allocations;
504
505 // Allocation index can't be larger than the number of allocations.
506 eigen_assert(m_allocation_index <= num_allocations);
507
508 // If we have existing allocation, and its size is larger or equal to
509 // requested size, we do nothing.
510
511 // If current allocation can't fit requested size, we deallocate it, and
512 // replace with a larger allocation.
513 if (has_allocation && m_allocations[m_allocation_index].size < size) {
514 m_device.deallocate(m_allocations[m_allocation_index].ptr);
515 m_allocations[m_allocation_index].ptr = m_device.allocate(size);
516 m_allocations[m_allocation_index].size = size;
517 }
518
519 // Make a new allocation if we don't have an existing one.
520 if (!has_allocation) {
521 Allocation allocation;
522 allocation.ptr = m_device.allocate(size);
523 allocation.size = size;
524 m_allocations.push_back(allocation);
525 }
526
527 eigen_assert(m_allocations[m_allocation_index].ptr != nullptr);
528 eigen_assert(m_allocations[m_allocation_index].size >= size);
529
530 return m_allocations[m_allocation_index++].ptr;
531 }
532
533 void reset() { m_allocation_index = 0; }
534
535 private:
536 struct Allocation {
537 void* ptr;
538 size_t size;
539 };
540
541 const Device& m_device;
542 int m_allocation_index;
543 // TODO(ezhulenev): This should be an inlined vector.
544 std::vector<Allocation> m_allocations;
545};
546
547// -------------------------------------------------------------------------- //
548// TensorBlockKind represents all possible block kinds, that can be produced by
549// TensorEvaluator::evalBlock function.
550enum TensorBlockKind {
551 // Tensor block that is a lazy expression that must be assigned to a
552 // destination using TensorBlockAssign.
553 kExpr,
554
555 // Tensor block that is a view into a memory buffer owned by an underlying
556 // Tensor expression (e.g. it can be a view into a Tensor buffer).
557 kView,
558
559 // Tensor block that was materialized in a scratch memory buffer, allocated
560 // with TensorBlockScratchAllocator. This block must be copied to a
561 // destination, similar to a block of `kExpr` type.
562 kMaterializedInScratch,
563
564 // Tensor block that was materialized directly into the final output memory
565 // buffer. For example if the left side of an assignment is a Tensor, we can
566 // directly materialize the block in the destination memory.
567 //
568 // If strides in the output buffer do not match tensor block strides, the
569 // Tensor expression will be invalid, and should not be used by
570 // TensorBlockAssign or for constructing another block expression.
571 kMaterializedInOutput
572};
573
574// -------------------------------------------------------------------------- //
575// TensorBlockNotImplemented should be used to define TensorBlock typedef in
576// TensorEvaluators that do not support block evaluation.
577
578class TensorBlockNotImplemented {
579 public:
580 using XprType = void;
581};
582
583template <typename Scalar, int NumDims, int Layout, typename IndexType>
584class TensorBlockView;
585
586template <typename Scalar, int NumDims, int Layout, typename IndexType>
587struct traits<TensorBlockView<Scalar, NumDims, Layout, IndexType>>
588 : traits<Tensor<Scalar, NumDims, Layout, IndexType>> {
589 static constexpr unsigned int Flags = 0;
590};
591
592// A block's inner runs are contiguous, but successive runs can belong to different
593// rows, columns, or planes of the underlying tensor.
594template <typename Scalar_, int NumDims, int Layout, typename IndexType>
595class TensorBlockView : public TensorBase<TensorBlockView<Scalar_, NumDims, Layout, IndexType>> {
596 public:
597 using Scalar = Scalar_;
598 using Index = IndexType;
599 using Dimensions = DSizes<Index, NumDims>;
600 using Nested = TensorBlockView;
601 using StorageKind = Dense;
602 using CoeffReturnType = Scalar;
603
604 TensorBlockView(const Scalar* data, const Dimensions& dimensions)
605 : TensorBlockView(data, dimensions, internal::strides<Layout>(dimensions)) {}
606
607 TensorBlockView(const Scalar* data, const Dimensions& dimensions, const Dimensions& strides)
608 : m_data(data), m_dimensions(dimensions), m_strides(strides), m_contiguous(true) {
609 eigen_assert(NumDims == 0 || strides[Layout == ColMajor ? 0 : NumDims - 1] == 1);
610 Index stride = 1;
611 for (int i = 0; i < NumDims; ++i) {
612 const int dim = Layout == ColMajor ? i : NumDims - 1 - i;
613 if (dimensions[dim] > 1 && strides[dim] != stride) m_contiguous = false;
614 stride *= dimensions[dim];
615 }
616 }
617
618 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_dimensions; }
619 EIGEN_DEVICE_FUNC const Dimensions& strides() const { return m_strides; }
620 EIGEN_DEVICE_FUNC const Scalar* data() const { return m_contiguous ? m_data : nullptr; }
621 EIGEN_DEVICE_FUNC const Scalar* rawData() const { return m_data; }
622
623 private:
624 const Scalar* m_data;
625 Dimensions m_dimensions;
626 Dimensions m_strides;
627 bool m_contiguous;
628};
629
630} // namespace internal
631
632template <typename Scalar_, int NumDims, int Layout_, typename IndexType, typename Device>
633struct TensorEvaluator<const internal::TensorBlockView<Scalar_, NumDims, Layout_, IndexType>, Device> {
634 using XprType = internal::TensorBlockView<Scalar_, NumDims, Layout_, IndexType>;
635 using Scalar = Scalar_;
636 using Index = IndexType;
637 using Dimensions = DSizes<Index, NumDims>;
638 using CoeffReturnType = Scalar;
639 using PacketReturnType = typename PacketType<Scalar, Device>::type;
640 using EvaluatorPointerType = const Scalar*;
641 using TensorBlock = internal::TensorBlockNotImplemented;
642 static constexpr int Layout = Layout_;
643 static constexpr bool IsAligned = false;
644 static constexpr bool PacketAccess = internal::packet_traits<Scalar>::Vectorizable;
645 static constexpr bool BlockAccess = false;
646 static constexpr bool PreferBlockAccess = false;
647 static constexpr bool CoordAccess = false;
648 static constexpr bool RawAccess = false;
649
650 TensorEvaluator(const XprType& expression, const Device&)
651 : m_expression(expression), m_output_strides(internal::strides<Layout>(expression.dimensions())) {
652 if (!expression.data()) {
653 for (int i = 0; i < NumDims; ++i) {
654 m_divisors[i] = internal::TensorIntDivisor<Index>(numext::maxi(Index(1), m_output_strides[i]));
655 }
656 }
657 }
658
659 EIGEN_DEVICE_FUNC const Dimensions& dimensions() const { return m_expression.dimensions(); }
660 EIGEN_DEVICE_FUNC bool evalSubExprsIfNeeded(EvaluatorPointerType) { return true; }
661 EIGEN_DEVICE_FUNC void cleanup() {}
662 EIGEN_DEVICE_FUNC const Scalar* data() const { return m_expression.data(); }
663
664 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index srcCoeff(Index index) const {
665 if (m_expression.data()) return index;
666 Index offset = 0;
667 for (int i = NumDims - 1; i > 0; --i) {
668 const int dim = Layout == ColMajor ? i : NumDims - 1 - i;
669 const Index coordinate = index / m_divisors[dim];
670 offset += coordinate * m_expression.strides()[dim];
671 index -= coordinate * m_output_strides[dim];
672 }
673 return offset + index;
674 }
675
676 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
677 return m_expression.rawData()[srcCoeff(index)];
678 }
679
680 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar* coeffAddress(Index index) const {
681 return m_expression.rawData() + srcCoeff(index);
682 }
683
684 template <int LoadMode>
685 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
686 constexpr int PacketSize = PacketType<Scalar, Device>::size;
687 const Index first = srcCoeff(index);
688 if (m_expression.data() || srcCoeff(index + PacketSize - 1) == first + PacketSize - 1) {
689 return internal::ploadu<PacketReturnType>(m_expression.rawData() + first);
690 }
691 EIGEN_ALIGN_MAX Scalar values[PacketSize];
692 for (int i = 0; i < PacketSize; ++i) values[i] = coeff(index + i);
693 return internal::ploadu<PacketReturnType>(values);
694 }
695
696 EIGEN_DEVICE_FUNC TensorOpCost costPerCoeff(bool vectorized) const {
697 return TensorOpCost(sizeof(Scalar), 0, m_expression.data() ? 0 : NumDims, vectorized,
698 PacketType<Scalar, Device>::size);
699 }
700
701 private:
702 XprType m_expression;
703 Dimensions m_output_strides;
704 array<internal::TensorIntDivisor<Index>, NumDims> m_divisors;
705};
706
707template <typename Scalar, int Layout, typename Index, typename Device>
708struct TensorEvaluator<const internal::TensorBlockView<Scalar, 1, Layout, Index>, Device>
709 : TensorEvaluator<const TensorMap<const Tensor<Scalar, 1, Layout, Index>>, Device> {
710 using XprType = internal::TensorBlockView<Scalar, 1, Layout, Index>;
711 using MapType = TensorMap<const Tensor<Scalar, 1, Layout, Index>>;
712 using Base = TensorEvaluator<const MapType, Device>;
713 TensorEvaluator(const XprType& expression, const Device& device)
714 : Base(MapType(expression.rawData(), expression.dimensions()), device) {}
715 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar* coeffAddress(Index index) const { return this->data() + index; }
716};
717
718namespace internal {
719
720// -------------------------------------------------------------------------- //
721// XprScalar extracts Scalar type from the Eigen expressions (if expression type
722// is not void). It's required to be able to define lazy block expression for
723// argument types, that do not support block evaluation.
724
725template <typename XprType>
726struct XprScalar {
727 using type = typename XprType::Scalar;
728};
729template <>
730struct XprScalar<void> {
731 using type = void;
732};
733
734// -------------------------------------------------------------------------- //
735// TensorMaterializedBlock is a fully evaluated block of the original tensor,
736// and XprType is a view over its data, allowing non-unit outer strides. It is
737// used to materialize blocks of tensor expressions, that can't be efficiently
738// represented as lazy Tensor expressions with fast coeff/packet operations,
739// e.g. we materialize all broadcasts into evaluated blocks.
740//
741// TensorMaterializedBlock does not own its memory buffer, it's either a memory
742// buffer that backs the original expression (e.g. block is just a view into a
743// Tensor), or a memory buffer allocated with scratch allocator, and in this
744// case the scratch allocator will deallocate it at the end of block based
745// expression execution.
746//
747// If the block was evaluated directly into the output buffer, and strides in
748// the output buffer do not match block strides, the dense block expression will
749// be invalid, and should never be used in block assignment or any other tensor
750// expression.
751
752template <typename Scalar, int NumDims, int Layout, typename IndexType = Eigen::Index>
753class TensorMaterializedBlock {
754 public:
755 using Dimensions = DSizes<IndexType, NumDims>;
756 using XprType = TensorBlockView<Scalar, NumDims, Layout, IndexType>;
757
758 TensorMaterializedBlock(TensorBlockKind kind, const Scalar* data, const Dimensions& dimensions,
759 bool valid_expr = true)
760 : m_kind(kind), m_data(data), m_dimensions(dimensions), m_expr(m_data, m_dimensions), m_valid_expr(valid_expr) {
761 eigen_assert(m_kind == internal::TensorBlockKind::kView ||
762 m_kind == internal::TensorBlockKind::kMaterializedInScratch ||
763 m_kind == internal::TensorBlockKind::kMaterializedInOutput);
764 }
765
766 TensorBlockKind kind() const { return m_kind; }
767 const XprType& expr() const {
768 eigen_assert(m_valid_expr);
769 return m_expr;
770 }
771 // Consumers of data() require a dense buffer; strided views must use expr().
772 const Scalar* data() const { return m_valid_expr ? m_expr.data() : m_data; }
773 void cleanup() {}
774
775 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, IndexType>;
776
777 // TensorMaterializedBlock can be backed by different types of storage:
778 //
779 // (1) Contiguous block of memory allocated with scratch allocator.
780 // (2) Contiguous block of memory reused from tensor block descriptor
781 // destination buffer.
782 // (3) Strided block of memory reused from tensor block descriptor
783 // destination buffer.
784 //
785 class Storage {
786 public:
787 Scalar* data() const { return m_data; }
788 const Dimensions& dimensions() const { return m_dimensions; }
789 const Dimensions& strides() const { return m_strides; }
790
791 TensorMaterializedBlock AsTensorMaterializedBlock() const {
792 return TensorMaterializedBlock(m_materialized_in_output ? internal::TensorBlockKind::kMaterializedInOutput
793 : internal::TensorBlockKind::kMaterializedInScratch,
794 m_data, m_dimensions, !m_strided_storage);
795 }
796
797 private:
798 friend class TensorMaterializedBlock<Scalar, NumDims, Layout, IndexType>;
799
800 Storage(Scalar* data, const Dimensions& dimensions, const Dimensions& strides, bool materialized_in_output,
801 bool strided_storage)
802 : m_data(data),
803 m_dimensions(dimensions),
804 m_strides(strides),
805 m_materialized_in_output(materialized_in_output),
806 m_strided_storage(strided_storage) {}
807
808 Scalar* m_data;
809 Dimensions m_dimensions;
810 Dimensions m_strides;
811 bool m_materialized_in_output;
812 bool m_strided_storage;
813 };
814
815 // Creates a storage for materialized block either from the block descriptor
816 // destination buffer, or allocates a new buffer with scratch allocator.
817 template <typename TensorBlockScratch>
818 EIGEN_STRONG_INLINE static Storage prepareStorage(TensorBlockDesc& desc, TensorBlockScratch& scratch,
819 bool allow_strided_storage = false) {
820 // Try to reuse destination as an output block buffer.
821 using DestinationBuffer = typename TensorBlockDesc::DestinationBuffer;
822
823 if (desc.destination().kind() == DestinationBuffer::kContiguous) {
824 Scalar* buffer = desc.destination().template data<Scalar>();
825 desc.DropDestinationBuffer();
826 return Storage(buffer, desc.dimensions(), internal::strides<Layout>(desc.dimensions()),
827 /*materialized_in_output=*/true,
828 /*strided_storage=*/false);
829
830 } else if (desc.destination().kind() == DestinationBuffer::kStrided && allow_strided_storage) {
831 Scalar* buffer = desc.destination().template data<Scalar>();
832 desc.DropDestinationBuffer();
833 return Storage(buffer, desc.dimensions(), desc.destination().strides(),
834 /*materialized_in_output=*/true, /*strided_storage=*/true);
835
836 } else {
837 void* mem = scratch.allocate(desc.size() * sizeof(Scalar));
838 return Storage(static_cast<Scalar*>(mem), desc.dimensions(), internal::strides<Layout>(desc.dimensions()),
839 /*materialized_in_output=*/false,
840 /*strided_storage=*/false);
841 }
842 }
843
844 // Creates a materialized block for the given descriptor from a memory buffer.
845 template <typename DataDimensions, typename TensorBlockScratch>
846 EIGEN_STRONG_INLINE static TensorMaterializedBlock materialize(const Scalar* data, const DataDimensions& data_dims,
847 TensorBlockDesc& desc,
848 TensorBlockScratch& /*scratch*/) {
849 eigen_assert(array_size<DataDimensions>::value == desc.dimensions().size());
850
851 TensorMaterializedBlock block(internal::TensorBlockKind::kView, data + desc.offset(), desc.dimensions());
852 block.m_expr = XprType(data + desc.offset(), desc.dimensions(), internal::strides<Layout>(Dimensions(data_dims)));
853 return block;
854 }
855
856 private:
857 TensorBlockKind m_kind;
858 const Scalar* m_data;
859 Dimensions m_dimensions;
860 XprType m_expr;
861 bool m_valid_expr;
862};
863
864// -------------------------------------------------------------------------- //
865// TensorCwiseUnaryBlock is a lazy tensor expression block that applies UnaryOp
866// functor to the blocks produced by the underlying Tensor expression.
867
868template <typename UnaryOp, typename ArgTensorBlock>
869class TensorCwiseUnaryBlock {
870 static constexpr bool NoArgBlockAccess = std::is_void<typename ArgTensorBlock::XprType>::value;
871
872 public:
873 using XprType =
874 std::conditional_t<NoArgBlockAccess, void, TensorCwiseUnaryOp<UnaryOp, const typename ArgTensorBlock::XprType>>;
875
876 using Scalar = typename XprScalar<XprType>::type;
877
878 TensorCwiseUnaryBlock(const ArgTensorBlock& arg_block, const UnaryOp& functor)
879 : m_arg_block(arg_block), m_functor(functor) {}
880
881 TensorBlockKind kind() const { return internal::TensorBlockKind::kExpr; }
882
883 XprType expr() const { return XprType(m_arg_block.expr(), m_functor); }
884 const Scalar* data() const { return nullptr; }
885 void cleanup() { m_arg_block.cleanup(); }
886
887 private:
888 ArgTensorBlock m_arg_block;
889 UnaryOp m_functor;
890};
891
892// -------------------------------------------------------------------------- //
893// TensorCwiseBinaryBlock is a lazy tensor expression block that applies BinaryOp
894// functor to the blocks produced by the underlying Tensor expression.
895
896template <typename BinaryOp, typename LhsTensorBlock, typename RhsTensorBlock>
897class TensorCwiseBinaryBlock {
898 static constexpr bool NoArgBlockAccess =
899 std::is_void<typename LhsTensorBlock::XprType>::value || std::is_void<typename RhsTensorBlock::XprType>::value;
900
901 public:
902 using XprType = std::conditional_t<
903 NoArgBlockAccess, void,
904 TensorCwiseBinaryOp<BinaryOp, const typename LhsTensorBlock::XprType, const typename RhsTensorBlock::XprType>>;
905
906 using Scalar = typename XprScalar<XprType>::type;
907
908 TensorCwiseBinaryBlock(const LhsTensorBlock& left_block, const RhsTensorBlock& right_block, const BinaryOp& functor)
909 : m_left_block(left_block), m_right_block(right_block), m_functor(functor) {}
910
911 TensorBlockKind kind() const { return internal::TensorBlockKind::kExpr; }
912
913 XprType expr() const { return XprType(m_left_block.expr(), m_right_block.expr(), m_functor); }
914
915 const Scalar* data() const { return nullptr; }
916
917 void cleanup() {
918 m_left_block.cleanup();
919 m_right_block.cleanup();
920 }
921
922 private:
923 LhsTensorBlock m_left_block;
924 RhsTensorBlock m_right_block;
925 BinaryOp m_functor;
926};
927
928// -------------------------------------------------------------------------- //
929// TensorUnaryExprBlock is a lazy tensor expression block that can construct
930// an arbitrary tensor expression from a block of the underlying type (this is a
931// generalization of the TensorCwiseUnaryBlock for arbitrary expressions).
932
933template <typename BlockFactory, typename ArgTensorBlock>
934class TensorUnaryExprBlock {
935 using ArgXprType = typename ArgTensorBlock::XprType;
936 static constexpr bool NoArgBlockAccess = std::is_void<ArgXprType>::value;
937
938 public:
939 using XprType = std::conditional_t<NoArgBlockAccess, void, typename BlockFactory::template XprType<ArgXprType>::type>;
940
941 using Scalar = typename XprScalar<XprType>::type;
942
943 TensorUnaryExprBlock(const ArgTensorBlock& arg_block, const BlockFactory& factory)
944 : m_arg_block(arg_block), m_factory(factory) {}
945
946 TensorBlockKind kind() const { return internal::TensorBlockKind::kExpr; }
947 XprType expr() const { return m_factory.expr(m_arg_block.expr()); }
948 const Scalar* data() const { return nullptr; }
949 void cleanup() { m_arg_block.cleanup(); }
950
951 private:
952 ArgTensorBlock m_arg_block;
953 BlockFactory m_factory;
954};
955
956// -------------------------------------------------------------------------- //
957// TensorTernaryExprBlock is a lazy tensor expression block that can construct
958// an arbitrary tensor expression from three blocks of the underlying type.
959
960template <typename BlockFactory, typename Arg1TensorBlock, typename Arg2TensorBlock, typename Arg3TensorBlock>
961class TensorTernaryExprBlock {
962 using Arg1XprType = typename Arg1TensorBlock::XprType;
963 using Arg2XprType = typename Arg2TensorBlock::XprType;
964 using Arg3XprType = typename Arg3TensorBlock::XprType;
965
966 static constexpr bool NoArgBlockAccess =
967 std::is_void<Arg1XprType>::value || std::is_void<Arg2XprType>::value || std::is_void<Arg3XprType>::value;
968
969 public:
970 using XprType =
971 std::conditional_t<NoArgBlockAccess, void,
972 typename BlockFactory::template XprType<Arg1XprType, Arg2XprType, Arg3XprType>::type>;
973
974 using Scalar = typename XprScalar<XprType>::type;
975
976 TensorTernaryExprBlock(const Arg1TensorBlock& arg1_block, const Arg2TensorBlock& arg2_block,
977 const Arg3TensorBlock& arg3_block, const BlockFactory& factory)
978 : m_arg1_block(arg1_block), m_arg2_block(arg2_block), m_arg3_block(arg3_block), m_factory(factory) {}
979
980 TensorBlockKind kind() const { return internal::TensorBlockKind::kExpr; }
981 XprType expr() const { return m_factory.expr(m_arg1_block.expr(), m_arg2_block.expr(), m_arg3_block.expr()); }
982 const Scalar* data() const { return nullptr; }
983 void cleanup() {
984 m_arg1_block.cleanup();
985 m_arg2_block.cleanup();
986 m_arg3_block.cleanup();
987 }
988
989 private:
990 Arg1TensorBlock m_arg1_block;
991 Arg2TensorBlock m_arg2_block;
992 Arg3TensorBlock m_arg3_block;
993 BlockFactory m_factory;
994};
995
996// -------------------------------------------------------------------------- //
997// StridedLinearBufferCopy provides a method to copy data between two linear
998// buffers with different strides, with optimized paths for scatter/gather.
999
1000template <typename Scalar, typename IndexType>
1001class StridedLinearBufferCopy {
1002 using Packet = typename packet_traits<Scalar>::type;
1003 using HalfPacket = typename unpacket_traits<Packet>::half;
1004 enum {
1005 Vectorizable = packet_traits<Scalar>::Vectorizable,
1006 PacketSize = packet_traits<Scalar>::size,
1007 HalfPacketSize = unpacket_traits<HalfPacket>::size,
1008 HasHalfPacket = static_cast<int>(HalfPacketSize) < static_cast<int>(PacketSize)
1009 };
1010
1011 public:
1012 // Specifying linear copy kind statically gives ~30% speedup for small sizes.
1013 enum class Kind {
1014 Linear = 0, // src_stride == 1 && dst_stride == 1
1015 Scatter = 1, // src_stride == 1 && dst_stride != 1 && dst_stride != -1
1016 FillLinear = 2, // src_stride == 0 && dst_stride == 1
1017 FillScatter = 3, // src_stride == 0 && dst_stride != 1
1018 Gather = 4, // dst_stride == 1 && src_stride != -1
1019 Random = 5, // everything else
1020 ReverseStore = 6, // src_stride == 1 && dst_stride == -1
1021 ReverseLoad = 7, // src_stride == -1 && dst_stride == 1
1022 ReverseBoth = 8 // src_stride == -1 && dst_stride == -1
1023 };
1024
1025 struct Dst {
1026 Dst(IndexType o, IndexType s, Scalar* d) : offset(o), stride(s), data(d) {}
1027
1028 IndexType offset;
1029 IndexType stride;
1030 Scalar* data;
1031 };
1032
1033 struct Src {
1034 Src(IndexType o, IndexType s, const Scalar* d) : offset(o), stride(s), data(d) {}
1035
1036 IndexType offset;
1037 IndexType stride;
1038 const Scalar* data;
1039 };
1040
1041 template <typename StridedLinearBufferCopy::Kind kind>
1042 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run(const Dst& dst, const Src& src, const size_t count) {
1043 Run<kind>(count, dst.offset, dst.stride, dst.data, src.offset, src.stride, src.data);
1044 }
1045
1046 private:
1047 template <typename StridedLinearBufferCopy::Kind kind>
1048 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run(const IndexType count, const IndexType dst_offset,
1049 const IndexType dst_stride, Scalar* EIGEN_RESTRICT dst_data,
1050 const IndexType src_offset, const IndexType src_stride,
1051 const Scalar* EIGEN_RESTRICT src_data) {
1052 const Scalar* src = &src_data[src_offset];
1053 Scalar* dst = &dst_data[dst_offset];
1054
1055 EIGEN_IF_CONSTEXPR (!Vectorizable) {
1056 for (Index i = 0; i < count; ++i) {
1057 dst[i * dst_stride] = src[i * src_stride];
1058 }
1059 return;
1060 }
1061
1062 const IndexType vectorized_size = PacketSize * (count / PacketSize);
1063 IndexType i = 0;
1064
1065 EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Linear ||
1066 kind == StridedLinearBufferCopy::Kind::ReverseBoth) {
1067 // ******************************************************************** //
1068 // Linear copy from `src` to `dst`. `ReverseBoth` walks both runs
1069 // backwards, which leaves the elements contiguous and in the same order
1070 // in both buffers, so it is this same copy once each pointer is moved to
1071 // the low end of its run. No evaluator produces a reversed run on both
1072 // sides today; the kind exists so that such a run does not fall back to
1073 // `Random`.
1074 constexpr IndexType run_stride = kind == StridedLinearBufferCopy::Kind::ReverseBoth ? -1 : 1;
1075 eigen_assert(src_stride == run_stride && dst_stride == run_stride);
1076 const IndexType run_offset = run_stride == 1 ? 0 : count - 1;
1077 const Scalar* run_src = src - run_offset;
1078 Scalar* run_dst = dst - run_offset;
1079 const IndexType unrolled_size = (4 * PacketSize) * (count / (4 * PacketSize));
1080 for (; i < unrolled_size; i += 4 * PacketSize) {
1081 for (int j = 0; j < 4; ++j) {
1082 Packet p = ploadu<Packet>(run_src + i + j * PacketSize);
1083 pstoreu<Scalar, Packet>(run_dst + i + j * PacketSize, p);
1084 }
1085 }
1086 for (; i < vectorized_size; i += PacketSize) {
1087 Packet p = ploadu<Packet>(run_src + i);
1088 pstoreu<Scalar, Packet>(run_dst + i, p);
1089 }
1090 EIGEN_IF_CONSTEXPR (HasHalfPacket) {
1091 const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
1092 if (i < vectorized_half_size) {
1093 HalfPacket p = ploadu<HalfPacket>(run_src + i);
1094 pstoreu<Scalar, HalfPacket>(run_dst + i, p);
1095 i += HalfPacketSize;
1096 }
1097 }
1098 for (; i < count; ++i) {
1099 run_dst[i] = run_src[i];
1100 }
1101 // ******************************************************************** //
1102 } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Scatter) {
1103 // Scatter from `src` to `dst`.
1104 eigen_assert(src_stride == 1 && dst_stride != 1);
1105 for (; i < vectorized_size; i += PacketSize) {
1106 Packet p = ploadu<Packet>(src + i);
1107 pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride);
1108 }
1109 EIGEN_IF_CONSTEXPR (HasHalfPacket) {
1110 const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
1111 if (i < vectorized_half_size) {
1112 HalfPacket p = ploadu<HalfPacket>(src + i);
1113 pscatter<Scalar, HalfPacket>(dst + i * dst_stride, p, dst_stride);
1114 i += HalfPacketSize;
1115 }
1116 }
1117 for (; i < count; ++i) {
1118 dst[i * dst_stride] = src[i];
1119 }
1120 // ******************************************************************** //
1121 } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillLinear) {
1122 // Fill `dst` with value at `*src`.
1123 eigen_assert(src_stride == 0 && dst_stride == 1);
1124
1125 const IndexType unrolled_size = (4 * PacketSize) * (count / (4 * PacketSize));
1126 Scalar s = *src;
1127 Packet p = pset1<Packet>(s);
1128 for (; i < unrolled_size; i += 4 * PacketSize) {
1129 for (int j = 0; j < 4; ++j) {
1130 pstoreu<Scalar, Packet>(dst + i + j * PacketSize, p);
1131 }
1132 }
1133 for (; i < vectorized_size; i += PacketSize) {
1134 pstoreu<Scalar, Packet>(dst + i, p);
1135 }
1136 EIGEN_IF_CONSTEXPR (HasHalfPacket) {
1137 const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
1138 if (i < vectorized_half_size) {
1139 HalfPacket hp = pset1<HalfPacket>(s);
1140 pstoreu<Scalar, HalfPacket>(dst + i, hp);
1141 i += HalfPacketSize;
1142 }
1143 }
1144 for (; i < count; ++i) {
1145 dst[i] = s;
1146 }
1147 // ******************************************************************** //
1148 } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::FillScatter) {
1149 // Scatter `*src` into `dst`.
1150 eigen_assert(src_stride == 0 && dst_stride != 1);
1151 Scalar s = *src;
1152 Packet p = pset1<Packet>(s);
1153 for (; i < vectorized_size; i += PacketSize) {
1154 pscatter<Scalar, Packet>(dst + i * dst_stride, p, dst_stride);
1155 }
1156 EIGEN_IF_CONSTEXPR (HasHalfPacket) {
1157 const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
1158 if (i < vectorized_half_size) {
1159 HalfPacket hp = pset1<HalfPacket>(s);
1160 pscatter<Scalar, HalfPacket>(dst + i * dst_stride, hp, dst_stride);
1161 i += HalfPacketSize;
1162 }
1163 }
1164 for (; i < count; ++i) {
1165 dst[i * dst_stride] = s;
1166 }
1167 // ******************************************************************** //
1168 } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Gather) {
1169 // Gather from `src` into `dst`.
1170 eigen_assert(dst_stride == 1);
1171 for (; i < vectorized_size; i += PacketSize) {
1172 Packet p = pgather<Scalar, Packet>(src + i * src_stride, src_stride);
1173 pstoreu<Scalar, Packet>(dst + i, p);
1174 }
1175 EIGEN_IF_CONSTEXPR (HasHalfPacket) {
1176 const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
1177 if (i < vectorized_half_size) {
1178 HalfPacket p = pgather<Scalar, HalfPacket>(src + i * src_stride, src_stride);
1179 pstoreu<Scalar, HalfPacket>(dst + i, p);
1180 i += HalfPacketSize;
1181 }
1182 }
1183 for (; i < count; ++i) {
1184 dst[i] = src[i * src_stride];
1185 }
1186 // ******************************************************************** //
1187 } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::ReverseStore) {
1188 // ******************************************************************** //
1189 // Contiguous read, reversed write: `dst[-i] = src[i]`. The destination
1190 // run covers [dst - count + 1, dst], so a packet is one contiguous load,
1191 // one `preverse` and one contiguous store -- instead of the `pscatter`
1192 // that a stride of -1 would otherwise fall into.
1193 eigen_assert(src_stride == 1 && dst_stride == -1);
1194 for (; i < vectorized_size; i += PacketSize) {
1195 Packet p = ploadu<Packet>(src + i);
1196 pstoreu<Scalar, Packet>(dst - i - (PacketSize - 1), preverse(p));
1197 }
1198 EIGEN_IF_CONSTEXPR (HasHalfPacket) {
1199 const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
1200 if (i < vectorized_half_size) {
1201 HalfPacket p = ploadu<HalfPacket>(src + i);
1202 pstoreu<Scalar, HalfPacket>(dst - i - (HalfPacketSize - 1), preverse(p));
1203 i += HalfPacketSize;
1204 }
1205 }
1206 for (; i < count; ++i) {
1207 dst[-i] = src[i];
1208 }
1209 // ******************************************************************** //
1210 } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::ReverseLoad) {
1211 // Reversed read, contiguous write: `dst[i] = src[-i]`.
1212 eigen_assert(src_stride == -1 && dst_stride == 1);
1213 for (; i < vectorized_size; i += PacketSize) {
1214 Packet p = ploadu<Packet>(src - i - (PacketSize - 1));
1215 pstoreu<Scalar, Packet>(dst + i, preverse(p));
1216 }
1217 EIGEN_IF_CONSTEXPR (HasHalfPacket) {
1218 const IndexType vectorized_half_size = HalfPacketSize * (count / HalfPacketSize);
1219 if (i < vectorized_half_size) {
1220 HalfPacket p = ploadu<HalfPacket>(src - i - (HalfPacketSize - 1));
1221 pstoreu<Scalar, HalfPacket>(dst + i, preverse(p));
1222 i += HalfPacketSize;
1223 }
1224 }
1225 for (; i < count; ++i) {
1226 dst[i] = src[-i];
1227 }
1228 // ******************************************************************** //
1229 } else EIGEN_IF_CONSTEXPR (kind == StridedLinearBufferCopy::Kind::Random) {
1230 // Random.
1231 for (; i < count; ++i) {
1232 dst[i * dst_stride] = src[i * src_stride];
1233 }
1234 } else {
1235 eigen_assert(false);
1236 }
1237 }
1238};
1239
1240// -------------------------------------------------------------------------- //
1241// TensorBlockIO copies data from `src` tensor block, to the `dst` tensor block.
1242// It's possible to specify src->dst dimension mapping for the copy operation.
1243// Dimensions of `dst` specify how many elements have to be copied, for the
1244// `src` we need to know only stride to navigate through source memory buffer.
1245//
1246// Strides may be non-unit (strided/dilated views), negative (reversed views),
1247// or, on the `src` side only, zero (the broadcasting trick). Inner dimensions
1248// are fused into one copy only while the elements keep forming a single
1249// arithmetic progression at the inner stride on both sides.
1250
1251template <typename Scalar, typename IndexType, int NumDims, int Layout>
1252class TensorBlockIO {
1253 static constexpr bool IsColMajor = Layout == ColMajor;
1254
1255 using LinCopy = StridedLinearBufferCopy<Scalar, IndexType>;
1256
1257 public:
1258 using Dimensions = DSizes<IndexType, NumDims>;
1259 using DimensionsMap = DSizes<int, NumDims>;
1260
1261 struct Dst {
1262 Dst(const Dimensions& dst_dims, const Dimensions& dst_strides, Scalar* dst, IndexType dst_offset = 0)
1263 : dims(dst_dims), strides(dst_strides), data(dst), offset(dst_offset) {}
1264
1265 Dimensions dims;
1266 Dimensions strides;
1267 Scalar* data;
1268 IndexType offset;
1269 };
1270
1271 struct Src {
1272 Src(const Dimensions& src_strides, const Scalar* src, IndexType src_offset = 0)
1273 : strides(src_strides), data(src), offset(src_offset) {}
1274
1275 Dimensions strides;
1276 const Scalar* data;
1277 IndexType offset;
1278 };
1279
1280 // Copies data to `dst` from `src`, using provided dimensions mapping:
1281 //
1282 // src_dimension_index = dst_to_src_dim_map[dst_dimension_index]
1283 //
1284 // Returns the number of copied elements.
1285 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE IndexType Copy(const Dst& dst, const Src& src,
1286 const DimensionsMap& dst_to_src_dim_map) {
1287 // Copy single scalar value from `src` to `dst`.
1288 EIGEN_IF_CONSTEXPR (NumDims == 0) {
1289 *(dst.data + dst.offset) = *(src.data + src.offset);
1290 return 1;
1291 }
1292
1293 // Give a shorter name to `dst_to_src_dim_map`.
1294 const DimensionsMap& dim_map = dst_to_src_dim_map;
1295
1296 // Do not squeeze reordered inner dimensions.
1297 int num_squeezable_dims = NumSqueezableInnerDims(dim_map);
1298
1299 // NOTE: We find the innermost dimension (contiguous in memory) in the dst
1300 // block, and we write data linearly into that dimension, reading it from
1301 // the src. If dimensions are reordered, we might end up reading data from
1302 // the src with `stride != 1`.
1303 //
1304 // NOTE: Random-Read/Linear-Write can be up to ~2X faster than
1305 // Linear-Read/Random-Write: https://stackoverflow.com/a/54935680
1306
1307 // Find the innermost dimension in the dst whose size is not 1. This is the
1308 // effective inner dim.
1309 int num_size_one_inner_dims = 0;
1310 for (int i = 0; i < num_squeezable_dims; ++i) {
1311 const int dst_dim = IsColMajor ? i : NumDims - i - 1;
1312 if (dst.dims[dst_dim] != 1) break;
1313 num_size_one_inner_dims++;
1314 }
1315
1316 // If all dimensions are of size 1, just copy a scalar from `src` to `dst`.
1317 if (num_size_one_inner_dims == NumDims) {
1318 *(dst.data + dst.offset) = *(src.data + src.offset);
1319 return 1;
1320 }
1321
1322 // Innermost dimension in the dst that still has to be copied. Its stride
1323 // need not be 1: the run may be dilated or reversed.
1324 const int dst_inner_dim = IsColMajor ? num_size_one_inner_dims : NumDims - num_size_one_inner_dims - 1;
1325
1326 // Dimension in the src that corresponds to the dst innermost dimension.
1327 const int src_dim_for_dst_inner_dim = NumDims == 0 ? 1 : dim_map[dst_inner_dim];
1328
1329 // Number of elements copied per line.
1330 IndexType dst_inner_dim_size = NumDims == 0 ? 1 : dst.dims[dst_inner_dim];
1331
1332 // Squeeze multiple inner dims into one if the elements keep forming a
1333 // single arithmetic progression at the inner stride across the dimension
1334 // boundary in both `dst` and `src` memory, so we can do less linear copy
1335 // calls.
1336 const IndexType output_stride = NumDims == 0 ? 1 : dst.strides[dst_inner_dim];
1337 const IndexType input_stride = NumDims == 0 ? 1 : src.strides[src_dim_for_dst_inner_dim];
1338 for (int i = num_size_one_inner_dims + 1; i < num_squeezable_dims; ++i) {
1339 const int dst_dim = IsColMajor ? i : NumDims - i - 1;
1340 const IndexType dst_stride = dst.strides[dst_dim];
1341 const IndexType src_stride = src.strides[dim_map[dst_dim]];
1342 if (dst_stride == dst_inner_dim_size * output_stride && src_stride == dst_inner_dim_size * input_stride) {
1343 dst_inner_dim_size *= dst.dims[dst_dim];
1344 ++num_size_one_inner_dims;
1345 } else {
1346 break;
1347 }
1348 }
1349
1350 // Setup strides to read data from `src` and write to `dst`.
1351 IndexType input_offset = src.offset;
1352 IndexType output_offset = dst.offset;
1353
1354 constexpr int at_least_1_dim = NumDims <= 1 ? 1 : NumDims - 1;
1355 array<BlockIteratorState, at_least_1_dim> it;
1356
1357 // Initialize block iterator state. Squeeze away any dimension of size 1.
1358 int idx = 0; // currently initialized iterator state index
1359 for (int i = num_size_one_inner_dims; i < NumDims - 1; ++i) {
1360 const int dst_dim = IsColMajor ? i + 1 : NumDims - i - 2;
1361 if (dst.dims[dst_dim] == 1) continue;
1362
1363 it[idx].size = dst.dims[dst_dim];
1364 it[idx].input_stride = src.strides[dim_map[dst_dim]];
1365 it[idx].output_stride = dst.strides[dst_dim];
1366
1367 it[idx].input_span = it[idx].input_stride * (it[idx].size - 1);
1368 it[idx].output_span = it[idx].output_stride * (it[idx].size - 1);
1369
1370 idx++;
1371 }
1372
1373 // Iterate copying data from src to dst.
1374 const IndexType block_total_size = NumDims == 0 ? 1 : dst.dims.TotalSize();
1375
1376#define COPY_INNER_DIM(KIND) \
1377 IndexType num_copied = 0; \
1378 for (num_copied = 0; num_copied < block_total_size; num_copied += dst_inner_dim_size) { \
1379 LinCopy::template Run<KIND>(typename LinCopy::Dst(output_offset, output_stride, dst.data), \
1380 typename LinCopy::Src(input_offset, input_stride, src.data), dst_inner_dim_size); \
1381 \
1382 for (int j = 0; j < idx; ++j) { \
1383 if (++it[j].count < it[j].size) { \
1384 input_offset += it[j].input_stride; \
1385 output_offset += it[j].output_stride; \
1386 break; \
1387 } \
1388 it[j].count = 0; \
1389 input_offset -= it[j].input_span; \
1390 output_offset -= it[j].output_span; \
1391 } \
1392 } \
1393 return num_copied;
1394
1395 if (input_stride == 1 && output_stride == 1) {
1396 COPY_INNER_DIM(LinCopy::Kind::Linear);
1397 } else if (input_stride == 1 && output_stride == -1) {
1398 COPY_INNER_DIM(LinCopy::Kind::ReverseStore);
1399 } else if (input_stride == -1 && output_stride == 1) {
1400 COPY_INNER_DIM(LinCopy::Kind::ReverseLoad);
1401 } else if (input_stride == -1 && output_stride == -1) {
1402 COPY_INNER_DIM(LinCopy::Kind::ReverseBoth);
1403 } else if (input_stride == 1 && output_stride != 1) {
1404 COPY_INNER_DIM(LinCopy::Kind::Scatter);
1405 } else if (input_stride == 0 && output_stride == 1) {
1406 COPY_INNER_DIM(LinCopy::Kind::FillLinear);
1407 } else if (input_stride == 0 && output_stride != 1) {
1408 COPY_INNER_DIM(LinCopy::Kind::FillScatter);
1409 } else if (output_stride == 1) {
1410 COPY_INNER_DIM(LinCopy::Kind::Gather);
1411 } else {
1412 COPY_INNER_DIM(LinCopy::Kind::Random);
1413 }
1414
1415#undef COPY_INNER_DIM
1416 }
1417
1418 // Copy from `src` to `dst` with an identity src->dst dimension map. Returns
1419 // the number of copied elements.
1420 static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE IndexType Copy(const Dst& dst, const Src& src) {
1421 DimensionsMap dst_to_src_map;
1422 for (int i = 0; i < NumDims; ++i) dst_to_src_map[i] = i;
1423 return Copy(dst, src, dst_to_src_map);
1424 }
1425
1426 private:
1427 struct BlockIteratorState {
1428 BlockIteratorState() = default;
1429
1430 IndexType size = 0;
1431 IndexType count = 0;
1432 IndexType input_stride = 0;
1433 IndexType output_stride = 0;
1434 IndexType input_span = 0;
1435 IndexType output_span = 0;
1436 };
1437
1438 // Compute how many inner dimensions it's allowed to squeeze when doing IO
1439 // between two tensor blocks. It's safe to squeeze inner dimensions, only
1440 // if they are not reordered.
1441 static int NumSqueezableInnerDims(const DimensionsMap& dim_map) {
1442 int num_squeezable_dims = 0;
1443 for (int i = 0; i < NumDims; ++i) {
1444 const int dim = IsColMajor ? i : NumDims - i - 1;
1445 if (dim_map[dim] != dim) break;
1446 num_squeezable_dims++;
1447 }
1448 return num_squeezable_dims;
1449 }
1450};
1451
1452// -------------------------------------------------------------------------- //
1453// Bind coefficient-wise block expressions to one contiguous inner run at a time.
1454// This moves strided source index calculations out of the coefficient/packet loop.
1455template <typename XprType>
1456struct TensorBlockRead {
1457 static constexpr bool Supported = false;
1458 using Expression = XprType;
1459 using Index = typename XprType::Index;
1460 explicit TensorBlockRead(const XprType& expression) : m_expression(expression) {}
1461 Index innerSize() const { return NumTraits<Index>::highest(); }
1462 const Expression& expr(Index, Index) const { return m_expression; }
1463
1464 private:
1465 const XprType& m_expression;
1466};
1467
1468template <typename Scalar, int NumDims, int Layout, typename Index>
1469struct TensorBlockRead<TensorBlockView<Scalar, NumDims, Layout, Index>> {
1470 static constexpr bool Supported = true;
1471 using XprType = TensorBlockView<Scalar, NumDims, Layout, Index>;
1472 using Expression = TensorBlockView<Scalar, 1, Layout, Index>;
1473 explicit TensorBlockRead(const XprType& expression) : m_evaluator(expression, m_device), m_inner_size(1) {
1474 for (int i = 0; i < NumDims; ++i) {
1475 const int dim = Layout == ColMajor ? i : NumDims - 1 - i;
1476 if (expression.dimensions()[dim] > 1 && expression.strides()[dim] != m_inner_size) break;
1477 m_inner_size *= expression.dimensions()[dim];
1478 }
1479 }
1480 Index innerSize() const { return m_inner_size; }
1481 Expression expr(Index offset, Index size) const {
1482 return Expression(m_evaluator.coeffAddress(offset), DSizes<Index, 1>(size));
1483 }
1484
1485 private:
1486 DefaultDevice m_device;
1487 TensorEvaluator<const XprType, DefaultDevice> m_evaluator;
1488 Index m_inner_size;
1489};
1490
1491// Rebinding a run must not restart a functor's mutable state.
1492template <typename Functor>
1493class TensorBlockReadFunctor {
1494 public:
1495 explicit TensorBlockReadFunctor(const Functor& functor) : m_functor(&functor) {}
1496 template <typename... Args>
1497 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto operator()(Args&&... args) const
1498 -> decltype(std::declval<const Functor&>()(std::forward<Args>(args)...)) {
1499 return (*m_functor)(std::forward<Args>(args)...);
1500 }
1501 template <typename... Args, typename F = Functor>
1502 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE auto packetOp(Args&&... args) const
1503 -> decltype(std::declval<const F&>().packetOp(std::forward<Args>(args)...)) {
1504 return m_functor->packetOp(std::forward<Args>(args)...);
1505 }
1506
1507 private:
1508 const Functor* m_functor;
1509};
1510
1511template <typename Functor>
1512struct functor_traits<TensorBlockReadFunctor<Functor>> : functor_traits<Functor> {};
1513
1514template <typename UnaryOp, typename Arg>
1515struct TensorBlockRead<TensorCwiseUnaryOp<UnaryOp, Arg>> {
1516 using ArgRead = TensorBlockRead<remove_all_t<Arg>>;
1517 static constexpr bool Supported = ArgRead::Supported;
1518 using XprType = TensorCwiseUnaryOp<UnaryOp, Arg>;
1519 using Expression = TensorCwiseUnaryOp<TensorBlockReadFunctor<UnaryOp>, const typename ArgRead::Expression>;
1520 using Index = typename XprType::Index;
1521 explicit TensorBlockRead(const XprType& expression)
1522 : m_arg(expression.nestedExpression()), m_functor(expression.functor()) {}
1523 Index innerSize() const { return m_arg.innerSize(); }
1524 Expression expr(Index offset, Index size) const {
1525 return Expression(m_arg.expr(offset, size), TensorBlockReadFunctor<UnaryOp>(m_functor));
1526 }
1527
1528 private:
1529 ArgRead m_arg;
1530 UnaryOp m_functor;
1531};
1532
1533template <typename BinaryOp, typename Left, typename Right>
1534struct TensorBlockRead<TensorCwiseBinaryOp<BinaryOp, Left, Right>> {
1535 using LeftRead = TensorBlockRead<remove_all_t<Left>>;
1536 using RightRead = TensorBlockRead<remove_all_t<Right>>;
1537 static constexpr bool Supported = LeftRead::Supported && RightRead::Supported;
1538 using XprType = TensorCwiseBinaryOp<BinaryOp, Left, Right>;
1539 using Expression = TensorCwiseBinaryOp<TensorBlockReadFunctor<BinaryOp>, const typename LeftRead::Expression,
1540 const typename RightRead::Expression>;
1541 using Index = typename XprType::Index;
1542 explicit TensorBlockRead(const XprType& expression)
1543 : m_left(expression.lhsExpression()), m_right(expression.rhsExpression()), m_functor(expression.functor()) {}
1544 Index innerSize() const { return numext::mini(m_left.innerSize(), m_right.innerSize()); }
1545 Expression expr(Index offset, Index size) const {
1546 return Expression(m_left.expr(offset, size), m_right.expr(offset, size),
1547 TensorBlockReadFunctor<BinaryOp>(m_functor));
1548 }
1549
1550 private:
1551 LeftRead m_left;
1552 RightRead m_right;
1553 BinaryOp m_functor;
1554};
1555
1556template <typename NullaryOp, typename Arg>
1557struct TensorBlockRead<TensorCwiseNullaryOp<NullaryOp, Arg>> {
1558 using XprType = TensorCwiseNullaryOp<NullaryOp, Arg>;
1559 using Evaluator = TensorEvaluator<const XprType, DefaultDevice>;
1560 static constexpr bool Supported = Evaluator::IndexIndependentFunctor;
1561 using Index = typename XprType::Index;
1562 using RunView = TensorBlockView<typename XprType::Scalar, 1, traits<XprType>::Layout, Index>;
1563 using Expression = TensorCwiseNullaryOp<NullaryOp, const RunView>;
1564 explicit TensorBlockRead(const XprType& expression) : m_functor(expression.functor()) {}
1565 Index innerSize() const { return NumTraits<Index>::highest(); }
1566 Expression expr(Index, Index size) const { return Expression(RunView(nullptr, DSizes<Index, 1>(size)), m_functor); }
1567
1568 private:
1569 NullaryOp m_functor;
1570};
1571
1572template <typename TernaryOp, typename Arg1, typename Arg2, typename Arg3>
1573struct TensorBlockRead<TensorCwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>> {
1574 using Arg1Read = TensorBlockRead<remove_all_t<Arg1>>;
1575 using Arg2Read = TensorBlockRead<remove_all_t<Arg2>>;
1576 using Arg3Read = TensorBlockRead<remove_all_t<Arg3>>;
1577 static constexpr bool Supported = Arg1Read::Supported && Arg2Read::Supported && Arg3Read::Supported;
1578 using XprType = TensorCwiseTernaryOp<TernaryOp, Arg1, Arg2, Arg3>;
1579 using Expression = TensorCwiseTernaryOp<TensorBlockReadFunctor<TernaryOp>, const typename Arg1Read::Expression,
1580 const typename Arg2Read::Expression, const typename Arg3Read::Expression>;
1581 using Index = typename XprType::Index;
1582 explicit TensorBlockRead(const XprType& expression)
1583 : m_arg1(expression.arg1Expression()),
1584 m_arg2(expression.arg2Expression()),
1585 m_arg3(expression.arg3Expression()),
1586 m_functor(expression.functor()) {}
1587 Index innerSize() const {
1588 return numext::mini(m_arg1.innerSize(), numext::mini(m_arg2.innerSize(), m_arg3.innerSize()));
1589 }
1590 Expression expr(Index offset, Index size) const {
1591 return Expression(m_arg1.expr(offset, size), m_arg2.expr(offset, size), m_arg3.expr(offset, size),
1592 TensorBlockReadFunctor<TernaryOp>(m_functor));
1593 }
1594
1595 private:
1596 Arg1Read m_arg1;
1597 Arg2Read m_arg2;
1598 Arg3Read m_arg3;
1599 TernaryOp m_functor;
1600};
1601
1602template <typename Cond, typename Then, typename Else>
1603struct TensorBlockRead<TensorSelectOp<Cond, Then, Else>> {
1604 using CondRead = TensorBlockRead<remove_all_t<Cond>>;
1605 using ThenRead = TensorBlockRead<remove_all_t<Then>>;
1606 using ElseRead = TensorBlockRead<remove_all_t<Else>>;
1607 static constexpr bool Supported = CondRead::Supported && ThenRead::Supported && ElseRead::Supported;
1608 using XprType = TensorSelectOp<Cond, Then, Else>;
1609 using Expression = TensorSelectOp<const typename CondRead::Expression, const typename ThenRead::Expression,
1610 const typename ElseRead::Expression>;
1611 using Index = typename XprType::Index;
1612 explicit TensorBlockRead(const XprType& expression)
1613 : m_cond(expression.ifExpression()), m_then(expression.thenExpression()), m_else(expression.elseExpression()) {}
1614 Index innerSize() const {
1615 return numext::mini(m_cond.innerSize(), numext::mini(m_then.innerSize(), m_else.innerSize()));
1616 }
1617 Expression expr(Index offset, Index size) const {
1618 return Expression(m_cond.expr(offset, size), m_then.expr(offset, size), m_else.expr(offset, size));
1619 }
1620
1621 private:
1622 CondRead m_cond;
1623 ThenRead m_then;
1624 ElseRead m_else;
1625};
1626
1627template <typename Scalar, typename Arg>
1628struct TensorBlockRead<TensorConversionOp<Scalar, Arg>> {
1629 using ArgRead = TensorBlockRead<remove_all_t<Arg>>;
1630 static constexpr bool Supported = ArgRead::Supported;
1631 using XprType = TensorConversionOp<Scalar, Arg>;
1632 using Expression = TensorConversionOp<Scalar, const typename ArgRead::Expression>;
1633 using Index = typename XprType::Index;
1634 explicit TensorBlockRead(const XprType& expression) : m_arg(expression.expression()) {}
1635 Index innerSize() const { return m_arg.innerSize(); }
1636 Expression expr(Index offset, Index size) const { return Expression(m_arg.expr(offset, size)); }
1637
1638 private:
1639 ArgRead m_arg;
1640};
1641
1642// TensorBlockAssignment assigns a block expression of type `TensorBlockExpr` to
1643// a Tensor block defined by `desc`, backed by a memory buffer at `target`.
1644//
1645// Currently there is no way to write from a Tensor expression to a block of
1646// memory, if dimensions are reordered. If you need to do that, you should
1647// materialize a Tensor block expression into a memory buffer, and then use
1648// TensorBlockIO to copy data between two memory buffers with a custom
1649// `target->src` dimension map (see definition above).
1650//
1651// Also currently the innermost dimension of `target` must have a stride '1'
1652// (contiguous in memory). This restriction could be lifted with a `pscatter`,
1653// but in practice it's never needed, and there is a similar TensorBlockIO
1654// workaround for that.
1655//
1656// TODO(ezhulenev): TensorBlockAssignment is a special case of TensorBlockIO
1657// where `src` is a tensor expression. Explore if it is possible to rewrite IO
1658// to use expressions instead of pointers, and after that TensorBlockAssignment
1659// will become an alias to IO.
1660template <typename Scalar, int NumDims, typename TensorBlockExpr, typename IndexType = Eigen::Index>
1661class TensorBlockAssignment {
1662 // We will use coeff/packet path to evaluate block expressions.
1663 using TensorBlockEvaluator = TensorEvaluator<const TensorBlockExpr, DefaultDevice>;
1664
1665 using Dimensions = DSizes<IndexType, NumDims>;
1666 using BlockRead = TensorBlockRead<TensorBlockExpr>;
1667
1668 enum { Vectorizable = packet_traits<Scalar>::Vectorizable, PacketSize = packet_traits<Scalar>::size };
1669
1670 template <bool Vectorizable, typename Evaluator>
1671 struct InnerDimAssign {
1672 EIGEN_ALWAYS_INLINE static void Run(Scalar* target, IndexType count, const Evaluator& eval, IndexType eval_offset) {
1673 for (IndexType i = 0; i < count; ++i) {
1674 target[i] = eval.coeff(eval_offset + i);
1675 }
1676 }
1677 };
1678
1679 template <typename Evaluator>
1680 struct InnerDimAssign<true, Evaluator> {
1681 EIGEN_ALWAYS_INLINE static void Run(Scalar* target, IndexType count, const Evaluator& eval, IndexType eval_offset) {
1682 using Packet = typename packet_traits<Scalar>::type;
1683
1684 const IndexType unrolled_size = (4 * PacketSize) * (count / (4 * PacketSize));
1685 const IndexType vectorized_size = PacketSize * (count / PacketSize);
1686 IndexType i = 0;
1687
1688 for (; i < unrolled_size; i += 4 * PacketSize) {
1689 for (int j = 0; j < 4; ++j) {
1690 const IndexType idx = eval_offset + i + j * PacketSize;
1691 Packet p = eval.template packet<Unaligned>(idx);
1692 pstoreu<Scalar>(target + i + j * PacketSize, p);
1693 }
1694 }
1695
1696 for (; i < vectorized_size; i += PacketSize) {
1697 Packet p = eval.template packet<Unaligned>(eval_offset + i);
1698 pstoreu<Scalar>(target + i, p);
1699 }
1700
1701 for (; i < count; ++i) {
1702 target[i] = eval.coeff(eval_offset + i);
1703 }
1704 }
1705 };
1706
1707 template <typename Evaluator>
1708 static EIGEN_STRONG_INLINE void AssignInner(Scalar* target, IndexType count, const Evaluator& eval, const BlockRead&,
1709 IndexType offset, std::false_type) {
1710 InnerDimAssign<Vectorizable && Evaluator::PacketAccess, Evaluator>::Run(target, count, eval, offset);
1711 }
1712
1713 static EIGEN_STRONG_INLINE void AssignInner(Scalar* target, IndexType count, const TensorBlockEvaluator&,
1714 const BlockRead& reader, IndexType offset, std::true_type) {
1715 const auto expression = reader.expr(offset, count);
1716 using RunEvaluator = TensorEvaluator<const typename BlockRead::Expression, DefaultDevice>;
1717 const DefaultDevice device;
1718 const RunEvaluator eval(expression, device);
1719 InnerDimAssign<Vectorizable && RunEvaluator::PacketAccess, RunEvaluator>::Run(target, count, eval, 0);
1720 }
1721
1722 public:
1723 struct Target {
1724 Target(const Dimensions& target_dims, const Dimensions& target_strides, Scalar* target_data,
1725 IndexType target_offset = 0)
1726 : dims(target_dims), strides(target_strides), data(target_data), offset(target_offset) {}
1727
1728 Dimensions dims;
1729 Dimensions strides;
1730 Scalar* data;
1731 IndexType offset;
1732 };
1733
1734 static Target target(const Dimensions& target_dims, const Dimensions& target_strides, Scalar* target_data,
1735 IndexType target_offset = 0) {
1736 return Target(target_dims, target_strides, target_data, target_offset);
1737 }
1738
1739 template <typename TargetDimsIndexType, typename TargetStridesIndexType>
1740 static Target target(const DSizes<TargetDimsIndexType, NumDims>& target_dims,
1741 const DSizes<TargetStridesIndexType, NumDims>& target_strides, Scalar* target_data,
1742 IndexType target_offset = 0) {
1743 // DSizes constructor will do index type promotion if it's safe.
1744 return Target(Dimensions(target_dims), Dimensions(target_strides), target_data, target_offset);
1745 }
1746
1747 static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void Run(const Target& target, const TensorBlockExpr& expr) {
1748 // Prepare evaluator for block expression.
1749 DefaultDevice default_device;
1750 TensorBlockEvaluator eval(expr, default_device);
1751 const BlockRead reader(expr);
1752
1753 // Tensor block expression dimension should match destination dimensions.
1754 eigen_assert(dimensions_match(target.dims, eval.dimensions()));
1755
1756 Run(target, eval, reader, bool_constant<BlockRead::Supported>());
1757 }
1758
1759 private:
1760 static void Run(const Target& target, const TensorBlockEvaluator& eval, const BlockRead& reader, std::false_type) {
1761 RunImpl<false>(target, eval, reader);
1762 }
1763
1764 static void Run(const Target& target, const TensorBlockEvaluator& eval, const BlockRead& reader, std::true_type) {
1765 const IndexType size = NumDims == 0 ? 1 : target.dims.TotalSize();
1766 if (reader.innerSize() >= size) {
1767 // Bind a dense source once, even when the destination has strided rows.
1768 const auto expression = reader.expr(0, size);
1769 using RunEvaluator = TensorEvaluator<const typename BlockRead::Expression, DefaultDevice>;
1770 const DefaultDevice device;
1771 const RunEvaluator dense_eval(expression, device);
1772 RunImpl<false>(target, dense_eval, reader);
1773 } else {
1774 RunImpl<true>(target, eval, reader);
1775 }
1776 }
1777
1778 template <bool UseInnerRuns, typename Evaluator>
1779 static EIGEN_STRONG_INLINE void RunImpl(const Target& target, const Evaluator& eval, const BlockRead& reader) {
1780 static constexpr int Layout = Evaluator::Layout;
1781 static constexpr bool is_col_major = Layout == ColMajor;
1782
1783 // Initialize output inner dimension size based on a layout.
1784 const IndexType output_size = NumDims == 0 ? 1 : target.dims.TotalSize();
1785 constexpr int inner_dim_idx = NumDims == 0 ? 0 : (is_col_major ? 0 : NumDims - 1);
1786 IndexType output_inner_dim_size = NumDims == 0 ? 1 : target.dims[inner_dim_idx];
1787
1788 // Target inner dimension stride must be '1'.
1789 EIGEN_IF_CONSTEXPR (NumDims > 0) {
1790 eigen_assert(target.strides[inner_dim_idx] == 1);
1791 }
1792
1793 // Squeeze multiple inner dims into one if they are contiguous in `target`.
1794 IndexType num_squeezed_dims = 0;
1795 for (Index i = 1; i < NumDims; ++i) {
1796 const Index dim = is_col_major ? i : NumDims - i - 1;
1797 const IndexType target_stride = target.strides[dim];
1798
1799 if (output_inner_dim_size == target_stride &&
1800 (!UseInnerRuns || output_inner_dim_size * target.dims[dim] <= reader.innerSize())) {
1801 output_inner_dim_size *= target.dims[dim];
1802 num_squeezed_dims++;
1803 } else {
1804 break;
1805 }
1806 }
1807
1808 // Initialize output block iterator state. Dimension in this array are
1809 // always in inner_most -> outer_most order (col major layout).
1810 array<BlockIteratorState, NumDims> it;
1811
1812 int idx = 0; // currently initialized iterator state index
1813 for (Index i = num_squeezed_dims; i < NumDims - 1; ++i) {
1814 const Index dim = is_col_major ? i + 1 : NumDims - i - 2;
1815
1816 it[idx].count = 0;
1817 it[idx].size = target.dims[dim];
1818 it[idx].output_stride = target.strides[dim];
1819 it[idx].output_span = it[idx].output_stride * (it[idx].size - 1);
1820 idx++;
1821 }
1822
1823 // We read block expression from the beginning, and start writing data to
1824 // `target` at given offset.
1825 IndexType input_offset = 0;
1826 IndexType output_offset = target.offset;
1827
1828 // Iterate copying data from `eval` to `target`.
1829 for (IndexType i = 0; i < output_size; i += output_inner_dim_size) {
1830 // Assign to `target` at current offset.
1831 AssignInner(target.data + output_offset, output_inner_dim_size, eval, reader, input_offset,
1832 bool_constant<UseInnerRuns>());
1833
1834 // Move input offset forward by the number of assigned coefficients.
1835 input_offset += output_inner_dim_size;
1836
1837 // Update index.
1838 for (int j = 0; j < idx; ++j) {
1839 if (++it[j].count < it[j].size) {
1840 output_offset += it[j].output_stride;
1841 break;
1842 }
1843 it[j].count = 0;
1844 output_offset -= it[j].output_span;
1845 }
1846 }
1847 }
1848
1849 private:
1850 struct BlockIteratorState {
1851 BlockIteratorState() = default;
1852
1853 IndexType count = 0;
1854 IndexType size = 0;
1855 IndexType output_stride = 0;
1856 IndexType output_span = 0;
1857 };
1858};
1859
1860// -------------------------------------------------------------------------- //
1861
1862} // namespace internal
1863} // namespace Eigen
1864
1865#endif // EIGEN_TENSOR_TENSOR_BLOCK_H
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47