Eigen-Contrib  5.0.1-dev+master
 
Loading...
Searching...
No Matches
TensorConcatenation.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2014 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9// SPDX-License-Identifier: MPL-2.0
10
11#ifndef EIGEN_TENSOR_TENSOR_CONCATENATION_H
12#define EIGEN_TENSOR_TENSOR_CONCATENATION_H
13
14// IWYU pragma: private
15#include "./InternalHeaderCheck.h"
16
17namespace Eigen {
18
19namespace internal {
20template <typename Axis, typename LhsXprType, typename RhsXprType>
21struct traits<TensorConcatenationOp<Axis, LhsXprType, RhsXprType>> {
22 // Type promotion to handle the case where the types of the lhs and the rhs are different.
23 using Scalar = typename promote_storage_type<typename LhsXprType::Scalar, typename RhsXprType::Scalar>::ret;
24 using StorageKind = typename promote_storage_type<typename traits<LhsXprType>::StorageKind,
25 typename traits<RhsXprType>::StorageKind>::ret;
26 using Index =
27 typename promote_index_type<typename traits<LhsXprType>::Index, typename traits<RhsXprType>::Index>::type;
28 static constexpr int NumDimensions = traits<LhsXprType>::NumDimensions;
29 static constexpr int Layout = traits<LhsXprType>::Layout;
30 enum { Flags = 0 };
31 using PointerType =
32 std::conditional_t<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val,
33 typename traits<LhsXprType>::PointerType, typename traits<RhsXprType>::PointerType>;
34};
35
36template <typename Axis, typename LhsXprType, typename RhsXprType>
37struct eval<TensorConcatenationOp<Axis, LhsXprType, RhsXprType>, Eigen::Dense> {
38 using type = const TensorConcatenationOp<Axis, LhsXprType, RhsXprType>&;
39};
40
41} // end namespace internal
42
48template <typename Axis, typename LhsXprType, typename RhsXprType>
49class TensorConcatenationOp : public TensorBase<TensorConcatenationOp<Axis, LhsXprType, RhsXprType>, WriteAccessors> {
50 public:
52 using Scalar = typename internal::traits<TensorConcatenationOp>::Scalar;
53 using StorageKind = typename internal::traits<TensorConcatenationOp>::StorageKind;
54 using Index = typename internal::traits<TensorConcatenationOp>::Index;
55 using Nested = typename internal::ref_selector<TensorConcatenationOp>::type;
56 using CoeffReturnType = typename internal::promote_storage_type<typename LhsXprType::CoeffReturnType,
57 typename RhsXprType::CoeffReturnType>::ret;
58 using RealScalar = typename NumTraits<Scalar>::Real;
59
60 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorConcatenationOp(const LhsXprType& lhs, const RhsXprType& rhs, Axis axis)
61 : m_lhs_xpr(lhs), m_rhs_xpr(rhs), m_axis(axis) {}
62
63 EIGEN_DEVICE_FUNC const internal::remove_all_t<typename LhsXprType::Nested>& lhsExpression() const {
64 return m_lhs_xpr;
65 }
66
67 EIGEN_DEVICE_FUNC const internal::remove_all_t<typename RhsXprType::Nested>& rhsExpression() const {
68 return m_rhs_xpr;
69 }
70
71 EIGEN_DEVICE_FUNC const Axis& axis() const { return m_axis; }
72
73 EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TensorConcatenationOp)
74 protected:
75 typename LhsXprType::Nested m_lhs_xpr;
76 typename RhsXprType::Nested m_rhs_xpr;
77 const Axis m_axis;
78};
79
80// Eval as rvalue
81template <typename Axis, typename LeftArgType, typename RightArgType, typename Device>
82struct TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgType>, Device> {
84 using Index = typename XprType::Index;
85 static constexpr int NumDims = internal::array_size<typename TensorEvaluator<LeftArgType, Device>::Dimensions>::value;
86 static constexpr int RightNumDims =
87 internal::array_size<typename TensorEvaluator<RightArgType, Device>::Dimensions>::value;
88 using Dimensions = DSizes<Index, NumDims>;
89 using Scalar = typename XprType::Scalar;
90 using CoeffReturnType = typename XprType::CoeffReturnType;
91 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
92 using Storage = StorageMemory<CoeffReturnType, Device>;
93 using EvaluatorPointerType = typename Storage::Type;
94 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
95 enum {
96 IsAligned = false,
97 PacketAccess =
98 TensorEvaluator<LeftArgType, Device>::PacketAccess && TensorEvaluator<RightArgType, Device>::PacketAccess,
99 // block() reads each operand's data() pointer directly, so both must
100 // expose raw storage. Scalar-changing block consumers
101 // (TensorCwiseUnaryOp, TensorConversionOp) drop the forwarded destination
102 // buffer before reaching us, so prepareStorage always sees either a
103 // matching-Scalar buffer or no buffer at all.
105 // Matches TensorShuffling / TensorBroadcasting / TensorPadding: bulk copy
106 // wins over the per-element coeff/packet path (which pays div/mod per
107 // access) at every size we benchmarked, so always prefer block.
108 PreferBlockAccess = true,
109 RawAccess = false
110 };
111
112 using ScalarNoConst = std::remove_const_t<Scalar>;
113
114 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
115 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
116 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
117
118 using TensorBlock = typename internal::TensorMaterializedBlock<ScalarNoConst, NumDims, Layout, Index>;
119 //===--------------------------------------------------------------------===//
120
121 EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device)
122 : m_leftImpl(op.lhsExpression(), device),
123 m_rightImpl(op.rhsExpression(), device),
124 m_device(device),
125 m_axis(op.axis()) {
126 EIGEN_STATIC_ASSERT((static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) ==
127 static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout) ||
128 NumDims == 1),
129 YOU_MADE_A_PROGRAMMING_MISTAKE);
130 // TensorConcatenationOp requires both operands to have the same static
131 // rank. Reshape the lower-rank operand explicitly if you need to mix
132 // ranks; see the concatenate() entry in contrib/Eigen/src/Tensor/README.md.
133 EIGEN_STATIC_ASSERT((NumDims == RightNumDims), YOU_MADE_A_PROGRAMMING_MISTAKE);
134 EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE);
135
136 eigen_assert(0 <= m_axis && m_axis < NumDims);
137 m_leftAxisSize = m_leftImpl.dimensions()[m_axis];
138 const Dimensions& lhs_dims = m_leftImpl.dimensions();
139 const Dimensions& rhs_dims = m_rightImpl.dimensions();
140 {
141 int i = 0;
142 for (; i < m_axis; ++i) {
143 eigen_assert(lhs_dims[i] >= 0);
144 eigen_assert(lhs_dims[i] == rhs_dims[i]);
145 m_dimensions[i] = lhs_dims[i];
146 }
147 eigen_assert(lhs_dims[i] >= 0); // Now i == m_axis.
148 eigen_assert(rhs_dims[i] >= 0);
149 m_dimensions[i] = lhs_dims[i] + rhs_dims[i];
150 for (++i; i < NumDims; ++i) {
151 eigen_assert(lhs_dims[i] >= 0);
152 eigen_assert(lhs_dims[i] == rhs_dims[i]);
153 m_dimensions[i] = lhs_dims[i];
154 }
155 }
156
157 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
158 m_leftStrides[0] = 1;
159 m_rightStrides[0] = 1;
160 m_outputStrides[0] = 1;
161
162 for (int j = 1; j < NumDims; ++j) {
163 m_leftStrides[j] = m_leftStrides[j - 1] * lhs_dims[j - 1];
164 m_rightStrides[j] = m_rightStrides[j - 1] * rhs_dims[j - 1];
165 m_outputStrides[j] = m_outputStrides[j - 1] * m_dimensions[j - 1];
166 }
167 } else {
168 m_leftStrides[NumDims - 1] = 1;
169 m_rightStrides[NumDims - 1] = 1;
170 m_outputStrides[NumDims - 1] = 1;
171
172 for (int j = NumDims - 2; j >= 0; --j) {
173 m_leftStrides[j] = m_leftStrides[j + 1] * lhs_dims[j + 1];
174 m_rightStrides[j] = m_rightStrides[j + 1] * rhs_dims[j + 1];
175 m_outputStrides[j] = m_outputStrides[j + 1] * m_dimensions[j + 1];
176 }
177 }
178 }
179
180 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Dimensions& dimensions() const { return m_dimensions; }
181
182 // TODO(phli): Add short-circuit memcpy evaluation if underlying data are linear.
183 EIGEN_STRONG_INLINE bool evalSubExprsIfNeeded(EvaluatorPointerType) {
184 m_leftImpl.evalSubExprsIfNeeded(nullptr);
185 // The executor does not call cleanup() when evaluation throws, so release the operand evaluated first.
186 EIGEN_TRY { m_rightImpl.evalSubExprsIfNeeded(nullptr); }
187 EIGEN_CATCH(...) {
188 m_leftImpl.cleanup();
189 EIGEN_THROW;
190 }
191 return true;
192 }
193
194 EIGEN_STRONG_INLINE void cleanup() {
195 m_leftImpl.cleanup();
196 m_rightImpl.cleanup();
197 }
198
199 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements() const {
200 // Target the L1 cache. A block straddling the concat axis is materialized
201 // into a merged scratch slab that the cwise consumer reads straight back;
202 // sizing the block to L1 keeps that round-trip out of the last-level
203 // cache. It also splits an otherwise cache-resident output into many
204 // blocks, so the off-axis ones qualify for the zero-copy view path in
205 // block(). Matches TensorBroadcasting, which likewise targets L1 for its
206 // materialized block path.
207 const size_t target_size = m_device.firstLevelCacheSize();
208 return internal::TensorBlockResourceRequirements::merge(
209 internal::TensorBlockResourceRequirements::skewed<Scalar>(target_size),
210 internal::TensorBlockResourceRequirements::merge(m_leftImpl.getResourceRequirements(),
211 m_rightImpl.getResourceRequirements()));
212 }
213
214 // True when a block of shape `block_dims` is a contiguous run of an operand
215 // whose shape is `operand_dims` -- i.e. it can be addressed as a plain
216 // pointer offset with no per-row stride walk. Mirrors the direct-access test
217 // in TensorMaterializedBlock::materialize().
218 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool isContiguousOperandSlab(const Dimensions& operand_dims,
219 const Dimensions& block_dims) const {
220 static constexpr bool IsColMajor = Layout == static_cast<int>(ColMajor);
221 int matching_inner_dims = 0;
222 for (int i = 0; i < NumDims; ++i) {
223 const int dim = IsColMajor ? i : NumDims - i - 1;
224 if (operand_dims[dim] != block_dims[dim]) break;
225 ++matching_inner_dims;
226 }
227 // Every dimension above the single partial dimension must be of size 1.
228 for (int i = matching_inner_dims + 1; i < NumDims; ++i) {
229 const int dim = IsColMajor ? i : NumDims - i - 1;
230 if (block_dims[dim] != 1) return false;
231 }
232 return true;
233 }
234
235 // Returns a zero-copy view when the block lies within a single operand and
236 // is contiguous there; otherwise materializes the slab(s) with
237 // TensorBlockIO::Copy, which collapses to a memcpy per contiguous slab.
238 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
239 bool root_of_expr_ast = false) const {
240 static constexpr bool IsColMajor = Layout == static_cast<int>(ColMajor);
241
242 if (desc.size() == 0) {
243 return TensorBlock(internal::TensorBlockKind::kView, nullptr, desc.dimensions());
244 }
245
246 Index remaining = desc.offset();
247 DSizes<Index, NumDims> out_coords;
248 EIGEN_IF_CONSTEXPR (IsColMajor) {
249 for (int i = NumDims - 1; i > 0; --i) {
250 out_coords[i] = remaining / m_outputStrides[i];
251 remaining -= out_coords[i] * m_outputStrides[i];
252 }
253 out_coords[0] = remaining;
254 } else {
255 for (int i = 0; i < NumDims - 1; ++i) {
256 out_coords[i] = remaining / m_outputStrides[i];
257 remaining -= out_coords[i] * m_outputStrides[i];
258 }
259 out_coords[NumDims - 1] = remaining;
260 }
261
262 const Index axis_start = out_coords[m_axis];
263 const Index axis_size = desc.dimension(static_cast<int>(m_axis));
264 const Index axis_end = axis_start + axis_size;
265
266 // Fast path: a block that lies entirely within one operand and forms a
267 // contiguous run of its storage needs no copy at all -- return a view
268 // straight into A / B. This is what keeps a cwise expression that reads
269 // the concatenation (e.g. `(A.concatenate(B, axis) + C)`) streaming: a
270 // cwise consumer drops our destination buffer before calling block() (see
271 // TensorCwiseBinaryOp::block), so without the view the slab would be
272 // bounced through a scratch buffer and read straight back, doubling cache
273 // traffic. Straddling blocks still need the merged buffer materialized
274 // below.
275 if (axis_end <= m_leftAxisSize) {
276 if (isContiguousOperandSlab(m_leftImpl.dimensions(), desc.dimensions())) {
277 Index left_src_offset = 0;
278 for (int i = 0; i < NumDims; ++i) {
279 left_src_offset += out_coords[i] * m_leftStrides[i];
280 }
281 return TensorBlock(internal::TensorBlockKind::kView, m_leftImpl.data() + left_src_offset, desc.dimensions());
282 }
283 } else if (axis_start >= m_leftAxisSize) {
284 if (isContiguousOperandSlab(m_rightImpl.dimensions(), desc.dimensions())) {
285 Index right_src_offset = (axis_start - m_leftAxisSize) * m_rightStrides[m_axis];
286 for (int i = 0; i < NumDims; ++i) {
287 if (i != m_axis) {
288 right_src_offset += out_coords[i] * m_rightStrides[i];
289 }
290 }
291 return TensorBlock(internal::TensorBlockKind::kView, m_rightImpl.data() + right_src_offset, desc.dimensions());
292 }
293 }
294
295 using TensorBlockIO = internal::TensorBlockIO<ScalarNoConst, Index, NumDims, Layout>;
296 using TensorBlockIODst = typename TensorBlockIO::Dst;
297 using TensorBlockIOSrc = typename TensorBlockIO::Src;
298
299 // Strided destination buffers are safe here because we only ever write
300 // dense slabs into them; allowing strided storage lets a root-of-AST
301 // assignment materialize directly into the output tensor.
302 typename TensorBlock::Storage block_storage =
303 TensorBlock::prepareStorage(desc, scratch, /*allow_strided_storage=*/root_of_expr_ast);
304
305 if (axis_start < m_leftAxisSize) {
306 const Index left_rows_in_block = numext::mini(m_leftAxisSize, axis_end) - axis_start;
307 DSizes<Index, NumDims> left_sub_dims = desc.dimensions();
308 left_sub_dims[m_axis] = left_rows_in_block;
309
310 Index left_src_offset = 0;
311 for (int i = 0; i < NumDims; ++i) {
312 left_src_offset += out_coords[i] * m_leftStrides[i];
313 }
314
315 typename TensorBlockIO::Dimensions left_strides(m_leftStrides);
316 TensorBlockIOSrc src(left_strides, m_leftImpl.data(), left_src_offset);
317 TensorBlockIODst dst(left_sub_dims, block_storage.strides(), block_storage.data(),
318 /*dst_offset=*/0);
319 TensorBlockIO::Copy(dst, src);
320 }
321
322 if (axis_end > m_leftAxisSize) {
323 const Index right_rows_in_block = axis_end - numext::maxi(m_leftAxisSize, axis_start);
324 DSizes<Index, NumDims> right_sub_dims = desc.dimensions();
325 right_sub_dims[m_axis] = right_rows_in_block;
326
327 // Right operand has the same per-dim coords as out_coords except along
328 // the concat axis, where it starts at max(0, axis_start - left_size).
329 const Index right_axis_start = numext::maxi(Index(0), axis_start - m_leftAxisSize);
330 Index right_src_offset = right_axis_start * m_rightStrides[m_axis];
331 for (int i = 0; i < NumDims; ++i) {
332 if (i != m_axis) {
333 right_src_offset += out_coords[i] * m_rightStrides[i];
334 }
335 }
336
337 // Offset within the materialized buffer where the right-side slab starts.
338 const Index dst_axis_offset = numext::maxi(Index(0), m_leftAxisSize - axis_start);
339 const Index dst_offset = dst_axis_offset * block_storage.strides()[m_axis];
340
341 typename TensorBlockIO::Dimensions right_strides(m_rightStrides);
342 TensorBlockIOSrc src(right_strides, m_rightImpl.data(), right_src_offset);
343 TensorBlockIODst dst(right_sub_dims, block_storage.strides(), block_storage.data(), dst_offset);
344 TensorBlockIO::Copy(dst, src);
345 }
346
347 return block_storage.AsTensorMaterializedBlock();
348 }
349
350 // The per-dim integer div/mod in this loop are the obvious "slow" candidates,
351 // but the same TensorIntDivisor substitution was measured net-negative on
352 // Intel Raptor Lake when prototyped for TensorBroadcasting (see the comment
353 // on TensorBroadcasting::indexColMajor). Modern x86 hardware div (~20
354 // cycles) amortizes well and the mul+shifts don't win back the added state.
355 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index) const {
356 // Collect dimension-wise indices (subs).
357 array<Index, NumDims> subs;
358 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
359 for (int i = NumDims - 1; i > 0; --i) {
360 subs[i] = index / m_outputStrides[i];
361 index -= subs[i] * m_outputStrides[i];
362 }
363 subs[0] = index;
364 } else {
365 for (int i = 0; i < NumDims - 1; ++i) {
366 subs[i] = index / m_outputStrides[i];
367 index -= subs[i] * m_outputStrides[i];
368 }
369 subs[NumDims - 1] = index;
370 }
371
372 const Dimensions& left_dims = m_leftImpl.dimensions();
373 if (subs[m_axis] < left_dims[m_axis]) {
374 Index left_index;
375 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
376 left_index = subs[0];
377 EIGEN_UNROLL_LOOP
378 for (int i = 1; i < NumDims; ++i) {
379 left_index += (subs[i] % left_dims[i]) * m_leftStrides[i];
380 }
381 } else {
382 left_index = subs[NumDims - 1];
383 EIGEN_UNROLL_LOOP
384 for (int i = NumDims - 2; i >= 0; --i) {
385 left_index += (subs[i] % left_dims[i]) * m_leftStrides[i];
386 }
387 }
388 return m_leftImpl.coeff(left_index);
389 } else {
390 subs[m_axis] -= left_dims[m_axis];
391 const Dimensions& right_dims = m_rightImpl.dimensions();
392 Index right_index;
393 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
394 right_index = subs[0];
395 EIGEN_UNROLL_LOOP
396 for (int i = 1; i < NumDims; ++i) {
397 right_index += (subs[i] % right_dims[i]) * m_rightStrides[i];
398 }
399 } else {
400 right_index = subs[NumDims - 1];
401 EIGEN_UNROLL_LOOP
402 for (int i = NumDims - 2; i >= 0; --i) {
403 right_index += (subs[i] % right_dims[i]) * m_rightStrides[i];
404 }
405 }
406 return m_rightImpl.coeff(right_index);
407 }
408 }
409
410 // When the packet sits entirely on one side of the concat boundary, delegate
411 // to that operand's packet<>() rather than assembling PacketSize coeff()
412 // calls. The packet stays on one side iff only the innermost dim varies
413 // across the packet -- i.e. all other subs match between the first and last
414 // index. When that holds, subs[m_axis] is either constant (m_axis is not
415 // innermost) or monotonic non-decreasing (m_axis is innermost), so checking
416 // just the endpoints decides the side. Otherwise subs[m_axis] can wrap back
417 // through the boundary mid-packet (as when the inner dim has fewer than
418 // PacketSize elements and the packet spills past the concat axis), so fall
419 // back to scalars.
420 template <int LoadMode>
421 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index) const {
422 const int packetSize = PacketType<CoeffReturnType, Device>::size;
423 EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE)
424 eigen_assert(index + packetSize - 1 < dimensions().TotalSize());
425
426 array<Index, NumDims> subs;
427 array<Index, NumDims> subs_end;
428 Index remaining = index;
429 Index remaining_end = index + packetSize - 1;
430 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
431 for (int i = NumDims - 1; i > 0; --i) {
432 subs[i] = remaining / m_outputStrides[i];
433 remaining -= subs[i] * m_outputStrides[i];
434 subs_end[i] = remaining_end / m_outputStrides[i];
435 remaining_end -= subs_end[i] * m_outputStrides[i];
436 }
437 subs[0] = remaining;
438 subs_end[0] = remaining_end;
439 } else {
440 for (int i = 0; i < NumDims - 1; ++i) {
441 subs[i] = remaining / m_outputStrides[i];
442 remaining -= subs[i] * m_outputStrides[i];
443 subs_end[i] = remaining_end / m_outputStrides[i];
444 remaining_end -= subs_end[i] * m_outputStrides[i];
445 }
446 subs[NumDims - 1] = remaining;
447 subs_end[NumDims - 1] = remaining_end;
448 }
449
450 const Dimensions& left_dims = m_leftImpl.dimensions();
451 const Index left_axis_size = left_dims[m_axis];
452
453 constexpr int innermost = (static_cast<int>(Layout) == static_cast<int>(ColMajor)) ? 0 : NumDims - 1;
454 bool packet_in_single_inner_row = true;
455 EIGEN_UNROLL_LOOP
456 for (int i = 0; i < NumDims; ++i) {
457 if (i != innermost && subs[i] != subs_end[i]) {
458 packet_in_single_inner_row = false;
459 }
460 }
461
462 const bool on_left =
463 packet_in_single_inner_row && subs[m_axis] < left_axis_size && subs_end[m_axis] < left_axis_size;
464 const bool on_right =
465 packet_in_single_inner_row && subs[m_axis] >= left_axis_size && subs_end[m_axis] >= left_axis_size;
466
467 if (on_left) {
468 Index left_index;
469 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
470 left_index = subs[0];
471 EIGEN_UNROLL_LOOP
472 for (int i = 1; i < NumDims; ++i) {
473 left_index += subs[i] * m_leftStrides[i];
474 }
475 } else {
476 left_index = subs[NumDims - 1];
477 EIGEN_UNROLL_LOOP
478 for (int i = NumDims - 2; i >= 0; --i) {
479 left_index += subs[i] * m_leftStrides[i];
480 }
481 }
482 return m_leftImpl.template packet<LoadMode>(left_index);
483 }
484 if (on_right) {
485 subs[m_axis] -= left_axis_size;
486 Index right_index;
487 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
488 right_index = subs[0];
489 EIGEN_UNROLL_LOOP
490 for (int i = 1; i < NumDims; ++i) {
491 right_index += subs[i] * m_rightStrides[i];
492 }
493 } else {
494 right_index = subs[NumDims - 1];
495 EIGEN_UNROLL_LOOP
496 for (int i = NumDims - 2; i >= 0; --i) {
497 right_index += subs[i] * m_rightStrides[i];
498 }
499 }
500 return m_rightImpl.template packet<LoadMode>(right_index);
501 }
502
503 // The packet straddles the boundary or spans multiple inner rows: fall
504 // back to assembling scalars.
505 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) CoeffReturnType values[packetSize];
506 EIGEN_UNROLL_LOOP
507 for (int i = 0; i < packetSize; ++i) {
508 values[i] = coeff(index + i);
509 }
510 PacketReturnType rslt = internal::pload<PacketReturnType>(values);
511 return rslt;
512 }
513
514 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(bool vectorized) const {
515 const double compute_cost = NumDims * (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() +
516 TensorOpCost::DivCost<Index>() + TensorOpCost::ModCost<Index>());
517 const double lhs_size = m_leftImpl.dimensions().TotalSize();
518 const double rhs_size = m_rightImpl.dimensions().TotalSize();
519 return (lhs_size / (lhs_size + rhs_size)) * m_leftImpl.costPerCoeff(vectorized) +
520 (rhs_size / (lhs_size + rhs_size)) * m_rightImpl.costPerCoeff(vectorized) + TensorOpCost(0, 0, compute_cost);
521 }
522
523 EIGEN_DEVICE_FUNC EvaluatorPointerType data() const { return nullptr; }
524
525 protected:
526 Dimensions m_dimensions;
527 array<Index, NumDims> m_outputStrides;
528 array<Index, NumDims> m_leftStrides;
529 array<Index, NumDims> m_rightStrides;
530 TensorEvaluator<LeftArgType, Device> m_leftImpl;
531 TensorEvaluator<RightArgType, Device> m_rightImpl;
532 const Device EIGEN_DEVICE_REF m_device;
533 const Axis m_axis;
534 Index m_leftAxisSize;
535};
536
537// Eval as lvalue
538template <typename Axis, typename LeftArgType, typename RightArgType, typename Device>
539struct TensorEvaluator<TensorConcatenationOp<Axis, LeftArgType, RightArgType>, Device>
540 : public TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgType>, Device> {
541 using Base = TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgType>, Device>;
542 using XprType = TensorConcatenationOp<Axis, LeftArgType, RightArgType>;
543 using Dimensions = typename Base::Dimensions;
544 static constexpr int NumDims = Base::NumDims;
545 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
546 enum {
547 IsAligned = false,
548 PacketAccess =
549 TensorEvaluator<LeftArgType, Device>::PacketAccess && TensorEvaluator<RightArgType, Device>::PacketAccess,
550 // writeBlock() splits the block at the concat axis and copies each piece
551 // straight into the operand's buffer, so both must expose raw storage.
552 BlockAccess = TensorEvaluator<LeftArgType, Device>::RawAccess && TensorEvaluator<RightArgType, Device>::RawAccess,
553 // The coeff/packet write path pays a div/mod cascade plus a per-dim mod
554 // for every scalar; the block path is a pair of bulk copies. Mirrors the
555 // rvalue evaluator.
556 PreferBlockAccess = true,
557 RawAccess = false
558 };
559
560 using ScalarNoConst = std::remove_const_t<typename XprType::Scalar>;
561
562 //===- Tensor block evaluation strategy (see TensorBlock.h) -------------===//
563 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, typename XprType::Index>;
564 //===--------------------------------------------------------------------===//
565
566 // The ColMajor-only static_assert lives in coeffRef/writePacket rather than
567 // here so that passthrough evaluators (e.g. TensorSlicingOp's) can
568 // instantiate this type for RowMajor concat operands without ever calling
569 // its lvalue methods. writeBlock() below is layout-generic, so tiled
570 // assignments through a RowMajor concatenation are supported.
571 EIGEN_STRONG_INLINE TensorEvaluator(const XprType& op, const Device& device) : Base(op, device) {}
572
573 using Index = typename XprType::Index;
574 using Scalar = typename XprType::Scalar;
575 using CoeffReturnType = typename XprType::CoeffReturnType;
576 using PacketReturnType = typename PacketType<CoeffReturnType, Device>::type;
577
578 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index) const {
579 EIGEN_STATIC_ASSERT((static_cast<int>(Layout) == static_cast<int>(ColMajor)), YOU_MADE_A_PROGRAMMING_MISTAKE);
580 // Collect dimension-wise indices (subs).
581 array<Index, Base::NumDims> subs;
582 for (int i = Base::NumDims - 1; i > 0; --i) {
583 subs[i] = index / this->m_outputStrides[i];
584 index -= subs[i] * this->m_outputStrides[i];
585 }
586 subs[0] = index;
587
588 const Dimensions& left_dims = this->m_leftImpl.dimensions();
589 if (subs[this->m_axis] < left_dims[this->m_axis]) {
590 Index left_index = subs[0];
591 for (int i = 1; i < Base::NumDims; ++i) {
592 left_index += (subs[i] % left_dims[i]) * this->m_leftStrides[i];
593 }
594 return this->m_leftImpl.coeffRef(left_index);
595 } else {
596 subs[this->m_axis] -= left_dims[this->m_axis];
597 const Dimensions& right_dims = this->m_rightImpl.dimensions();
598 Index right_index = subs[0];
599 for (int i = 1; i < Base::NumDims; ++i) {
600 right_index += (subs[i] % right_dims[i]) * this->m_rightStrides[i];
601 }
602 return this->m_rightImpl.coeffRef(right_index);
603 }
604 }
605
606 template <int StoreMode>
607 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void writePacket(Index index, const PacketReturnType& x) const {
608 EIGEN_STATIC_ASSERT((static_cast<int>(Layout) == static_cast<int>(ColMajor)), YOU_MADE_A_PROGRAMMING_MISTAKE);
609 const int packetSize = PacketType<CoeffReturnType, Device>::size;
610 EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE)
611 eigen_assert(index + packetSize - 1 < this->dimensions().TotalSize());
612
613 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) CoeffReturnType values[packetSize];
614 internal::pstore<CoeffReturnType, PacketReturnType>(values, x);
615 for (int i = 0; i < packetSize; ++i) {
616 coeffRef(index + i) = values[i];
617 }
618 }
619
620 // Mirror of the rvalue block(): split the block at the concat axis and copy
621 // each piece into the corresponding operand's buffer.
622 template <typename TensorBlock>
623 EIGEN_STRONG_INLINE void writeBlock(const TensorBlockDesc& desc, const TensorBlock& block) {
624 if (desc.size() == 0) return;
625 eigen_assert(this->m_leftImpl.data() != nullptr && this->m_rightImpl.data() != nullptr);
626
627 const DSizes<Index, NumDims> block_strides = internal::strides<Layout>(desc.dimensions());
628
629 // Materialize the block into a temporary buffer if it is lazy.
630 const ScalarNoConst* block_buffer = block.data();
631 void* mem = nullptr;
632 if (block_buffer == nullptr) {
633 mem = this->m_device.allocate(desc.size() * sizeof(Scalar));
634 ScalarNoConst* buf = static_cast<ScalarNoConst*>(mem);
635
636 using TensorBlockAssignment =
637 internal::TensorBlockAssignment<ScalarNoConst, NumDims, typename TensorBlock::XprType, Index>;
638 TensorBlockAssignment::Run(TensorBlockAssignment::target(desc.dimensions(), block_strides, buf), block.expr());
639
640 block_buffer = buf;
641 }
642
643 // Decompose the block's offset into output coordinates.
644 Index remaining = desc.offset();
645 DSizes<Index, NumDims> out_coords;
646 EIGEN_IF_CONSTEXPR (static_cast<int>(Layout) == static_cast<int>(ColMajor)) {
647 for (int i = NumDims - 1; i > 0; --i) {
648 out_coords[i] = remaining / this->m_outputStrides[i];
649 remaining -= out_coords[i] * this->m_outputStrides[i];
650 }
651 out_coords[0] = remaining;
652 } else {
653 for (int i = 0; i < NumDims - 1; ++i) {
654 out_coords[i] = remaining / this->m_outputStrides[i];
655 remaining -= out_coords[i] * this->m_outputStrides[i];
656 }
657 out_coords[NumDims - 1] = remaining;
658 }
659
660 const Index axis_start = out_coords[this->m_axis];
661 const Index axis_size = desc.dimension(static_cast<int>(this->m_axis));
662 const Index axis_end = axis_start + axis_size;
663 const Index left_axis_size = this->m_leftAxisSize;
664
665 using TensorBlockIO = internal::TensorBlockIO<ScalarNoConst, Index, NumDims, Layout>;
666 using TensorBlockIODst = typename TensorBlockIO::Dst;
667 using TensorBlockIOSrc = typename TensorBlockIO::Src;
668
669 if (axis_start < left_axis_size) {
670 DSizes<Index, NumDims> left_sub_dims = desc.dimensions();
671 left_sub_dims[this->m_axis] = numext::mini(left_axis_size, axis_end) - axis_start;
672
673 Index left_dst_offset = 0;
674 for (int i = 0; i < NumDims; ++i) {
675 left_dst_offset += out_coords[i] * this->m_leftStrides[i];
676 }
677
678 TensorBlockIOSrc src(block_strides, block_buffer, /*src_offset=*/0);
679 TensorBlockIODst dst(left_sub_dims, typename TensorBlockIO::Dimensions(this->m_leftStrides),
680 this->m_leftImpl.data(), left_dst_offset);
681 TensorBlockIO::Copy(dst, src);
682 }
683
684 if (axis_end > left_axis_size) {
685 DSizes<Index, NumDims> right_sub_dims = desc.dimensions();
686 right_sub_dims[this->m_axis] = axis_end - numext::maxi(left_axis_size, axis_start);
687
688 const Index right_axis_start = numext::maxi(Index(0), axis_start - left_axis_size);
689 Index right_dst_offset = right_axis_start * this->m_rightStrides[this->m_axis];
690 for (int i = 0; i < NumDims; ++i) {
691 if (i != this->m_axis) {
692 right_dst_offset += out_coords[i] * this->m_rightStrides[i];
693 }
694 }
695
696 // Offset within the block buffer where the right-side piece starts.
697 const Index src_offset = numext::maxi(Index(0), left_axis_size - axis_start) * block_strides[this->m_axis];
698
699 TensorBlockIOSrc src(block_strides, block_buffer, src_offset);
700 TensorBlockIODst dst(right_sub_dims, typename TensorBlockIO::Dimensions(this->m_rightStrides),
701 this->m_rightImpl.data(), right_dst_offset);
702 TensorBlockIO::Copy(dst, src);
703 }
704
705 // Deallocate temporary buffer used for the block materialization.
706 if (mem != nullptr) this->m_device.deallocate(mem);
707 }
708};
709
710} // end namespace Eigen
711
712#endif // EIGEN_TENSOR_TENSOR_CONCATENATION_H
The tensor base class.
Definition TensorForwardDeclarations.h:69
Tensor concatenation class.
Definition TensorConcatenation.h:49
WriteAccessors
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47