11#ifndef EIGEN_TENSOR_TENSOR_CONCATENATION_H
12#define EIGEN_TENSOR_TENSOR_CONCATENATION_H
15#include "./InternalHeaderCheck.h"
20template <
typename Axis,
typename LhsXprType,
typename RhsXprType>
21struct traits<TensorConcatenationOp<Axis, LhsXprType, RhsXprType>> {
23 using Scalar =
typename promote_storage_type<typename LhsXprType::Scalar, typename RhsXprType::Scalar>::ret;
24 using StorageKind =
typename promote_storage_type<typename traits<LhsXprType>::StorageKind,
25 typename traits<RhsXprType>::StorageKind>::ret;
27 typename promote_index_type<typename traits<LhsXprType>::Index,
typename traits<RhsXprType>::Index>::type;
28 static constexpr int NumDimensions = traits<LhsXprType>::NumDimensions;
29 static constexpr int Layout = traits<LhsXprType>::Layout;
32 std::conditional_t<Pointer_type_promotion<typename LhsXprType::Scalar, Scalar>::val,
33 typename traits<LhsXprType>::PointerType,
typename traits<RhsXprType>::PointerType>;
36template <
typename Axis,
typename LhsXprType,
typename RhsXprType>
37struct eval<TensorConcatenationOp<Axis, LhsXprType, RhsXprType>, Eigen::Dense> {
38 using type =
const TensorConcatenationOp<Axis, LhsXprType, RhsXprType>&;
48template <
typename Axis,
typename LhsXprType,
typename RhsXprType>
49class TensorConcatenationOp :
public TensorBase<TensorConcatenationOp<Axis, LhsXprType, RhsXprType>, WriteAccessors> {
52 using Scalar =
typename internal::traits<TensorConcatenationOp>::Scalar;
53 using StorageKind =
typename internal::traits<TensorConcatenationOp>::StorageKind;
54 using Index =
typename internal::traits<TensorConcatenationOp>::Index;
55 using Nested =
typename internal::ref_selector<TensorConcatenationOp>::type;
56 using CoeffReturnType =
typename internal::promote_storage_type<
typename LhsXprType::CoeffReturnType,
57 typename RhsXprType::CoeffReturnType>::ret;
60 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorConcatenationOp(
const LhsXprType& lhs,
const RhsXprType& rhs, Axis axis)
61 : m_lhs_xpr(lhs), m_rhs_xpr(rhs), m_axis(axis) {}
63 EIGEN_DEVICE_FUNC
const internal::remove_all_t<typename LhsXprType::Nested>& lhsExpression()
const {
67 EIGEN_DEVICE_FUNC
const internal::remove_all_t<typename RhsXprType::Nested>& rhsExpression()
const {
71 EIGEN_DEVICE_FUNC
const Axis& axis()
const {
return m_axis; }
73 EIGEN_INHERIT_ASSIGNMENT_OPERATORS(TensorConcatenationOp)
75 typename LhsXprType::Nested m_lhs_xpr;
76 typename RhsXprType::Nested m_rhs_xpr;
81template <
typename Axis,
typename LeftArgType,
typename RightArgType,
typename Device>
84 using Index =
typename XprType::Index;
85 static constexpr int NumDims = internal::array_size<typename TensorEvaluator<LeftArgType, Device>::Dimensions>::value;
86 static constexpr int RightNumDims =
87 internal::array_size<typename TensorEvaluator<RightArgType, Device>::Dimensions>::value;
88 using Dimensions = DSizes<Index, NumDims>;
89 using Scalar =
typename XprType::Scalar;
90 using CoeffReturnType =
typename XprType::CoeffReturnType;
91 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
92 using Storage = StorageMemory<CoeffReturnType, Device>;
93 using EvaluatorPointerType =
typename Storage::Type;
94 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
98 TensorEvaluator<LeftArgType, Device>::PacketAccess && TensorEvaluator<RightArgType, Device>::PacketAccess,
108 PreferBlockAccess =
true,
112 using ScalarNoConst = std::remove_const_t<Scalar>;
115 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, Index>;
116 using TensorBlockScratch = internal::TensorBlockScratchAllocator<Device>;
118 using TensorBlock =
typename internal::TensorMaterializedBlock<ScalarNoConst, NumDims, Layout, Index>;
121 EIGEN_STRONG_INLINE TensorEvaluator(
const XprType& op,
const Device& device)
122 : m_leftImpl(op.lhsExpression(), device),
123 m_rightImpl(op.rhsExpression(), device),
126 EIGEN_STATIC_ASSERT((
static_cast<int>(TensorEvaluator<LeftArgType, Device>::Layout) ==
127 static_cast<int>(TensorEvaluator<RightArgType, Device>::Layout) ||
129 YOU_MADE_A_PROGRAMMING_MISTAKE);
133 EIGEN_STATIC_ASSERT((NumDims == RightNumDims), YOU_MADE_A_PROGRAMMING_MISTAKE);
134 EIGEN_STATIC_ASSERT((NumDims > 0), YOU_MADE_A_PROGRAMMING_MISTAKE);
136 eigen_assert(0 <= m_axis && m_axis < NumDims);
137 m_leftAxisSize = m_leftImpl.dimensions()[m_axis];
138 const Dimensions& lhs_dims = m_leftImpl.dimensions();
139 const Dimensions& rhs_dims = m_rightImpl.dimensions();
142 for (; i < m_axis; ++i) {
143 eigen_assert(lhs_dims[i] >= 0);
144 eigen_assert(lhs_dims[i] == rhs_dims[i]);
145 m_dimensions[i] = lhs_dims[i];
147 eigen_assert(lhs_dims[i] >= 0);
148 eigen_assert(rhs_dims[i] >= 0);
149 m_dimensions[i] = lhs_dims[i] + rhs_dims[i];
150 for (++i; i < NumDims; ++i) {
151 eigen_assert(lhs_dims[i] >= 0);
152 eigen_assert(lhs_dims[i] == rhs_dims[i]);
153 m_dimensions[i] = lhs_dims[i];
157 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
158 m_leftStrides[0] = 1;
159 m_rightStrides[0] = 1;
160 m_outputStrides[0] = 1;
162 for (
int j = 1; j < NumDims; ++j) {
163 m_leftStrides[j] = m_leftStrides[j - 1] * lhs_dims[j - 1];
164 m_rightStrides[j] = m_rightStrides[j - 1] * rhs_dims[j - 1];
165 m_outputStrides[j] = m_outputStrides[j - 1] * m_dimensions[j - 1];
168 m_leftStrides[NumDims - 1] = 1;
169 m_rightStrides[NumDims - 1] = 1;
170 m_outputStrides[NumDims - 1] = 1;
172 for (
int j = NumDims - 2; j >= 0; --j) {
173 m_leftStrides[j] = m_leftStrides[j + 1] * lhs_dims[j + 1];
174 m_rightStrides[j] = m_rightStrides[j + 1] * rhs_dims[j + 1];
175 m_outputStrides[j] = m_outputStrides[j + 1] * m_dimensions[j + 1];
180 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Dimensions& dimensions()
const {
return m_dimensions; }
183 EIGEN_STRONG_INLINE
bool evalSubExprsIfNeeded(EvaluatorPointerType) {
184 m_leftImpl.evalSubExprsIfNeeded(
nullptr);
186 EIGEN_TRY { m_rightImpl.evalSubExprsIfNeeded(
nullptr); }
188 m_leftImpl.cleanup();
194 EIGEN_STRONG_INLINE
void cleanup() {
195 m_leftImpl.cleanup();
196 m_rightImpl.cleanup();
199 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE internal::TensorBlockResourceRequirements getResourceRequirements()
const {
207 const size_t target_size = m_device.firstLevelCacheSize();
208 return internal::TensorBlockResourceRequirements::merge(
209 internal::TensorBlockResourceRequirements::skewed<Scalar>(target_size),
210 internal::TensorBlockResourceRequirements::merge(m_leftImpl.getResourceRequirements(),
211 m_rightImpl.getResourceRequirements()));
218 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
bool isContiguousOperandSlab(
const Dimensions& operand_dims,
219 const Dimensions& block_dims)
const {
220 static constexpr bool IsColMajor = Layout ==
static_cast<int>(
ColMajor);
221 int matching_inner_dims = 0;
222 for (
int i = 0; i < NumDims; ++i) {
223 const int dim = IsColMajor ? i : NumDims - i - 1;
224 if (operand_dims[dim] != block_dims[dim])
break;
225 ++matching_inner_dims;
228 for (
int i = matching_inner_dims + 1; i < NumDims; ++i) {
229 const int dim = IsColMajor ? i : NumDims - i - 1;
230 if (block_dims[dim] != 1)
return false;
238 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorBlock block(TensorBlockDesc& desc, TensorBlockScratch& scratch,
239 bool root_of_expr_ast =
false)
const {
240 static constexpr bool IsColMajor = Layout ==
static_cast<int>(
ColMajor);
242 if (desc.size() == 0) {
243 return TensorBlock(internal::TensorBlockKind::kView,
nullptr, desc.dimensions());
246 Index remaining = desc.offset();
247 DSizes<Index, NumDims> out_coords;
248 EIGEN_IF_CONSTEXPR (IsColMajor) {
249 for (
int i = NumDims - 1; i > 0; --i) {
250 out_coords[i] = remaining / m_outputStrides[i];
251 remaining -= out_coords[i] * m_outputStrides[i];
253 out_coords[0] = remaining;
255 for (
int i = 0; i < NumDims - 1; ++i) {
256 out_coords[i] = remaining / m_outputStrides[i];
257 remaining -= out_coords[i] * m_outputStrides[i];
259 out_coords[NumDims - 1] = remaining;
262 const Index axis_start = out_coords[m_axis];
263 const Index axis_size = desc.dimension(
static_cast<int>(m_axis));
264 const Index axis_end = axis_start + axis_size;
275 if (axis_end <= m_leftAxisSize) {
276 if (isContiguousOperandSlab(m_leftImpl.dimensions(), desc.dimensions())) {
277 Index left_src_offset = 0;
278 for (
int i = 0; i < NumDims; ++i) {
279 left_src_offset += out_coords[i] * m_leftStrides[i];
281 return TensorBlock(internal::TensorBlockKind::kView, m_leftImpl.data() + left_src_offset, desc.dimensions());
283 }
else if (axis_start >= m_leftAxisSize) {
284 if (isContiguousOperandSlab(m_rightImpl.dimensions(), desc.dimensions())) {
285 Index right_src_offset = (axis_start - m_leftAxisSize) * m_rightStrides[m_axis];
286 for (
int i = 0; i < NumDims; ++i) {
288 right_src_offset += out_coords[i] * m_rightStrides[i];
291 return TensorBlock(internal::TensorBlockKind::kView, m_rightImpl.data() + right_src_offset, desc.dimensions());
295 using TensorBlockIO = internal::TensorBlockIO<ScalarNoConst, Index, NumDims, Layout>;
296 using TensorBlockIODst =
typename TensorBlockIO::Dst;
297 using TensorBlockIOSrc =
typename TensorBlockIO::Src;
302 typename TensorBlock::Storage block_storage =
303 TensorBlock::prepareStorage(desc, scratch, root_of_expr_ast);
305 if (axis_start < m_leftAxisSize) {
306 const Index left_rows_in_block = numext::mini(m_leftAxisSize, axis_end) - axis_start;
307 DSizes<Index, NumDims> left_sub_dims = desc.dimensions();
308 left_sub_dims[m_axis] = left_rows_in_block;
310 Index left_src_offset = 0;
311 for (
int i = 0; i < NumDims; ++i) {
312 left_src_offset += out_coords[i] * m_leftStrides[i];
315 typename TensorBlockIO::Dimensions left_strides(m_leftStrides);
316 TensorBlockIOSrc src(left_strides, m_leftImpl.data(), left_src_offset);
317 TensorBlockIODst dst(left_sub_dims, block_storage.strides(), block_storage.data(),
319 TensorBlockIO::Copy(dst, src);
322 if (axis_end > m_leftAxisSize) {
323 const Index right_rows_in_block = axis_end - numext::maxi(m_leftAxisSize, axis_start);
324 DSizes<Index, NumDims> right_sub_dims = desc.dimensions();
325 right_sub_dims[m_axis] = right_rows_in_block;
329 const Index right_axis_start = numext::maxi(Index(0), axis_start - m_leftAxisSize);
330 Index right_src_offset = right_axis_start * m_rightStrides[m_axis];
331 for (
int i = 0; i < NumDims; ++i) {
333 right_src_offset += out_coords[i] * m_rightStrides[i];
338 const Index dst_axis_offset = numext::maxi(Index(0), m_leftAxisSize - axis_start);
339 const Index dst_offset = dst_axis_offset * block_storage.strides()[m_axis];
341 typename TensorBlockIO::Dimensions right_strides(m_rightStrides);
342 TensorBlockIOSrc src(right_strides, m_rightImpl.data(), right_src_offset);
343 TensorBlockIODst dst(right_sub_dims, block_storage.strides(), block_storage.data(), dst_offset);
344 TensorBlockIO::Copy(dst, src);
347 return block_storage.AsTensorMaterializedBlock();
355 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType coeff(Index index)
const {
357 array<Index, NumDims> subs;
358 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
359 for (
int i = NumDims - 1; i > 0; --i) {
360 subs[i] = index / m_outputStrides[i];
361 index -= subs[i] * m_outputStrides[i];
365 for (
int i = 0; i < NumDims - 1; ++i) {
366 subs[i] = index / m_outputStrides[i];
367 index -= subs[i] * m_outputStrides[i];
369 subs[NumDims - 1] = index;
372 const Dimensions& left_dims = m_leftImpl.dimensions();
373 if (subs[m_axis] < left_dims[m_axis]) {
375 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
376 left_index = subs[0];
378 for (
int i = 1; i < NumDims; ++i) {
379 left_index += (subs[i] % left_dims[i]) * m_leftStrides[i];
382 left_index = subs[NumDims - 1];
384 for (
int i = NumDims - 2; i >= 0; --i) {
385 left_index += (subs[i] % left_dims[i]) * m_leftStrides[i];
388 return m_leftImpl.coeff(left_index);
390 subs[m_axis] -= left_dims[m_axis];
391 const Dimensions& right_dims = m_rightImpl.dimensions();
393 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
394 right_index = subs[0];
396 for (
int i = 1; i < NumDims; ++i) {
397 right_index += (subs[i] % right_dims[i]) * m_rightStrides[i];
400 right_index = subs[NumDims - 1];
402 for (
int i = NumDims - 2; i >= 0; --i) {
403 right_index += (subs[i] % right_dims[i]) * m_rightStrides[i];
406 return m_rightImpl.coeff(right_index);
420 template <
int LoadMode>
421 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketReturnType packet(Index index)
const {
422 const int packetSize = PacketType<CoeffReturnType, Device>::size;
423 EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE)
424 eigen_assert(index + packetSize - 1 < dimensions().TotalSize());
426 array<Index, NumDims> subs;
427 array<Index, NumDims> subs_end;
428 Index remaining = index;
429 Index remaining_end = index + packetSize - 1;
430 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
431 for (
int i = NumDims - 1; i > 0; --i) {
432 subs[i] = remaining / m_outputStrides[i];
433 remaining -= subs[i] * m_outputStrides[i];
434 subs_end[i] = remaining_end / m_outputStrides[i];
435 remaining_end -= subs_end[i] * m_outputStrides[i];
438 subs_end[0] = remaining_end;
440 for (
int i = 0; i < NumDims - 1; ++i) {
441 subs[i] = remaining / m_outputStrides[i];
442 remaining -= subs[i] * m_outputStrides[i];
443 subs_end[i] = remaining_end / m_outputStrides[i];
444 remaining_end -= subs_end[i] * m_outputStrides[i];
446 subs[NumDims - 1] = remaining;
447 subs_end[NumDims - 1] = remaining_end;
450 const Dimensions& left_dims = m_leftImpl.dimensions();
451 const Index left_axis_size = left_dims[m_axis];
453 constexpr int innermost = (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) ? 0 : NumDims - 1;
454 bool packet_in_single_inner_row =
true;
456 for (
int i = 0; i < NumDims; ++i) {
457 if (i != innermost && subs[i] != subs_end[i]) {
458 packet_in_single_inner_row =
false;
463 packet_in_single_inner_row && subs[m_axis] < left_axis_size && subs_end[m_axis] < left_axis_size;
464 const bool on_right =
465 packet_in_single_inner_row && subs[m_axis] >= left_axis_size && subs_end[m_axis] >= left_axis_size;
469 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
470 left_index = subs[0];
472 for (
int i = 1; i < NumDims; ++i) {
473 left_index += subs[i] * m_leftStrides[i];
476 left_index = subs[NumDims - 1];
478 for (
int i = NumDims - 2; i >= 0; --i) {
479 left_index += subs[i] * m_leftStrides[i];
482 return m_leftImpl.template packet<LoadMode>(left_index);
485 subs[m_axis] -= left_axis_size;
487 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
488 right_index = subs[0];
490 for (
int i = 1; i < NumDims; ++i) {
491 right_index += subs[i] * m_rightStrides[i];
494 right_index = subs[NumDims - 1];
496 for (
int i = NumDims - 2; i >= 0; --i) {
497 right_index += subs[i] * m_rightStrides[i];
500 return m_rightImpl.template packet<LoadMode>(right_index);
505 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) CoeffReturnType values[packetSize];
507 for (
int i = 0; i < packetSize; ++i) {
508 values[i] = coeff(index + i);
510 PacketReturnType rslt = internal::pload<PacketReturnType>(values);
514 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorOpCost costPerCoeff(
bool vectorized)
const {
515 const double compute_cost = NumDims * (2 * TensorOpCost::AddCost<Index>() + 2 * TensorOpCost::MulCost<Index>() +
516 TensorOpCost::DivCost<Index>() + TensorOpCost::ModCost<Index>());
517 const double lhs_size = m_leftImpl.dimensions().TotalSize();
518 const double rhs_size = m_rightImpl.dimensions().TotalSize();
519 return (lhs_size / (lhs_size + rhs_size)) * m_leftImpl.costPerCoeff(vectorized) +
520 (rhs_size / (lhs_size + rhs_size)) * m_rightImpl.costPerCoeff(vectorized) + TensorOpCost(0, 0, compute_cost);
523 EIGEN_DEVICE_FUNC EvaluatorPointerType data()
const {
return nullptr; }
526 Dimensions m_dimensions;
527 array<Index, NumDims> m_outputStrides;
528 array<Index, NumDims> m_leftStrides;
529 array<Index, NumDims> m_rightStrides;
530 TensorEvaluator<LeftArgType, Device> m_leftImpl;
531 TensorEvaluator<RightArgType, Device> m_rightImpl;
532 const Device EIGEN_DEVICE_REF m_device;
534 Index m_leftAxisSize;
538template <
typename Axis,
typename LeftArgType,
typename RightArgType,
typename Device>
540 :
public TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgType>, Device> {
541 using Base = TensorEvaluator<const TensorConcatenationOp<Axis, LeftArgType, RightArgType>, Device>;
542 using XprType = TensorConcatenationOp<Axis, LeftArgType, RightArgType>;
543 using Dimensions =
typename Base::Dimensions;
544 static constexpr int NumDims = Base::NumDims;
545 static constexpr int Layout = TensorEvaluator<LeftArgType, Device>::Layout;
549 TensorEvaluator<LeftArgType, Device>::PacketAccess && TensorEvaluator<RightArgType, Device>::PacketAccess,
552 BlockAccess = TensorEvaluator<LeftArgType, Device>::RawAccess && TensorEvaluator<RightArgType, Device>::RawAccess,
556 PreferBlockAccess =
true,
560 using ScalarNoConst = std::remove_const_t<typename XprType::Scalar>;
563 using TensorBlockDesc = internal::TensorBlockDescriptor<NumDims, typename XprType::Index>;
571 EIGEN_STRONG_INLINE TensorEvaluator(
const XprType& op,
const Device& device) : Base(op, device) {}
573 using Index =
typename XprType::Index;
574 using Scalar =
typename XprType::Scalar;
575 using CoeffReturnType =
typename XprType::CoeffReturnType;
576 using PacketReturnType =
typename PacketType<CoeffReturnType, Device>::type;
578 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE CoeffReturnType& coeffRef(Index index)
const {
579 EIGEN_STATIC_ASSERT((
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)), YOU_MADE_A_PROGRAMMING_MISTAKE);
581 array<Index, Base::NumDims> subs;
582 for (
int i = Base::NumDims - 1; i > 0; --i) {
583 subs[i] = index / this->m_outputStrides[i];
584 index -= subs[i] * this->m_outputStrides[i];
588 const Dimensions& left_dims = this->m_leftImpl.dimensions();
589 if (subs[this->m_axis] < left_dims[this->m_axis]) {
590 Index left_index = subs[0];
591 for (
int i = 1; i < Base::NumDims; ++i) {
592 left_index += (subs[i] % left_dims[i]) * this->m_leftStrides[i];
594 return this->m_leftImpl.coeffRef(left_index);
596 subs[this->m_axis] -= left_dims[this->m_axis];
597 const Dimensions& right_dims = this->m_rightImpl.dimensions();
598 Index right_index = subs[0];
599 for (
int i = 1; i < Base::NumDims; ++i) {
600 right_index += (subs[i] % right_dims[i]) * this->m_rightStrides[i];
602 return this->m_rightImpl.coeffRef(right_index);
606 template <
int StoreMode>
607 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void writePacket(Index index,
const PacketReturnType& x)
const {
608 EIGEN_STATIC_ASSERT((
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)), YOU_MADE_A_PROGRAMMING_MISTAKE);
609 const int packetSize = PacketType<CoeffReturnType, Device>::size;
610 EIGEN_STATIC_ASSERT((packetSize > 1), YOU_MADE_A_PROGRAMMING_MISTAKE)
611 eigen_assert(index + packetSize - 1 < this->dimensions().TotalSize());
613 EIGEN_ALIGN_TO_BOUNDARY(internal::unpacket_traits<PacketReturnType>::alignment) CoeffReturnType values[packetSize];
614 internal::pstore<CoeffReturnType, PacketReturnType>(values, x);
615 for (
int i = 0; i < packetSize; ++i) {
616 coeffRef(index + i) = values[i];
622 template <
typename TensorBlock>
623 EIGEN_STRONG_INLINE
void writeBlock(
const TensorBlockDesc& desc,
const TensorBlock& block) {
624 if (desc.size() == 0)
return;
625 eigen_assert(this->m_leftImpl.data() !=
nullptr && this->m_rightImpl.data() !=
nullptr);
627 const DSizes<Index, NumDims> block_strides = internal::strides<Layout>(desc.dimensions());
630 const ScalarNoConst* block_buffer = block.data();
632 if (block_buffer ==
nullptr) {
633 mem = this->m_device.allocate(desc.size() *
sizeof(Scalar));
634 ScalarNoConst* buf =
static_cast<ScalarNoConst*
>(mem);
636 using TensorBlockAssignment =
637 internal::TensorBlockAssignment<ScalarNoConst, NumDims, typename TensorBlock::XprType, Index>;
638 TensorBlockAssignment::Run(TensorBlockAssignment::target(desc.dimensions(), block_strides, buf), block.expr());
644 Index remaining = desc.offset();
645 DSizes<Index, NumDims> out_coords;
646 EIGEN_IF_CONSTEXPR (
static_cast<int>(Layout) ==
static_cast<int>(
ColMajor)) {
647 for (
int i = NumDims - 1; i > 0; --i) {
648 out_coords[i] = remaining / this->m_outputStrides[i];
649 remaining -= out_coords[i] * this->m_outputStrides[i];
651 out_coords[0] = remaining;
653 for (
int i = 0; i < NumDims - 1; ++i) {
654 out_coords[i] = remaining / this->m_outputStrides[i];
655 remaining -= out_coords[i] * this->m_outputStrides[i];
657 out_coords[NumDims - 1] = remaining;
660 const Index axis_start = out_coords[this->m_axis];
661 const Index axis_size = desc.dimension(
static_cast<int>(this->m_axis));
662 const Index axis_end = axis_start + axis_size;
663 const Index left_axis_size = this->m_leftAxisSize;
665 using TensorBlockIO = internal::TensorBlockIO<ScalarNoConst, Index, NumDims, Layout>;
666 using TensorBlockIODst =
typename TensorBlockIO::Dst;
667 using TensorBlockIOSrc =
typename TensorBlockIO::Src;
669 if (axis_start < left_axis_size) {
670 DSizes<Index, NumDims> left_sub_dims = desc.dimensions();
671 left_sub_dims[this->m_axis] = numext::mini(left_axis_size, axis_end) - axis_start;
673 Index left_dst_offset = 0;
674 for (
int i = 0; i < NumDims; ++i) {
675 left_dst_offset += out_coords[i] * this->m_leftStrides[i];
678 TensorBlockIOSrc src(block_strides, block_buffer, 0);
679 TensorBlockIODst dst(left_sub_dims,
typename TensorBlockIO::Dimensions(this->m_leftStrides),
680 this->m_leftImpl.data(), left_dst_offset);
681 TensorBlockIO::Copy(dst, src);
684 if (axis_end > left_axis_size) {
685 DSizes<Index, NumDims> right_sub_dims = desc.dimensions();
686 right_sub_dims[this->m_axis] = axis_end - numext::maxi(left_axis_size, axis_start);
688 const Index right_axis_start = numext::maxi(Index(0), axis_start - left_axis_size);
689 Index right_dst_offset = right_axis_start * this->m_rightStrides[this->m_axis];
690 for (
int i = 0; i < NumDims; ++i) {
691 if (i != this->m_axis) {
692 right_dst_offset += out_coords[i] * this->m_rightStrides[i];
697 const Index src_offset = numext::maxi(Index(0), left_axis_size - axis_start) * block_strides[this->m_axis];
699 TensorBlockIOSrc src(block_strides, block_buffer, src_offset);
700 TensorBlockIODst dst(right_sub_dims,
typename TensorBlockIO::Dimensions(this->m_rightStrides),
701 this->m_rightImpl.data(), right_dst_offset);
702 TensorBlockIO::Copy(dst, src);
706 if (mem !=
nullptr) this->m_device.deallocate(mem);
The tensor base class.
Definition TensorForwardDeclarations.h:69
Tensor concatenation class.
Definition TensorConcatenation.h:49
Namespace containing all symbols from the Eigen library.
The tensor evaluator class.
Definition TensorEvaluator.h:47